1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/malloc.h> 37 #include <sys/queue.h> 38 #include <sys/rman.h> 39 #include <sys/smp.h> 40 #include <sys/syslog.h> 41 #include <sys/taskqueue.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 46 #include <dev/pci/pcireg.h> 47 #include <dev/pci/pcivar.h> 48 49 #include "common/efx.h" 50 51 #include "sfxge.h" 52 53 static int 54 sfxge_intr_line_filter(void *arg) 55 { 56 struct sfxge_evq *evq; 57 struct sfxge_softc *sc; 58 efx_nic_t *enp; 59 struct sfxge_intr *intr; 60 boolean_t fatal; 61 uint32_t qmask; 62 63 evq = (struct sfxge_evq *)arg; 64 sc = evq->sc; 65 enp = sc->enp; 66 intr = &sc->intr; 67 68 KASSERT(intr != NULL, ("intr == NULL")); 69 KASSERT(intr->type == EFX_INTR_LINE, 70 ("intr->type != EFX_INTR_LINE")); 71 72 if (intr->state != SFXGE_INTR_STARTED) 73 return (FILTER_STRAY); 74 75 (void)efx_intr_status_line(enp, &fatal, &qmask); 76 77 if (fatal) { 78 (void) efx_intr_disable(enp); 79 (void) efx_intr_fatal(enp); 80 return (FILTER_HANDLED); 81 } 82 83 if (qmask != 0) { 84 intr->zero_count = 0; 85 return (FILTER_SCHEDULE_THREAD); 86 } 87 88 /* SF bug 15783: If the function is not asserting its IRQ and 89 * we read the queue mask on the cycle before a flag is added 90 * to the mask, this inhibits the function from asserting the 91 * IRQ even though we don't see the flag set. To work around 92 * this, we must re-prime all event queues and report the IRQ 93 * as handled when we see a mask of zero. To allow for shared 94 * IRQs, we don't repeat this if we see a mask of zero twice 95 * or more in a row. 96 */ 97 if (intr->zero_count++ == 0) { 98 if (evq->init_state == SFXGE_EVQ_STARTED) { 99 if (efx_ev_qpending(evq->common, evq->read_ptr)) 100 return (FILTER_SCHEDULE_THREAD); 101 efx_ev_qprime(evq->common, evq->read_ptr); 102 return (FILTER_HANDLED); 103 } 104 } 105 106 return (FILTER_STRAY); 107 } 108 109 static void 110 sfxge_intr_line(void *arg) 111 { 112 struct sfxge_evq *evq = arg; 113 114 (void)sfxge_ev_qpoll(evq); 115 } 116 117 static void 118 sfxge_intr_message(void *arg) 119 { 120 struct sfxge_evq *evq; 121 struct sfxge_softc *sc; 122 efx_nic_t *enp; 123 struct sfxge_intr *intr; 124 unsigned int index; 125 boolean_t fatal; 126 127 evq = (struct sfxge_evq *)arg; 128 sc = evq->sc; 129 enp = sc->enp; 130 intr = &sc->intr; 131 index = evq->index; 132 133 KASSERT(intr != NULL, ("intr == NULL")); 134 KASSERT(intr->type == EFX_INTR_MESSAGE, 135 ("intr->type != EFX_INTR_MESSAGE")); 136 137 if (__predict_false(intr->state != SFXGE_INTR_STARTED)) 138 return; 139 140 (void)efx_intr_status_message(enp, index, &fatal); 141 142 if (fatal) { 143 (void)efx_intr_disable(enp); 144 (void)efx_intr_fatal(enp); 145 return; 146 } 147 148 (void)sfxge_ev_qpoll(evq); 149 } 150 151 static int 152 sfxge_intr_bus_enable(struct sfxge_softc *sc) 153 { 154 struct sfxge_intr *intr; 155 struct sfxge_intr_hdl *table; 156 driver_filter_t *filter; 157 driver_intr_t *handler; 158 int index; 159 int err; 160 161 intr = &sc->intr; 162 table = intr->table; 163 164 switch (intr->type) { 165 case EFX_INTR_MESSAGE: 166 filter = NULL; /* not shared */ 167 handler = sfxge_intr_message; 168 break; 169 170 case EFX_INTR_LINE: 171 filter = sfxge_intr_line_filter; 172 handler = sfxge_intr_line; 173 break; 174 175 default: 176 KASSERT(0, ("Invalid interrupt type")); 177 return (EINVAL); 178 } 179 180 /* Try to add the handlers */ 181 for (index = 0; index < intr->n_alloc; index++) { 182 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 183 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 184 sc->evq[index], &table[index].eih_tag)) != 0) { 185 goto fail; 186 } 187 #ifdef SFXGE_HAVE_DESCRIBE_INTR 188 if (intr->n_alloc > 1) 189 bus_describe_intr(sc->dev, table[index].eih_res, 190 table[index].eih_tag, "%d", index); 191 #endif 192 bus_bind_intr(sc->dev, table[index].eih_res, index); 193 194 } 195 196 return (0); 197 198 fail: 199 /* Remove remaining handlers */ 200 while (--index >= 0) 201 bus_teardown_intr(sc->dev, table[index].eih_res, 202 table[index].eih_tag); 203 204 return (err); 205 } 206 207 static void 208 sfxge_intr_bus_disable(struct sfxge_softc *sc) 209 { 210 struct sfxge_intr *intr; 211 struct sfxge_intr_hdl *table; 212 int i; 213 214 intr = &sc->intr; 215 table = intr->table; 216 217 /* Remove all handlers */ 218 for (i = 0; i < intr->n_alloc; i++) 219 bus_teardown_intr(sc->dev, table[i].eih_res, 220 table[i].eih_tag); 221 } 222 223 static int 224 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 225 { 226 device_t dev; 227 struct sfxge_intr_hdl *table; 228 struct sfxge_intr *intr; 229 struct resource *res; 230 int rid; 231 int error; 232 int i; 233 234 dev = sc->dev; 235 intr = &sc->intr; 236 error = 0; 237 238 table = malloc(count * sizeof(struct sfxge_intr_hdl), 239 M_SFXGE, M_WAITOK); 240 intr->table = table; 241 242 for (i = 0; i < count; i++) { 243 rid = i + 1; 244 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 245 RF_SHAREABLE | RF_ACTIVE); 246 if (res == NULL) { 247 device_printf(dev, "Couldn't allocate interrupts for " 248 "message %d\n", rid); 249 error = ENOMEM; 250 break; 251 } 252 table[i].eih_rid = rid; 253 table[i].eih_res = res; 254 } 255 256 if (error != 0) { 257 count = i - 1; 258 for (i = 0; i < count; i++) 259 bus_release_resource(dev, SYS_RES_IRQ, 260 table[i].eih_rid, table[i].eih_res); 261 } 262 263 return (error); 264 } 265 266 static void 267 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 268 { 269 device_t dev; 270 struct resource *resp; 271 int rid; 272 273 dev = sc->dev; 274 resp = sc->intr.msix_res; 275 276 rid = rman_get_rid(resp); 277 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 278 } 279 280 static int 281 sfxge_intr_setup_msix(struct sfxge_softc *sc) 282 { 283 struct sfxge_intr *intr; 284 struct resource *resp; 285 device_t dev; 286 int count; 287 int rid; 288 289 dev = sc->dev; 290 intr = &sc->intr; 291 292 /* Check if MSI-X is available. */ 293 count = pci_msix_count(dev); 294 if (count == 0) 295 return (EINVAL); 296 297 /* Limit the number of interrupts to the number of CPUs. */ 298 if (count > mp_ncpus) 299 count = mp_ncpus; 300 301 /* Not very likely these days... */ 302 if (count > EFX_MAXRSS) 303 count = EFX_MAXRSS; 304 305 if (sc->max_rss_channels > 0 && count > sc->max_rss_channels) 306 count = sc->max_rss_channels; 307 308 rid = PCIR_BAR(4); 309 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 310 if (resp == NULL) 311 return (ENOMEM); 312 313 if (pci_alloc_msix(dev, &count) != 0) { 314 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 315 return (ENOMEM); 316 } 317 318 /* Allocate interrupt handlers. */ 319 if (sfxge_intr_alloc(sc, count) != 0) { 320 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 321 pci_release_msi(dev); 322 return (ENOMEM); 323 } 324 325 intr->type = EFX_INTR_MESSAGE; 326 intr->n_alloc = count; 327 intr->msix_res = resp; 328 329 return (0); 330 } 331 332 static int 333 sfxge_intr_setup_msi(struct sfxge_softc *sc) 334 { 335 struct sfxge_intr_hdl *table; 336 struct sfxge_intr *intr; 337 device_t dev; 338 int count; 339 int error; 340 341 dev = sc->dev; 342 intr = &sc->intr; 343 table = intr->table; 344 345 /* 346 * Check if MSI is available. All messages must be written to 347 * the same address and on x86 this means the IRQs have the 348 * same CPU affinity. So we only ever allocate 1. 349 */ 350 count = pci_msi_count(dev) ? 1 : 0; 351 if (count == 0) 352 return (EINVAL); 353 354 if ((error = pci_alloc_msi(dev, &count)) != 0) 355 return (ENOMEM); 356 357 /* Allocate interrupt handler. */ 358 if (sfxge_intr_alloc(sc, count) != 0) { 359 pci_release_msi(dev); 360 return (ENOMEM); 361 } 362 363 intr->type = EFX_INTR_MESSAGE; 364 intr->n_alloc = count; 365 366 return (0); 367 } 368 369 static int 370 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 371 { 372 struct sfxge_intr_hdl *table; 373 struct sfxge_intr *intr; 374 struct resource *res; 375 device_t dev; 376 int rid; 377 378 dev = sc->dev; 379 intr = &sc->intr; 380 381 rid = 0; 382 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 383 RF_SHAREABLE | RF_ACTIVE); 384 if (res == NULL) 385 return (ENOMEM); 386 387 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 388 table[0].eih_rid = rid; 389 table[0].eih_res = res; 390 391 intr->type = EFX_INTR_LINE; 392 intr->n_alloc = 1; 393 intr->table = table; 394 395 return (0); 396 } 397 398 static const char *const __sfxge_err[] = { 399 "", 400 "SRAM out-of-bounds", 401 "Buffer ID out-of-bounds", 402 "Internal memory parity", 403 "Receive buffer ownership", 404 "Transmit buffer ownership", 405 "Receive descriptor ownership", 406 "Transmit descriptor ownership", 407 "Event queue ownership", 408 "Event queue FIFO overflow", 409 "Illegal address", 410 "SRAM parity" 411 }; 412 413 void 414 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 415 uint32_t dword1) 416 { 417 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 418 device_t dev = sc->dev; 419 420 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 421 device_get_name(dev), device_get_unit(dev), 422 __sfxge_err[code], dword1, dword0); 423 } 424 425 void 426 sfxge_intr_stop(struct sfxge_softc *sc) 427 { 428 struct sfxge_intr *intr; 429 430 intr = &sc->intr; 431 432 KASSERT(intr->state == SFXGE_INTR_STARTED, 433 ("Interrupts not started")); 434 435 intr->state = SFXGE_INTR_INITIALIZED; 436 437 /* Disable interrupts at the NIC */ 438 efx_intr_disable(sc->enp); 439 440 /* Disable interrupts at the bus */ 441 sfxge_intr_bus_disable(sc); 442 443 /* Tear down common code interrupt bits. */ 444 efx_intr_fini(sc->enp); 445 } 446 447 int 448 sfxge_intr_start(struct sfxge_softc *sc) 449 { 450 struct sfxge_intr *intr; 451 efsys_mem_t *esmp; 452 int rc; 453 454 intr = &sc->intr; 455 esmp = &intr->status; 456 457 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 458 ("Interrupts not initialized")); 459 460 /* Zero the memory. */ 461 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 462 463 /* Initialize common code interrupt bits. */ 464 (void)efx_intr_init(sc->enp, intr->type, esmp); 465 466 /* Enable interrupts at the bus */ 467 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 468 goto fail; 469 470 intr->state = SFXGE_INTR_STARTED; 471 472 /* Enable interrupts at the NIC */ 473 efx_intr_enable(sc->enp); 474 475 return (0); 476 477 fail: 478 /* Tear down common code interrupt bits. */ 479 efx_intr_fini(sc->enp); 480 481 intr->state = SFXGE_INTR_INITIALIZED; 482 483 return (rc); 484 } 485 486 void 487 sfxge_intr_fini(struct sfxge_softc *sc) 488 { 489 struct sfxge_intr_hdl *table; 490 struct sfxge_intr *intr; 491 efsys_mem_t *esmp; 492 device_t dev; 493 int i; 494 495 dev = sc->dev; 496 intr = &sc->intr; 497 esmp = &intr->status; 498 table = intr->table; 499 500 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 501 ("intr->state != SFXGE_INTR_INITIALIZED")); 502 503 /* Free DMA memory. */ 504 sfxge_dma_free(esmp); 505 506 /* Free interrupt handles. */ 507 for (i = 0; i < intr->n_alloc; i++) 508 bus_release_resource(dev, SYS_RES_IRQ, 509 table[i].eih_rid, table[i].eih_res); 510 511 if (table[0].eih_rid != 0) 512 pci_release_msi(dev); 513 514 if (intr->msix_res != NULL) 515 sfxge_intr_teardown_msix(sc); 516 517 /* Free the handle table */ 518 free(table, M_SFXGE); 519 intr->table = NULL; 520 intr->n_alloc = 0; 521 522 /* Clear the interrupt type */ 523 intr->type = EFX_INTR_INVALID; 524 525 intr->state = SFXGE_INTR_UNINITIALIZED; 526 } 527 528 int 529 sfxge_intr_init(struct sfxge_softc *sc) 530 { 531 device_t dev; 532 struct sfxge_intr *intr; 533 efsys_mem_t *esmp; 534 int rc; 535 536 dev = sc->dev; 537 intr = &sc->intr; 538 esmp = &intr->status; 539 540 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 541 ("Interrupts already initialized")); 542 543 /* Try to setup MSI-X or MSI interrupts if available. */ 544 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 545 device_printf(dev, "Using MSI-X interrupts\n"); 546 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 547 device_printf(dev, "Using MSI interrupts\n"); 548 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 549 device_printf(dev, "Using fixed interrupts\n"); 550 } else { 551 device_printf(dev, "Couldn't setup interrupts\n"); 552 return (ENOMEM); 553 } 554 555 /* Set up DMA for interrupts. */ 556 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 557 return (ENOMEM); 558 559 intr->state = SFXGE_INTR_INITIALIZED; 560 561 return (0); 562 } 563