1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/kernel.h> 36 #include <sys/malloc.h> 37 #include <sys/queue.h> 38 #include <sys/rman.h> 39 #include <sys/smp.h> 40 #include <sys/syslog.h> 41 #include <sys/taskqueue.h> 42 43 #include <machine/bus.h> 44 #include <machine/resource.h> 45 46 #include <dev/pci/pcireg.h> 47 #include <dev/pci/pcivar.h> 48 49 #include "common/efx.h" 50 51 #include "sfxge.h" 52 53 static int 54 sfxge_intr_line_filter(void *arg) 55 { 56 struct sfxge_evq *evq; 57 struct sfxge_softc *sc; 58 efx_nic_t *enp; 59 struct sfxge_intr *intr; 60 boolean_t fatal; 61 uint32_t qmask; 62 63 evq = (struct sfxge_evq *)arg; 64 sc = evq->sc; 65 enp = sc->enp; 66 intr = &sc->intr; 67 68 KASSERT(intr != NULL, ("intr == NULL")); 69 KASSERT(intr->type == EFX_INTR_LINE, 70 ("intr->type != EFX_INTR_LINE")); 71 72 if (intr->state != SFXGE_INTR_STARTED) 73 return FILTER_STRAY; 74 75 (void)efx_intr_status_line(enp, &fatal, &qmask); 76 77 if (fatal) { 78 (void) efx_intr_disable(enp); 79 (void) efx_intr_fatal(enp); 80 return FILTER_HANDLED; 81 } 82 83 if (qmask != 0) { 84 intr->zero_count = 0; 85 return FILTER_SCHEDULE_THREAD; 86 } 87 88 /* SF bug 15783: If the function is not asserting its IRQ and 89 * we read the queue mask on the cycle before a flag is added 90 * to the mask, this inhibits the function from asserting the 91 * IRQ even though we don't see the flag set. To work around 92 * this, we must re-prime all event queues and report the IRQ 93 * as handled when we see a mask of zero. To allow for shared 94 * IRQs, we don't repeat this if we see a mask of zero twice 95 * or more in a row. 96 */ 97 if (intr->zero_count++ == 0) { 98 if (evq->init_state == SFXGE_EVQ_STARTED) { 99 if (efx_ev_qpending(evq->common, evq->read_ptr)) 100 return FILTER_SCHEDULE_THREAD; 101 efx_ev_qprime(evq->common, evq->read_ptr); 102 return FILTER_HANDLED; 103 } 104 } 105 106 return FILTER_STRAY; 107 } 108 109 static void 110 sfxge_intr_line(void *arg) 111 { 112 struct sfxge_evq *evq = arg; 113 struct sfxge_softc *sc = evq->sc; 114 115 (void)sfxge_ev_qpoll(sc, 0); 116 } 117 118 static void 119 sfxge_intr_message(void *arg) 120 { 121 struct sfxge_evq *evq; 122 struct sfxge_softc *sc; 123 efx_nic_t *enp; 124 struct sfxge_intr *intr; 125 unsigned int index; 126 boolean_t fatal; 127 128 evq = (struct sfxge_evq *)arg; 129 sc = evq->sc; 130 enp = sc->enp; 131 intr = &sc->intr; 132 index = evq->index; 133 134 KASSERT(intr != NULL, ("intr == NULL")); 135 KASSERT(intr->type == EFX_INTR_MESSAGE, 136 ("intr->type != EFX_INTR_MESSAGE")); 137 138 if (intr->state != SFXGE_INTR_STARTED) 139 return; 140 141 (void)efx_intr_status_message(enp, index, &fatal); 142 143 if (fatal) { 144 (void)efx_intr_disable(enp); 145 (void)efx_intr_fatal(enp); 146 return; 147 } 148 149 (void)sfxge_ev_qpoll(sc, index); 150 } 151 152 static int 153 sfxge_intr_bus_enable(struct sfxge_softc *sc) 154 { 155 struct sfxge_intr *intr; 156 struct sfxge_intr_hdl *table; 157 driver_filter_t *filter; 158 driver_intr_t *handler; 159 int index; 160 int err; 161 162 intr = &sc->intr; 163 table = intr->table; 164 165 switch (intr->type) { 166 case EFX_INTR_MESSAGE: 167 filter = NULL; /* not shared */ 168 handler = sfxge_intr_message; 169 break; 170 171 case EFX_INTR_LINE: 172 filter = sfxge_intr_line_filter; 173 handler = sfxge_intr_line; 174 break; 175 176 default: 177 KASSERT(0, ("Invalid interrupt type")); 178 return EINVAL; 179 } 180 181 /* Try to add the handlers */ 182 for (index = 0; index < intr->n_alloc; index++) { 183 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 184 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 185 sc->evq[index], &table[index].eih_tag)) != 0) { 186 goto fail; 187 } 188 #ifdef SFXGE_HAVE_DESCRIBE_INTR 189 if (intr->n_alloc > 1) 190 bus_describe_intr(sc->dev, table[index].eih_res, 191 table[index].eih_tag, "%d", index); 192 #endif 193 bus_bind_intr(sc->dev, table[index].eih_res, index); 194 195 } 196 197 return (0); 198 199 fail: 200 /* Remove remaining handlers */ 201 while (--index >= 0) 202 bus_teardown_intr(sc->dev, table[index].eih_res, 203 table[index].eih_tag); 204 205 return (err); 206 } 207 208 static void 209 sfxge_intr_bus_disable(struct sfxge_softc *sc) 210 { 211 struct sfxge_intr *intr; 212 struct sfxge_intr_hdl *table; 213 int i; 214 215 intr = &sc->intr; 216 table = intr->table; 217 218 /* Remove all handlers */ 219 for (i = 0; i < intr->n_alloc; i++) 220 bus_teardown_intr(sc->dev, table[i].eih_res, 221 table[i].eih_tag); 222 } 223 224 static int 225 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 226 { 227 device_t dev; 228 struct sfxge_intr_hdl *table; 229 struct sfxge_intr *intr; 230 struct resource *res; 231 int rid; 232 int error; 233 int i; 234 235 dev = sc->dev; 236 intr = &sc->intr; 237 error = 0; 238 239 table = malloc(count * sizeof(struct sfxge_intr_hdl), 240 M_SFXGE, M_WAITOK); 241 intr->table = table; 242 243 for (i = 0; i < count; i++) { 244 rid = i + 1; 245 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 246 RF_SHAREABLE | RF_ACTIVE); 247 if (res == NULL) { 248 device_printf(dev, "Couldn't allocate interrupts for " 249 "message %d\n", rid); 250 error = ENOMEM; 251 break; 252 } 253 table[i].eih_rid = rid; 254 table[i].eih_res = res; 255 } 256 257 if (error) { 258 count = i - 1; 259 for (i = 0; i < count; i++) 260 bus_release_resource(dev, SYS_RES_IRQ, 261 table[i].eih_rid, table[i].eih_res); 262 } 263 264 return (error); 265 } 266 267 static void 268 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 269 { 270 device_t dev; 271 struct resource *resp; 272 int rid; 273 274 dev = sc->dev; 275 resp = sc->intr.msix_res; 276 277 rid = rman_get_rid(resp); 278 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 279 } 280 281 static int 282 sfxge_intr_setup_msix(struct sfxge_softc *sc) 283 { 284 struct sfxge_intr *intr; 285 struct resource *resp; 286 device_t dev; 287 int count; 288 int rid; 289 290 dev = sc->dev; 291 intr = &sc->intr; 292 293 /* Check if MSI-X is available. */ 294 count = pci_msix_count(dev); 295 if (count == 0) 296 return (EINVAL); 297 298 /* Limit the number of interrupts to the number of CPUs. */ 299 if (count > mp_ncpus) 300 count = mp_ncpus; 301 302 /* Not very likely these days... */ 303 if (count > EFX_MAXRSS) 304 count = EFX_MAXRSS; 305 306 rid = PCIR_BAR(4); 307 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 308 if (resp == NULL) 309 return (ENOMEM); 310 311 if (pci_alloc_msix(dev, &count) != 0) { 312 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 313 return (ENOMEM); 314 } 315 316 /* Allocate interrupt handlers. */ 317 if (sfxge_intr_alloc(sc, count) != 0) { 318 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 319 pci_release_msi(dev); 320 return (ENOMEM); 321 } 322 323 intr->type = EFX_INTR_MESSAGE; 324 intr->n_alloc = count; 325 intr->msix_res = resp; 326 327 return (0); 328 } 329 330 static int 331 sfxge_intr_setup_msi(struct sfxge_softc *sc) 332 { 333 struct sfxge_intr_hdl *table; 334 struct sfxge_intr *intr; 335 device_t dev; 336 int count; 337 int error; 338 339 dev = sc->dev; 340 intr = &sc->intr; 341 table = intr->table; 342 343 /* 344 * Check if MSI is available. All messages must be written to 345 * the same address and on x86 this means the IRQs have the 346 * same CPU affinity. So we only ever allocate 1. 347 */ 348 count = pci_msi_count(dev) ? 1 : 0; 349 if (count == 0) 350 return (EINVAL); 351 352 if ((error = pci_alloc_msi(dev, &count)) != 0) 353 return (ENOMEM); 354 355 /* Allocate interrupt handler. */ 356 if (sfxge_intr_alloc(sc, count) != 0) { 357 pci_release_msi(dev); 358 return (ENOMEM); 359 } 360 361 intr->type = EFX_INTR_MESSAGE; 362 intr->n_alloc = count; 363 364 return (0); 365 } 366 367 static int 368 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 369 { 370 struct sfxge_intr_hdl *table; 371 struct sfxge_intr *intr; 372 struct resource *res; 373 device_t dev; 374 int rid; 375 376 dev = sc->dev; 377 intr = &sc->intr; 378 379 rid = 0; 380 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 381 RF_SHAREABLE | RF_ACTIVE); 382 if (res == NULL) 383 return (ENOMEM); 384 385 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 386 table[0].eih_rid = rid; 387 table[0].eih_res = res; 388 389 intr->type = EFX_INTR_LINE; 390 intr->n_alloc = 1; 391 intr->table = table; 392 393 return (0); 394 } 395 396 static const char *const __sfxge_err[] = { 397 "", 398 "SRAM out-of-bounds", 399 "Buffer ID out-of-bounds", 400 "Internal memory parity", 401 "Receive buffer ownership", 402 "Transmit buffer ownership", 403 "Receive descriptor ownership", 404 "Transmit descriptor ownership", 405 "Event queue ownership", 406 "Event queue FIFO overflow", 407 "Illegal address", 408 "SRAM parity" 409 }; 410 411 void 412 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 413 uint32_t dword1) 414 { 415 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 416 device_t dev = sc->dev; 417 418 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 419 device_get_name(dev), device_get_unit(dev), 420 __sfxge_err[code], dword1, dword0); 421 } 422 423 void 424 sfxge_intr_stop(struct sfxge_softc *sc) 425 { 426 struct sfxge_intr *intr; 427 428 intr = &sc->intr; 429 430 KASSERT(intr->state == SFXGE_INTR_STARTED, 431 ("Interrupts not started")); 432 433 intr->state = SFXGE_INTR_INITIALIZED; 434 435 /* Disable interrupts at the NIC */ 436 efx_intr_disable(sc->enp); 437 438 /* Disable interrupts at the bus */ 439 sfxge_intr_bus_disable(sc); 440 441 /* Tear down common code interrupt bits. */ 442 efx_intr_fini(sc->enp); 443 } 444 445 int 446 sfxge_intr_start(struct sfxge_softc *sc) 447 { 448 struct sfxge_intr *intr; 449 efsys_mem_t *esmp; 450 int rc; 451 452 intr = &sc->intr; 453 esmp = &intr->status; 454 455 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 456 ("Interrupts not initialized")); 457 458 /* Zero the memory. */ 459 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 460 461 /* Initialize common code interrupt bits. */ 462 (void)efx_intr_init(sc->enp, intr->type, esmp); 463 464 /* Enable interrupts at the bus */ 465 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 466 goto fail; 467 468 intr->state = SFXGE_INTR_STARTED; 469 470 /* Enable interrupts at the NIC */ 471 efx_intr_enable(sc->enp); 472 473 return (0); 474 475 fail: 476 /* Tear down common code interrupt bits. */ 477 efx_intr_fini(sc->enp); 478 479 intr->state = SFXGE_INTR_INITIALIZED; 480 481 return (rc); 482 } 483 484 void 485 sfxge_intr_fini(struct sfxge_softc *sc) 486 { 487 struct sfxge_intr_hdl *table; 488 struct sfxge_intr *intr; 489 efsys_mem_t *esmp; 490 device_t dev; 491 int i; 492 493 dev = sc->dev; 494 intr = &sc->intr; 495 esmp = &intr->status; 496 table = intr->table; 497 498 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 499 ("intr->state != SFXGE_INTR_INITIALIZED")); 500 501 /* Free DMA memory. */ 502 sfxge_dma_free(esmp); 503 504 /* Free interrupt handles. */ 505 for (i = 0; i < intr->n_alloc; i++) 506 bus_release_resource(dev, SYS_RES_IRQ, 507 table[i].eih_rid, table[i].eih_res); 508 509 if (table[0].eih_rid != 0) 510 pci_release_msi(dev); 511 512 if (intr->msix_res != NULL) 513 sfxge_intr_teardown_msix(sc); 514 515 /* Free the handle table */ 516 free(table, M_SFXGE); 517 intr->table = NULL; 518 intr->n_alloc = 0; 519 520 /* Clear the interrupt type */ 521 intr->type = EFX_INTR_INVALID; 522 523 intr->state = SFXGE_INTR_UNINITIALIZED; 524 } 525 526 int 527 sfxge_intr_init(struct sfxge_softc *sc) 528 { 529 device_t dev; 530 struct sfxge_intr *intr; 531 efsys_mem_t *esmp; 532 int rc; 533 534 dev = sc->dev; 535 intr = &sc->intr; 536 esmp = &intr->status; 537 538 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 539 ("Interrupts already initialized")); 540 541 /* Try to setup MSI-X or MSI interrupts if available. */ 542 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 543 device_printf(dev, "Using MSI-X interrupts\n"); 544 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 545 device_printf(dev, "Using MSI interrupts\n"); 546 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 547 device_printf(dev, "Using fixed interrupts\n"); 548 } else { 549 device_printf(dev, "Couldn't setup interrupts\n"); 550 return (ENOMEM); 551 } 552 553 /* Set up DMA for interrupts. */ 554 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 555 return (ENOMEM); 556 557 intr->state = SFXGE_INTR_INITIALIZED; 558 559 return (0); 560 } 561