1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/bus.h> 35 #include <sys/rman.h> 36 #include <sys/smp.h> 37 #include <sys/syslog.h> 38 39 #include <machine/bus.h> 40 #include <machine/resource.h> 41 42 #include <dev/pci/pcireg.h> 43 #include <dev/pci/pcivar.h> 44 45 #include "common/efx.h" 46 47 #include "sfxge.h" 48 49 static int 50 sfxge_intr_line_filter(void *arg) 51 { 52 struct sfxge_evq *evq; 53 struct sfxge_softc *sc; 54 efx_nic_t *enp; 55 struct sfxge_intr *intr; 56 boolean_t fatal; 57 uint32_t qmask; 58 59 evq = (struct sfxge_evq *)arg; 60 sc = evq->sc; 61 enp = sc->enp; 62 intr = &sc->intr; 63 64 KASSERT(intr != NULL, ("intr == NULL")); 65 KASSERT(intr->type == EFX_INTR_LINE, 66 ("intr->type != EFX_INTR_LINE")); 67 68 if (intr->state != SFXGE_INTR_STARTED) 69 return FILTER_STRAY; 70 71 (void)efx_intr_status_line(enp, &fatal, &qmask); 72 73 if (fatal) { 74 (void) efx_intr_disable(enp); 75 (void) efx_intr_fatal(enp); 76 return FILTER_HANDLED; 77 } 78 79 if (qmask != 0) { 80 intr->zero_count = 0; 81 return FILTER_SCHEDULE_THREAD; 82 } 83 84 /* SF bug 15783: If the function is not asserting its IRQ and 85 * we read the queue mask on the cycle before a flag is added 86 * to the mask, this inhibits the function from asserting the 87 * IRQ even though we don't see the flag set. To work around 88 * this, we must re-prime all event queues and report the IRQ 89 * as handled when we see a mask of zero. To allow for shared 90 * IRQs, we don't repeat this if we see a mask of zero twice 91 * or more in a row. 92 */ 93 if (intr->zero_count++ == 0) { 94 if (evq->init_state == SFXGE_EVQ_STARTED) { 95 if (efx_ev_qpending(evq->common, evq->read_ptr)) 96 return FILTER_SCHEDULE_THREAD; 97 efx_ev_qprime(evq->common, evq->read_ptr); 98 return FILTER_HANDLED; 99 } 100 } 101 102 return FILTER_STRAY; 103 } 104 105 static void 106 sfxge_intr_line(void *arg) 107 { 108 struct sfxge_evq *evq = arg; 109 struct sfxge_softc *sc = evq->sc; 110 111 (void)sfxge_ev_qpoll(sc, 0); 112 } 113 114 static void 115 sfxge_intr_message(void *arg) 116 { 117 struct sfxge_evq *evq; 118 struct sfxge_softc *sc; 119 efx_nic_t *enp; 120 struct sfxge_intr *intr; 121 unsigned int index; 122 boolean_t fatal; 123 124 evq = (struct sfxge_evq *)arg; 125 sc = evq->sc; 126 enp = sc->enp; 127 intr = &sc->intr; 128 index = evq->index; 129 130 KASSERT(intr != NULL, ("intr == NULL")); 131 KASSERT(intr->type == EFX_INTR_MESSAGE, 132 ("intr->type != EFX_INTR_MESSAGE")); 133 134 if (intr->state != SFXGE_INTR_STARTED) 135 return; 136 137 (void)efx_intr_status_message(enp, index, &fatal); 138 139 if (fatal) { 140 (void)efx_intr_disable(enp); 141 (void)efx_intr_fatal(enp); 142 return; 143 } 144 145 (void)sfxge_ev_qpoll(sc, index); 146 } 147 148 static int 149 sfxge_intr_bus_enable(struct sfxge_softc *sc) 150 { 151 struct sfxge_intr *intr; 152 struct sfxge_intr_hdl *table; 153 driver_filter_t *filter; 154 driver_intr_t *handler; 155 int index; 156 int err; 157 158 intr = &sc->intr; 159 table = intr->table; 160 161 switch (intr->type) { 162 case EFX_INTR_MESSAGE: 163 filter = NULL; /* not shared */ 164 handler = sfxge_intr_message; 165 break; 166 167 case EFX_INTR_LINE: 168 filter = sfxge_intr_line_filter; 169 handler = sfxge_intr_line; 170 break; 171 172 default: 173 KASSERT(0, ("Invalid interrupt type")); 174 return EINVAL; 175 } 176 177 /* Try to add the handlers */ 178 for (index = 0; index < intr->n_alloc; index++) { 179 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 180 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 181 sc->evq[index], &table[index].eih_tag)) != 0) { 182 goto fail; 183 } 184 #ifdef SFXGE_HAVE_DESCRIBE_INTR 185 if (intr->n_alloc > 1) 186 bus_describe_intr(sc->dev, table[index].eih_res, 187 table[index].eih_tag, "%d", index); 188 #endif 189 bus_bind_intr(sc->dev, table[index].eih_res, index); 190 191 } 192 193 return (0); 194 195 fail: 196 /* Remove remaining handlers */ 197 while (--index >= 0) 198 bus_teardown_intr(sc->dev, table[index].eih_res, 199 table[index].eih_tag); 200 201 return (err); 202 } 203 204 static void 205 sfxge_intr_bus_disable(struct sfxge_softc *sc) 206 { 207 struct sfxge_intr *intr; 208 struct sfxge_intr_hdl *table; 209 int i; 210 211 intr = &sc->intr; 212 table = intr->table; 213 214 /* Remove all handlers */ 215 for (i = 0; i < intr->n_alloc; i++) 216 bus_teardown_intr(sc->dev, table[i].eih_res, 217 table[i].eih_tag); 218 } 219 220 static int 221 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 222 { 223 device_t dev; 224 struct sfxge_intr_hdl *table; 225 struct sfxge_intr *intr; 226 struct resource *res; 227 int rid; 228 int error; 229 int i; 230 231 dev = sc->dev; 232 intr = &sc->intr; 233 error = 0; 234 235 table = malloc(count * sizeof(struct sfxge_intr_hdl), 236 M_SFXGE, M_WAITOK); 237 intr->table = table; 238 239 for (i = 0; i < count; i++) { 240 rid = i + 1; 241 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 242 RF_SHAREABLE | RF_ACTIVE); 243 if (res == NULL) { 244 device_printf(dev, "Couldn't allocate interrupts for " 245 "message %d\n", rid); 246 error = ENOMEM; 247 break; 248 } 249 table[i].eih_rid = rid; 250 table[i].eih_res = res; 251 } 252 253 if (error) { 254 count = i - 1; 255 for (i = 0; i < count; i++) 256 bus_release_resource(dev, SYS_RES_IRQ, 257 table[i].eih_rid, table[i].eih_res); 258 } 259 260 return (error); 261 } 262 263 static void 264 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 265 { 266 device_t dev; 267 struct resource *resp; 268 int rid; 269 270 dev = sc->dev; 271 resp = sc->intr.msix_res; 272 273 rid = rman_get_rid(resp); 274 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 275 } 276 277 static int 278 sfxge_intr_setup_msix(struct sfxge_softc *sc) 279 { 280 struct sfxge_intr *intr; 281 struct resource *resp; 282 device_t dev; 283 int count; 284 int rid; 285 286 dev = sc->dev; 287 intr = &sc->intr; 288 289 /* Check if MSI-X is available. */ 290 count = pci_msix_count(dev); 291 if (count == 0) 292 return (EINVAL); 293 294 /* Limit the number of interrupts to the number of CPUs. */ 295 if (count > mp_ncpus) 296 count = mp_ncpus; 297 298 /* Not very likely these days... */ 299 if (count > EFX_MAXRSS) 300 count = EFX_MAXRSS; 301 302 rid = PCIR_BAR(4); 303 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 304 if (resp == NULL) 305 return (ENOMEM); 306 307 if (pci_alloc_msix(dev, &count) != 0) { 308 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 309 return (ENOMEM); 310 } 311 312 /* Allocate interrupt handlers. */ 313 if (sfxge_intr_alloc(sc, count) != 0) { 314 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 315 pci_release_msi(dev); 316 return (ENOMEM); 317 } 318 319 intr->type = EFX_INTR_MESSAGE; 320 intr->n_alloc = count; 321 intr->msix_res = resp; 322 323 return (0); 324 } 325 326 static int 327 sfxge_intr_setup_msi(struct sfxge_softc *sc) 328 { 329 struct sfxge_intr_hdl *table; 330 struct sfxge_intr *intr; 331 device_t dev; 332 int count; 333 int error; 334 335 dev = sc->dev; 336 intr = &sc->intr; 337 table = intr->table; 338 339 /* 340 * Check if MSI is available. All messages must be written to 341 * the same address and on x86 this means the IRQs have the 342 * same CPU affinity. So we only ever allocate 1. 343 */ 344 count = pci_msi_count(dev) ? 1 : 0; 345 if (count == 0) 346 return (EINVAL); 347 348 if ((error = pci_alloc_msi(dev, &count)) != 0) 349 return (ENOMEM); 350 351 /* Allocate interrupt handler. */ 352 if (sfxge_intr_alloc(sc, count) != 0) { 353 pci_release_msi(dev); 354 return (ENOMEM); 355 } 356 357 intr->type = EFX_INTR_MESSAGE; 358 intr->n_alloc = count; 359 360 return (0); 361 } 362 363 static int 364 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 365 { 366 struct sfxge_intr_hdl *table; 367 struct sfxge_intr *intr; 368 struct resource *res; 369 device_t dev; 370 int rid; 371 372 dev = sc->dev; 373 intr = &sc->intr; 374 375 rid = 0; 376 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 377 RF_SHAREABLE | RF_ACTIVE); 378 if (res == NULL) 379 return (ENOMEM); 380 381 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 382 table[0].eih_rid = rid; 383 table[0].eih_res = res; 384 385 intr->type = EFX_INTR_LINE; 386 intr->n_alloc = 1; 387 intr->table = table; 388 389 return (0); 390 } 391 392 static const char *const __sfxge_err[] = { 393 "", 394 "SRAM out-of-bounds", 395 "Buffer ID out-of-bounds", 396 "Internal memory parity", 397 "Receive buffer ownership", 398 "Transmit buffer ownership", 399 "Receive descriptor ownership", 400 "Transmit descriptor ownership", 401 "Event queue ownership", 402 "Event queue FIFO overflow", 403 "Illegal address", 404 "SRAM parity" 405 }; 406 407 void 408 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 409 uint32_t dword1) 410 { 411 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 412 device_t dev = sc->dev; 413 414 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 415 device_get_name(dev), device_get_unit(dev), 416 __sfxge_err[code], dword1, dword0); 417 } 418 419 void 420 sfxge_intr_stop(struct sfxge_softc *sc) 421 { 422 struct sfxge_intr *intr; 423 424 intr = &sc->intr; 425 426 KASSERT(intr->state == SFXGE_INTR_STARTED, 427 ("Interrupts not started")); 428 429 intr->state = SFXGE_INTR_INITIALIZED; 430 431 /* Disable interrupts at the NIC */ 432 efx_intr_disable(sc->enp); 433 434 /* Disable interrupts at the bus */ 435 sfxge_intr_bus_disable(sc); 436 437 /* Tear down common code interrupt bits. */ 438 efx_intr_fini(sc->enp); 439 } 440 441 int 442 sfxge_intr_start(struct sfxge_softc *sc) 443 { 444 struct sfxge_intr *intr; 445 efsys_mem_t *esmp; 446 int rc; 447 448 intr = &sc->intr; 449 esmp = &intr->status; 450 451 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 452 ("Interrupts not initialized")); 453 454 /* Zero the memory. */ 455 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 456 457 /* Initialize common code interrupt bits. */ 458 (void)efx_intr_init(sc->enp, intr->type, esmp); 459 460 /* Enable interrupts at the bus */ 461 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 462 goto fail; 463 464 intr->state = SFXGE_INTR_STARTED; 465 466 /* Enable interrupts at the NIC */ 467 efx_intr_enable(sc->enp); 468 469 return (0); 470 471 fail: 472 /* Tear down common code interrupt bits. */ 473 efx_intr_fini(sc->enp); 474 475 intr->state = SFXGE_INTR_INITIALIZED; 476 477 return (rc); 478 } 479 480 void 481 sfxge_intr_fini(struct sfxge_softc *sc) 482 { 483 struct sfxge_intr_hdl *table; 484 struct sfxge_intr *intr; 485 efsys_mem_t *esmp; 486 device_t dev; 487 int i; 488 489 dev = sc->dev; 490 intr = &sc->intr; 491 esmp = &intr->status; 492 table = intr->table; 493 494 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 495 ("intr->state != SFXGE_INTR_INITIALIZED")); 496 497 /* Free DMA memory. */ 498 sfxge_dma_free(esmp); 499 500 /* Free interrupt handles. */ 501 for (i = 0; i < intr->n_alloc; i++) 502 bus_release_resource(dev, SYS_RES_IRQ, 503 table[i].eih_rid, table[i].eih_res); 504 505 if (table[0].eih_rid != 0) 506 pci_release_msi(dev); 507 508 if (intr->msix_res != NULL) 509 sfxge_intr_teardown_msix(sc); 510 511 /* Free the handle table */ 512 free(table, M_SFXGE); 513 intr->table = NULL; 514 intr->n_alloc = 0; 515 516 /* Clear the interrupt type */ 517 intr->type = EFX_INTR_INVALID; 518 519 intr->state = SFXGE_INTR_UNINITIALIZED; 520 } 521 522 int 523 sfxge_intr_init(struct sfxge_softc *sc) 524 { 525 device_t dev; 526 struct sfxge_intr *intr; 527 efsys_mem_t *esmp; 528 int rc; 529 530 dev = sc->dev; 531 intr = &sc->intr; 532 esmp = &intr->status; 533 534 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 535 ("Interrupts already initialized")); 536 537 /* Try to setup MSI-X or MSI interrupts if available. */ 538 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 539 device_printf(dev, "Using MSI-X interrupts\n"); 540 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 541 device_printf(dev, "Using MSI interrupts\n"); 542 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 543 device_printf(dev, "Using fixed interrupts\n"); 544 } else { 545 device_printf(dev, "Couldn't setup interrupts\n"); 546 return (ENOMEM); 547 } 548 549 /* Set up DMA for interrupts. */ 550 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 551 return (ENOMEM); 552 553 intr->state = SFXGE_INTR_INITIALIZED; 554 555 return (0); 556 } 557