1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/queue.h> 42 #include <sys/rman.h> 43 #include <sys/syslog.h> 44 #include <sys/taskqueue.h> 45 46 #include <machine/bus.h> 47 #include <machine/resource.h> 48 49 #include <dev/pci/pcireg.h> 50 #include <dev/pci/pcivar.h> 51 52 #include "common/efx.h" 53 54 #include "sfxge.h" 55 56 static int 57 sfxge_intr_line_filter(void *arg) 58 { 59 struct sfxge_evq *evq; 60 struct sfxge_softc *sc; 61 efx_nic_t *enp; 62 struct sfxge_intr *intr; 63 boolean_t fatal; 64 uint32_t qmask; 65 66 evq = (struct sfxge_evq *)arg; 67 sc = evq->sc; 68 enp = sc->enp; 69 intr = &sc->intr; 70 71 KASSERT(intr != NULL, ("intr == NULL")); 72 KASSERT(intr->type == EFX_INTR_LINE, 73 ("intr->type != EFX_INTR_LINE")); 74 75 if (intr->state != SFXGE_INTR_STARTED) 76 return (FILTER_STRAY); 77 78 (void)efx_intr_status_line(enp, &fatal, &qmask); 79 80 if (fatal) { 81 (void) efx_intr_disable(enp); 82 (void) efx_intr_fatal(enp); 83 return (FILTER_HANDLED); 84 } 85 86 if (qmask != 0) { 87 intr->zero_count = 0; 88 return (FILTER_SCHEDULE_THREAD); 89 } 90 91 /* SF bug 15783: If the function is not asserting its IRQ and 92 * we read the queue mask on the cycle before a flag is added 93 * to the mask, this inhibits the function from asserting the 94 * IRQ even though we don't see the flag set. To work around 95 * this, we must re-prime all event queues and report the IRQ 96 * as handled when we see a mask of zero. To allow for shared 97 * IRQs, we don't repeat this if we see a mask of zero twice 98 * or more in a row. 99 */ 100 if (intr->zero_count++ == 0) { 101 if (evq->init_state == SFXGE_EVQ_STARTED) { 102 if (efx_ev_qpending(evq->common, evq->read_ptr)) 103 return (FILTER_SCHEDULE_THREAD); 104 efx_ev_qprime(evq->common, evq->read_ptr); 105 return (FILTER_HANDLED); 106 } 107 } 108 109 return (FILTER_STRAY); 110 } 111 112 static void 113 sfxge_intr_line(void *arg) 114 { 115 struct sfxge_evq *evq = arg; 116 117 (void)sfxge_ev_qpoll(evq); 118 } 119 120 static void 121 sfxge_intr_message(void *arg) 122 { 123 struct sfxge_evq *evq; 124 struct sfxge_softc *sc; 125 efx_nic_t *enp; 126 struct sfxge_intr *intr; 127 unsigned int index; 128 boolean_t fatal; 129 130 evq = (struct sfxge_evq *)arg; 131 sc = evq->sc; 132 enp = sc->enp; 133 intr = &sc->intr; 134 index = evq->index; 135 136 KASSERT(intr != NULL, ("intr == NULL")); 137 KASSERT(intr->type == EFX_INTR_MESSAGE, 138 ("intr->type != EFX_INTR_MESSAGE")); 139 140 if (__predict_false(intr->state != SFXGE_INTR_STARTED)) 141 return; 142 143 (void)efx_intr_status_message(enp, index, &fatal); 144 145 if (fatal) { 146 (void)efx_intr_disable(enp); 147 (void)efx_intr_fatal(enp); 148 return; 149 } 150 151 (void)sfxge_ev_qpoll(evq); 152 } 153 154 static int 155 sfxge_intr_bus_enable(struct sfxge_softc *sc) 156 { 157 struct sfxge_intr *intr; 158 struct sfxge_intr_hdl *table; 159 driver_filter_t *filter; 160 driver_intr_t *handler; 161 int index; 162 int err; 163 164 intr = &sc->intr; 165 table = intr->table; 166 167 switch (intr->type) { 168 case EFX_INTR_MESSAGE: 169 filter = NULL; /* not shared */ 170 handler = sfxge_intr_message; 171 break; 172 173 case EFX_INTR_LINE: 174 filter = sfxge_intr_line_filter; 175 handler = sfxge_intr_line; 176 break; 177 178 default: 179 KASSERT(0, ("Invalid interrupt type")); 180 return (EINVAL); 181 } 182 183 /* Try to add the handlers */ 184 for (index = 0; index < intr->n_alloc; index++) { 185 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 186 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 187 sc->evq[index], &table[index].eih_tag)) != 0) { 188 goto fail; 189 } 190 #ifdef SFXGE_HAVE_DESCRIBE_INTR 191 if (intr->n_alloc > 1) 192 bus_describe_intr(sc->dev, table[index].eih_res, 193 table[index].eih_tag, "%d", index); 194 #endif 195 bus_bind_intr(sc->dev, table[index].eih_res, index); 196 197 } 198 199 return (0); 200 201 fail: 202 /* Remove remaining handlers */ 203 while (--index >= 0) 204 bus_teardown_intr(sc->dev, table[index].eih_res, 205 table[index].eih_tag); 206 207 return (err); 208 } 209 210 static void 211 sfxge_intr_bus_disable(struct sfxge_softc *sc) 212 { 213 struct sfxge_intr *intr; 214 struct sfxge_intr_hdl *table; 215 int i; 216 217 intr = &sc->intr; 218 table = intr->table; 219 220 /* Remove all handlers */ 221 for (i = 0; i < intr->n_alloc; i++) 222 bus_teardown_intr(sc->dev, table[i].eih_res, 223 table[i].eih_tag); 224 } 225 226 static int 227 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 228 { 229 device_t dev; 230 struct sfxge_intr_hdl *table; 231 struct sfxge_intr *intr; 232 struct resource *res; 233 int rid; 234 int error; 235 int i; 236 237 dev = sc->dev; 238 intr = &sc->intr; 239 error = 0; 240 241 table = malloc(count * sizeof(struct sfxge_intr_hdl), 242 M_SFXGE, M_WAITOK); 243 intr->table = table; 244 245 for (i = 0; i < count; i++) { 246 rid = i + 1; 247 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 248 RF_SHAREABLE | RF_ACTIVE); 249 if (res == NULL) { 250 device_printf(dev, "Couldn't allocate interrupts for " 251 "message %d\n", rid); 252 error = ENOMEM; 253 break; 254 } 255 table[i].eih_rid = rid; 256 table[i].eih_res = res; 257 } 258 259 if (error != 0) { 260 count = i - 1; 261 for (i = 0; i < count; i++) 262 bus_release_resource(dev, SYS_RES_IRQ, 263 table[i].eih_rid, table[i].eih_res); 264 } 265 266 return (error); 267 } 268 269 static void 270 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 271 { 272 device_t dev; 273 struct resource *resp; 274 int rid; 275 276 dev = sc->dev; 277 resp = sc->intr.msix_res; 278 279 rid = rman_get_rid(resp); 280 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 281 } 282 283 static int 284 sfxge_intr_setup_msix(struct sfxge_softc *sc) 285 { 286 struct sfxge_intr *intr; 287 struct resource *resp; 288 device_t dev; 289 int count; 290 int rid; 291 292 dev = sc->dev; 293 intr = &sc->intr; 294 295 /* Check if MSI-X is available. */ 296 count = pci_msix_count(dev); 297 if (count == 0) 298 return (EINVAL); 299 300 /* Do not try to allocate more than already estimated EVQ maximum */ 301 KASSERT(sc->evq_max > 0, ("evq_max is zero")); 302 count = MIN(count, sc->evq_max); 303 304 rid = PCIR_BAR(4); 305 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 306 if (resp == NULL) 307 return (ENOMEM); 308 309 if (pci_alloc_msix(dev, &count) != 0) { 310 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 311 return (ENOMEM); 312 } 313 314 /* Allocate interrupt handlers. */ 315 if (sfxge_intr_alloc(sc, count) != 0) { 316 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 317 pci_release_msi(dev); 318 return (ENOMEM); 319 } 320 321 intr->type = EFX_INTR_MESSAGE; 322 intr->n_alloc = count; 323 intr->msix_res = resp; 324 325 return (0); 326 } 327 328 static int 329 sfxge_intr_setup_msi(struct sfxge_softc *sc) 330 { 331 struct sfxge_intr_hdl *table; 332 struct sfxge_intr *intr; 333 device_t dev; 334 int count; 335 int error; 336 337 dev = sc->dev; 338 intr = &sc->intr; 339 table = intr->table; 340 341 /* 342 * Check if MSI is available. All messages must be written to 343 * the same address and on x86 this means the IRQs have the 344 * same CPU affinity. So we only ever allocate 1. 345 */ 346 count = pci_msi_count(dev) ? 1 : 0; 347 if (count == 0) 348 return (EINVAL); 349 350 if ((error = pci_alloc_msi(dev, &count)) != 0) 351 return (ENOMEM); 352 353 /* Allocate interrupt handler. */ 354 if (sfxge_intr_alloc(sc, count) != 0) { 355 pci_release_msi(dev); 356 return (ENOMEM); 357 } 358 359 intr->type = EFX_INTR_MESSAGE; 360 intr->n_alloc = count; 361 362 return (0); 363 } 364 365 static int 366 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 367 { 368 struct sfxge_intr_hdl *table; 369 struct sfxge_intr *intr; 370 struct resource *res; 371 device_t dev; 372 int rid; 373 374 dev = sc->dev; 375 intr = &sc->intr; 376 377 rid = 0; 378 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 379 RF_SHAREABLE | RF_ACTIVE); 380 if (res == NULL) 381 return (ENOMEM); 382 383 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 384 table[0].eih_rid = rid; 385 table[0].eih_res = res; 386 387 intr->type = EFX_INTR_LINE; 388 intr->n_alloc = 1; 389 intr->table = table; 390 391 return (0); 392 } 393 394 static const char *const __sfxge_err[] = { 395 "", 396 "SRAM out-of-bounds", 397 "Buffer ID out-of-bounds", 398 "Internal memory parity", 399 "Receive buffer ownership", 400 "Transmit buffer ownership", 401 "Receive descriptor ownership", 402 "Transmit descriptor ownership", 403 "Event queue ownership", 404 "Event queue FIFO overflow", 405 "Illegal address", 406 "SRAM parity" 407 }; 408 409 void 410 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 411 uint32_t dword1) 412 { 413 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 414 device_t dev = sc->dev; 415 416 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 417 device_get_name(dev), device_get_unit(dev), 418 __sfxge_err[code], dword1, dword0); 419 } 420 421 void 422 sfxge_intr_stop(struct sfxge_softc *sc) 423 { 424 struct sfxge_intr *intr; 425 426 intr = &sc->intr; 427 428 KASSERT(intr->state == SFXGE_INTR_STARTED, 429 ("Interrupts not started")); 430 431 intr->state = SFXGE_INTR_INITIALIZED; 432 433 /* Disable interrupts at the NIC */ 434 efx_intr_disable(sc->enp); 435 436 /* Disable interrupts at the bus */ 437 sfxge_intr_bus_disable(sc); 438 439 /* Tear down common code interrupt bits. */ 440 efx_intr_fini(sc->enp); 441 } 442 443 int 444 sfxge_intr_start(struct sfxge_softc *sc) 445 { 446 struct sfxge_intr *intr; 447 efsys_mem_t *esmp; 448 int rc; 449 450 intr = &sc->intr; 451 esmp = &intr->status; 452 453 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 454 ("Interrupts not initialized")); 455 456 /* Zero the memory. */ 457 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 458 459 /* Initialize common code interrupt bits. */ 460 (void)efx_intr_init(sc->enp, intr->type, esmp); 461 462 /* Enable interrupts at the bus */ 463 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 464 goto fail; 465 466 intr->state = SFXGE_INTR_STARTED; 467 468 /* Enable interrupts at the NIC */ 469 efx_intr_enable(sc->enp); 470 471 return (0); 472 473 fail: 474 /* Tear down common code interrupt bits. */ 475 efx_intr_fini(sc->enp); 476 477 intr->state = SFXGE_INTR_INITIALIZED; 478 479 return (rc); 480 } 481 482 void 483 sfxge_intr_fini(struct sfxge_softc *sc) 484 { 485 struct sfxge_intr_hdl *table; 486 struct sfxge_intr *intr; 487 efsys_mem_t *esmp; 488 device_t dev; 489 int i; 490 491 dev = sc->dev; 492 intr = &sc->intr; 493 esmp = &intr->status; 494 table = intr->table; 495 496 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 497 ("intr->state != SFXGE_INTR_INITIALIZED")); 498 499 /* Free DMA memory. */ 500 sfxge_dma_free(esmp); 501 502 /* Free interrupt handles. */ 503 for (i = 0; i < intr->n_alloc; i++) 504 bus_release_resource(dev, SYS_RES_IRQ, 505 table[i].eih_rid, table[i].eih_res); 506 507 if (table[0].eih_rid != 0) 508 pci_release_msi(dev); 509 510 if (intr->msix_res != NULL) 511 sfxge_intr_teardown_msix(sc); 512 513 /* Free the handle table */ 514 free(table, M_SFXGE); 515 intr->table = NULL; 516 intr->n_alloc = 0; 517 518 /* Clear the interrupt type */ 519 intr->type = EFX_INTR_INVALID; 520 521 intr->state = SFXGE_INTR_UNINITIALIZED; 522 } 523 524 int 525 sfxge_intr_init(struct sfxge_softc *sc) 526 { 527 device_t dev; 528 struct sfxge_intr *intr; 529 efsys_mem_t *esmp; 530 int rc; 531 532 dev = sc->dev; 533 intr = &sc->intr; 534 esmp = &intr->status; 535 536 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 537 ("Interrupts already initialized")); 538 539 /* Try to setup MSI-X or MSI interrupts if available. */ 540 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 541 device_printf(dev, "Using MSI-X interrupts\n"); 542 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 543 device_printf(dev, "Using MSI interrupts\n"); 544 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 545 device_printf(dev, "Using fixed interrupts\n"); 546 } else { 547 device_printf(dev, "Couldn't setup interrupts\n"); 548 return (ENOMEM); 549 } 550 551 /* Set up DMA for interrupts. */ 552 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 553 return (ENOMEM); 554 555 intr->state = SFXGE_INTR_INITIALIZED; 556 557 return (0); 558 } 559