1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 #include "opt_rss.h" 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/kernel.h> 42 #include <sys/malloc.h> 43 #include <sys/queue.h> 44 #include <sys/rman.h> 45 #include <sys/syslog.h> 46 #include <sys/taskqueue.h> 47 48 #include <machine/bus.h> 49 #include <machine/resource.h> 50 51 #include <dev/pci/pcireg.h> 52 #include <dev/pci/pcivar.h> 53 54 #ifdef RSS 55 #include <net/rss_config.h> 56 #endif 57 58 #include "common/efx.h" 59 60 #include "sfxge.h" 61 62 static int 63 sfxge_intr_line_filter(void *arg) 64 { 65 struct sfxge_evq *evq; 66 struct sfxge_softc *sc; 67 efx_nic_t *enp; 68 struct sfxge_intr *intr; 69 boolean_t fatal; 70 uint32_t qmask; 71 72 evq = (struct sfxge_evq *)arg; 73 sc = evq->sc; 74 enp = sc->enp; 75 intr = &sc->intr; 76 77 KASSERT(intr != NULL, ("intr == NULL")); 78 KASSERT(intr->type == EFX_INTR_LINE, 79 ("intr->type != EFX_INTR_LINE")); 80 81 if (intr->state != SFXGE_INTR_STARTED) 82 return (FILTER_STRAY); 83 84 (void)efx_intr_status_line(enp, &fatal, &qmask); 85 86 if (fatal) { 87 (void) efx_intr_disable(enp); 88 (void) efx_intr_fatal(enp); 89 return (FILTER_HANDLED); 90 } 91 92 if (qmask != 0) { 93 intr->zero_count = 0; 94 return (FILTER_SCHEDULE_THREAD); 95 } 96 97 /* SF bug 15783: If the function is not asserting its IRQ and 98 * we read the queue mask on the cycle before a flag is added 99 * to the mask, this inhibits the function from asserting the 100 * IRQ even though we don't see the flag set. To work around 101 * this, we must re-prime all event queues and report the IRQ 102 * as handled when we see a mask of zero. To allow for shared 103 * IRQs, we don't repeat this if we see a mask of zero twice 104 * or more in a row. 105 */ 106 if (intr->zero_count++ == 0) { 107 if (evq->init_state == SFXGE_EVQ_STARTED) { 108 if (efx_ev_qpending(evq->common, evq->read_ptr)) 109 return (FILTER_SCHEDULE_THREAD); 110 efx_ev_qprime(evq->common, evq->read_ptr); 111 return (FILTER_HANDLED); 112 } 113 } 114 115 return (FILTER_STRAY); 116 } 117 118 static void 119 sfxge_intr_line(void *arg) 120 { 121 struct sfxge_evq *evq = arg; 122 123 (void)sfxge_ev_qpoll(evq); 124 } 125 126 static void 127 sfxge_intr_message(void *arg) 128 { 129 struct sfxge_evq *evq; 130 struct sfxge_softc *sc; 131 efx_nic_t *enp; 132 struct sfxge_intr *intr; 133 unsigned int index; 134 boolean_t fatal; 135 136 evq = (struct sfxge_evq *)arg; 137 sc = evq->sc; 138 enp = sc->enp; 139 intr = &sc->intr; 140 index = evq->index; 141 142 KASSERT(intr != NULL, ("intr == NULL")); 143 KASSERT(intr->type == EFX_INTR_MESSAGE, 144 ("intr->type != EFX_INTR_MESSAGE")); 145 146 if (__predict_false(intr->state != SFXGE_INTR_STARTED)) 147 return; 148 149 (void)efx_intr_status_message(enp, index, &fatal); 150 151 if (fatal) { 152 (void)efx_intr_disable(enp); 153 (void)efx_intr_fatal(enp); 154 return; 155 } 156 157 (void)sfxge_ev_qpoll(evq); 158 } 159 160 static int 161 sfxge_intr_bus_enable(struct sfxge_softc *sc) 162 { 163 struct sfxge_intr *intr; 164 struct sfxge_intr_hdl *table; 165 driver_filter_t *filter; 166 driver_intr_t *handler; 167 int index; 168 int err; 169 170 intr = &sc->intr; 171 table = intr->table; 172 173 switch (intr->type) { 174 case EFX_INTR_MESSAGE: 175 filter = NULL; /* not shared */ 176 handler = sfxge_intr_message; 177 break; 178 179 case EFX_INTR_LINE: 180 filter = sfxge_intr_line_filter; 181 handler = sfxge_intr_line; 182 break; 183 184 default: 185 KASSERT(0, ("Invalid interrupt type")); 186 return (EINVAL); 187 } 188 189 /* Try to add the handlers */ 190 for (index = 0; index < intr->n_alloc; index++) { 191 if ((err = bus_setup_intr(sc->dev, table[index].eih_res, 192 INTR_MPSAFE|INTR_TYPE_NET, filter, handler, 193 sc->evq[index], &table[index].eih_tag)) != 0) { 194 goto fail; 195 } 196 if (intr->n_alloc > 1) 197 bus_describe_intr(sc->dev, table[index].eih_res, 198 table[index].eih_tag, "%d", index); 199 #ifdef RSS 200 bus_bind_intr(sc->dev, table[index].eih_res, 201 rss_getcpu(index)); 202 #else 203 bus_bind_intr(sc->dev, table[index].eih_res, index); 204 #endif 205 } 206 207 return (0); 208 209 fail: 210 /* Remove remaining handlers */ 211 while (--index >= 0) 212 bus_teardown_intr(sc->dev, table[index].eih_res, 213 table[index].eih_tag); 214 215 return (err); 216 } 217 218 static void 219 sfxge_intr_bus_disable(struct sfxge_softc *sc) 220 { 221 struct sfxge_intr *intr; 222 struct sfxge_intr_hdl *table; 223 int i; 224 225 intr = &sc->intr; 226 table = intr->table; 227 228 /* Remove all handlers */ 229 for (i = 0; i < intr->n_alloc; i++) 230 bus_teardown_intr(sc->dev, table[i].eih_res, 231 table[i].eih_tag); 232 } 233 234 static int 235 sfxge_intr_alloc(struct sfxge_softc *sc, int count) 236 { 237 device_t dev; 238 struct sfxge_intr_hdl *table; 239 struct sfxge_intr *intr; 240 struct resource *res; 241 int rid; 242 int error; 243 int i; 244 245 dev = sc->dev; 246 intr = &sc->intr; 247 error = 0; 248 249 table = malloc(count * sizeof(struct sfxge_intr_hdl), 250 M_SFXGE, M_WAITOK); 251 intr->table = table; 252 253 for (i = 0; i < count; i++) { 254 rid = i + 1; 255 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 256 RF_SHAREABLE | RF_ACTIVE); 257 if (res == NULL) { 258 device_printf(dev, "Couldn't allocate interrupts for " 259 "message %d\n", rid); 260 error = ENOMEM; 261 break; 262 } 263 table[i].eih_rid = rid; 264 table[i].eih_res = res; 265 } 266 267 if (error != 0) { 268 count = i - 1; 269 for (i = 0; i < count; i++) 270 bus_release_resource(dev, SYS_RES_IRQ, 271 table[i].eih_rid, table[i].eih_res); 272 } 273 274 return (error); 275 } 276 277 static void 278 sfxge_intr_teardown_msix(struct sfxge_softc *sc) 279 { 280 device_t dev; 281 struct resource *resp; 282 int rid; 283 284 dev = sc->dev; 285 resp = sc->intr.msix_res; 286 287 rid = rman_get_rid(resp); 288 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 289 } 290 291 static int 292 sfxge_intr_setup_msix(struct sfxge_softc *sc) 293 { 294 struct sfxge_intr *intr; 295 struct resource *resp; 296 device_t dev; 297 int count; 298 int rid; 299 300 dev = sc->dev; 301 intr = &sc->intr; 302 303 /* Check if MSI-X is available. */ 304 count = pci_msix_count(dev); 305 if (count == 0) 306 return (EINVAL); 307 308 /* Do not try to allocate more than already estimated EVQ maximum */ 309 KASSERT(sc->evq_max > 0, ("evq_max is zero")); 310 count = MIN(count, sc->evq_max); 311 312 rid = PCIR_BAR(4); 313 resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE); 314 if (resp == NULL) 315 return (ENOMEM); 316 317 if (pci_alloc_msix(dev, &count) != 0) { 318 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 319 return (ENOMEM); 320 } 321 322 /* Allocate interrupt handlers. */ 323 if (sfxge_intr_alloc(sc, count) != 0) { 324 bus_release_resource(dev, SYS_RES_MEMORY, rid, resp); 325 pci_release_msi(dev); 326 return (ENOMEM); 327 } 328 329 intr->type = EFX_INTR_MESSAGE; 330 intr->n_alloc = count; 331 intr->msix_res = resp; 332 333 return (0); 334 } 335 336 static int 337 sfxge_intr_setup_msi(struct sfxge_softc *sc) 338 { 339 struct sfxge_intr *intr; 340 device_t dev; 341 int count; 342 int error; 343 344 dev = sc->dev; 345 intr = &sc->intr; 346 347 /* 348 * Check if MSI is available. All messages must be written to 349 * the same address and on x86 this means the IRQs have the 350 * same CPU affinity. So we only ever allocate 1. 351 */ 352 count = pci_msi_count(dev) ? 1 : 0; 353 if (count == 0) 354 return (EINVAL); 355 356 if ((error = pci_alloc_msi(dev, &count)) != 0) 357 return (ENOMEM); 358 359 /* Allocate interrupt handler. */ 360 if (sfxge_intr_alloc(sc, count) != 0) { 361 pci_release_msi(dev); 362 return (ENOMEM); 363 } 364 365 intr->type = EFX_INTR_MESSAGE; 366 intr->n_alloc = count; 367 368 return (0); 369 } 370 371 static int 372 sfxge_intr_setup_fixed(struct sfxge_softc *sc) 373 { 374 struct sfxge_intr_hdl *table; 375 struct sfxge_intr *intr; 376 struct resource *res; 377 device_t dev; 378 int rid; 379 380 dev = sc->dev; 381 intr = &sc->intr; 382 383 rid = 0; 384 res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 385 RF_SHAREABLE | RF_ACTIVE); 386 if (res == NULL) 387 return (ENOMEM); 388 389 table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK); 390 table[0].eih_rid = rid; 391 table[0].eih_res = res; 392 393 intr->type = EFX_INTR_LINE; 394 intr->n_alloc = 1; 395 intr->table = table; 396 397 return (0); 398 } 399 400 static const char *const __sfxge_err[] = { 401 "", 402 "SRAM out-of-bounds", 403 "Buffer ID out-of-bounds", 404 "Internal memory parity", 405 "Receive buffer ownership", 406 "Transmit buffer ownership", 407 "Receive descriptor ownership", 408 "Transmit descriptor ownership", 409 "Event queue ownership", 410 "Event queue FIFO overflow", 411 "Illegal address", 412 "SRAM parity" 413 }; 414 415 void 416 sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0, 417 uint32_t dword1) 418 { 419 struct sfxge_softc *sc = (struct sfxge_softc *)arg; 420 device_t dev = sc->dev; 421 422 log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)", 423 device_get_name(dev), device_get_unit(dev), 424 __sfxge_err[code], dword1, dword0); 425 } 426 427 void 428 sfxge_intr_stop(struct sfxge_softc *sc) 429 { 430 struct sfxge_intr *intr; 431 432 intr = &sc->intr; 433 434 KASSERT(intr->state == SFXGE_INTR_STARTED, 435 ("Interrupts not started")); 436 437 intr->state = SFXGE_INTR_INITIALIZED; 438 439 /* Disable interrupts at the NIC */ 440 efx_intr_disable(sc->enp); 441 442 /* Disable interrupts at the bus */ 443 sfxge_intr_bus_disable(sc); 444 445 /* Tear down common code interrupt bits. */ 446 efx_intr_fini(sc->enp); 447 } 448 449 int 450 sfxge_intr_start(struct sfxge_softc *sc) 451 { 452 struct sfxge_intr *intr; 453 efsys_mem_t *esmp; 454 int rc; 455 456 intr = &sc->intr; 457 esmp = &intr->status; 458 459 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 460 ("Interrupts not initialized")); 461 462 /* Zero the memory. */ 463 (void)memset(esmp->esm_base, 0, EFX_INTR_SIZE); 464 465 /* Initialize common code interrupt bits. */ 466 (void)efx_intr_init(sc->enp, intr->type, esmp); 467 468 /* Enable interrupts at the bus */ 469 if ((rc = sfxge_intr_bus_enable(sc)) != 0) 470 goto fail; 471 472 intr->state = SFXGE_INTR_STARTED; 473 474 /* Enable interrupts at the NIC */ 475 efx_intr_enable(sc->enp); 476 477 return (0); 478 479 fail: 480 /* Tear down common code interrupt bits. */ 481 efx_intr_fini(sc->enp); 482 483 intr->state = SFXGE_INTR_INITIALIZED; 484 485 return (rc); 486 } 487 488 void 489 sfxge_intr_fini(struct sfxge_softc *sc) 490 { 491 struct sfxge_intr_hdl *table; 492 struct sfxge_intr *intr; 493 efsys_mem_t *esmp; 494 device_t dev; 495 int i; 496 497 dev = sc->dev; 498 intr = &sc->intr; 499 esmp = &intr->status; 500 table = intr->table; 501 502 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 503 ("intr->state != SFXGE_INTR_INITIALIZED")); 504 505 /* Free DMA memory. */ 506 sfxge_dma_free(esmp); 507 508 /* Free interrupt handles. */ 509 for (i = 0; i < intr->n_alloc; i++) 510 bus_release_resource(dev, SYS_RES_IRQ, 511 table[i].eih_rid, table[i].eih_res); 512 513 if (table[0].eih_rid != 0) 514 pci_release_msi(dev); 515 516 if (intr->msix_res != NULL) 517 sfxge_intr_teardown_msix(sc); 518 519 /* Free the handle table */ 520 free(table, M_SFXGE); 521 intr->table = NULL; 522 intr->n_alloc = 0; 523 524 /* Clear the interrupt type */ 525 intr->type = EFX_INTR_INVALID; 526 527 intr->state = SFXGE_INTR_UNINITIALIZED; 528 } 529 530 int 531 sfxge_intr_init(struct sfxge_softc *sc) 532 { 533 device_t dev; 534 struct sfxge_intr *intr; 535 efsys_mem_t *esmp; 536 int rc; 537 538 dev = sc->dev; 539 intr = &sc->intr; 540 esmp = &intr->status; 541 542 KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED, 543 ("Interrupts already initialized")); 544 545 /* Try to setup MSI-X or MSI interrupts if available. */ 546 if ((rc = sfxge_intr_setup_msix(sc)) == 0) 547 device_printf(dev, "Using MSI-X interrupts\n"); 548 else if ((rc = sfxge_intr_setup_msi(sc)) == 0) 549 device_printf(dev, "Using MSI interrupts\n"); 550 else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) { 551 device_printf(dev, "Using fixed interrupts\n"); 552 } else { 553 device_printf(dev, "Couldn't setup interrupts\n"); 554 return (ENOMEM); 555 } 556 557 /* Set up DMA for interrupts. */ 558 if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0) 559 return (ENOMEM); 560 561 intr->state = SFXGE_INTR_INITIALIZED; 562 563 return (0); 564 } 565