dev/sfxge/sfxge_intr.c

/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2010-2016 Solarflare Communications Inc.
 * All rights reserved.
 *
 * This software was developed in part by Philip Paeps under contract for
 * Solarflare Communications, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 *
 * The views and conclusions contained in the software and documentation are
 * those of the authors and should not be interpreted as representing official
 * policies, either expressed or implied, of the FreeBSD Project.
 */

#include <sys/cdefs.h>
#include "opt_rss.h"

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/malloc.h>
#include <sys/queue.h>
#include <sys/rman.h>
#include <sys/syslog.h>
#include <sys/taskqueue.h>

#include <machine/bus.h>
#include <machine/resource.h>

#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>

#ifdef RSS
#include <net/rss_config.h>
#endif

#include "common/efx.h"

#include "sfxge.h"

static int
sfxge_intr_line_filter(void *arg)
{
	struct sfxge_evq *evq;
	struct sfxge_softc *sc;
	efx_nic_t *enp;
	struct sfxge_intr *intr;
	boolean_t fatal;
	uint32_t qmask;

	evq = (struct sfxge_evq *)arg;
	sc = evq->sc;
	enp = sc->enp;
	intr = &sc->intr;

	KASSERT(intr != NULL, ("intr == NULL"));
	KASSERT(intr->type == EFX_INTR_LINE,
	    ("intr->type != EFX_INTR_LINE"));

	if (intr->state != SFXGE_INTR_STARTED)
		return (FILTER_STRAY);

	(void)efx_intr_status_line(enp, &fatal, &qmask);

	if (fatal) {
		(void) efx_intr_disable(enp);
		(void) efx_intr_fatal(enp);
		return (FILTER_HANDLED);
	}

	if (qmask != 0) {
		intr->zero_count = 0;
		return (FILTER_SCHEDULE_THREAD);
	}

	/* SF bug 15783: If the function is not asserting its IRQ and
	 * we read the queue mask on the cycle before a flag is added
	 * to the mask, this inhibits the function from asserting the
	 * IRQ even though we don't see the flag set.  To work around
	 * this, we must re-prime all event queues and report the IRQ
	 * as handled when we see a mask of zero.  To allow for shared
	 * IRQs, we don't repeat this if we see a mask of zero twice
	 * or more in a row.
	 */
	if (intr->zero_count++ == 0) {
		if (evq->init_state == SFXGE_EVQ_STARTED) {
			if (efx_ev_qpending(evq->common, evq->read_ptr))
				return (FILTER_SCHEDULE_THREAD);
			efx_ev_qprime(evq->common, evq->read_ptr);
			return (FILTER_HANDLED);
		}
	}

	return (FILTER_STRAY);
}

static void
sfxge_intr_line(void *arg)
{
	struct sfxge_evq *evq = arg;

	(void)sfxge_ev_qpoll(evq);
}

static void
sfxge_intr_message(void *arg)
{
	struct sfxge_evq *evq;
	struct sfxge_softc *sc;
	efx_nic_t *enp;
	struct sfxge_intr *intr;
	unsigned int index;
	boolean_t fatal;

	evq = (struct sfxge_evq *)arg;
	sc = evq->sc;
	enp = sc->enp;
	intr = &sc->intr;
	index = evq->index;

	KASSERT(intr != NULL, ("intr == NULL"));
	KASSERT(intr->type == EFX_INTR_MESSAGE,
	    ("intr->type != EFX_INTR_MESSAGE"));

	if (__predict_false(intr->state != SFXGE_INTR_STARTED))
		return;

	(void)efx_intr_status_message(enp, index, &fatal);

	if (fatal) {
		(void)efx_intr_disable(enp);
		(void)efx_intr_fatal(enp);
		return;
	}

	(void)sfxge_ev_qpoll(evq);
}

static int
sfxge_intr_bus_enable(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;
	struct sfxge_intr_hdl *table;
	driver_filter_t *filter;
	driver_intr_t *handler;
	int index;
	int err;

	intr = &sc->intr;
	table = intr->table;

	switch (intr->type) {
	case EFX_INTR_MESSAGE:
		filter = NULL; /* not shared */
		handler = sfxge_intr_message;
		break;

	case EFX_INTR_LINE:
		filter = sfxge_intr_line_filter;
		handler = sfxge_intr_line;
		break;

	default:
		KASSERT(0, ("Invalid interrupt type"));
		return (EINVAL);
	}

	/* Try to add the handlers */
	for (index = 0; index < intr->n_alloc; index++) {
		if ((err = bus_setup_intr(sc->dev, table[index].eih_res,
			    INTR_MPSAFE|INTR_TYPE_NET, filter, handler,
			    sc->evq[index], &table[index].eih_tag)) != 0) {
			goto fail;
		}
		if (intr->n_alloc > 1)
			bus_describe_intr(sc->dev, table[index].eih_res,
			    table[index].eih_tag, "%d", index);
#ifdef RSS
		bus_bind_intr(sc->dev, table[index].eih_res,
			      rss_getcpu(index));
#else
		bus_bind_intr(sc->dev, table[index].eih_res, index);
#endif
	}

	return (0);

fail:
	/* Remove remaining handlers */
	while (--index >= 0)
		bus_teardown_intr(sc->dev, table[index].eih_res,
		    table[index].eih_tag);

	return (err);
}

static void
sfxge_intr_bus_disable(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;
	struct sfxge_intr_hdl *table;
	int i;

	intr = &sc->intr;
	table = intr->table;

	/* Remove all handlers */
	for (i = 0; i < intr->n_alloc; i++)
		bus_teardown_intr(sc->dev, table[i].eih_res,
		    table[i].eih_tag);
}

static int
sfxge_intr_alloc(struct sfxge_softc *sc, int count)
{
	device_t dev;
	struct sfxge_intr_hdl *table;
	struct sfxge_intr *intr;
	struct resource *res;
	int rid;
	int error;
	int i;

	dev = sc->dev;
	intr = &sc->intr;
	error = 0;

	table = malloc(count * sizeof(struct sfxge_intr_hdl),
	    M_SFXGE, M_WAITOK);
	intr->table = table;

	for (i = 0; i < count; i++) {
		rid = i + 1;
		res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
		    RF_SHAREABLE | RF_ACTIVE);
		if (res == NULL) {
			device_printf(dev, "Couldn't allocate interrupts for "
			    "message %d\n", rid);
			error = ENOMEM;
			break;
		}
		table[i].eih_rid = rid;
		table[i].eih_res = res;
	}

	if (error != 0) {
		count = i - 1;
		for (i = 0; i < count; i++)
			bus_release_resource(dev, SYS_RES_IRQ,
			    table[i].eih_rid, table[i].eih_res);
	}

	return (error);
}

static void
sfxge_intr_teardown_msix(struct sfxge_softc *sc)
{
	device_t dev;
	struct resource *resp;
	int rid;

	dev = sc->dev;
	resp = sc->intr.msix_res;

	rid = rman_get_rid(resp);
	bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
}

static int
sfxge_intr_setup_msix(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;
	struct resource *resp;
	device_t dev;
	int count;
	int rid;

	dev = sc->dev;
	intr = &sc->intr;

	/* Check if MSI-X is available. */
	count = pci_msix_count(dev);
	if (count == 0)
		return (EINVAL);

	/* Do not try to allocate more than already estimated EVQ maximum */
	KASSERT(sc->evq_max > 0, ("evq_max is zero"));
	count = MIN(count, sc->evq_max);

	rid = PCIR_BAR(4);
	resp = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, RF_ACTIVE);
	if (resp == NULL)
		return (ENOMEM);

	if (pci_alloc_msix(dev, &count) != 0) {
		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
		return (ENOMEM);
	}

	/* Allocate interrupt handlers. */
	if (sfxge_intr_alloc(sc, count) != 0) {
		bus_release_resource(dev, SYS_RES_MEMORY, rid, resp);
		pci_release_msi(dev);
		return (ENOMEM);
	}

	intr->type = EFX_INTR_MESSAGE;
	intr->n_alloc = count;
	intr->msix_res = resp;

	return (0);
}

static int
sfxge_intr_setup_msi(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;
	device_t dev;
	int count;
	int error;

	dev = sc->dev;
	intr = &sc->intr;

	/*
	 * Check if MSI is available.  All messages must be written to
	 * the same address and on x86 this means the IRQs have the
	 * same CPU affinity.  So we only ever allocate 1.
	 */
	count = pci_msi_count(dev) ? 1 : 0;
	if (count == 0)
		return (EINVAL);

	if ((error = pci_alloc_msi(dev, &count)) != 0)
		return (ENOMEM);

	/* Allocate interrupt handler. */
	if (sfxge_intr_alloc(sc, count) != 0) {
		pci_release_msi(dev);
		return (ENOMEM);
	}

	intr->type = EFX_INTR_MESSAGE;
	intr->n_alloc = count;

	return (0);
}

static int
sfxge_intr_setup_fixed(struct sfxge_softc *sc)
{
	struct sfxge_intr_hdl *table;
	struct sfxge_intr *intr;
	struct resource *res;
	device_t dev;
	int rid;

	dev = sc->dev;
	intr = &sc->intr;

	rid = 0;
	res =  bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
	    RF_SHAREABLE | RF_ACTIVE);
	if (res == NULL)
		return (ENOMEM);

	table = malloc(sizeof(struct sfxge_intr_hdl), M_SFXGE, M_WAITOK);
	table[0].eih_rid = rid;
	table[0].eih_res = res;

	intr->type = EFX_INTR_LINE;
	intr->n_alloc = 1;
	intr->table = table;

	return (0);
}

static const char *const __sfxge_err[] = {
	"",
	"SRAM out-of-bounds",
	"Buffer ID out-of-bounds",
	"Internal memory parity",
	"Receive buffer ownership",
	"Transmit buffer ownership",
	"Receive descriptor ownership",
	"Transmit descriptor ownership",
	"Event queue ownership",
	"Event queue FIFO overflow",
	"Illegal address",
	"SRAM parity"
};

void
sfxge_err(efsys_identifier_t *arg, unsigned int code, uint32_t dword0,
	  uint32_t dword1)
{
	struct sfxge_softc *sc = (struct sfxge_softc *)arg;
	device_t dev = sc->dev;

	log(LOG_WARNING, "[%s%d] FATAL ERROR: %s (0x%08x%08x)",
	    device_get_name(dev), device_get_unit(dev),
		__sfxge_err[code], dword1, dword0);
}

void
sfxge_intr_stop(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;

	intr = &sc->intr;

	KASSERT(intr->state == SFXGE_INTR_STARTED,
	    ("Interrupts not started"));

	intr->state = SFXGE_INTR_INITIALIZED;

	/* Disable interrupts at the NIC */
	efx_intr_disable(sc->enp);

	/* Disable interrupts at the bus */
	sfxge_intr_bus_disable(sc);

	/* Tear down common code interrupt bits. */
	efx_intr_fini(sc->enp);
}

int
sfxge_intr_start(struct sfxge_softc *sc)
{
	struct sfxge_intr *intr;
	efsys_mem_t *esmp;
	int rc;

	intr = &sc->intr;
	esmp = &intr->status;

	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
	    ("Interrupts not initialized"));

	/* Zero the memory. */
	(void)memset(esmp->esm_base, 0, EFX_INTR_SIZE);

	/* Initialize common code interrupt bits. */
	(void)efx_intr_init(sc->enp, intr->type, esmp);

	/* Enable interrupts at the bus */
	if ((rc = sfxge_intr_bus_enable(sc)) != 0)
		goto fail;

	intr->state = SFXGE_INTR_STARTED;

	/* Enable interrupts at the NIC */
	efx_intr_enable(sc->enp);

	return (0);

fail:
	/* Tear down common code interrupt bits. */
	efx_intr_fini(sc->enp);

	intr->state = SFXGE_INTR_INITIALIZED;

	return (rc);
}

void
sfxge_intr_fini(struct sfxge_softc *sc)
{
	struct sfxge_intr_hdl *table;
	struct sfxge_intr *intr;
	efsys_mem_t *esmp;
	device_t dev;
	int i;

	dev = sc->dev;
	intr = &sc->intr;
	esmp = &intr->status;
	table = intr->table;

	KASSERT(intr->state == SFXGE_INTR_INITIALIZED,
	    ("intr->state != SFXGE_INTR_INITIALIZED"));

	/* Free DMA memory. */
	sfxge_dma_free(esmp);

	/* Free interrupt handles. */
	for (i = 0; i < intr->n_alloc; i++)
		bus_release_resource(dev, SYS_RES_IRQ,
		    table[i].eih_rid, table[i].eih_res);

	if (table[0].eih_rid != 0)
		pci_release_msi(dev);

	if (intr->msix_res != NULL)
		sfxge_intr_teardown_msix(sc);

	/* Free the handle table */
	free(table, M_SFXGE);
	intr->table = NULL;
	intr->n_alloc = 0;

	/* Clear the interrupt type */
	intr->type = EFX_INTR_INVALID;

	intr->state = SFXGE_INTR_UNINITIALIZED;
}

int
sfxge_intr_init(struct sfxge_softc *sc)
{
	device_t dev;
	struct sfxge_intr *intr;
	efsys_mem_t *esmp;
	int rc;

	dev = sc->dev;
	intr = &sc->intr;
	esmp = &intr->status;

	KASSERT(intr->state == SFXGE_INTR_UNINITIALIZED,
	    ("Interrupts already initialized"));

	/* Try to setup MSI-X or MSI interrupts if available. */
	if ((rc = sfxge_intr_setup_msix(sc)) == 0)
		device_printf(dev, "Using MSI-X interrupts\n");
	else if ((rc = sfxge_intr_setup_msi(sc)) == 0)
		device_printf(dev, "Using MSI interrupts\n");
	else if ((rc = sfxge_intr_setup_fixed(sc)) == 0) {
		device_printf(dev, "Using fixed interrupts\n");
	} else {
		device_printf(dev, "Couldn't setup interrupts\n");
		return (ENOMEM);
	}

	/* Set up DMA for interrupts. */
	if ((rc = sfxge_dma_alloc(sc, EFX_INTR_SIZE, esmp)) != 0)
		return (ENOMEM);

	intr->state = SFXGE_INTR_INITIALIZED;

	return (0);
}