xref: /freebsd/sys/dev/dpaa2/dpaa2_swp.c (revision 58983e4b0253ad38a3e1ef2166fedd3133fdb552)

/*-
 * SPDX-License-Identifier: BSD-3-Clause AND BSD-2-Clause
 *
 * Copyright © 2014-2016 Freescale Semiconductor, Inc.
 * Copyright © 2016-2019 NXP
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice,
 *    this list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its
 *    contributors may be used to endorse or promote products derived from this
 *    software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Original source file obtained from:
 * drivers/soc/fsl/dpio/qbman-portal.c
 *
 * Commit: 4c86114194e644b6da9107d75910635c9e87179e
 * Repository: https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
 */

/*
 * Copyright © 2021-2022 Dmitry Salychev
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#include <sys/cdefs.h>
/*
 * DPAA2 QBMan software portal.
 */

#include <sys/param.h>
#include <sys/kernel.h>
#include <sys/bus.h>
#include <sys/rman.h>
#include <sys/module.h>
#include <sys/malloc.h>
#include <sys/mutex.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/systm.h>
#include <sys/lock.h>

#include <machine/bus.h>
#include <machine/resource.h>
#include <machine/atomic.h>

#include "pcib_if.h"
#include "pci_if.h"

#include "dpaa2_swp.h"
#include "dpaa2_mc.h"
#include "dpaa2_bp.h"

#define CMD_SPIN_TIMEOUT		100u	/* us */
#define CMD_SPIN_ATTEMPTS		2000u	/* 200 ms max. */

#define CMD_VERB_MASK			0x7Fu

/* Shifts in the VERB byte of the enqueue command descriptor. */
#define ENQ_CMD_ORP_ENABLE_SHIFT	2
#define ENQ_CMD_IRQ_ON_DISPATCH_SHIFT	3
#define ENQ_CMD_TARGET_TYPE_SHIFT	4
#define ENQ_CMD_DCA_EN_SHIFT		7
/* VERB byte options of the enqueue command descriptor. */
#define ENQ_CMD_EMPTY			0u
#define ENQ_CMD_RESPONSE_ALWAYS		1u
#define ENQ_CMD_REJECTS_TO_FQ		2u

#define ENQ_DESC_FD_OFFSET		32u

#define ENQ_DCA_IDXMASK			0x0Fu
#define ENQ_FLAG_DCA			(1ull << 31)

/* QBMan portal command codes. */
#define CMDID_SWP_MC_ACQUIRE		0x30
#define CMDID_SWP_BP_QUERY		0x32
#define CMDID_SWP_WQCHAN_CONFIGURE	0x46

/* QBMan portal command result codes. */
#define QBMAN_CMD_RC_OK			0xF0

/* SDQCR attribute codes */
#define QB_SDQCR_FC_SHIFT 		29u
#define QB_SDQCR_FC_MASK		0x1u
#define QB_SDQCR_DCT_SHIFT		24u
#define QB_SDQCR_DCT_MASK		0x3u
#define QB_SDQCR_TOK_SHIFT		16u
#define QB_SDQCR_TOK_MASK		0xFFu
#define QB_SDQCR_SRC_SHIFT		0u
#define QB_SDQCR_SRC_MASK		0xFFFFu

/* Shifts in the VERB byte of the volatile dequeue command. */
#define QB_VDQCR_VERB_DCT0_SHIFT	0
#define QB_VDQCR_VERB_DCT1_SHIFT	1
#define QB_VDQCR_VERB_DT0_SHIFT		2
#define QB_VDQCR_VERB_DT1_SHIFT		3
#define QB_VDQCR_VERB_RLS_SHIFT		4
#define QB_VDQCR_VERB_WAE_SHIFT		5
#define QB_VDQCR_VERB_RAD_SHIFT		6

/* Maximum timeout period for the DQRR interrupt. */
#define DQRR_MAX_ITP			4096u
#define DQRR_PI_MASK			0x0Fu

/* Release Array Allocation register helpers. */
#define RAR_IDX(rar)			((rar) & 0x7u)
#define RAR_VB(rar)			((rar) & 0x80u)
#define RAR_SUCCESS(rar)		((rar) & 0x100u)

MALLOC_DEFINE(M_DPAA2_SWP, "dpaa2_swp", "DPAA2 QBMan Software Portal");

enum qbman_sdqcr_dct {
	qbman_sdqcr_dct_null = 0,
	qbman_sdqcr_dct_prio_ics,
	qbman_sdqcr_dct_active_ics,
	qbman_sdqcr_dct_active
};

enum qbman_sdqcr_fc {
	qbman_sdqcr_fc_one = 0,
	qbman_sdqcr_fc_up_to_3 = 1
};

/* Routines to execute software portal commands. */
static int dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *,
    struct dpaa2_swp_cmd *, struct dpaa2_swp_rsp *, uint8_t);
static int dpaa2_swp_exec_br_command(struct dpaa2_swp *, struct dpaa2_swp_cmd *,
    uint32_t);
static int dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *,
    struct dpaa2_swp_cmd *);

/* Management Commands helpers. */
static int dpaa2_swp_send_mgmt_command(struct dpaa2_swp *,
    struct dpaa2_swp_cmd *, uint8_t);
static int dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *,
    struct dpaa2_swp_rsp *);

/* Helper subroutines. */
static int dpaa2_swp_cyc_diff(uint8_t, uint8_t, uint8_t);

int
dpaa2_swp_init_portal(struct dpaa2_swp **swp, struct dpaa2_swp_desc *desc,
    uint16_t flags)
{
	struct dpaa2_swp *p;
	uint32_t reg, mask_size, eqcr_pi; /* EQCR producer index */

	if (!swp || !desc)
		return (DPAA2_SWP_STAT_EINVAL);

	p = malloc(sizeof(struct dpaa2_swp), M_DPAA2_SWP,
	    flags & DPAA2_SWP_NOWAIT_ALLOC
	    ? (M_NOWAIT | M_ZERO)
	    : (M_WAITOK | M_ZERO));
	if (!p)
		return (DPAA2_SWP_STAT_NO_MEMORY);

	mtx_init(&p->lock, "swp_sleep_lock", NULL, MTX_DEF);

	p->cfg.mem_backed = false;
	p->cfg.writes_cinh = true;

	p->desc = desc;
	p->flags = flags;
	p->mc.valid_bit = DPAA2_SWP_VALID_BIT;
	p->mr.valid_bit = DPAA2_SWP_VALID_BIT;

	/* FIXME: Memory-backed mode doesn't work now. Why? */
	p->cena_res = desc->cena_res;
	p->cena_map = desc->cena_map;
	p->cinh_res = desc->cinh_res;
	p->cinh_map = desc->cinh_map;

	/* Static Dequeue Command Register configuration. */
	p->sdq = 0;
	p->sdq |= qbman_sdqcr_dct_prio_ics << QB_SDQCR_DCT_SHIFT;
	p->sdq |= qbman_sdqcr_fc_up_to_3 << QB_SDQCR_FC_SHIFT;
	p->sdq |= DPAA2_SWP_SDQCR_TOKEN << QB_SDQCR_TOK_SHIFT;

	/* Volatile Dequeue Command configuration. */
	p->vdq.valid_bit = DPAA2_SWP_VALID_BIT;

	/* Dequeue Response Ring configuration */
	p->dqrr.next_idx = 0;
	p->dqrr.valid_bit = DPAA2_SWP_VALID_BIT;
	if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_4100) {
		p->dqrr.ring_size = 4;
		p->dqrr.reset_bug = 1;
	} else {
		p->dqrr.ring_size = 8;
		p->dqrr.reset_bug = 0;
	}

	if ((desc->swp_version & DPAA2_SWP_REV_MASK) < DPAA2_SWP_REV_5000) {
		reg = dpaa2_swp_set_cfg(
		    p->dqrr.ring_size, /* max. entries QMan writes to DQRR */
		    1, /* writes enabled in the CINH memory only */
		    0, /* EQCR_CI stashing threshold */
		    3, /* RPM: RCR in array mode */
		    2, /* DCM: Discrete consumption ack */
		    2, /* EPM: EQCR in ring mode (FIFO) */
		    1, /* mem stashing drop enable enable */
		    1, /* mem stashing priority enable */
		    1, /* mem stashing enable */
		    1, /* dequeue stashing priority enable */
		    0, /* dequeue stashing enable enable */
		    0  /* EQCR_CI stashing priority enable */
		);
		reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
	} else {
		bus_set_region_4(p->cena_map, 0, 0,
		    rman_get_size(p->cena_res) / 4);

		reg = dpaa2_swp_set_cfg(
		    p->dqrr.ring_size, /* max. entries QMan writes to DQRR */					/* DQRR_MF */
		    1, /* writes enabled in the CINH memory only */						/* WN */
		    0, /* EQCR_CI stashing is disabled */							/* EST */
		    3, /* RPM: RCR in array mode */								/* RPM */
		    2, /* DCM: Discrete consumption ack */							/* DCM */
		    2, /* EPM: EQCR in ring mode (FIFO) */							/* EPM */
		    1, /* Dequeued frame data, annotation, and FQ context stashing drop enable */		/* SD */
		    1, /* Dequeued frame data, annotation, and FQ context stashing priority */			/* SP */
		    1, /* Dequeued frame data, annotation, and FQ context stashing enable */			/* SE */
		    1, /* Dequeue response ring (DQRR) entry stashing priority */				/* DP */
		    0, /* Dequeue response ring (DQRR) entry, or cacheable portal area, stashing enable. */	/* DE */
		    0  /* EQCR_CI stashing priority */								/* EP */
		);
		/* TODO: Switch to memory-backed mode. */
		reg &= ~(1 << DPAA2_SWP_CFG_CPBS_SHIFT); /* QMan-backed mode */
	}
	dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_CFG, reg);
	reg = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_CFG);
	if (!reg) {
		free(p, M_DPAA2_SWP);
		return (DPAA2_SWP_STAT_PORTAL_DISABLED);
	}

	/*
	 * Static Dequeue Command Register needs to be initialized to 0 when no
	 * channels are being dequeued from or else the QMan HW will indicate an
	 * error. The values that were calculated above will be applied when
	 * dequeues from a specific channel are enabled.
	 */
	dpaa2_swp_write_reg(p, DPAA2_SWP_CINH_SDQCR, 0);

	p->eqcr.pi_ring_size = 8;
	/* if ((desc->swp_version & DPAA2_SWP_REV_MASK) >= DPAA2_SWP_REV_5000) */
	/* 	p->eqcr.pi_ring_size = 32; */

	for (mask_size = p->eqcr.pi_ring_size; mask_size > 0; mask_size >>= 1)
		p->eqcr.pi_ci_mask = (p->eqcr.pi_ci_mask << 1) + 1;

	eqcr_pi = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_PI);
	p->eqcr.pi = eqcr_pi & p->eqcr.pi_ci_mask;
	p->eqcr.pi_vb = eqcr_pi & DPAA2_SWP_VALID_BIT;
	p->eqcr.ci = dpaa2_swp_read_reg(p, DPAA2_SWP_CINH_EQCR_CI)
	    & p->eqcr.pi_ci_mask;
	p->eqcr.available = p->eqcr.pi_ring_size;

	/* TODO: sysctl(9) for the IRQ timeout? */
	/* Initialize the portal with an IRQ threshold and timeout of 120us. */
	dpaa2_swp_set_irq_coalescing(p, p->dqrr.ring_size - 1, 120);

	*swp = p;

	return (0);
}

void
dpaa2_swp_free_portal(struct dpaa2_swp *swp)
{
	uint16_t flags;

	KASSERT(swp != NULL, ("%s: swp is NULL", __func__));

	DPAA2_SWP_LOCK(swp, &flags);
	swp->flags |= DPAA2_SWP_DESTROYED;
	DPAA2_SWP_UNLOCK(swp);

	/* Let threads stop using this portal. */
	DELAY(DPAA2_SWP_TIMEOUT);

	mtx_destroy(&swp->lock);
	free(swp, M_DPAA2_SWP);
}

uint32_t
dpaa2_swp_set_cfg(uint8_t max_fill, uint8_t wn, uint8_t est, uint8_t rpm,
    uint8_t dcm, uint8_t epm, int sd, int sp, int se, int dp, int de, int ep)
{
	return (
	    max_fill	<< DPAA2_SWP_CFG_DQRR_MF_SHIFT |
	    est		<< DPAA2_SWP_CFG_EST_SHIFT |
	    wn		<< DPAA2_SWP_CFG_WN_SHIFT |
	    rpm		<< DPAA2_SWP_CFG_RPM_SHIFT |
	    dcm		<< DPAA2_SWP_CFG_DCM_SHIFT |
	    epm		<< DPAA2_SWP_CFG_EPM_SHIFT |
	    sd		<< DPAA2_SWP_CFG_SD_SHIFT |
	    sp		<< DPAA2_SWP_CFG_SP_SHIFT |
	    se		<< DPAA2_SWP_CFG_SE_SHIFT |
	    dp		<< DPAA2_SWP_CFG_DP_SHIFT |
	    de		<< DPAA2_SWP_CFG_DE_SHIFT |
	    ep		<< DPAA2_SWP_CFG_EP_SHIFT
	);
}

/* Read/write registers of a software portal. */

void
dpaa2_swp_write_reg(struct dpaa2_swp *swp, uint32_t o, uint32_t v)
{
	bus_write_4(swp->cinh_map, o, v);
}

uint32_t
dpaa2_swp_read_reg(struct dpaa2_swp *swp, uint32_t o)
{
	return (bus_read_4(swp->cinh_map, o));
}

/* Helper routines. */

/**
 * @brief Set enqueue descriptor without Order Point Record ID.
 *
 * ed:		Enqueue descriptor.
 * resp_always:	Enqueue with response always (1); FD from a rejected enqueue
 *		will be returned on a FQ (0).
 */
void
dpaa2_swp_set_ed_norp(struct dpaa2_eq_desc *ed, bool resp_always)
{
	ed->verb &= ~(1 << ENQ_CMD_ORP_ENABLE_SHIFT);
	if (resp_always)
		ed->verb |= ENQ_CMD_RESPONSE_ALWAYS;
	else
		ed->verb |= ENQ_CMD_REJECTS_TO_FQ;
}

/**
 * @brief Set FQ of the enqueue descriptor.
 */
void
dpaa2_swp_set_ed_fq(struct dpaa2_eq_desc *ed, uint32_t fqid)
{
	ed->verb &= ~(1 << ENQ_CMD_TARGET_TYPE_SHIFT);
	ed->tgtid = fqid;
}

/**
 * @brief Enable interrupts for a software portal.
 */
void
dpaa2_swp_set_intr_trigger(struct dpaa2_swp *swp, uint32_t mask)
{
	if (swp != NULL)
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_IER, mask);
}

/**
 * @brief Return the value in the SWP_IER register.
 */
uint32_t
dpaa2_swp_get_intr_trigger(struct dpaa2_swp *swp)
{
	if (swp != NULL)
		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_IER);
	return (0);
}

/**
 * @brief Return the value in the SWP_ISR register.
 */
uint32_t
dpaa2_swp_read_intr_status(struct dpaa2_swp *swp)
{
	if (swp != NULL)
		return dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_ISR);
	return (0);
}

/**
 * @brief Clear SWP_ISR register according to the given mask.
 */
void
dpaa2_swp_clear_intr_status(struct dpaa2_swp *swp, uint32_t mask)
{
	if (swp != NULL)
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ISR, mask);
}

/**
 * @brief Enable or disable push dequeue.
 *
 * swp:		the software portal object
 * chan_idx:	the channel index (0 to 15)
 * en:		enable or disable push dequeue
 */
void
dpaa2_swp_set_push_dequeue(struct dpaa2_swp *swp, uint8_t chan_idx, bool en)
{
	uint16_t dqsrc;

	if (swp != NULL) {
		if (chan_idx > 15u) {
			device_printf(swp->desc->dpio_dev, "channel index "
			    "should be <= 15: chan_idx=%d\n", chan_idx);
			return;
		}

		if (en)
			swp->sdq |= 1 << chan_idx;
		else
			swp->sdq &= ~(1 << chan_idx);
		/*
		 * Read make the complete src map. If no channels are enabled
		 * the SDQCR must be 0 or else QMan will assert errors.
		 */
		dqsrc = (swp->sdq >> DPAA2_SDQCR_SRC_SHIFT) &
		    DPAA2_SDQCR_SRC_MASK;
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_SDQCR, dqsrc != 0
		    ? swp->sdq : 0);
	}
}

/**
 * @brief Set new IRQ coalescing values.
 *
 * swp:		The software portal object.
 * threshold:	Threshold for DQRR interrupt generation. The DQRR interrupt
 *		asserts when the ring contains greater than "threshold" entries.
 * holdoff:	DQRR interrupt holdoff (timeout) period in us.
 */
int dpaa2_swp_set_irq_coalescing(struct dpaa2_swp *swp, uint32_t threshold,
    uint32_t holdoff)
{
	uint32_t itp; /* Interrupt Timeout Period */

	if (swp == NULL)
		return (EINVAL);

	/*
	 * Convert "holdoff" value from us to 256 QBMAN clock cycles
	 * increments. This depends on the QBMAN internal frequency.
	 */
	itp = (holdoff * 1000u) / swp->desc->swp_cycles_ratio;
	if (itp > DQRR_MAX_ITP)
		itp = DQRR_MAX_ITP;
	if (threshold >= swp->dqrr.ring_size)
		threshold = swp->dqrr.ring_size - 1;

	swp->dqrr.irq_threshold = threshold;
	swp->dqrr.irq_itp = itp;

	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_DQRR_ITR, threshold);
	dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_ITPR, itp);

	return (0);
}

/*
 * Software portal commands.
 */

/**
 * @brief Configure the channel data availability notification (CDAN)
 * in a particular WQ channel.
 */
int
dpaa2_swp_conf_wq_channel(struct dpaa2_swp *swp, uint16_t chan_id,
    uint8_t we_mask, bool cdan_en, uint64_t ctx)
{
	/* NOTE: 64 bytes command. */
	struct __packed {
		uint8_t		verb;
		uint8_t		result; /* in response only! */
		uint16_t	chan_id;
		uint8_t		we;
		uint8_t		ctrl;
		uint16_t	_reserved2;
		uint64_t	ctx;
		uint8_t		_reserved3[48];
	} cmd = {0};
	struct __packed {
		uint8_t		verb;
		uint8_t		result;
		uint16_t	chan_id;
		uint8_t		_reserved[60];
	} rsp;
	int error;

	if (swp == NULL)
		return (EINVAL);

	cmd.chan_id = chan_id;
	cmd.we = we_mask;
	cmd.ctrl = cdan_en ? 1u : 0u;
	cmd.ctx = ctx;

	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
	    (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_WQCHAN_CONFIGURE);
	if (error)
		return (error);

	if (rsp.result != QBMAN_CMD_RC_OK) {
		device_printf(swp->desc->dpio_dev, "WQ channel configuration "
		    "error: channel_id=%d, result=0x%02x\n", chan_id,
		    rsp.result);
		return (EIO);
	}

	return (0);
}

/**
 * @brief Query current configuration/state of the buffer pool.
 */
int
dpaa2_swp_query_bp(struct dpaa2_swp *swp, uint16_t bpid,
    struct dpaa2_bp_conf *conf)
{
	/* NOTE: 64 bytes command. */
	struct __packed {
		uint8_t		verb;
		uint8_t		_reserved1;
		uint16_t	bpid;
		uint8_t		_reserved2[60];
	} cmd = {0};
	struct __packed {
		uint8_t		verb;
		uint8_t		result;
		uint32_t	_reserved1;
		uint8_t		bdi;
		uint8_t		state;
		uint32_t	fill;
		/* TODO: Support the other fields as well. */
		uint8_t		_reserved2[52];
	} rsp;
	int error;

	if (swp == NULL || conf == NULL)
		return (EINVAL);

	cmd.bpid = bpid;

	error = dpaa2_swp_exec_mgmt_command(swp, (struct dpaa2_swp_cmd *) &cmd,
	    (struct dpaa2_swp_rsp *) &rsp, CMDID_SWP_BP_QUERY);
	if (error)
		return (error);

	if (rsp.result != QBMAN_CMD_RC_OK) {
		device_printf(swp->desc->dpio_dev, "BP query error: bpid=%d, "
		    "result=0x%02x\n", bpid, rsp.result);
		return (EIO);
	}

	conf->bdi = rsp.bdi;
	conf->state = rsp.state;
	conf->free_bufn = rsp.fill;

	return (0);
}

int
dpaa2_swp_release_bufs(struct dpaa2_swp *swp, uint16_t bpid, bus_addr_t *buf,
    uint32_t buf_num)
{
	/* NOTE: 64 bytes command. */
	struct __packed {
		uint8_t		verb;
		uint8_t		_reserved1;
		uint16_t	bpid;
		uint32_t	_reserved2;
		uint64_t	buf[DPAA2_SWP_BUFS_PER_CMD];
	} cmd = {0};
	int error;

	if (swp == NULL || buf == NULL || buf_num == 0u ||
	    buf_num > DPAA2_SWP_BUFS_PER_CMD)
		return (EINVAL);

	for (uint32_t i = 0; i < buf_num; i++)
		cmd.buf[i] = buf[i];
	cmd.bpid = bpid;
	cmd.verb |= 1 << 5; /* Switch release buffer command to valid. */

	error = dpaa2_swp_exec_br_command(swp, (struct dpaa2_swp_cmd *) &cmd,
	    buf_num);
	if (error) {
		device_printf(swp->desc->dpio_dev, "buffers release command "
		    "failed\n");
		return (error);
	}

	return (0);
}

int
dpaa2_swp_dqrr_next_locked(struct dpaa2_swp *swp, struct dpaa2_dq *dq,
    uint32_t *idx)
{
	struct resource_map *map = swp->cinh_map;
	struct dpaa2_swp_rsp *rsp = (struct dpaa2_swp_rsp *) dq;
	uint32_t verb, pi; /* producer index */
	uint32_t offset = swp->cfg.mem_backed
	    ? DPAA2_SWP_CENA_DQRR_MEM(swp->dqrr.next_idx)
	    : DPAA2_SWP_CENA_DQRR(swp->dqrr.next_idx);

	if (swp == NULL || dq == NULL)
		return (EINVAL);

	/*
	 * Before using valid-bit to detect if something is there, we have to
	 * handle the case of the DQRR reset bug...
	 */
	if (swp->dqrr.reset_bug) {
		/*
		 * We pick up new entries by cache-inhibited producer index,
		 * which means that a non-coherent mapping would require us to
		 * invalidate and read *only* once that PI has indicated that
		 * there's an entry here. The first trip around the DQRR ring
		 * will be much less efficient than all subsequent trips around
		 * it...
		 */
		pi = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_DQPI) & DQRR_PI_MASK;

		/* There are new entries if pi != next_idx */
		if (pi == swp->dqrr.next_idx)
			return (ENOENT);

		/*
		 * If next_idx is/was the last ring index, and 'pi' is
		 * different, we can disable the workaround as all the ring
		 * entries have now been DMA'd to so valid-bit checking is
		 * repaired.
		 *
		 * NOTE: This logic needs to be based on next_idx (which
		 *	 increments one at a time), rather than on pi (which
		 *	 can burst and wrap-around between our snapshots of it).
		 */
		if (swp->dqrr.next_idx == (swp->dqrr.ring_size - 1))
			swp->dqrr.reset_bug = 0;
	}

	verb = bus_read_4(map, offset);
	if ((verb & DPAA2_SWP_VALID_BIT) != swp->dqrr.valid_bit)
		return (ENOENT);

	/* Read dequeue response message. */
	for (int i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
		rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));

	/* Return index of the current entry (if requested). */
	if (idx != NULL)
		*idx = swp->dqrr.next_idx;

	/*
	 * There's something there. Move "next_idx" attention to the next ring
	 * entry before returning what we found.
	 */
	swp->dqrr.next_idx++;
	swp->dqrr.next_idx &= swp->dqrr.ring_size - 1; /* wrap around */
	if (swp->dqrr.next_idx == 0u)
		swp->dqrr.valid_bit ^= DPAA2_SWP_VALID_BIT;

	return (0);
}

int
dpaa2_swp_pull(struct dpaa2_swp *swp, uint16_t chan_id, struct dpaa2_buf *buf,
    uint32_t frames_n)
{
	/* NOTE: 64 bytes command. */
	struct __packed {
		uint8_t		verb;
		uint8_t		numf;
		uint8_t		tok;
		uint8_t		_reserved;
		uint32_t	dq_src;
		uint64_t	rsp_addr;
		uint64_t	_reserved1[6];
	} cmd = {0};
	struct dpaa2_dq *msg;
	uint16_t flags;
	int i, error;

	KASSERT(frames_n != 0u, ("%s: cannot pull zero frames", __func__));
	KASSERT(frames_n <= 16u, ("%s: too much frames to pull", __func__));

	cmd.numf = frames_n - 1;
	cmd.tok = DPAA2_SWP_VDQCR_TOKEN;
	cmd.dq_src = chan_id;
	cmd.rsp_addr = (uint64_t)buf->paddr;

	/* Dequeue command type */
	cmd.verb &= ~(1 << QB_VDQCR_VERB_DCT0_SHIFT);
	cmd.verb |=  (1 << QB_VDQCR_VERB_DCT1_SHIFT);
	/* Dequeue from a specific software portal channel (ID's in DQ_SRC). */
	cmd.verb &= ~(1 << QB_VDQCR_VERB_DT0_SHIFT);
	cmd.verb &= ~(1 << QB_VDQCR_VERB_DT1_SHIFT);
	/* Write the response to this command into memory (at the RSP_ADDR). */
	cmd.verb |=  (1 << QB_VDQCR_VERB_RLS_SHIFT);
	/* Response writes won't attempt to allocate into a cache. */
	cmd.verb &= ~(1 << QB_VDQCR_VERB_WAE_SHIFT);
	/* Allow the FQ to remain active in the portal after dequeue. */
	cmd.verb &= ~(1 << QB_VDQCR_VERB_RAD_SHIFT);

	DPAA2_SWP_LOCK(swp, &flags);
	if (flags & DPAA2_SWP_DESTROYED) {
		/* Terminate operation if portal is destroyed. */
		DPAA2_SWP_UNLOCK(swp);
		return (ENOENT);
	}

	error = dpaa2_swp_exec_vdc_command_locked(swp,
	    (struct dpaa2_swp_cmd *) &cmd);
	if (error != 0) {
		DPAA2_SWP_UNLOCK(swp);
		return (error);
	}

	/* Let's sync before reading VDQ response from QBMan. */
	bus_dmamap_sync(buf->dmat, buf->dmap, BUS_DMASYNC_POSTREAD);

	/* Read VDQ response from QBMan. */
	msg = (struct dpaa2_dq *)buf->vaddr;
	for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
		if ((msg->fdr.desc.stat & DPAA2_DQ_STAT_VOLATILE) &&
		    (msg->fdr.desc.tok == DPAA2_SWP_VDQCR_TOKEN)) {
			/* Reset token. */
			msg->fdr.desc.tok = 0;
			break;
		}
		DELAY(CMD_SPIN_TIMEOUT);
	}
	DPAA2_SWP_UNLOCK(swp);

	/* Return an error on expired timeout. */
	return (i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0);
}

/**
 * @brief Issue a command to enqueue a frame using one enqueue descriptor.
 *
 * swp:		Software portal used to send this command to.
 * ed:		Enqueue command descriptor.
 * fd:		Frame descriptor to enqueue.
 */
int
dpaa2_swp_enq(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
    struct dpaa2_fd *fd)
{
	uint32_t flags = 0;
	int rc = dpaa2_swp_enq_mult(swp, ed, fd, &flags, 1);

	return (rc >= 0 ? 0 : EBUSY);
}

/**
 * @brief Issue a command to enqueue frames using one enqueue descriptor.
 *
 * swp:		Software portal used to send this command to.
 * ed:		Enqueue command descriptor.
 * fd:		Frame descriptor to enqueue.
 * flags:	Table pointer of QBMAN_ENQUEUE_FLAG_DCA flags, not used if NULL.
 * frames_n:	Number of FDs to enqueue.
 *
 * NOTE: Enqueue command (64 bytes): 32 (eq. descriptor) + 32 (frame descriptor).
 */
int
dpaa2_swp_enq_mult(struct dpaa2_swp *swp, struct dpaa2_eq_desc *ed,
    struct dpaa2_fd *fd, uint32_t *flags, int frames_n)
{
	const uint8_t  *ed_pdat8 =  (const uint8_t *) ed;
	const uint32_t *ed_pdat32 = (const uint32_t *) ed;
	const uint64_t *ed_pdat64 = (const uint64_t *) ed;
	const uint64_t *fd_pdat64 = (const uint64_t *) fd;
	struct resource_map *map;
	uint32_t eqcr_ci, eqcr_pi; /* EQCR consumer/producer index */
	uint32_t half_mask, full_mask, val, ci_offset;
	uint16_t swp_flags;
	int num_enq = 0;

	if (swp == NULL || ed == NULL || fd == NULL || flags == NULL ||
	    frames_n == 0)
		return (EINVAL);

	DPAA2_SWP_LOCK(swp, &swp_flags);
	if (swp_flags & DPAA2_SWP_DESTROYED) {
		/* Terminate operation if portal is destroyed. */
		DPAA2_SWP_UNLOCK(swp);
		return (ENOENT);
	}

	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
	ci_offset = swp->cfg.mem_backed
	    ? DPAA2_SWP_CENA_EQCR_CI_MEMBACK
	    : DPAA2_SWP_CENA_EQCR_CI;

	half_mask = swp->eqcr.pi_ci_mask >> 1;
	full_mask = swp->eqcr.pi_ci_mask;

	if (swp->eqcr.available == 0) {
		val = dpaa2_swp_read_reg(swp, ci_offset);
		eqcr_ci = swp->eqcr.ci;
		swp->eqcr.ci = val & full_mask;

		swp->eqcr.available = dpaa2_swp_cyc_diff(swp->eqcr.pi_ring_size,
		    eqcr_ci, swp->eqcr.ci);

		if (swp->eqcr.available == 0) {
			DPAA2_SWP_UNLOCK(swp);
			return (0);
		}
	}

	eqcr_pi = swp->eqcr.pi;
	num_enq = swp->eqcr.available < frames_n
	    ? swp->eqcr.available : frames_n;
	swp->eqcr.available -= num_enq;

	KASSERT(num_enq >= 0 && num_enq <= swp->eqcr.pi_ring_size,
	    ("%s: unexpected num_enq=%d", __func__, num_enq));
	KASSERT(swp->eqcr.available >= 0 &&
	    swp->eqcr.available <= swp->eqcr.pi_ring_size,
	    ("%s: unexpected eqcr.available=%d", __func__, swp->eqcr.available));

	/* Fill in the EQCR ring. */
	for (int i = 0; i < num_enq; i++) {
		/* Write enq. desc. without the VERB, DCA, SEQNUM and OPRID. */
		for (int j = 1; j <= 3; j++)
			bus_write_8(map,
			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
			    sizeof(uint64_t) * j, ed_pdat64[j]);
		/* Write OPRID. */
		bus_write_4(map,
		    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + sizeof(uint32_t),
		    ed_pdat32[1]);
		/* Write DCA and SEQNUM without VERB byte. */
		for (int j = 1; j <= 3; j++)
			bus_write_1(map,
			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
			    sizeof(uint8_t) * j, ed_pdat8[j]);

		/* Write frame descriptor. */
		for (int j = 0; j <= 3; j++)
			bus_write_8(map,
			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) +
			    ENQ_DESC_FD_OFFSET +
			    sizeof(uint64_t) * j, fd_pdat64[j]);
		eqcr_pi++;
	}

	wmb();

	/* Write the VERB byte of enqueue descriptor. */
	eqcr_pi = swp->eqcr.pi;
	for (int i = 0; i < num_enq; i++) {
		bus_write_1(map,
		    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask),
		    ed_pdat8[0] | swp->eqcr.pi_vb);

		if (flags && (flags[i] & ENQ_FLAG_DCA)) {
			/* Update DCA byte. */
			bus_write_1(map,
			    DPAA2_SWP_CENA_EQCR(eqcr_pi & half_mask) + 1,
			    (1 << ENQ_CMD_DCA_EN_SHIFT) |
			    (flags[i] & ENQ_DCA_IDXMASK));
		}
		eqcr_pi++;
		if (!(eqcr_pi & half_mask))
			swp->eqcr.pi_vb ^= DPAA2_SWP_VALID_BIT;
	}
	swp->eqcr.pi = eqcr_pi & full_mask;

	DPAA2_SWP_UNLOCK(swp);

	return (num_enq);
}

static int
dpaa2_swp_cyc_diff(uint8_t ringsize, uint8_t first, uint8_t last)
{
	/* 'first' is included, 'last' is excluded */
	return ((first <= last)
	    ? (last - first) : ((2 * ringsize) - (first - last)));
}

/**
 * @brief Execute Buffer Release Command (BRC).
 */
static int
dpaa2_swp_exec_br_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
    uint32_t buf_num)
{
	struct __packed with_verb {
		uint8_t	verb;
		uint8_t	_reserved[63];
	} *c;
	const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
	const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
	struct resource_map *map;
	uint32_t offset, rar; /* Release Array Allocation register */
	uint16_t flags;

	if (!swp || !cmd)
		return (EINVAL);

	DPAA2_SWP_LOCK(swp, &flags);
	if (flags & DPAA2_SWP_DESTROYED) {
		/* Terminate operation if portal is destroyed. */
		DPAA2_SWP_UNLOCK(swp);
		return (ENOENT);
	}

	rar = dpaa2_swp_read_reg(swp, DPAA2_SWP_CINH_RAR);
	if (!RAR_SUCCESS(rar)) {
		DPAA2_SWP_UNLOCK(swp);
		return (EBUSY);
	}

	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
	offset = swp->cfg.mem_backed
	    ? DPAA2_SWP_CENA_RCR_MEM(RAR_IDX(rar))
	    : DPAA2_SWP_CENA_RCR(RAR_IDX(rar));
	c = (struct with_verb *) cmd;

	/* Write command bytes (without VERB byte). */
	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
	bus_write_4(map, offset + 4, cmd_pdat32[1]);
	for (uint32_t i = 1; i <= 3; i++)
		bus_write_1(map, offset + i, cmd_pdat8[i]);

	/* Write VERB byte and trigger command execution. */
	if (swp->cfg.mem_backed) {
		bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
		wmb();
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_RCR_AM_RT +
		    RAR_IDX(rar) * 4, DPAA2_SWP_RT_MODE);
	} else {
		wmb();
		bus_write_1(map, offset, c->verb | RAR_VB(rar) | buf_num);
	}

	DPAA2_SWP_UNLOCK(swp);

	return (0);
}

/**
 * @brief Execute Volatile Dequeue Command (VDC).
 *
 * This command will be executed by QBMan only once in order to deliver requested
 * number of frames (1-16 or 1-32 depending on QBMan version) to the driver via
 * DQRR or arbitrary DMA-mapped memory.
 *
 * NOTE: There is a counterpart to the volatile dequeue command called static
 *	 dequeue command (SDQC) which is executed periodically all the time the
 *	 command is present in the SDQCR register.
 */
static int
dpaa2_swp_exec_vdc_command_locked(struct dpaa2_swp *swp,
    struct dpaa2_swp_cmd *cmd)
{
	struct __packed with_verb {
		uint8_t	verb;
		uint8_t	_reserved[63];
	} *c;
	const uint8_t *p8 = (const uint8_t *) cmd->params;
	const uint32_t *p32 = (const uint32_t *) cmd->params;
	struct resource_map *map;
	uint32_t offset;

	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
	offset = swp->cfg.mem_backed
	    ? DPAA2_SWP_CENA_VDQCR_MEM : DPAA2_SWP_CENA_VDQCR;
	c = (struct with_verb *) cmd;

	/* Write command bytes (without VERB byte). */
	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
	bus_write_4(map, offset + 4, p32[1]);
	for (uint32_t i = 1; i <= 3; i++)
		bus_write_1(map, offset + i, p8[i]);

	/* Write VERB byte and trigger command execution. */
	if (swp->cfg.mem_backed) {
		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
		wmb();
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_VDQCR_RT,
		    DPAA2_SWP_RT_MODE);
	} else {
		wmb();
		bus_write_1(map, offset, c->verb | swp->vdq.valid_bit);
		swp->vdq.valid_bit ^= DPAA2_SWP_VALID_BIT;
	}

	return (0);
}

/**
 * @brief Execute a QBMan management command.
 */
static int
dpaa2_swp_exec_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
    struct dpaa2_swp_rsp *rsp, uint8_t cmdid)
{
#if (defined(_KERNEL) && defined(INVARIANTS))
	struct __packed with_verb {
		uint8_t	verb;
		uint8_t	_reserved[63];
	} *r;
#endif
	uint16_t flags;
	int error;

	if (swp == NULL || cmd == NULL || rsp == NULL)
		return (EINVAL);

	DPAA2_SWP_LOCK(swp, &flags);
	if (flags & DPAA2_SWP_DESTROYED) {
		/* Terminate operation if portal is destroyed. */
		DPAA2_SWP_UNLOCK(swp);
		return (ENOENT);
	}

	/*
	 * Send a command to QBMan using Management Command register and wait
	 * for response from the Management Response registers.
	 */
	dpaa2_swp_send_mgmt_command(swp, cmd, cmdid);
	error = dpaa2_swp_wait_for_mgmt_response(swp, rsp);
	if (error) {
		DPAA2_SWP_UNLOCK(swp);
		return (error);
	}
	DPAA2_SWP_UNLOCK(swp);

#if (defined(_KERNEL) && defined(INVARIANTS))
	r = (struct with_verb *) rsp;
	KASSERT((r->verb & CMD_VERB_MASK) == cmdid,
	    ("wrong VERB byte in response: resp=0x%02x, expected=0x%02x",
	    r->verb, cmdid));
#endif

	return (0);
}

static int
dpaa2_swp_send_mgmt_command(struct dpaa2_swp *swp, struct dpaa2_swp_cmd *cmd,
    uint8_t cmdid)
{
	const uint8_t *cmd_pdat8 = (const uint8_t *) cmd->params;
	const uint32_t *cmd_pdat32 = (const uint32_t *) cmd->params;
	struct resource_map *map;
	uint32_t offset;

	map = swp->cfg.writes_cinh ? swp->cinh_map : swp->cena_map;
	offset = swp->cfg.mem_backed ? DPAA2_SWP_CENA_CR_MEM : DPAA2_SWP_CENA_CR;

	/* Write command bytes (without VERB byte). */
	for (uint32_t i = 1; i < DPAA2_SWP_CMD_PARAMS_N; i++)
		bus_write_8(map, offset + sizeof(uint64_t) * i, cmd->params[i]);
	bus_write_4(map, offset + 4, cmd_pdat32[1]);
	for (uint32_t i = 1; i <= 3; i++)
		bus_write_1(map, offset + i, cmd_pdat8[i]);

	/* Write VERB byte and trigger command execution. */
	if (swp->cfg.mem_backed) {
		bus_write_1(map, offset, cmdid | swp->mr.valid_bit);
		wmb();
		dpaa2_swp_write_reg(swp, DPAA2_SWP_CINH_CR_RT,
		    DPAA2_SWP_RT_MODE);
	} else {
		wmb();
		bus_write_1(map, offset, cmdid | swp->mc.valid_bit);
	}

	return (0);
}

static int
dpaa2_swp_wait_for_mgmt_response(struct dpaa2_swp *swp, struct dpaa2_swp_rsp *rsp)
{
	struct resource_map *map = swp->cfg.mem_backed
	    ? swp->cena_map : swp->cinh_map;
	/* Management command response to be read from the only RR or RR0/RR1. */
	const uint32_t offset = swp->cfg.mem_backed
	    ? DPAA2_SWP_CENA_RR_MEM
	    : DPAA2_SWP_CENA_RR(swp->mc.valid_bit);
	uint32_t i, verb, ret;
	int rc;

	/* Wait for a command response from QBMan. */
	for (i = 1; i <= CMD_SPIN_ATTEMPTS; i++) {
		if (swp->cfg.mem_backed) {
			verb = (uint32_t) (bus_read_4(map, offset) & 0xFFu);
			if (swp->mr.valid_bit != (verb & DPAA2_SWP_VALID_BIT))
				goto wait;
			if (!(verb & ~DPAA2_SWP_VALID_BIT))
				goto wait;
			swp->mr.valid_bit ^= DPAA2_SWP_VALID_BIT;
		} else {
			ret = bus_read_4(map, offset);
			verb = ret & ~DPAA2_SWP_VALID_BIT; /* remove valid bit */
			if (verb == 0u)
				goto wait;
			swp->mc.valid_bit ^= DPAA2_SWP_VALID_BIT;
		}
		break;
 wait:
		DELAY(CMD_SPIN_TIMEOUT);
	}
	/* Return an error on expired timeout. */
	rc = i > CMD_SPIN_ATTEMPTS ? ETIMEDOUT : 0;

	/* Read command response. */
	for (i = 0; i < DPAA2_SWP_RSP_PARAMS_N; i++)
		rsp->params[i] = bus_read_8(map, offset + i * sizeof(uint64_t));

	return (rc);
}