xref: /illumos-gate/usr/src/uts/common/io/mlxcx/mlxcx.c (revision 5014e1fa5f21d62166ea238737adcb2691e91963)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2023, The University of Queensland
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2023 RackTop Systems, Inc.
 * Copyright 2023 MNX Cloud, Inc.
 */

/*
 * Mellanox Connect-X 4/5/6 driver.
 */

/*
 * The PRM for this family of parts was freely available at:
 *
 * https://www.mellanox.com/related-docs/user_manuals/ \
 *   Ethernet_Adapters_Programming_Manual.pdf
 *
 * but has since disappeared.
 */
/*
 * ConnectX glossary
 * -----------------
 *
 * WR		Work Request: something we've asked the hardware to do by
 *		creating a Work Queue Entry (WQE), e.g. send or recv a packet
 *
 * WQE		Work Queue Entry: a descriptor on a work queue descriptor ring
 *
 * WQ		Work Queue: a descriptor ring that we can place WQEs on, usually
 *		either a Send Queue (SQ) or Receive Queue (RQ). Different WQ
 *		types have different WQE structures, different commands for
 *		creating and destroying them, etc, but share a common context
 *		structure, counter setup and state graph.
 * SQ		Send Queue, a specific type of WQ that sends packets
 * RQ		Receive Queue, a specific type of WQ that receives packets
 *
 * CQ		Completion Queue: completion of WRs from a WQ are reported to
 *		one of these, as a CQE on its entry ring.
 * CQE		Completion Queue Entry: an entry in a CQ ring. Contains error
 *		info, as well as packet size, the ID of the WQ, and the index
 *		of the WQE which completed. Does not contain any packet data.
 *
 * EQ		Event Queue: a ring of event structs from the hardware informing
 *		us when particular events happen. Many events can point at a
 *		a particular CQ which we should then go look at.
 * EQE		Event Queue Entry: an entry on the EQ ring
 *
 * UAR		User Access Region, a page of the device's PCI BAR which is
 *		tied to particular EQ/CQ/WQ sets and contains doorbells to
 *		ring to arm them for interrupts or wake them up for new work
 *
 * RQT		RQ Table, a collection of indexed RQs used to refer to the group
 *		as a single unit (for e.g. hashing/RSS).
 *
 * TIR		Transport Interface Recieve, a bucket of resources for the
 *		reception of packets. TIRs have to point at either a single RQ
 *		or a table of RQs (RQT). They then serve as a target for flow
 *		table entries (FEs). TIRs that point at an RQT also contain the
 *		settings for hashing for RSS.
 *
 * TIS		Transport Interface Send, a bucket of resources associated with
 *		the transmission of packets. In particular, the temporary
 *		resources used for LSO internally in the card are accounted to
 *		a TIS.
 *
 * FT		Flow Table, a collection of FEs and FGs that can be referred to
 *		as a single entity (e.g. used as a target from another flow
 *		entry or set as the "root" table to handle incoming or outgoing
 *		packets). Packets arriving at a FT are matched against the
 *		FEs in the table until either one matches with a terminating
 *		action or all FEs are exhausted (it's first-match-wins but with
 *		some actions that are non-terminal, like counting actions).
 *
 * FG		Flow Group, a group of FEs which share a common "mask" (i.e.
 *		they match on the same attributes of packets coming into the
 *		flow).
 *
 * FE		Flow Entry, an individual set of values to match against
 *		packets entering the flow table, combined with an action to
 *		take upon a successful match. The action we use most is
 *		"forward", which sends the packets to a TIR or another flow
 *		table and then stops further processing within the FE's FT.
 *
 * lkey/mkey	A reference to something similar to a page table but in the
 *		device's internal onboard MMU. Since Connect-X parts double as
 *		IB cards (lots of RDMA) they have extensive onboard memory mgmt
 *		features which we try very hard not to use. For our WQEs we use
 *		the "reserved" lkey, which is a special value which indicates
 *		that addresses we give are linear addresses and should not be
 *		translated.
 *
 * PD		Protection Domain, an IB concept. We have to allocate one to
 *		provide as a parameter for new WQs, but we don't do anything
 *		with it.
 *
 * TDOM/TD	Transport Domain, an IB concept. We allocate one in order to
 *		provide it as a parameter to TIR/TIS creation, but we don't do
 *		anything with it.
 */
/*
 *
 * Data flow overview
 * ------------------
 *
 * This driver is a MAC ring-enabled driver which maps rings to send and recv
 * queues in hardware on the device.
 *
 * Each SQ and RQ is set up to report to its own individual CQ, to ensure
 * sufficient space, and simplify the logic needed to work out which buffer
 * was completed.
 *
 * The CQs are then round-robin allocated onto EQs, of which we set up one per
 * interrupt that the system gives us for the device. Normally this means we
 * have 8 EQs.
 *
 * When we have >= 8 EQs available, we try to allocate only RX or only TX
 * CQs on each one. The EQs are chosen for RX and TX in an alternating fashion.
 *
 * EQ #0 is reserved for all event types other than completion events, and has
 * no CQs associated with it at any time. EQs #1 and upwards are only used for
 * handling CQ completion events.
 *
 * +------+     +------+           +------+        +---------+
 * | SQ 0 |---->| CQ 0 |-----+     | EQ 0 |------> | MSI-X 0 |     mlxcx_intr_0
 * +------+     +------+     |     +------+        +---------+
 *                           |
 * +------+     +------+     |
 * | SQ 1 |---->| CQ 1 |---+ |     +------+
 * +------+     +------+   | +---> |      |
 *                         |       |      |
 * +------+     +------+   |       | EQ 1 |        +---------+
 * | SQ 2 |---->| CQ 2 |---------> |      |------> | MSI-X 1 |     mlxcx_intr_n
 * +------+     +------+   | +---> |      |        +---------+
 *                         | |     +------+
 *                         | |
 *   ...                   | |
 *                         | |     +------+
 * +------+     +------+   +-----> |      |
 * | RQ 0 |---->| CQ 3 |---------> |      |        +---------+
 * +------+     +------+     |     | EQ 2 |------> | MSI-X 2 |     mlxcx_intr_n
 *                           |     |      |        +---------+
 * +------+     +------+     | +-> |      |
 * | RQ 1 |---->| CQ 4 |-----+ |   +------+
 * +------+     +------+       |
 *                             |     ....
 * +------+     +------+       |
 * | RQ 2 |---->| CQ 5 |-------+
 * +------+     +------+
 *
 *   ... (note this diagram does not show RX-only or TX-only EQs)
 *
 * For TX, we advertise all of the SQs we create as plain rings to MAC with
 * no TX groups. This puts MAC in "virtual group" mode where it will allocate
 * and use the rings as it sees fit.
 *
 * For RX, we advertise actual groups in order to make use of hardware
 * classification.
 *
 * The hardware classification we use is based around Flow Tables, and we
 * currently ignore all of the eswitch features of the card. The NIC VPORT
 * is always set to promisc mode so that the eswitch sends us all of the
 * traffic that arrives on the NIC, and we use flow entries to manage
 * everything.
 *
 * We use 2 layers of flow tables for classification: traffic arrives at the
 * root RX flow table which contains MAC address filters. Those then send
 * matched traffic to the per-group L1 VLAN filter tables which contain VLAN
 * presence and VID filters.
 *
 * Since these parts only support doing RSS hashing on a single protocol at a
 * time, we have to use a third layer of flow tables as well to break traffic
 * down by L4 and L3 protocol (TCPv6, TCPv4, UDPv6, UDPv4, IPv6, IPv4 etc)
 * so that it can be sent to the appropriate TIR for hashing.
 *
 * Incoming packets
 *        +           +---------+      +---------+
 *        |        +->| group 0 |      | group 0 |
 *        |        |  | vlan ft |  +-->| hash ft |
 *        v        |  |   L1    |  |   |   L2    |
 *   +----+----+   |  +---------+  |   +---------+    +-----+    +-----+------+
 *   | eswitch |   |  |         |  |   |  TCPv6  |--->| TIR |--->|     |  RQ0 |
 *   +----+----+   |  |         |  |   +---------+    +-----+    |     +------+
 *        |        |  |         |  |   |  UDPv6  |--->| TIR |--->|     |  RQ1 |
 *        |        |  |         |  |   +---------+    +-----+    |     +------+
 *        |        |  |         |  |   |  TCPv4  |--->| TIR |--->|     |  RQ2 |
 *        v        |  |         |  |   +---------+    +-----+    | RQT +------+
 *   +----+----+   |  +---------+  |   |  UDPv4  |--->| TIR |--->|     |  ... |
 *   | root rx |   |  | default |--+   +---------+    +-----+    |     |      |
 *   | flow tb |   |  +---------+  |   |  IPv6   |--->| TIR |--->|     |      |
 *   |    L0   |   |  | promisc |--+   +---------+    +-----+    |     |      |
 *   +---------+   |  +---------+  ^   |  IPv4   |--->| TIR |--->|     |      |
 *   |  bcast  |---|---------------+   +---------+    +-----+    +-----+------+
 *   +---------+   |               ^   |  other  |-+
 *   |  MAC 0  |---+               |   +---------+ |  +-----+    +-----+
 *   +---------+                   |               +->| TIR |--->| RQ0 |
 *   |  MAC 1  |-+                 |                  +-----+    +-----+
 *   +---------+ | +---------------+
 *   |  MAC 2  |-+ |               ^
 *   +---------+ | |               |
 *   |  MAC 3  |-+ |  +---------+  |   +---------+
 *   +---------+ | |  | group 1 |  |   | group 1 |
 *   |  .....  | +--->| vlan ft |  | +>| hash ft |
 *   |         |   |  |   L1    |  | | |   L2    |
 *   +---------+   |  +---------+  | | +---------+    +-----+    +-----+------+
 *   | promisc |---+  | VLAN 0  |----+ |  TCPv6  |--->| TIR |--->|     |  RQ3 |
 *   +---------+      +---------+  |   +---------+    +-----+    |     +------+
 *                    |  .....  |  |   |  UDPv6  |--->| TIR |--->|     |  RQ4 |
 *                    |         |  |   +---------+    +-----+    |     +------+
 *                    |         |  |   |  TCPv4  |--->| TIR |--->|     |  RQ5 |
 *                    |         |  |   +---------+    +-----+    | RQT +------+
 *                    +---------+  |   |  UDPv4  |--->| TIR |--->|     |  ... |
 *                    |         |  |   +---------+    +-----+    |     |      |
 *                    +---------+  |   |  IPv6   |--->| TIR |--->|     |      |
 *                    | promisc |--+   +---------+    +-----+    |     |      |
 *                    +---------+      |  IPv4   |--->| TIR |--->|     |      |
 *                                     +---------+    +-----+    +-----+------+
 *                                     |  other  |-+
 *                                     +---------+ |
 *                      .......                    |  +-----+    +-----+
 *                                                 +->| TIR |--->| RQ3 |
 *                                                    +-----+    +-----+
 *
 * Note that the "promisc" flow entries are only set/enabled when promisc
 * mode is enabled for the NIC. All promisc flow entries point directly at
 * group 0's hashing flowtable (so all promisc-only traffic lands on group 0,
 * the "default group" in MAC).
 *
 * The "default" entry in the L1 VLAN filter flow tables is used when there
 * are no VLANs set for the group, to accept any traffic regardless of tag. It
 * is deleted as soon as a VLAN filter is added (and re-instated if the
 * last VLAN filter is removed).
 *
 * The actual descriptor ring structures for RX on Connect-X4 don't contain any
 * space for packet data (they're a collection of scatter pointers only). TX
 * descriptors contain some space for "inline headers" (and the card requires
 * us to put at least the L2 Ethernet headers there for the eswitch to look at)
 * but all the rest of the data comes from the gather pointers.
 *
 * When we get completions back they simply contain the ring index number of
 * the WR (work request) which completed. So, we manage the buffers for actual
 * packet data completely independently of the descriptors in this driver. When
 * a WR is enqueued in a WQE (work queue entry), we stamp the packet data buffer
 * with the WQE index that we put it at, and therefore don't have to look at
 * the original descriptor at all when handling completions.
 *
 * For RX, we create sufficient packet data buffers to fill 150% of the
 * available descriptors for each ring. These all are pre-set-up for DMA and
 * have an mblk_t associated with them (with desballoc()).
 *
 * For TX we either borrow the mblk's memory and DMA bind it (if the packet is
 * large enough), or we copy it into a pre-allocated buffer set up in the same
 * as as for RX.
 */

/*
 * Buffer lifecycle: RX
 * --------------------
 *
 * The lifecycle of an mlxcx_buffer_t (packet buffer) used for RX is pretty
 * straightforward.
 *
 * It is created (and has all its memory allocated) at the time of starting up
 * the RX ring it belongs to. Then it is placed on the "free" list in the
 * mlxcx_buffer_shard_t associated with its RQ. When mlxcx_rq_refill() wants
 * more buffers to add to the RQ, it takes one off and marks it as "on WQ"
 * before making a WQE for it.
 *
 * After a completion event occurs, the packet is either discarded (and the
 * buffer_t returned to the free list), or it is readied for loaning to MAC
 * and placed on the "loaned" list in the mlxcx_buffer_shard_t.
 *
 * Once MAC and the rest of the system have finished with the packet, they call
 * freemsg() on its mblk, which will call mlxcx_buf_mp_return. At this point
 * the fate of the buffer_t is determined by the state of the
 * mlxcx_buffer_shard_t. When the shard is in its normal state the buffer_t
 * will be returned to the free list, potentially to be recycled and used
 * again. But if the shard is draining (E.g. after a ring stop) there will be
 * no recycling and the buffer_t is immediately destroyed.
 *
 * At detach/teardown time, buffers are only every destroyed from the free list.
 *
 *
 *                         +
 *                         |
 *                         | mlxcx_buf_create
 *                         |
 *                         v
 *                    +----+----+
 *                    | created |
 *                    +----+----+                        +------+
 *                         |                             | dead |
 *                         |                             +------+
 *                         | mlxcx_buf_return                ^
 *                         |                                 |
 *                         v                                 | mlxcx_buf_destroy
 * mlxcx_buf_destroy  +----+----+          +-----------+     |
 *          +---------|  free   |<------no-| draining? |-yes-+
 *          |         +----+----+          +-----------+
 *          |              |                     ^
 *          |              |                     |
 *          v              | mlxcx_buf_take      | mlxcx_buf_return
 *      +---+--+           v                     |
 *      | dead |       +---+---+                 |
 *      +------+       | on WQ |- - - - - - - - >O
 *                     +---+---+                 ^
 *                         |                     |
 *                         |                     |
 *                         | mlxcx_buf_loan      | mlxcx_buf_mp_return
 *                         v                     |
 *                 +-------+--------+            |
 *                 | on loan to MAC |----------->O
 *                 +----------------+  freemsg()
 *
 */

/*
 * Buffer lifecycle: TX
 * --------------------
 *
 * mlxcx_buffer_ts used for TX are divided into two kinds: regular buffers, and
 * "foreign" buffers.
 *
 * The former have their memory allocated and DMA bound by this driver, while
 * the latter (the "foreign" buffers) are on loan from MAC. Their memory is
 * not owned by us, though we do DMA bind it (and take responsibility for
 * un-binding it when we're done with them).
 *
 * We use separate mlxcx_buf_shard_ts for foreign and local buffers on each
 * SQ. Thus, there is a separate free list and mutex for each kind.
 *
 * Since a TX packet might consist of multiple mblks, we translate each mblk
 * into exactly one buffer_t. The buffer_ts are chained together in the same
 * order as the mblks, using the mlb_tx_chain/mlb_tx_chain_entry list_t.
 *
 * Each chain of TX buffers may consist of foreign or driver buffers, in any
 * mixture.
 *
 * The head of a TX buffer chain has mlb_tx_head == itself, which distinguishes
 * it from the rest of the chain buffers.
 *
 * TX buffer chains are always returned to the free list by
 * mlxcx_buf_return_chain(), which takes care of walking the mlb_tx_chain and
 * freeing all of the members.
 *
 * We only call freemsg() once, on the head of the TX buffer chain's original
 * mblk. This is true whether we copied it or bound it in a foreign buffer.
 */

/*
 * Startup and command interface
 * -----------------------------
 *
 * The command interface is the primary way in which we give control orders to
 * the hardware (e.g. actions like "create this queue" or "delete this flow
 * entry"). The command interface is never used to transmit or receive packets
 * -- that takes place only on the queues that are set up through it.
 *
 * In mlxcx_cmd.c we implement our use of the command interface on top of a
 * simple taskq. As commands are submitted from the taskq they choose a
 * "slot", if there are no free slots then execution of the command will
 * be paused until one is free. The hardware permits up to 32 independent
 * slots for concurrent command execution.
 *
 * Before interrupts are enabled, command completion is polled, once
 * interrupts are up command completions become asynchronous and are
 * wired to EQ 0. A caveat to this is commands can not be submitted
 * directly from EQ 0's completion handler, and any processing resulting from
 * an asynchronous event which requires further use of the command interface
 * is posted through a taskq.
 *
 * The startup/attach process for this card involves a bunch of different steps
 * which are summarised pretty well in the PRM. We have to send a number of
 * commands which do different things to start the card up, give it some pages
 * of our own memory for it to use, then start creating all the entities that
 * we need to use like EQs, CQs, WQs, as well as their dependencies like PDs
 * and TDoms.
 */

/*
 * UARs
 * ----
 *
 * The pages of the PCI BAR other than the first few are reserved for use as
 * "UAR" sections in this device. Each UAR section can be used as a set of
 * doorbells for our queues.
 *
 * Currently we just make one single UAR for all of our queues. It doesn't
 * seem to be a major limitation yet.
 *
 * When we're sending packets through an SQ, the PRM is not awful clear about
 * exactly how we're meant to use the first 16 bytes of the Blueflame buffers
 * (it's clear on the pattern of alternation you're expected to use between
 * even and odd for Blueflame sends, but not for regular doorbells).
 *
 * Currently we don't do the even-odd alternating pattern for ordinary
 * doorbells, and we don't use Blueflame at all. This seems to work fine, at
 * least on Connect-X4 Lx.
 */

/*
 * Lock ordering
 * -------------
 *
 * Interrupt side:
 *
 *  - mleq_mtx
 *    - mlcq_arm_mtx
 *      - mlcq_mtx
 *        - mlcq_bufbmtx
 *        - mlwq_mtx
 *          - mlbs_mtx
 *    - mlp_mtx
 *
 * GLD side:
 *
 *  - mlp_mtx
 *    - mlg_mtx
 *      - mlg_*.mlft_mtx
 *    - mlp_*.mlft_mtx
 *    - mlwq_mtx
 *      - mlbs_mtx
 *      - mlcq_bufbmtx
 *  - mleq_mtx
 *    - mlcq_arm_mtx
 *      - mlcq_mtx
 *
 */

#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/sysmacros.h>
#include <sys/time.h>
#include <sys/pci.h>
#include <sys/mac_provider.h>

#include <mlxcx.h>

CTASSERT((1 << MLXCX_RX_HASH_FT_SIZE_SHIFT) >= MLXCX_TIRS_PER_GROUP);

#define	MLXCX_MODULE_NAME	"mlxcx"
/*
 * We give this to the firmware, so it has to be in a fixed format that it
 * understands.
 */
#define	MLXCX_DRIVER_VERSION	"illumos,mlxcx,1.0.0,1,000,000000"

/*
 * Firmware may take a while to reclaim pages. Try a set number of times.
 */
clock_t mlxcx_reclaim_delay = 1000 * 50; /* 50 ms in us */
uint_t mlxcx_reclaim_tries = 100; /* Wait at most 5000ms */

static void *mlxcx_softstate;

/*
 * Fault detection thresholds.
 */
uint_t mlxcx_doorbell_tries = MLXCX_DOORBELL_TRIES_DFLT;
uint_t mlxcx_stuck_intr_count = MLXCX_STUCK_INTR_COUNT_DFLT;

static void
mlxcx_load_prop_defaults(mlxcx_t *mlxp)
{
	mlxcx_drv_props_t *p = &mlxp->mlx_props;
	mlxcx_port_t *port = &mlxp->mlx_ports[0];

	VERIFY((mlxp->mlx_attach & MLXCX_ATTACH_PORTS) != 0);
	VERIFY((mlxp->mlx_attach & (MLXCX_ATTACH_CQS | MLXCX_ATTACH_WQS)) == 0);

	/*
	 * Currently we have different queue size defaults for two
	 * categories of queues. One set for devices which support a
	 * maximum speed of 10Gb/s, and another for those above that.
	 */
	if ((port->mlp_max_proto & (MLXCX_PROTO_25G | MLXCX_PROTO_40G |
	    MLXCX_PROTO_50G | MLXCX_PROTO_100G)) != 0 ||
	    (port->mlp_ext_max_proto & (MLXCX_EXTPROTO_25G |
	    MLXCX_EXTPROTO_40G | MLXCX_EXTPROTO_50G | MLXCX_EXTPROTO_100G |
	    MLXCX_EXTPROTO_200G | MLXCX_EXTPROTO_400G)) != 0) {
		p->mldp_cq_size_shift_default = MLXCX_CQ_SIZE_SHIFT_25G;
		p->mldp_rq_size_shift_default = MLXCX_RQ_SIZE_SHIFT_25G;
		p->mldp_sq_size_shift_default = MLXCX_SQ_SIZE_SHIFT_25G;
	} else if ((port->mlp_max_proto & (MLXCX_PROTO_100M | MLXCX_PROTO_1G |
	    MLXCX_PROTO_10G)) != 0 ||
	    (port->mlp_ext_max_proto & (MLXCX_EXTPROTO_100M |
	    MLXCX_EXTPROTO_5G | MLXCX_EXTPROTO_1G | MLXCX_EXTPROTO_10G)) != 0) {
		p->mldp_cq_size_shift_default = MLXCX_CQ_SIZE_SHIFT_DFLT;
		p->mldp_rq_size_shift_default = MLXCX_RQ_SIZE_SHIFT_DFLT;
		p->mldp_sq_size_shift_default = MLXCX_SQ_SIZE_SHIFT_DFLT;
	} else {
		mlxcx_warn(mlxp, "Encountered a port with a speed we don't "
		    "recognize. Proto: 0x%x", port->mlp_max_proto);
		p->mldp_cq_size_shift_default = MLXCX_CQ_SIZE_SHIFT_DFLT;
		p->mldp_rq_size_shift_default = MLXCX_RQ_SIZE_SHIFT_DFLT;
		p->mldp_sq_size_shift_default = MLXCX_SQ_SIZE_SHIFT_DFLT;
	}
}

/*
 * Properties which may have different defaults based on hardware
 * characteristics.
 */
static void
mlxcx_load_model_props(mlxcx_t *mlxp)
{
	mlxcx_drv_props_t *p = &mlxp->mlx_props;

	mlxcx_load_prop_defaults(mlxp);

	p->mldp_cq_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "cq_size_shift",
	    p->mldp_cq_size_shift_default);
	p->mldp_sq_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "sq_size_shift",
	    p->mldp_sq_size_shift_default);
	p->mldp_rq_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rq_size_shift",
	    p->mldp_rq_size_shift_default);
}

static void
mlxcx_load_props(mlxcx_t *mlxp)
{
	mlxcx_drv_props_t *p = &mlxp->mlx_props;

	p->mldp_eq_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "eq_size_shift",
	    MLXCX_EQ_SIZE_SHIFT_DFLT);
	p->mldp_cqemod_period_usec = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "cqemod_period_usec",
	    MLXCX_CQEMOD_PERIOD_USEC_DFLT);
	p->mldp_cqemod_count = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "cqemod_count",
	    MLXCX_CQEMOD_COUNT_DFLT);
	p->mldp_intrmod_period_usec = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "intrmod_period_usec",
	    MLXCX_INTRMOD_PERIOD_USEC_DFLT);

	p->mldp_tx_ngroups = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "tx_ngroups",
	    MLXCX_TX_NGROUPS_DFLT);
	p->mldp_tx_nrings_per_group = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "tx_nrings_per_group",
	    MLXCX_TX_NRINGS_PER_GROUP_DFLT);

	p->mldp_rx_ngroups_large = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rx_ngroups_large",
	    MLXCX_RX_NGROUPS_LARGE_DFLT);
	p->mldp_rx_ngroups_small = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rx_ngroups_small",
	    MLXCX_RX_NGROUPS_SMALL_DFLT);
	p->mldp_rx_nrings_per_large_group = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "rx_nrings_per_large_group", MLXCX_RX_NRINGS_PER_LARGE_GROUP_DFLT);
	p->mldp_rx_nrings_per_small_group = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "rx_nrings_per_small_group", MLXCX_RX_NRINGS_PER_SMALL_GROUP_DFLT);

	p->mldp_ftbl_root_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "ftbl_root_size_shift",
	    MLXCX_FTBL_ROOT_SIZE_SHIFT_DFLT);

	p->mldp_tx_bind_threshold = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "tx_bind_threshold",
	    MLXCX_TX_BIND_THRESHOLD_DFLT);

	p->mldp_ftbl_vlan_size_shift = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "ftbl_vlan_size_shift",
	    MLXCX_FTBL_VLAN_SIZE_SHIFT_DFLT);

	p->mldp_eq_check_interval_sec = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "eq_check_interval_sec", MLXCX_EQ_CHECK_INTERVAL_SEC_DFLT);
	p->mldp_cq_check_interval_sec = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "cq_check_interval_sec", MLXCX_CQ_CHECK_INTERVAL_SEC_DFLT);
	p->mldp_wq_check_interval_sec = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "wq_check_interval_sec", MLXCX_WQ_CHECK_INTERVAL_SEC_DFLT);

	p->mldp_rx_per_cq = ddi_getprop(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "rx_limit_per_completion",
	    MLXCX_RX_PER_CQ_DEFAULT);

	if (p->mldp_rx_per_cq < MLXCX_RX_PER_CQ_MIN ||
	    p->mldp_rx_per_cq > MLXCX_RX_PER_CQ_MAX) {
		mlxcx_warn(mlxp, "!rx_limit_per_completion = %u is "
		    "out of range. Defaulting to: %d. Valid values are from "
		    "%d to %d", p->mldp_rx_per_cq, MLXCX_RX_PER_CQ_DEFAULT,
		    MLXCX_RX_PER_CQ_MIN, MLXCX_RX_PER_CQ_MAX);
		p->mldp_rx_per_cq = MLXCX_RX_PER_CQ_DEFAULT;
	}

	p->mldp_rx_p50_loan_min_size = ddi_getprop(DDI_DEV_T_ANY,
	    mlxp->mlx_dip, DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS,
	    "rx_p50_loan_min_size", MLXCX_P50_LOAN_MIN_SIZE_DFLT);
}

void
mlxcx_note(mlxcx_t *mlxp, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	if (mlxp != NULL && mlxp->mlx_dip != NULL) {
		vdev_err(mlxp->mlx_dip, CE_NOTE, fmt, ap);
	} else {
		vcmn_err(CE_NOTE, fmt, ap);
	}
	va_end(ap);
}

void
mlxcx_warn(mlxcx_t *mlxp, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	if (mlxp != NULL && mlxp->mlx_dip != NULL) {
		vdev_err(mlxp->mlx_dip, CE_WARN, fmt, ap);
	} else {
		vcmn_err(CE_WARN, fmt, ap);
	}
	va_end(ap);
}

void
mlxcx_panic(mlxcx_t *mlxp, const char *fmt, ...)
{
	va_list ap;

	va_start(ap, fmt);
	if (mlxp != NULL && mlxp->mlx_dip != NULL) {
		vdev_err(mlxp->mlx_dip, CE_PANIC, fmt, ap);
	} else {
		vcmn_err(CE_PANIC, fmt, ap);
	}
	va_end(ap);
}

uint16_t
mlxcx_get16(mlxcx_t *mlxp, uintptr_t off)
{
	uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
	return (ddi_get16(mlxp->mlx_regs_handle, (void *)addr));
}

uint32_t
mlxcx_get32(mlxcx_t *mlxp, uintptr_t off)
{
	uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
	return (ddi_get32(mlxp->mlx_regs_handle, (void *)addr));
}

uint64_t
mlxcx_get64(mlxcx_t *mlxp, uintptr_t off)
{
	uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
	return (ddi_get64(mlxp->mlx_regs_handle, (void *)addr));
}

void
mlxcx_put32(mlxcx_t *mlxp, uintptr_t off, uint32_t val)
{
	uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
	ddi_put32(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_put64(mlxcx_t *mlxp, uintptr_t off, uint64_t val)
{
	uintptr_t addr = off + (uintptr_t)mlxp->mlx_regs_base;
	ddi_put64(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_uar_put32(mlxcx_t *mlxp, mlxcx_uar_t *mlu, uintptr_t off, uint32_t val)
{
	/*
	 * The UAR is always inside the first BAR, which we mapped as
	 * mlx_regs
	 */
	uintptr_t addr = off + (uintptr_t)mlu->mlu_base +
	    (uintptr_t)mlxp->mlx_regs_base;
	ddi_put32(mlxp->mlx_regs_handle, (void *)addr, val);
}

void
mlxcx_uar_put64(mlxcx_t *mlxp, mlxcx_uar_t *mlu, uintptr_t off, uint64_t val)
{
	uintptr_t addr = off + (uintptr_t)mlu->mlu_base +
	    (uintptr_t)mlxp->mlx_regs_base;
	ddi_put64(mlxp->mlx_regs_handle, (void *)addr, val);
}

static void
mlxcx_fm_fini(mlxcx_t *mlxp)
{
	if (mlxp->mlx_fm_caps == 0)
		return;

	if (DDI_FM_ERRCB_CAP(mlxp->mlx_fm_caps))
		ddi_fm_handler_unregister(mlxp->mlx_dip);

	if (DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps) ||
	    DDI_FM_ERRCB_CAP(mlxp->mlx_fm_caps))
		pci_ereport_teardown(mlxp->mlx_dip);

	ddi_fm_fini(mlxp->mlx_dip);

	mlxp->mlx_fm_caps = 0;
}

void
mlxcx_fm_ereport(mlxcx_t *mlxp, const char *detail)
{
	uint64_t ena;
	char buf[FM_MAX_CLASS];

	if (!DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps))
		return;

	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
	ena = fm_ena_generate(0, FM_ENA_FMT1);
	ddi_fm_ereport_post(mlxp->mlx_dip, buf, ena, DDI_NOSLEEP,
	    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0,
	    NULL);
}

static int
mlxcx_fm_errcb(dev_info_t *dip, ddi_fm_error_t *err, const void *arg)
{
	/*
	 * as the driver can always deal with an error in any dma or
	 * access handle, we can just return the fme_status value.
	 */
	pci_ereport_post(dip, err, NULL);
	return (err->fme_status);
}

static void
mlxcx_fm_init(mlxcx_t *mlxp)
{
	ddi_iblock_cookie_t iblk;
	int def = DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE;

	mlxp->mlx_fm_caps = ddi_prop_get_int(DDI_DEV_T_ANY, mlxp->mlx_dip,
	    DDI_PROP_DONTPASS, "fm_capable", def);

	if (mlxp->mlx_fm_caps < 0) {
		mlxp->mlx_fm_caps = 0;
	}
	mlxp->mlx_fm_caps &= def;

	if (mlxp->mlx_fm_caps == 0)
		return;

	ddi_fm_init(mlxp->mlx_dip, &mlxp->mlx_fm_caps, &iblk);
	if (DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps) ||
	    DDI_FM_ERRCB_CAP(mlxp->mlx_fm_caps)) {
		pci_ereport_setup(mlxp->mlx_dip);
	}
	if (DDI_FM_ERRCB_CAP(mlxp->mlx_fm_caps)) {
		ddi_fm_handler_register(mlxp->mlx_dip, mlxcx_fm_errcb,
		    (void *)mlxp);
	}
}

static void
mlxcx_mlbs_teardown(mlxcx_t *mlxp, mlxcx_buf_shard_t *s)
{
	mlxcx_buffer_t *buf;

	mutex_enter(&s->mlbs_mtx);

	while (!list_is_empty(&s->mlbs_busy))
		cv_wait(&s->mlbs_free_nonempty, &s->mlbs_mtx);

	while (!list_is_empty(&s->mlbs_loaned))
		cv_wait(&s->mlbs_free_nonempty, &s->mlbs_mtx);

	while ((buf = list_head(&s->mlbs_free)) != NULL)
		mlxcx_buf_destroy(mlxp, buf);

	list_destroy(&s->mlbs_free);
	list_destroy(&s->mlbs_busy);
	list_destroy(&s->mlbs_loaned);
	mutex_exit(&s->mlbs_mtx);

	cv_destroy(&s->mlbs_free_nonempty);
	mutex_destroy(&s->mlbs_mtx);
}

static void
mlxcx_teardown_bufs(mlxcx_t *mlxp)
{
	mlxcx_buf_shard_t *s;

	while ((s = list_remove_head(&mlxp->mlx_buf_shards)) != NULL) {
		mlxcx_mlbs_teardown(mlxp, s);
		kmem_free(s, sizeof (mlxcx_buf_shard_t));
	}
	list_destroy(&mlxp->mlx_buf_shards);

	kmem_cache_destroy(mlxp->mlx_bufs_cache);
}

static void
mlxcx_teardown_pages(mlxcx_t *mlxp)
{
	uint_t nzeros = 0;
	uint64_t *pas;

	pas = kmem_alloc(sizeof (*pas) * MLXCX_MANAGE_PAGES_MAX_PAGES,
	    KM_SLEEP);

	mutex_enter(&mlxp->mlx_pagemtx);

	while (mlxp->mlx_npages > 0) {
		int32_t req, ret;

		ASSERT0(avl_is_empty(&mlxp->mlx_pages));
		req = MIN(mlxp->mlx_npages, MLXCX_MANAGE_PAGES_MAX_PAGES);

		if (!mlxcx_cmd_return_pages(mlxp, req, pas, &ret)) {
			mlxcx_warn(mlxp, "hardware refused to return pages, "
			    "leaking %u remaining pages", mlxp->mlx_npages);
			goto out;
		}

		for (int32_t i = 0; i < ret; i++) {
			mlxcx_dev_page_t *mdp, probe;
			bzero(&probe, sizeof (probe));
			probe.mxdp_pa = pas[i];

			mdp = avl_find(&mlxp->mlx_pages, &probe, NULL);

			if (mdp != NULL) {
				avl_remove(&mlxp->mlx_pages, mdp);
				mlxp->mlx_npages--;
				mlxcx_dma_free(&mdp->mxdp_dma);
				kmem_free(mdp, sizeof (mlxcx_dev_page_t));
			} else {
				mlxcx_panic(mlxp, "hardware returned a page "
				    "with PA 0x%" PRIx64 " but we have no "
				    "record of giving out such a page", pas[i]);
			}
		}

		/*
		 * If no pages were returned, note that fact.
		 */
		if (ret == 0) {
			nzeros++;
			if (nzeros > mlxcx_reclaim_tries) {
				mlxcx_warn(mlxp, "hardware refused to return "
				    "pages, leaking %u remaining pages",
				    mlxp->mlx_npages);
				goto out;
			}
			delay(drv_usectohz(mlxcx_reclaim_delay));
		}
	}

	avl_destroy(&mlxp->mlx_pages);

out:
	mutex_exit(&mlxp->mlx_pagemtx);
	mutex_destroy(&mlxp->mlx_pagemtx);

	kmem_free(pas, sizeof (*pas) * MLXCX_MANAGE_PAGES_MAX_PAGES);
}

static boolean_t
mlxcx_eq_alloc_dma(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
{
	ddi_device_acc_attr_t acc;
	ddi_dma_attr_t attr;
	boolean_t ret;
	size_t sz, i;

	VERIFY0(mleq->mleq_state & MLXCX_EQ_ALLOC);

	mleq->mleq_entshift = mlxp->mlx_props.mldp_eq_size_shift;
	mleq->mleq_nents = (1 << mleq->mleq_entshift);
	sz = mleq->mleq_nents * sizeof (mlxcx_eventq_ent_t);
	ASSERT3U(sz & (MLXCX_HW_PAGE_SIZE - 1), ==, 0);

	mlxcx_dma_acc_attr(mlxp, &acc);
	mlxcx_dma_queue_attr(mlxp, &attr);

	ret = mlxcx_dma_alloc(mlxp, &mleq->mleq_dma, &attr, &acc,
	    B_TRUE, sz, B_TRUE);
	if (!ret) {
		mlxcx_warn(mlxp, "failed to allocate EQ memory");
		return (B_FALSE);
	}

	mleq->mleq_ent = (mlxcx_eventq_ent_t *)mleq->mleq_dma.mxdb_va;

	for (i = 0; i < mleq->mleq_nents; ++i)
		mleq->mleq_ent[i].mleqe_owner = MLXCX_EQ_OWNER_INIT;

	mleq->mleq_state |= MLXCX_EQ_ALLOC;

	return (B_TRUE);
}

static void
mlxcx_eq_rele_dma(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq)
{
	VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC);
	if (mleq->mleq_state & MLXCX_EQ_CREATED)
		VERIFY(mleq->mleq_state & MLXCX_EQ_DESTROYED);

	mlxcx_dma_free(&mleq->mleq_dma);
	mleq->mleq_ent = NULL;

	mleq->mleq_state &= ~MLXCX_EQ_ALLOC;
}

void
mlxcx_teardown_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *ft)
{
	mlxcx_flow_group_t *fg;
	mlxcx_flow_entry_t *fe;
	int i;

	ASSERT(mutex_owned(&ft->mlft_mtx));

	for (i = ft->mlft_nents - 1; i >= 0; --i) {
		fe = &ft->mlft_ent[i];
		if (fe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) {
			if (!mlxcx_cmd_delete_flow_table_entry(mlxp, fe)) {
				mlxcx_panic(mlxp, "failed to delete flow "
				    "entry %u on table %u", i,
				    ft->mlft_num);
			}
		}
	}

	while ((fg = list_remove_head(&ft->mlft_groups)) != NULL) {
		if (fg->mlfg_state & MLXCX_FLOW_GROUP_CREATED &&
		    !(fg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED)) {
			if (!mlxcx_cmd_destroy_flow_group(mlxp, fg)) {
				mlxcx_panic(mlxp, "failed to destroy flow "
				    "group %u", fg->mlfg_num);
			}
		}
		kmem_free(fg, sizeof (mlxcx_flow_group_t));
	}
	list_destroy(&ft->mlft_groups);
	if (ft->mlft_state & MLXCX_FLOW_TABLE_CREATED &&
	    !(ft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED)) {
		if (!mlxcx_cmd_destroy_flow_table(mlxp, ft)) {
			mlxcx_panic(mlxp, "failed to destroy flow table %u",
			    ft->mlft_num);
		}
	}
	kmem_free(ft->mlft_ent, ft->mlft_entsize);
	ft->mlft_ent = NULL;
	mutex_exit(&ft->mlft_mtx);
	mutex_destroy(&ft->mlft_mtx);
	kmem_free(ft, sizeof (mlxcx_flow_table_t));
}

static void
mlxcx_teardown_ports(mlxcx_t *mlxp)
{
	uint_t i;
	mlxcx_port_t *p;
	mlxcx_flow_table_t *ft;

	for (i = 0; i < mlxp->mlx_nports; ++i) {
		p = &mlxp->mlx_ports[i];
		if (!(p->mlp_init & MLXCX_PORT_INIT))
			continue;
		mutex_enter(&p->mlp_mtx);
		if ((ft = p->mlp_rx_flow) != NULL) {
			mutex_enter(&ft->mlft_mtx);
			/*
			 * teardown_flow_table() will destroy the mutex, so
			 * we don't release it here.
			 */
			mlxcx_teardown_flow_table(mlxp, ft);
		}
		mutex_exit(&p->mlp_mtx);
		mutex_destroy(&p->mlp_mtx);
		mutex_destroy(&p->mlx_port_event.mla_mtx);
		p->mlx_port_event.mla_mlx = NULL;
		p->mlx_port_event.mla_port = NULL;
		p->mlp_init &= ~MLXCX_PORT_INIT;
	}

	kmem_free(mlxp->mlx_ports, mlxp->mlx_ports_size);
	mlxp->mlx_ports = NULL;
}

static void
mlxcx_teardown_wqs(mlxcx_t *mlxp)
{
	mlxcx_work_queue_t *mlwq;

	while ((mlwq = list_head(&mlxp->mlx_wqs)) != NULL) {
		mlxcx_wq_teardown(mlxp, mlwq);
	}
	list_destroy(&mlxp->mlx_wqs);
}

static void
mlxcx_teardown_cqs(mlxcx_t *mlxp)
{
	mlxcx_completion_queue_t *mlcq;

	while ((mlcq = list_head(&mlxp->mlx_cqs)) != NULL) {
		mlxcx_cq_teardown(mlxp, mlcq);
	}
	list_destroy(&mlxp->mlx_cqs);
}

static void
mlxcx_teardown_eqs(mlxcx_t *mlxp)
{
	mlxcx_event_queue_t *mleq;
	uint_t i;

	for (i = 0; i < mlxp->mlx_intr_count; ++i) {
		mleq = &mlxp->mlx_eqs[i];
		mutex_enter(&mleq->mleq_mtx);
		if ((mleq->mleq_state & MLXCX_EQ_CREATED) &&
		    !(mleq->mleq_state & MLXCX_EQ_DESTROYED)) {
			if (!mlxcx_cmd_destroy_eq(mlxp, mleq)) {
				mlxcx_warn(mlxp, "failed to destroy "
				    "event queue idx %u eqn %u",
				    i, mleq->mleq_num);
			}
		}
		if (mleq->mleq_state & MLXCX_EQ_ALLOC) {
			mlxcx_eq_rele_dma(mlxp, mleq);
		}
		mutex_exit(&mleq->mleq_mtx);
	}
}

static void
mlxcx_teardown_checktimers(mlxcx_t *mlxp)
{
	if (mlxp->mlx_props.mldp_eq_check_interval_sec > 0)
		ddi_periodic_delete(mlxp->mlx_eq_checktimer);
	if (mlxp->mlx_props.mldp_cq_check_interval_sec > 0)
		ddi_periodic_delete(mlxp->mlx_cq_checktimer);
	if (mlxp->mlx_props.mldp_wq_check_interval_sec > 0)
		ddi_periodic_delete(mlxp->mlx_wq_checktimer);
}

static void
mlxcx_teardown(mlxcx_t *mlxp)
{
	uint_t i;
	dev_info_t *dip = mlxp->mlx_dip;

	if (mlxp->mlx_attach & MLXCX_ATTACH_INTRS) {
		/*
		 * Disable interrupts and let any active vectors quiesce.
		 */
		mlxcx_intr_disable(mlxp);
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_SENSORS) {
		mlxcx_teardown_sensors(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_SENSORS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_CHKTIMERS) {
		mlxcx_teardown_checktimers(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_CHKTIMERS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_GROUPS) {
		mlxcx_teardown_groups(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_GROUPS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_WQS) {
		mlxcx_teardown_wqs(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_WQS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_CQS) {
		mlxcx_teardown_cqs(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_CQS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_BUFS) {
		mlxcx_teardown_bufs(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_BUFS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_PORTS) {
		mlxcx_teardown_ports(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_PORTS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_INTRS) {
		mlxcx_teardown_eqs(mlxp);
		mlxcx_intr_teardown(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_INTRS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_UAR_PD_TD) {
		if (mlxp->mlx_uar.mlu_allocated) {
			if (!mlxcx_cmd_dealloc_uar(mlxp, &mlxp->mlx_uar)) {
				mlxcx_warn(mlxp, "failed to release UAR");
			}
			for (i = 0; i < MLXCX_BF_PER_UAR; ++i)
				mutex_destroy(&mlxp->mlx_uar.mlu_bf[i].mbf_mtx);
		}
		if (mlxp->mlx_pd.mlpd_allocated &&
		    !mlxcx_cmd_dealloc_pd(mlxp, &mlxp->mlx_pd)) {
			mlxcx_warn(mlxp, "failed to release PD");
		}
		if (mlxp->mlx_tdom.mltd_allocated &&
		    !mlxcx_cmd_dealloc_tdom(mlxp, &mlxp->mlx_tdom)) {
			mlxcx_warn(mlxp, "failed to release TDOM");
		}
		mlxp->mlx_attach &= ~MLXCX_ATTACH_UAR_PD_TD;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_INIT_HCA) {
		if (!mlxcx_cmd_teardown_hca(mlxp)) {
			mlxcx_warn(mlxp, "failed to send teardown HCA "
			    "command during device detach");
		}
		mlxp->mlx_attach &= ~MLXCX_ATTACH_INIT_HCA;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_PAGE_LIST) {
		mlxcx_teardown_pages(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_PAGE_LIST;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_ASYNC_TQ) {
		for (i = 0; i <= MLXCX_FUNC_ID_MAX; i++) {
			mlxp->mlx_npages_req[i].mla_mlx = NULL;
			mutex_destroy(&mlxp->mlx_npages_req[i].mla_mtx);
		}
		taskq_destroy(mlxp->mlx_async_tq);
		mlxp->mlx_async_tq = NULL;
		mlxp->mlx_attach &= ~MLXCX_ATTACH_ASYNC_TQ;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_ENABLE_HCA) {
		if (!mlxcx_cmd_disable_hca(mlxp)) {
			mlxcx_warn(mlxp, "failed to send DISABLE HCA command "
			    "during device detach");
		}
		mlxp->mlx_attach &= ~MLXCX_ATTACH_ENABLE_HCA;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_CMD) {
		mlxcx_cmd_queue_fini(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_CMD;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_CAPS) {
		kmem_free(mlxp->mlx_caps, sizeof (mlxcx_caps_t));
		mlxp->mlx_caps = NULL;
		mlxp->mlx_attach &= ~MLXCX_ATTACH_CAPS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_REGS) {
		ddi_regs_map_free(&mlxp->mlx_regs_handle);
		mlxp->mlx_regs_handle = NULL;
		mlxp->mlx_attach &= ~MLXCX_ATTACH_REGS;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_PCI_CONFIG) {
		pci_config_teardown(&mlxp->mlx_cfg_handle);
		mlxp->mlx_cfg_handle = NULL;
		mlxp->mlx_attach &= ~MLXCX_ATTACH_PCI_CONFIG;
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_FM) {
		mlxcx_fm_fini(mlxp);
		mlxp->mlx_attach &= ~MLXCX_ATTACH_FM;
	}

	VERIFY3S(mlxp->mlx_attach, ==, 0);
	ddi_soft_state_free(mlxcx_softstate, mlxp->mlx_inst);
	ddi_set_driver_private(dip, NULL);
}

static void
mlxcx_get_model(mlxcx_t *mlxp)
{
	uint16_t venid;
	uint16_t devid;

	venid = pci_config_get16(mlxp->mlx_cfg_handle, PCI_CONF_VENID);
	if (venid != MLXCX_VENDOR_ID) {
		/* Currently, all supported cards have a Mellanox vendor id. */
		mlxp->mlx_type = MLXCX_DEV_UNKNOWN;
		return;
	}

	devid = pci_config_get16(mlxp->mlx_cfg_handle, PCI_CONF_DEVID);
	switch (devid) {
	case MLXCX_CX4_DEVID:
	case MLXCX_CX4_VF_DEVID:
	case MLXCX_CX4_LX_VF_DEVID:
		mlxp->mlx_type = MLXCX_DEV_CX4;
		break;
	case MLXCX_CX5_DEVID:
	case MLXCX_CX5_VF_DEVID:
	case MLXCX_CX5_EX_DEVID:
	case MLXCX_CX5_EX_VF_DEVID:
	case MLXCX_CX5_GEN_VF_DEVID:
		mlxp->mlx_type = MLXCX_DEV_CX5;
		break;
	case MLXCX_CX6_DEVID:
	case MLXCX_CX6_VF_DEVID:
	case MLXCX_CX6_DF_DEVID:
	case MLXCX_CX6_LX_DEVID:
		mlxp->mlx_type = MLXCX_DEV_CX6;
		break;
	default:
		mlxp->mlx_type = MLXCX_DEV_UNKNOWN;
	}
}

static boolean_t
mlxcx_regs_map(mlxcx_t *mlxp)
{
	off_t memsize;
	int ret;
	ddi_device_acc_attr_t da;

	if (ddi_dev_regsize(mlxp->mlx_dip, MLXCX_REG_NUMBER, &memsize) !=
	    DDI_SUCCESS) {
		mlxcx_warn(mlxp, "failed to get register set size");
		return (B_FALSE);
	}

	/*
	 * All data in the main BAR is kept in big-endian even though it's a PCI
	 * device.
	 */
	bzero(&da, sizeof (ddi_device_acc_attr_t));
	da.devacc_attr_version = DDI_DEVICE_ATTR_V0;
	da.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
	da.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
	if (DDI_FM_ACC_ERR_CAP(mlxp->mlx_fm_caps)) {
		da.devacc_attr_access = DDI_FLAGERR_ACC;
	} else {
		da.devacc_attr_access = DDI_DEFAULT_ACC;
	}

	ret = ddi_regs_map_setup(mlxp->mlx_dip, MLXCX_REG_NUMBER,
	    &mlxp->mlx_regs_base, 0, memsize, &da, &mlxp->mlx_regs_handle);

	if (ret != DDI_SUCCESS) {
		mlxcx_warn(mlxp, "failed to map device registers: %d", ret);
		return (B_FALSE);
	}

	return (B_TRUE);
}

static boolean_t
mlxcx_check_issi(mlxcx_t *mlxp)
{
	uint32_t issi;

	if (!mlxcx_cmd_query_issi(mlxp, &issi)) {
		mlxcx_warn(mlxp, "failed to get ISSI");
		return (B_FALSE);
	}

	if ((issi & (1 << MLXCX_CURRENT_ISSI)) == 0) {
		mlxcx_warn(mlxp, "hardware does not support software ISSI, "
		    "hw vector 0x%x, sw version %u", issi, MLXCX_CURRENT_ISSI);
		return (B_FALSE);
	}

	if (!mlxcx_cmd_set_issi(mlxp, MLXCX_CURRENT_ISSI)) {
		mlxcx_warn(mlxp, "failed to set ISSI to %u",
		    MLXCX_CURRENT_ISSI);
		return (B_FALSE);
	}

	return (B_TRUE);
}

boolean_t
mlxcx_give_pages(mlxcx_t *mlxp, int32_t npages, int32_t *ngiven)
{
	ddi_device_acc_attr_t acc;
	ddi_dma_attr_t attr;
	int32_t i;
	list_t plist;
	mlxcx_dev_page_t *mdp;
	mlxcx_dev_page_t **pages;
	const ddi_dma_cookie_t *ck;

	/*
	 * If there are no pages required, then we're done here.
	 */
	if (npages <= 0) {
		*ngiven = 0;
		return (B_TRUE);
	}

	npages = MIN(npages, MLXCX_MANAGE_PAGES_MAX_PAGES);

	pages = kmem_alloc(sizeof (*pages) * npages, KM_SLEEP);

	list_create(&plist, sizeof (mlxcx_dev_page_t),
	    offsetof(mlxcx_dev_page_t, mxdp_list));

	for (i = 0; i < npages; i++) {
		mdp = kmem_zalloc(sizeof (mlxcx_dev_page_t), KM_SLEEP);
		mlxcx_dma_acc_attr(mlxp, &acc);
		mlxcx_dma_page_attr(mlxp, &attr);
		if (!mlxcx_dma_alloc(mlxp, &mdp->mxdp_dma, &attr, &acc,
		    B_TRUE, MLXCX_HW_PAGE_SIZE, B_TRUE)) {
			mlxcx_warn(mlxp, "failed to allocate 4k page %u/%u", i,
			    npages);
			kmem_free(mdp, sizeof (mlxcx_dev_page_t));
			goto cleanup_npages;
		}
		ck = mlxcx_dma_cookie_one(&mdp->mxdp_dma);
		mdp->mxdp_pa = ck->dmac_laddress;

		list_insert_tail(&plist, mdp);
	}

	/*
	 * Now that all of the pages have been allocated, given them to hardware
	 * in chunks.
	 */
	for (i = 0; i < npages; i++) {
		pages[i] = list_remove_head(&plist);
	}

	if (!mlxcx_cmd_give_pages(mlxp,
	    MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES, npages, pages)) {
		mlxcx_warn(mlxp, "!hardware refused our gift of %u "
		    "pages!", npages);
		for (i = 0; i < npages; i++) {
			list_insert_tail(&plist, pages[i]);
		}
		goto cleanup_npages;
	}

	mutex_enter(&mlxp->mlx_pagemtx);
	for (i = 0; i < npages; i++) {
		avl_add(&mlxp->mlx_pages, pages[i]);
	}
	mlxp->mlx_npages += npages;
	mutex_exit(&mlxp->mlx_pagemtx);

	list_destroy(&plist);
	kmem_free(pages, sizeof (*pages) * npages);

	*ngiven = npages;

	return (B_TRUE);

cleanup_npages:
	kmem_free(pages, sizeof (*pages) * npages);
	while ((mdp = list_remove_head(&plist)) != NULL) {
		mlxcx_dma_free(&mdp->mxdp_dma);
		kmem_free(mdp, sizeof (mlxcx_dev_page_t));
	}
	list_destroy(&plist);
	return (B_FALSE);
}

static boolean_t
mlxcx_init_pages(mlxcx_t *mlxp, uint_t type)
{
	int32_t npages, given;

	if (!mlxcx_cmd_query_pages(mlxp, type, &npages)) {
		mlxcx_warn(mlxp, "failed to determine boot pages");
		return (B_FALSE);
	}

	while (npages > 0) {
		if (!mlxcx_give_pages(mlxp, npages, &given))
			return (B_FALSE);

		npages -= given;
	}

	return (B_TRUE);
}

static int
mlxcx_bufs_cache_constr(void *arg, void *cookie, int kmflags)
{
	mlxcx_t *mlxp = cookie;
	mlxcx_buffer_t *b = arg;

	bzero(b, sizeof (mlxcx_buffer_t));
	b->mlb_mlx = mlxp;
	b->mlb_state = MLXCX_BUFFER_INIT;
	list_create(&b->mlb_tx_chain, sizeof (mlxcx_buffer_t),
	    offsetof(mlxcx_buffer_t, mlb_tx_chain_entry));

	return (0);
}

static void
mlxcx_bufs_cache_destr(void *arg, void *cookie)
{
	mlxcx_t *mlxp = cookie;
	mlxcx_buffer_t *b = arg;
	VERIFY3P(b->mlb_mlx, ==, mlxp);
	VERIFY(b->mlb_state == MLXCX_BUFFER_INIT);
	list_destroy(&b->mlb_tx_chain);
}

mlxcx_buf_shard_t *
mlxcx_mlbs_create(mlxcx_t *mlxp)
{
	mlxcx_buf_shard_t *s;

	s = kmem_zalloc(sizeof (mlxcx_buf_shard_t), KM_SLEEP);

	mutex_init(&s->mlbs_mtx, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(mlxp->mlx_intr_pri));
	list_create(&s->mlbs_busy, sizeof (mlxcx_buffer_t),
	    offsetof(mlxcx_buffer_t, mlb_entry));
	list_create(&s->mlbs_free, sizeof (mlxcx_buffer_t),
	    offsetof(mlxcx_buffer_t, mlb_entry));
	list_create(&s->mlbs_loaned, sizeof (mlxcx_buffer_t),
	    offsetof(mlxcx_buffer_t, mlb_entry));
	cv_init(&s->mlbs_free_nonempty, NULL, CV_DRIVER, NULL);

	list_insert_tail(&mlxp->mlx_buf_shards, s);

	return (s);
}

static boolean_t
mlxcx_setup_bufs(mlxcx_t *mlxp)
{
	char namebuf[KSTAT_STRLEN];

	(void) snprintf(namebuf, KSTAT_STRLEN, "mlxcx%d_bufs_cache",
	    ddi_get_instance(mlxp->mlx_dip));
	mlxp->mlx_bufs_cache = kmem_cache_create(namebuf,
	    sizeof (mlxcx_buffer_t), sizeof (uint64_t),
	    mlxcx_bufs_cache_constr, mlxcx_bufs_cache_destr,
	    NULL, mlxp, NULL, 0);

	list_create(&mlxp->mlx_buf_shards, sizeof (mlxcx_buf_shard_t),
	    offsetof(mlxcx_buf_shard_t, mlbs_entry));

	return (B_TRUE);
}

static void
mlxcx_fm_qstate_ereport(mlxcx_t *mlxp, const char *qtype, uint32_t qnum,
    const char *state, uint8_t statenum)
{
	uint64_t ena;
	char buf[FM_MAX_CLASS];

	if (!DDI_FM_EREPORT_CAP(mlxp->mlx_fm_caps))
		return;

	(void) snprintf(buf, FM_MAX_CLASS, "%s.%s",
	    MLXCX_FM_SERVICE_MLXCX, "qstate.err");
	ena = fm_ena_generate(0, FM_ENA_FMT1);

	ddi_fm_ereport_post(mlxp->mlx_dip, buf, ena, DDI_NOSLEEP,
	    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0,
	    "state", DATA_TYPE_STRING, state,
	    "state_num", DATA_TYPE_UINT8, statenum,
	    "qtype", DATA_TYPE_STRING, qtype,
	    "qnum", DATA_TYPE_UINT32, qnum,
	    NULL);
	ddi_fm_service_impact(mlxp->mlx_dip, DDI_SERVICE_DEGRADED);
}

/*
 * The following set of routines are for monitoring the health of
 * event, completion and work queues. They run infrequently peeking at
 * the structs to catch stalls and inconsistent state.
 *
 * They peek at the structs *without* acquiring locks - we don't want
 * to impede flow of data. Driver start up and shutdown semantics
 * guarantee the structs are present and won't disappear underneath
 * these routines.
 *
 * As previously noted, the routines peek at active data in the structs and
 * they will store some values for comparison on next invocation. To
 * maintain integrity of the saved values, these values are only modified
 * within these routines.
 */
static void
mlxcx_eq_check(void *arg)
{
	mlxcx_t *mlxp = (mlxcx_t *)arg;
	mlxcx_event_queue_t *eq;
	mlxcx_eventq_ctx_t ctx;
	const char *str;

	uint_t i;

	for (i = 0; i < mlxp->mlx_intr_count; ++i) {
		eq = &mlxp->mlx_eqs[i];

		if ((eq->mleq_state & MLXCX_EQ_CREATED) == 0)
			continue;

		/*
		 * If the event queue was successfully created in the HCA,
		 * then initialization and shutdown sequences guarantee
		 * the queue exists.
		 */
		ASSERT0(eq->mleq_state & MLXCX_EQ_DESTROYED);

		if (!mlxcx_cmd_query_eq(mlxp, eq, &ctx))
			continue;

		str = "???";
		switch (ctx.mleqc_status) {
		case MLXCX_EQ_STATUS_OK:
			break;
		case MLXCX_EQ_STATUS_WRITE_FAILURE:
			str = "WRITE_FAILURE";
			break;
		}

		if (ctx.mleqc_status != MLXCX_EQ_STATUS_OK) {
			mlxcx_fm_qstate_ereport(mlxp, "event",
			    eq->mleq_num, str, ctx.mleqc_status);
			mlxcx_warn(mlxp, "EQ %u is in bad status: %x (%s)",
			    eq->mleq_intr_index, ctx.mleqc_status, str);
		}

		if (ctx.mleqc_state != MLXCX_EQ_ST_ARMED &&
		    (eq->mleq_state & MLXCX_EQ_ARMED)) {
			if (eq->mleq_cc == eq->mleq_check_disarm_cc &&
			    ++eq->mleq_check_disarm_cnt >= 3) {
				mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_STALL);
				mlxcx_warn(mlxp, "EQ %u isn't armed",
				    eq->mleq_intr_index);
			}
			eq->mleq_check_disarm_cc = eq->mleq_cc;
		} else {
			eq->mleq_check_disarm_cc = 0;
			eq->mleq_check_disarm_cnt = 0;
		}
	}
}

static void
mlxcx_cq_check(void *arg)
{
	mlxcx_t *mlxp = (mlxcx_t *)arg;
	mlxcx_completion_queue_t *cq;
	mlxcx_completionq_ctx_t ctx;
	const char *str, *type;
	uint_t v;

	for (cq = list_head(&mlxp->mlx_cqs); cq != NULL;
	    cq = list_next(&mlxp->mlx_cqs, cq)) {

		if ((cq->mlcq_state & MLXCX_CQ_CREATED) == 0)
			continue;

		/*
		 * If the completion queue was successfully created in the HCA,
		 * then initialization and shutdown sequences guarantee
		 * the queue exists.
		 */
		ASSERT0(cq->mlcq_state & MLXCX_CQ_DESTROYED);
		ASSERT0(cq->mlcq_state & MLXCX_CQ_TEARDOWN);

		if (cq->mlcq_fm_repd_qstate)
			continue;

		if (!mlxcx_cmd_query_cq(mlxp, cq, &ctx))
			continue;

		if (cq->mlcq_wq != NULL) {
			mlxcx_work_queue_t *wq = cq->mlcq_wq;
			if (wq->mlwq_type == MLXCX_WQ_TYPE_RECVQ)
				type = "rx ";
			else if (wq->mlwq_type == MLXCX_WQ_TYPE_SENDQ)
				type = "tx ";
			else
				type = "";
		} else {
			type = "";
		}

		str = "???";
		v = get_bits32(ctx.mlcqc_flags, MLXCX_CQ_CTX_STATUS);
		switch (v) {
		case MLXCX_CQC_STATUS_OK:
			break;
		case MLXCX_CQC_STATUS_OVERFLOW:
			str = "OVERFLOW";
			break;
		case MLXCX_CQC_STATUS_WRITE_FAIL:
			str = "WRITE_FAIL";
			break;
		case MLXCX_CQC_STATUS_INVALID:
			str = "INVALID";
			break;
		}

		if (v != MLXCX_CQC_STATUS_OK) {
			mlxcx_fm_qstate_ereport(mlxp, "completion",
			    cq->mlcq_num, str, v);
			mlxcx_warn(mlxp, "%sCQ 0x%x is in bad status: %x (%s)",
			    type, cq->mlcq_num, v, str);
			cq->mlcq_fm_repd_qstate = B_TRUE;
		}

		v = get_bits32(ctx.mlcqc_flags, MLXCX_CQ_CTX_STATE);
		if (v != MLXCX_CQC_STATE_ARMED &&
		    (cq->mlcq_state & MLXCX_CQ_ARMED) &&
		    !(cq->mlcq_state & MLXCX_CQ_POLLING)) {
			if (cq->mlcq_cc == cq->mlcq_check_disarm_cc &&
			    ++cq->mlcq_check_disarm_cnt >= 3) {
				mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_STALL);
				mlxcx_warn(mlxp, "%sCQ 0x%x (%p) isn't armed",
				    type, cq->mlcq_num, cq);
			}
			cq->mlcq_check_disarm_cc = cq->mlcq_cc;
		} else {
			cq->mlcq_check_disarm_cnt = 0;
			cq->mlcq_check_disarm_cc = 0;
		}
	}
}

void
mlxcx_check_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *sq)
{
	mlxcx_sq_ctx_t ctx;
	mlxcx_sq_state_t state;

	if (!mlxcx_cmd_query_sq(mlxp, sq, &ctx))
		return;

	ASSERT3U(from_be24(ctx.mlsqc_cqn), ==, sq->mlwq_cq->mlcq_num);
	state = get_bits32(ctx.mlsqc_flags, MLXCX_SQ_STATE);
	switch (state) {
	case MLXCX_SQ_STATE_RST:
		if (sq->mlwq_state & MLXCX_WQ_STARTED) {
			mlxcx_fm_qstate_ereport(mlxp, "send",
			    sq->mlwq_num, "RST", state);
			sq->mlwq_fm_repd_qstate = B_TRUE;
		}
		break;
	case MLXCX_SQ_STATE_RDY:
		if (!(sq->mlwq_state & MLXCX_WQ_STARTED)) {
			mlxcx_fm_qstate_ereport(mlxp, "send",
			    sq->mlwq_num, "RDY", state);
			sq->mlwq_fm_repd_qstate = B_TRUE;
		}
		break;
	case MLXCX_SQ_STATE_ERR:
		mlxcx_fm_qstate_ereport(mlxp, "send",
		    sq->mlwq_num, "ERR", state);
		sq->mlwq_fm_repd_qstate = B_TRUE;
		break;
	default:
		mlxcx_fm_qstate_ereport(mlxp, "send",
		    sq->mlwq_num, "???", state);
		sq->mlwq_fm_repd_qstate = B_TRUE;
		break;
	}
}

void
mlxcx_check_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *rq)
{
	mlxcx_rq_ctx_t ctx;
	mlxcx_rq_state_t state;


	if (!mlxcx_cmd_query_rq(mlxp, rq, &ctx))
		return;

	ASSERT3U(from_be24(ctx.mlrqc_cqn), ==, rq->mlwq_cq->mlcq_num);
	state = get_bits32(ctx.mlrqc_flags, MLXCX_RQ_STATE);
	switch (state) {
	case MLXCX_RQ_STATE_RST:
		if (rq->mlwq_state & MLXCX_WQ_STARTED) {
			mlxcx_fm_qstate_ereport(mlxp, "receive",
			    rq->mlwq_num, "RST", state);
			rq->mlwq_fm_repd_qstate = B_TRUE;
		}
		break;
	case MLXCX_RQ_STATE_RDY:
		if (!(rq->mlwq_state & MLXCX_WQ_STARTED)) {
			mlxcx_fm_qstate_ereport(mlxp, "receive",
			    rq->mlwq_num, "RDY", state);
			rq->mlwq_fm_repd_qstate = B_TRUE;
		}
		break;
	case MLXCX_RQ_STATE_ERR:
		mlxcx_fm_qstate_ereport(mlxp, "receive",
		    rq->mlwq_num, "ERR", state);
		rq->mlwq_fm_repd_qstate = B_TRUE;
		break;
	default:
		mlxcx_fm_qstate_ereport(mlxp, "receive",
		    rq->mlwq_num, "???", state);
		rq->mlwq_fm_repd_qstate = B_TRUE;
		break;
	}
}

static void
mlxcx_wq_check(void *arg)
{
	mlxcx_t *mlxp = (mlxcx_t *)arg;
	mlxcx_work_queue_t *wq;

	for (wq = list_head(&mlxp->mlx_wqs); wq != NULL;
	    wq = list_next(&mlxp->mlx_wqs, wq)) {

		if ((wq->mlwq_state & MLXCX_WQ_CREATED) == 0)
			continue;

		/*
		 * If the work queue was successfully created in the HCA,
		 * then initialization and shutdown sequences guarantee
		 * the queue exists.
		 */
		ASSERT0(wq->mlwq_state & MLXCX_WQ_DESTROYED);
		ASSERT0(wq->mlwq_state & MLXCX_WQ_TEARDOWN);

		if (wq->mlwq_fm_repd_qstate)
			continue;

		switch (wq->mlwq_type) {
		case MLXCX_WQ_TYPE_SENDQ:
			mlxcx_check_sq(mlxp, wq);
			break;
		case MLXCX_WQ_TYPE_RECVQ:
			mlxcx_check_rq(mlxp, wq);
			break;
		}
	}
}

static boolean_t
mlxcx_setup_checktimers(mlxcx_t *mlxp)
{
	if (mlxp->mlx_props.mldp_eq_check_interval_sec > 0) {
		mlxp->mlx_eq_checktimer = ddi_periodic_add(mlxcx_eq_check, mlxp,
		    mlxp->mlx_props.mldp_eq_check_interval_sec * NANOSEC,
		    DDI_IPL_0);
	}
	if (mlxp->mlx_props.mldp_cq_check_interval_sec > 0) {
		mlxp->mlx_cq_checktimer = ddi_periodic_add(mlxcx_cq_check, mlxp,
		    mlxp->mlx_props.mldp_cq_check_interval_sec * NANOSEC,
		    DDI_IPL_0);
	}
	if (mlxp->mlx_props.mldp_wq_check_interval_sec > 0) {
		mlxp->mlx_wq_checktimer = ddi_periodic_add(mlxcx_wq_check, mlxp,
		    mlxp->mlx_props.mldp_wq_check_interval_sec * NANOSEC,
		    DDI_IPL_0);
	}
	return (B_TRUE);
}

int
mlxcx_dmac_fe_compare(const void *arg0, const void *arg1)
{
	const mlxcx_flow_entry_t *left = arg0;
	const mlxcx_flow_entry_t *right = arg1;
	int bcmpr;

	bcmpr = memcmp(left->mlfe_dmac, right->mlfe_dmac,
	    sizeof (left->mlfe_dmac));
	if (bcmpr < 0)
		return (-1);
	if (bcmpr > 0)
		return (1);
	if (left->mlfe_vid < right->mlfe_vid)
		return (-1);
	if (left->mlfe_vid > right->mlfe_vid)
		return (1);
	return (0);
}

int
mlxcx_grmac_compare(const void *arg0, const void *arg1)
{
	const mlxcx_group_mac_t *left = arg0;
	const mlxcx_group_mac_t *right = arg1;
	int bcmpr;

	bcmpr = memcmp(left->mlgm_mac, right->mlgm_mac,
	    sizeof (left->mlgm_mac));
	if (bcmpr < 0)
		return (-1);
	if (bcmpr > 0)
		return (1);
	return (0);
}

int
mlxcx_page_compare(const void *arg0, const void *arg1)
{
	const mlxcx_dev_page_t *p0 = arg0;
	const mlxcx_dev_page_t *p1 = arg1;

	if (p0->mxdp_pa < p1->mxdp_pa)
		return (-1);
	if (p0->mxdp_pa > p1->mxdp_pa)
		return (1);
	return (0);
}

static boolean_t
mlxcx_setup_ports(mlxcx_t *mlxp)
{
	uint_t i, j;
	mlxcx_port_t *p;
	mlxcx_flow_table_t *ft;
	mlxcx_flow_group_t *fg;
	mlxcx_flow_entry_t *fe;

	VERIFY3U(mlxp->mlx_nports, >, 0);
	mlxp->mlx_ports_size = mlxp->mlx_nports * sizeof (mlxcx_port_t);
	mlxp->mlx_ports = kmem_zalloc(mlxp->mlx_ports_size, KM_SLEEP);

	for (i = 0; i < mlxp->mlx_nports; ++i) {
		p = &mlxp->mlx_ports[i];
		p->mlp_num = i;
		p->mlx_port_event.mla_mlx = mlxp;
		p->mlx_port_event.mla_port = p;
		mutex_init(&p->mlx_port_event.mla_mtx, NULL,
		    MUTEX_DRIVER, DDI_INTR_PRI(mlxp->mlx_async_intr_pri));
		p->mlp_init |= MLXCX_PORT_INIT;
		mutex_init(&p->mlp_mtx, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(mlxp->mlx_intr_pri));
		mutex_enter(&p->mlp_mtx);
		if (!mlxcx_cmd_query_nic_vport_ctx(mlxp, p)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		if (!mlxcx_cmd_query_port_mtu(mlxp, p)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		if (!mlxcx_cmd_query_port_status(mlxp, p)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		if (!mlxcx_cmd_query_port_speed(mlxp, p)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		if (!mlxcx_cmd_modify_nic_vport_ctx(mlxp, p,
		    MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		if (!mlxcx_cmd_query_port_fec(mlxp, p)) {
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		p->mlp_fec_requested = LINK_FEC_AUTO;

		mutex_exit(&p->mlp_mtx);
	}

	for (i = 0; i < mlxp->mlx_nports; ++i) {
		p = &mlxp->mlx_ports[i];
		mutex_enter(&p->mlp_mtx);
		p->mlp_rx_flow = (ft = kmem_zalloc(sizeof (mlxcx_flow_table_t),
		    KM_SLEEP));
		mutex_init(&ft->mlft_mtx, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(mlxp->mlx_intr_pri));

		mutex_enter(&ft->mlft_mtx);

		ft->mlft_type = MLXCX_FLOW_TABLE_NIC_RX;
		ft->mlft_port = p;
		ft->mlft_entshift = mlxp->mlx_props.mldp_ftbl_root_size_shift;
		if (ft->mlft_entshift > mlxp->mlx_caps->mlc_max_rx_ft_shift)
			ft->mlft_entshift = mlxp->mlx_caps->mlc_max_rx_ft_shift;
		ft->mlft_nents = (1 << ft->mlft_entshift);
		ft->mlft_entsize = ft->mlft_nents * sizeof (mlxcx_flow_entry_t);
		ft->mlft_ent = kmem_zalloc(ft->mlft_entsize, KM_SLEEP);
		list_create(&ft->mlft_groups, sizeof (mlxcx_flow_group_t),
		    offsetof(mlxcx_flow_group_t, mlfg_entry));

		for (j = 0; j < ft->mlft_nents; ++j) {
			ft->mlft_ent[j].mlfe_table = ft;
			ft->mlft_ent[j].mlfe_index = j;
		}

		if (!mlxcx_cmd_create_flow_table(mlxp, ft)) {
			mutex_exit(&ft->mlft_mtx);
			mutex_exit(&p->mlp_mtx);
			goto err;
		}

		if (!mlxcx_cmd_set_flow_table_root(mlxp, ft)) {
			mutex_exit(&ft->mlft_mtx);
			mutex_exit(&p->mlp_mtx);
			goto err;
		}

		/*
		 * We match broadcast at the top of the root flow table, then
		 * all multicast/unicast MACs, then the promisc entry is down
		 * the very bottom.
		 *
		 * This way when promisc is on, that entry simply catches any
		 * remaining traffic that earlier flows haven't matched.
		 */
		fg = kmem_zalloc(sizeof (mlxcx_flow_group_t), KM_SLEEP);
		list_insert_tail(&ft->mlft_groups, fg);
		fg->mlfg_table = ft;
		fg->mlfg_size = 1;
		fg->mlfg_mask |= MLXCX_FLOW_MATCH_DMAC;
		if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {
			mutex_exit(&ft->mlft_mtx);
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		p->mlp_bcast = fg;
		fe = list_head(&fg->mlfg_entries);
		fe->mlfe_action = MLXCX_FLOW_ACTION_FORWARD;
		(void) memset(fe->mlfe_dmac, 0xff, sizeof (fe->mlfe_dmac));
		fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;

		fg = kmem_zalloc(sizeof (mlxcx_flow_group_t), KM_SLEEP);
		list_insert_tail(&ft->mlft_groups, fg);
		fg->mlfg_table = ft;
		fg->mlfg_size = ft->mlft_nents - 2;
		fg->mlfg_mask |= MLXCX_FLOW_MATCH_DMAC;
		if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {
			mutex_exit(&ft->mlft_mtx);
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		p->mlp_umcast = fg;

		fg = kmem_zalloc(sizeof (mlxcx_flow_group_t), KM_SLEEP);
		list_insert_tail(&ft->mlft_groups, fg);
		fg->mlfg_table = ft;
		fg->mlfg_size = 1;
		if (!mlxcx_setup_flow_group(mlxp, ft, fg)) {
			mutex_exit(&ft->mlft_mtx);
			mutex_exit(&p->mlp_mtx);
			goto err;
		}
		p->mlp_promisc = fg;
		fe = list_head(&fg->mlfg_entries);
		fe->mlfe_action = MLXCX_FLOW_ACTION_FORWARD;
		fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;

		avl_create(&p->mlp_dmac_fe, mlxcx_dmac_fe_compare,
		    sizeof (mlxcx_flow_entry_t), offsetof(mlxcx_flow_entry_t,
		    mlfe_dmac_entry));

		mutex_exit(&ft->mlft_mtx);
		mutex_exit(&p->mlp_mtx);
	}

	return (B_TRUE);

err:
	mlxcx_teardown_ports(mlxp);
	return (B_FALSE);
}

void
mlxcx_remove_all_vlan_entries(mlxcx_t *mlxp, mlxcx_ring_group_t *g)
{
	mlxcx_flow_table_t *ft = g->mlg_rx_vlan_ft;
	mlxcx_flow_group_t *fg = g->mlg_rx_vlan_fg;
	mlxcx_flow_group_t *dfg = g->mlg_rx_vlan_def_fg;
	mlxcx_flow_entry_t *fe;
	mlxcx_group_vlan_t *v;

	ASSERT(mutex_owned(&g->mlg_mtx));

	mutex_enter(&ft->mlft_mtx);

	if (!list_is_empty(&g->mlg_rx_vlans)) {
		fe = list_head(&dfg->mlfg_entries);
		(void) mlxcx_cmd_set_flow_table_entry(mlxp, fe);
	}

	while ((v = list_remove_head(&g->mlg_rx_vlans)) != NULL) {
		fe = v->mlgv_fe;
		ASSERT3P(fe->mlfe_table, ==, ft);
		ASSERT3P(fe->mlfe_group, ==, fg);
		kmem_free(v, sizeof (mlxcx_group_vlan_t));

		(void) mlxcx_cmd_delete_flow_table_entry(mlxp, fe);
		fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;
	}

	mutex_exit(&ft->mlft_mtx);
}

boolean_t
mlxcx_remove_vlan_entry(mlxcx_t *mlxp, mlxcx_ring_group_t *g,
    boolean_t tagged, uint16_t vid)
{
	mlxcx_flow_table_t *ft = g->mlg_rx_vlan_ft;
	mlxcx_flow_group_t *fg = g->mlg_rx_vlan_fg;
	mlxcx_flow_group_t *dfg = g->mlg_rx_vlan_def_fg;
	mlxcx_flow_entry_t *fe;
	mlxcx_group_vlan_t *v;
	boolean_t found = B_FALSE;

	ASSERT(mutex_owned(&g->mlg_mtx));

	mutex_enter(&ft->mlft_mtx);

	for (v = list_head(&g->mlg_rx_vlans); v != NULL;
	    v = list_next(&g->mlg_rx_vlans, v)) {
		if (v->mlgv_tagged == tagged && v->mlgv_vid == vid) {
			found = B_TRUE;
			break;
		}
	}
	if (!found) {
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}

	list_remove(&g->mlg_rx_vlans, v);

	/*
	 * If this is the last VLAN entry, we have to go back to accepting
	 * any VLAN (which means re-enabling the default entry).
	 *
	 * Do this before we remove the flow entry for the last specific
	 * VLAN so that we don't lose any traffic in the transition.
	 */
	if (list_is_empty(&g->mlg_rx_vlans)) {
		fe = list_head(&dfg->mlfg_entries);
		if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {
			list_insert_tail(&g->mlg_rx_vlans, v);
			mutex_exit(&ft->mlft_mtx);
			return (B_FALSE);
		}
	}

	fe = v->mlgv_fe;
	ASSERT(fe->mlfe_state & MLXCX_FLOW_ENTRY_RESERVED);
	ASSERT(fe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED);
	ASSERT3P(fe->mlfe_table, ==, ft);
	ASSERT3P(fe->mlfe_group, ==, fg);

	if (!mlxcx_cmd_delete_flow_table_entry(mlxp, fe)) {
		list_insert_tail(&g->mlg_rx_vlans, v);
		fe = list_head(&dfg->mlfg_entries);
		if (fe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) {
			(void) mlxcx_cmd_delete_flow_table_entry(mlxp, fe);
		}
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}

	fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;

	kmem_free(v, sizeof (mlxcx_group_vlan_t));

	mutex_exit(&ft->mlft_mtx);
	return (B_TRUE);
}

boolean_t
mlxcx_add_vlan_entry(mlxcx_t *mlxp, mlxcx_ring_group_t *g, boolean_t tagged,
    uint16_t vid)
{
	mlxcx_flow_table_t *ft = g->mlg_rx_vlan_ft;
	mlxcx_flow_group_t *fg = g->mlg_rx_vlan_fg;
	mlxcx_flow_group_t *dfg = g->mlg_rx_vlan_def_fg;
	mlxcx_flow_entry_t *fe;
	mlxcx_group_vlan_t *v;
	boolean_t found = B_FALSE;
	boolean_t first = B_FALSE;

	ASSERT(mutex_owned(&g->mlg_mtx));

	mutex_enter(&ft->mlft_mtx);

	for (v = list_head(&g->mlg_rx_vlans); v != NULL;
	    v = list_next(&g->mlg_rx_vlans, v)) {
		if (v->mlgv_tagged == tagged && v->mlgv_vid == vid) {
			mutex_exit(&ft->mlft_mtx);
			return (B_TRUE);
		}
	}
	if (list_is_empty(&g->mlg_rx_vlans))
		first = B_TRUE;

	for (fe = list_head(&fg->mlfg_entries); fe != NULL;
	    fe = list_next(&fg->mlfg_entries, fe)) {
		if (!(fe->mlfe_state & MLXCX_FLOW_ENTRY_RESERVED)) {
			found = B_TRUE;
			break;
		}
	}
	if (!found) {
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}

	v = kmem_zalloc(sizeof (mlxcx_group_vlan_t), KM_SLEEP);
	v->mlgv_fe = fe;
	v->mlgv_tagged = tagged;
	v->mlgv_vid = vid;

	fe->mlfe_state |= MLXCX_FLOW_ENTRY_RESERVED;
	fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;
	fe->mlfe_vid = vid;
	if (tagged) {
		fe->mlfe_vlan_type = MLXCX_VLAN_TYPE_CVLAN;
	} else {
		fe->mlfe_vlan_type = MLXCX_VLAN_TYPE_NONE;
	}

	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {
		fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY;
		fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;
		kmem_free(v, sizeof (mlxcx_group_vlan_t));
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}

	list_insert_tail(&g->mlg_rx_vlans, v);

	/*
	 * If the vlan list was empty for this group before adding this one,
	 * then we no longer want the "default" entry to allow all VLANs
	 * through.
	 */
	if (first) {
		fe = list_head(&dfg->mlfg_entries);
		(void) mlxcx_cmd_delete_flow_table_entry(mlxp, fe);
	}

	mutex_exit(&ft->mlft_mtx);
	return (B_TRUE);
}

void
mlxcx_remove_all_umcast_entries(mlxcx_t *mlxp, mlxcx_port_t *port,
    mlxcx_ring_group_t *group)
{
	mlxcx_flow_entry_t *fe;
	mlxcx_flow_table_t *ft = port->mlp_rx_flow;
	mlxcx_group_mac_t *gm, *ngm;

	ASSERT(mutex_owned(&port->mlp_mtx));
	ASSERT(mutex_owned(&group->mlg_mtx));

	mutex_enter(&ft->mlft_mtx);

	gm = avl_first(&group->mlg_rx_macs);
	for (; gm != NULL; gm = ngm) {
		ngm = AVL_NEXT(&group->mlg_rx_macs, gm);

		ASSERT3P(gm->mlgm_group, ==, group);
		fe = gm->mlgm_fe;
		ASSERT3P(fe->mlfe_table, ==, ft);

		avl_remove(&group->mlg_rx_macs, gm);
		list_remove(&fe->mlfe_ring_groups, gm);
		kmem_free(gm, sizeof (mlxcx_group_mac_t));

		fe->mlfe_ndest = 0;
		for (gm = list_head(&fe->mlfe_ring_groups); gm != NULL;
		    gm = list_next(&fe->mlfe_ring_groups, gm)) {
			fe->mlfe_dest[fe->mlfe_ndest++].mlfed_flow =
			    gm->mlgm_group->mlg_rx_vlan_ft;
		}
		fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;

		if (fe->mlfe_ndest > 0) {
			(void) mlxcx_cmd_set_flow_table_entry(mlxp, fe);
			continue;
		}

		/*
		 * There are no more ring groups left for this MAC (it wasn't
		 * attached to any other groups since ndest == 0), so clean up
		 * its flow entry.
		 */
		avl_remove(&port->mlp_dmac_fe, fe);
		(void) mlxcx_cmd_delete_flow_table_entry(mlxp, fe);
		list_destroy(&fe->mlfe_ring_groups);
		fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;
	}

	mutex_exit(&ft->mlft_mtx);
}

boolean_t
mlxcx_remove_umcast_entry(mlxcx_t *mlxp, mlxcx_port_t *port,
    mlxcx_ring_group_t *group, const uint8_t *macaddr)
{
	mlxcx_flow_entry_t *fe;
	mlxcx_flow_table_t *ft = port->mlp_rx_flow;
	mlxcx_group_mac_t *gm, probe;

	ASSERT(mutex_owned(&port->mlp_mtx));
	ASSERT(mutex_owned(&group->mlg_mtx));

	bzero(&probe, sizeof (probe));
	bcopy(macaddr, probe.mlgm_mac, sizeof (probe.mlgm_mac));

	mutex_enter(&ft->mlft_mtx);

	gm = avl_find(&group->mlg_rx_macs, &probe, NULL);
	if (gm == NULL) {
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}
	ASSERT3P(gm->mlgm_group, ==, group);
	ASSERT0(bcmp(macaddr, gm->mlgm_mac, sizeof (gm->mlgm_mac)));

	fe = gm->mlgm_fe;
	ASSERT3P(fe->mlfe_table, ==, ft);
	ASSERT0(bcmp(macaddr, fe->mlfe_dmac, sizeof (fe->mlfe_dmac)));

	list_remove(&fe->mlfe_ring_groups, gm);
	avl_remove(&group->mlg_rx_macs, gm);
	kmem_free(gm, sizeof (mlxcx_group_mac_t));

	fe->mlfe_ndest = 0;
	for (gm = list_head(&fe->mlfe_ring_groups); gm != NULL;
	    gm = list_next(&fe->mlfe_ring_groups, gm)) {
		fe->mlfe_dest[fe->mlfe_ndest++].mlfed_flow =
		    gm->mlgm_group->mlg_rx_vlan_ft;
	}
	fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;

	if (fe->mlfe_ndest > 0) {
		if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {
			mutex_exit(&ft->mlft_mtx);
			return (B_FALSE);
		}
		mutex_exit(&ft->mlft_mtx);
		return (B_TRUE);
	}

	/*
	 * There are no more ring groups left for this MAC (it wasn't attached
	 * to any other groups since ndest == 0), so clean up its flow entry.
	 */
	avl_remove(&port->mlp_dmac_fe, fe);
	(void) mlxcx_cmd_delete_flow_table_entry(mlxp, fe);
	list_destroy(&fe->mlfe_ring_groups);

	fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;

	mutex_exit(&ft->mlft_mtx);

	return (B_TRUE);
}

boolean_t
mlxcx_add_umcast_entry(mlxcx_t *mlxp, mlxcx_port_t *port,
    mlxcx_ring_group_t *group, const uint8_t *macaddr)
{
	mlxcx_flow_group_t *fg;
	mlxcx_flow_entry_t *fe, probe;
	mlxcx_flow_table_t *ft = port->mlp_rx_flow;
	mlxcx_group_mac_t *gm;
	boolean_t found = B_FALSE;

	ASSERT(mutex_owned(&port->mlp_mtx));
	ASSERT(mutex_owned(&group->mlg_mtx));

	bzero(&probe, sizeof (probe));
	bcopy(macaddr, probe.mlfe_dmac, sizeof (probe.mlfe_dmac));

	mutex_enter(&ft->mlft_mtx);

	fe = avl_find(&port->mlp_dmac_fe, &probe, NULL);

	if (fe == NULL) {
		fg = port->mlp_umcast;
		for (fe = list_head(&fg->mlfg_entries); fe != NULL;
		    fe = list_next(&fg->mlfg_entries, fe)) {
			if (!(fe->mlfe_state & MLXCX_FLOW_ENTRY_RESERVED)) {
				found = B_TRUE;
				break;
			}
		}
		if (!found) {
			mutex_exit(&ft->mlft_mtx);
			return (B_FALSE);
		}
		list_create(&fe->mlfe_ring_groups, sizeof (mlxcx_group_mac_t),
		    offsetof(mlxcx_group_mac_t, mlgm_fe_entry));
		fe->mlfe_state |= MLXCX_FLOW_ENTRY_RESERVED;
		fe->mlfe_action = MLXCX_FLOW_ACTION_FORWARD;
		bcopy(macaddr, fe->mlfe_dmac, sizeof (fe->mlfe_dmac));

		avl_add(&port->mlp_dmac_fe, fe);
	}

	fe->mlfe_dest[fe->mlfe_ndest++].mlfed_flow = group->mlg_rx_vlan_ft;
	fe->mlfe_state |= MLXCX_FLOW_ENTRY_DIRTY;

	if (!mlxcx_cmd_set_flow_table_entry(mlxp, fe)) {
		fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY;
		if (--fe->mlfe_ndest == 0) {
			fe->mlfe_state &= ~MLXCX_FLOW_ENTRY_RESERVED;
		}
		mutex_exit(&ft->mlft_mtx);
		return (B_FALSE);
	}

	gm = kmem_zalloc(sizeof (mlxcx_group_mac_t), KM_SLEEP);
	gm->mlgm_group = group;
	gm->mlgm_fe = fe;
	bcopy(macaddr, gm->mlgm_mac, sizeof (gm->mlgm_mac));
	avl_add(&group->mlg_rx_macs, gm);
	list_insert_tail(&fe->mlfe_ring_groups, gm);

	mutex_exit(&ft->mlft_mtx);

	return (B_TRUE);
}

boolean_t
mlxcx_setup_flow_group(mlxcx_t *mlxp, mlxcx_flow_table_t *ft,
    mlxcx_flow_group_t *fg)
{
	mlxcx_flow_entry_t *fe;
	uint_t i, idx;

	ASSERT(mutex_owned(&ft->mlft_mtx));
	ASSERT(ft->mlft_state & MLXCX_FLOW_TABLE_CREATED);
	ASSERT3P(fg->mlfg_table, ==, ft);

	if (ft->mlft_next_ent + fg->mlfg_size > ft->mlft_nents)
		return (B_FALSE);
	fg->mlfg_start_idx = ft->mlft_next_ent;

	if (!mlxcx_cmd_create_flow_group(mlxp, fg)) {
		return (B_FALSE);
	}

	list_create(&fg->mlfg_entries, sizeof (mlxcx_flow_entry_t),
	    offsetof(mlxcx_flow_entry_t, mlfe_group_entry));
	for (i = 0; i < fg->mlfg_size; ++i) {
		idx = fg->mlfg_start_idx + i;
		fe = &ft->mlft_ent[idx];
		fe->mlfe_group = fg;
		list_insert_tail(&fg->mlfg_entries, fe);
	}
	fg->mlfg_avail = fg->mlfg_size;
	ft->mlft_next_ent += fg->mlfg_size;

	return (B_TRUE);
}

static boolean_t
mlxcx_setup_eq(mlxcx_t *mlxp, uint_t vec, uint64_t events)
{
	mlxcx_event_queue_t *mleq = &mlxp->mlx_eqs[vec];

	mutex_enter(&mleq->mleq_mtx);
	if (!mlxcx_eq_alloc_dma(mlxp, mleq)) {
		/* mlxcx_teardown_eqs() will clean this up */
		mutex_exit(&mleq->mleq_mtx);
		return (B_FALSE);
	}
	mleq->mleq_mlx = mlxp;
	mleq->mleq_uar = &mlxp->mlx_uar;
	mleq->mleq_events = events;
	mleq->mleq_intr_index = vec;

	if (!mlxcx_cmd_create_eq(mlxp, mleq)) {
		/* mlxcx_teardown_eqs() will clean this up */
		mutex_exit(&mleq->mleq_mtx);
		return (B_FALSE);
	}

	if (ddi_intr_enable(mlxp->mlx_intr_handles[vec]) != DDI_SUCCESS) {
		/*
		 * mlxcx_teardown_eqs() will handle calling cmd_destroy_eq and
		 * eq_rele_dma
		 */
		mutex_exit(&mleq->mleq_mtx);
		return (B_FALSE);
	}
	mleq->mleq_state |= MLXCX_EQ_INTR_ENABLED;
	mleq->mleq_state |= MLXCX_EQ_ATTACHING;
	mlxcx_arm_eq(mlxp, mleq);
	mutex_exit(&mleq->mleq_mtx);

	return (B_TRUE);
}

static void
mlxcx_eq_set_attached(mlxcx_t *mlxp)
{
	uint_t vec;
	mlxcx_event_queue_t *mleq;

	for (vec = 0; vec < mlxp->mlx_intr_count; ++vec) {
		mleq = &mlxp->mlx_eqs[vec];

		mutex_enter(&mleq->mleq_mtx);
		mleq->mleq_state &= ~MLXCX_EQ_ATTACHING;
		mutex_exit(&mleq->mleq_mtx);
	}
}

static boolean_t
mlxcx_setup_async_eqs(mlxcx_t *mlxp)
{
	boolean_t ret;

	ret = mlxcx_setup_eq(mlxp, 0,
	    (1ULL << MLXCX_EVENT_CMD_COMPLETION) |
	    (1ULL << MLXCX_EVENT_PAGE_REQUEST) |
	    (1ULL << MLXCX_EVENT_PORT_STATE) |
	    (1ULL << MLXCX_EVENT_INTERNAL_ERROR) |
	    (1ULL << MLXCX_EVENT_PORT_MODULE) |
	    (1ULL << MLXCX_EVENT_SENDQ_DRAIN) |
	    (1ULL << MLXCX_EVENT_LAST_WQE) |
	    (1ULL << MLXCX_EVENT_CQ_ERROR) |
	    (1ULL << MLXCX_EVENT_WQ_CATASTROPHE) |
	    (1ULL << MLXCX_EVENT_PAGE_FAULT) |
	    (1ULL << MLXCX_EVENT_WQ_INVALID_REQ) |
	    (1ULL << MLXCX_EVENT_WQ_ACCESS_VIOL) |
	    (1ULL << MLXCX_EVENT_NIC_VPORT) |
	    (1ULL << MLXCX_EVENT_DOORBELL_CONGEST));

	if (ret)
		mlxcx_cmd_eq_enable(mlxp);

	return (ret);
}

int
mlxcx_cq_compare(const void *arg0, const void *arg1)
{
	const mlxcx_completion_queue_t *left = arg0;
	const mlxcx_completion_queue_t *right = arg1;

	if (left->mlcq_num < right->mlcq_num) {
		return (-1);
	}
	if (left->mlcq_num > right->mlcq_num) {
		return (1);
	}
	return (0);
}

static boolean_t
mlxcx_setup_eqs(mlxcx_t *mlxp)
{
	uint_t i;
	mlxcx_event_queue_t *mleq;

	ASSERT3S(mlxp->mlx_intr_count, >, 0);

	for (i = mlxp->mlx_intr_cq0; i < mlxp->mlx_intr_count; ++i) {
		mleq = &mlxp->mlx_eqs[i];
		mutex_enter(&mleq->mleq_mtx);
		if (!mlxcx_eq_alloc_dma(mlxp, mleq)) {
			mutex_exit(&mleq->mleq_mtx);
			return (B_FALSE);
		}
		mleq->mleq_uar = &mlxp->mlx_uar;
		if (!mlxcx_cmd_create_eq(mlxp, mleq)) {
			/* mlxcx_teardown() will handle calling eq_rele_dma */
			mutex_exit(&mleq->mleq_mtx);
			return (B_FALSE);
		}
		if (mlxp->mlx_props.mldp_intrmod_period_usec != 0 &&
		    !mlxcx_cmd_set_int_mod(mlxp, i,
		    mlxp->mlx_props.mldp_intrmod_period_usec)) {
			mutex_exit(&mleq->mleq_mtx);
			return (B_FALSE);
		}
		if (ddi_intr_enable(mlxp->mlx_intr_handles[i]) != DDI_SUCCESS) {
			mutex_exit(&mleq->mleq_mtx);
			return (B_FALSE);
		}
		mleq->mleq_state |= MLXCX_EQ_INTR_ENABLED;
		mlxcx_arm_eq(mlxp, mleq);
		mutex_exit(&mleq->mleq_mtx);
	}

	mlxp->mlx_next_eq = mlxp->mlx_intr_cq0;

	return (B_TRUE);
}

/*
 * A more recent ConnectX part will have the Port CApability Mask register.
 * Explore it and note things here.
 */
static void
mlxcx_explore_pcam(mlxcx_t *mlxp, mlxcx_caps_t *c)
{
	mlxcx_register_data_t data;
	mlxcx_reg_pcam_t *pcam = &data.mlrd_pcam;

	ASSERT(c->mlc_pcam);
	bzero(&data, sizeof (data));

	/*
	 * Okay, so we have access the the Ports CApability Mask (PCAM).
	 * There are various things we need to check about it.
	 */

	VERIFY(mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ,
	    MLXCX_REG_PCAM, &data));

	/*
	 * NOTE: These ASSERT()s may change in future mlxcx(4D) parts.
	 * As of now, only 0 is valid, and 1-255 are reserved.  A future part
	 * may return non-zero in these fields.
	 */
	ASSERT0(pcam->mlrd_pcam_feature_group);
	ASSERT0(pcam->mlrd_pcam_access_reg_group);

	c->mlc_ext_ptys = get_bit64(pcam->mlrd_pcam_feature_cap_mask_low,
	    MLXCX_PCAM_LOW_FFLAGS_PTYS_EXTENDED);
}

/*
 * Snapshot all of the hardware capabilities that we care about and then modify
 * the HCA capabilities to get things moving.
 */
static boolean_t
mlxcx_init_caps(mlxcx_t *mlxp)
{
	mlxcx_caps_t *c;

	mlxp->mlx_caps = c = kmem_zalloc(sizeof (mlxcx_caps_t), KM_SLEEP);

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_GENERAL,
	    MLXCX_HCA_CAP_MODE_CURRENT, &c->mlc_hca_cur)) {
		mlxcx_warn(mlxp, "failed to obtain current HCA general caps");
	}

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_GENERAL,
	    MLXCX_HCA_CAP_MODE_MAX, &c->mlc_hca_max)) {
		mlxcx_warn(mlxp, "failed to obtain maximum HCA general caps");
	}

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_ETHERNET,
	    MLXCX_HCA_CAP_MODE_CURRENT, &c->mlc_ether_cur)) {
		mlxcx_warn(mlxp, "failed to obtain current HCA eth caps");
	}

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_ETHERNET,
	    MLXCX_HCA_CAP_MODE_MAX, &c->mlc_ether_max)) {
		mlxcx_warn(mlxp, "failed to obtain maximum HCA eth caps");
	}

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_NIC_FLOW,
	    MLXCX_HCA_CAP_MODE_CURRENT, &c->mlc_nic_flow_cur)) {
		mlxcx_warn(mlxp, "failed to obtain current HCA flow caps");
	}

	if (!mlxcx_cmd_query_hca_cap(mlxp, MLXCX_HCA_CAP_NIC_FLOW,
	    MLXCX_HCA_CAP_MODE_MAX, &c->mlc_nic_flow_max)) {
		mlxcx_warn(mlxp, "failed to obtain maximum HCA flow caps");
	}

	/*
	 * Check the caps meet our requirements.
	 */
	const mlxcx_hca_cap_general_caps_t *gen = &c->mlc_hca_cur.mhc_general;

	if (gen->mlcap_general_log_pg_sz != 12) {
		mlxcx_warn(mlxp, "!hardware has page size != 4k "
		    "(log_pg_sz = %u)", (uint_t)gen->mlcap_general_log_pg_sz);
		goto err;
	}
	if (gen->mlcap_general_cqe_version != 1) {
		mlxcx_warn(mlxp, "!hardware does not support CQE v1 "
		    "(cqe_ver = %u)", (uint_t)gen->mlcap_general_cqe_version);
		goto err;
	}
	if (gen->mlcap_general_port_type !=
	    MLXCX_CAP_GENERAL_PORT_TYPE_ETHERNET) {
		mlxcx_warn(mlxp, "!hardware has non-ethernet ports");
		goto err;
	}
	mlxp->mlx_nports = gen->mlcap_general_num_ports;
	mlxp->mlx_max_sdu = (1 << (gen->mlcap_general_log_max_msg & 0x1F));

	if (mlxp->mlx_type >= MLXCX_DEV_CX5 &&
	    get_bit16(gen->mlcap_general_flags_c,
	    MLXCX_CAP_GENERAL_FLAGS_C_PCAM_REG)) {
		c->mlc_pcam = B_TRUE;
	}

	c->mlc_max_tir = (1 << gen->mlcap_general_log_max_tir);

	c->mlc_checksum = get_bit32(c->mlc_ether_cur.mhc_eth.mlcap_eth_flags,
	    MLXCX_ETH_CAP_CSUM_CAP);
	c->mlc_vxlan = get_bit32(c->mlc_ether_cur.mhc_eth.mlcap_eth_flags,
	    MLXCX_ETH_CAP_TUNNEL_STATELESS_VXLAN);

	c->mlc_max_lso_size = (1 << get_bits32(c->mlc_ether_cur.mhc_eth.
	    mlcap_eth_flags, MLXCX_ETH_CAP_MAX_LSO_CAP));
	if (c->mlc_max_lso_size == 1) {
		c->mlc_max_lso_size = 0;
		c->mlc_lso = B_FALSE;
	} else {
		c->mlc_lso = B_TRUE;
	}

	c->mlc_max_rqt_size = (1 << get_bits32(c->mlc_ether_cur.mhc_eth.
	    mlcap_eth_flags, MLXCX_ETH_CAP_RSS_IND_TBL_CAP));

	if (!get_bit32(c->mlc_nic_flow_cur.mhc_flow.mlcap_flow_nic_rx.
	    mlcap_flow_prop_flags, MLXCX_FLOW_CAP_PROPS_SUPPORT)) {
		mlxcx_warn(mlxp, "!hardware does not support rx flow tables");
		goto err;
	}
	if (!get_bit32(c->mlc_nic_flow_cur.mhc_flow.mlcap_flow_nic_rx.
	    mlcap_flow_prop_flags, MLXCX_FLOW_CAP_PROPS_MODIFY)) {
		mlxcx_warn(mlxp, "!hardware does not support modifying rx "
		    "flow table entries");
		goto err;
	}

	c->mlc_max_rx_ft_shift = c->mlc_nic_flow_cur.mhc_flow.mlcap_flow_nic_rx.
	    mlcap_flow_prop_log_max_ft_size;
	c->mlc_max_rx_flows = (1 << c->mlc_nic_flow_cur.mhc_flow.
	    mlcap_flow_nic_rx.mlcap_flow_prop_log_max_flow);
	c->mlc_max_rx_ft = (1 << c->mlc_nic_flow_cur.mhc_flow.
	    mlcap_flow_nic_rx.mlcap_flow_prop_log_max_ft_num);
	c->mlc_max_rx_fe_dest = (1 << c->mlc_nic_flow_cur.mhc_flow.
	    mlcap_flow_nic_rx.mlcap_flow_prop_log_max_destination);

	return (B_TRUE);

err:
	kmem_free(mlxp->mlx_caps, sizeof (mlxcx_caps_t));
	return (B_FALSE);
}

static int
mlxcx_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	mlxcx_t *mlxp;

	if (cmd != DDI_DETACH)
		return (DDI_FAILURE);

	mlxp = ddi_get_driver_private(dip);
	if (mlxp == NULL) {
		mlxcx_warn(NULL, "asked to detach, but missing instance "
		    "private data");
		return (DDI_FAILURE);
	}

	if (mlxp->mlx_attach & MLXCX_ATTACH_MAC_HDL) {
		if (mac_unregister(mlxp->mlx_mac_hdl) != DDI_SUCCESS) {
			return (DDI_FAILURE);
		}
		mlxp->mlx_attach &= ~MLXCX_ATTACH_MAC_HDL;
	}

	mlxcx_teardown(mlxp);
	return (DDI_SUCCESS);
}

static size_t
mlxcx_calc_rx_ngroups(mlxcx_t *mlxp)
{
	size_t ngroups = mlxp->mlx_props.mldp_rx_ngroups_large +
	    mlxp->mlx_props.mldp_rx_ngroups_small;
	size_t tirlim, flowlim, gflowlim;

	tirlim = mlxp->mlx_caps->mlc_max_tir / MLXCX_TIRS_PER_GROUP;
	if (tirlim < ngroups) {
		mlxcx_note(mlxp, "limiting number of rx groups to %u based "
		    "on number of TIRs available", tirlim);
		ngroups = tirlim;
	}

	flowlim = (1 << mlxp->mlx_caps->mlc_max_rx_ft_shift) - 2;
	if (flowlim < ngroups) {
		mlxcx_note(mlxp, "limiting number of rx groups to %u based "
		    "on max size of RX flow tables", flowlim);
		ngroups = flowlim;
	}

	/*
	 * Restrict the number of groups not to exceed the max flow
	 * table number from the devices capabilities.
	 * There is one root table entry per port and 2 entries per
	 * group.
	 */
	flowlim = (mlxp->mlx_caps->mlc_max_rx_ft - mlxp->mlx_nports) / 2;
	if (flowlim < ngroups) {
		mlxcx_note(mlxp, "limiting number of rx groups to %u based "
		    "on max number of RX flow tables",
		    flowlim);
		ngroups = flowlim;
	}

	do {
		gflowlim = mlxp->mlx_caps->mlc_max_rx_flows - 16 * ngroups - 2;
		if (gflowlim < ngroups) {
			mlxcx_note(mlxp, "limiting number of rx groups to %u "
			    "based on max total RX flows", gflowlim);
			--ngroups;
		}
	} while (gflowlim < ngroups);

	return (ngroups);
}

static int
mlxcx_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	mlxcx_t *mlxp;
	char tq_name[TASKQ_NAMELEN];
	uint_t i;
	int inst, ret;

	if (cmd != DDI_ATTACH)
		return (DDI_FAILURE);

	inst = ddi_get_instance(dip);
	ret = ddi_soft_state_zalloc(mlxcx_softstate, inst);
	if (ret != 0)
		return (ret);

	mlxp = ddi_get_soft_state(mlxcx_softstate, inst);
	if (mlxp == NULL)
		return (DDI_FAILURE);
	mlxp->mlx_dip = dip;
	mlxp->mlx_inst = inst;
	ddi_set_driver_private(dip, mlxp);

	mlxcx_load_props(mlxp);

	mlxcx_fm_init(mlxp);
	mlxp->mlx_attach |= MLXCX_ATTACH_FM;

	if (pci_config_setup(mlxp->mlx_dip, &mlxp->mlx_cfg_handle) !=
	    DDI_SUCCESS) {
		mlxcx_warn(mlxp, "failed to initial PCI config space");
		goto err;
	}
	mlxcx_get_model(mlxp);
	mlxp->mlx_attach |= MLXCX_ATTACH_PCI_CONFIG;

	if (!mlxcx_regs_map(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_REGS;

	if (!mlxcx_cmd_queue_init(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_CMD;

	if (!mlxcx_cmd_enable_hca(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_ENABLE_HCA;

	if (!mlxcx_check_issi(mlxp)) {
		goto err;
	}

	/*
	 * We have to get our interrupts now so we know what priority to
	 * create pagemtx with.
	 */
	if (!mlxcx_intr_setup(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_INTRS;

	mutex_init(&mlxp->mlx_pagemtx, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(mlxp->mlx_intr_pri));
	avl_create(&mlxp->mlx_pages, mlxcx_page_compare,
	    sizeof (mlxcx_dev_page_t), offsetof(mlxcx_dev_page_t, mxdp_tree));
	mlxp->mlx_attach |= MLXCX_ATTACH_PAGE_LIST;

	/*
	 * Taskq for asynchronous events which may interact with the HCA
	 * via the command interface. Single threaded FIFO.
	 */
	(void) snprintf(tq_name, sizeof (tq_name), "%s_async_%d",
	    ddi_driver_name(mlxp->mlx_dip), mlxp->mlx_inst);
	mlxp->mlx_async_tq = taskq_create(tq_name, 1, minclsyspri, 1, INT_MAX,
	    TASKQ_PREPOPULATE);
	/*
	 * Initialize any pre-allocated taskq param structs.
	 */
	for (i = 0; i <= MLXCX_FUNC_ID_MAX; i++) {
		mlxp->mlx_npages_req[i].mla_mlx = mlxp;
		mutex_init(&mlxp->mlx_npages_req[i].mla_mtx, NULL,
		    MUTEX_DRIVER, DDI_INTR_PRI(mlxp->mlx_async_intr_pri));
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_ASYNC_TQ;

	if (!mlxcx_init_pages(mlxp, MLXCX_QUERY_PAGES_OPMOD_BOOT)) {
		goto err;
	}

	if (!mlxcx_init_caps(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_CAPS;

	if (!mlxcx_init_pages(mlxp, MLXCX_QUERY_PAGES_OPMOD_INIT)) {
		goto err;
	}

	if (!mlxcx_cmd_init_hca(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_INIT_HCA;

	if (!mlxcx_cmd_set_driver_version(mlxp, MLXCX_DRIVER_VERSION)) {
		goto err;
	}

	if (mlxp->mlx_caps->mlc_pcam) {
		mlxcx_explore_pcam(mlxp, mlxp->mlx_caps);
	}

	/*
	 * The User Access Region (UAR) is needed so we can ring EQ and CQ
	 * doorbells.
	 */
	if (!mlxcx_cmd_alloc_uar(mlxp, &mlxp->mlx_uar)) {
		goto err;
	}
	for (i = 0; i < MLXCX_BF_PER_UAR; ++i) {
		mutex_init(&mlxp->mlx_uar.mlu_bf[i].mbf_mtx, NULL,
		    MUTEX_DRIVER, DDI_INTR_PRI(mlxp->mlx_intr_pri));
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_UAR_PD_TD;

	/*
	 * Set up asynchronous event queue which handles control type events
	 * like PAGE_REQUEST and CMD completion events.
	 *
	 * This will enable and arm the interrupt on EQ 0. Note that only page
	 * reqs and cmd completions will be handled until we call
	 * mlxcx_eq_set_attached further down (this way we don't need an extra
	 * set of locks over the mlxcx_t sub-structs not allocated yet)
	 */
	if (!mlxcx_setup_async_eqs(mlxp)) {
		goto err;
	}

	/*
	 * Allocate a protection and transport domain. These don't really do
	 * anything for us (they're IB concepts), but we need to give their
	 * ID numbers in other commands.
	 */
	if (!mlxcx_cmd_alloc_pd(mlxp, &mlxp->mlx_pd)) {
		goto err;
	}
	if (!mlxcx_cmd_alloc_tdom(mlxp, &mlxp->mlx_tdom)) {
		goto err;
	}
	/*
	 * Fetch the "reserved" lkey that lets us give linear addresses in
	 * work queue entries, rather than having to mess with the NIC's
	 * internal MMU.
	 */
	if (!mlxcx_cmd_query_special_ctxs(mlxp)) {
		goto err;
	}

	/*
	 * Query our port information and current state, populate the
	 * mlxcx_port_t structs.
	 *
	 * This also sets up the root flow tables and flow groups.
	 */
	if (!mlxcx_setup_ports(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_PORTS;

	mlxcx_load_model_props(mlxp);

	/*
	 * Set up, enable and arm the rest of the interrupt EQs which will
	 * service events from CQs.
	 *
	 * The MLXCX_ATTACH_INTRS flag covers checking if these need to be
	 * cleaned up.
	 */
	if (!mlxcx_setup_eqs(mlxp)) {
		goto err;
	}

	/* Completion queues */
	list_create(&mlxp->mlx_cqs, sizeof (mlxcx_completion_queue_t),
	    offsetof(mlxcx_completion_queue_t, mlcq_entry));
	mlxp->mlx_attach |= MLXCX_ATTACH_CQS;

	/* Work queues (send queues, receive queues) */
	list_create(&mlxp->mlx_wqs, sizeof (mlxcx_work_queue_t),
	    offsetof(mlxcx_work_queue_t, mlwq_entry));
	mlxp->mlx_attach |= MLXCX_ATTACH_WQS;

	/*
	 * Construct our arrays of mlxcx_ring_group_ts, which represent the
	 * "groups" we advertise to MAC.
	 */
	mlxp->mlx_rx_ngroups = mlxcx_calc_rx_ngroups(mlxp);
	mlxp->mlx_rx_groups_size = mlxp->mlx_rx_ngroups *
	    sizeof (mlxcx_ring_group_t);
	mlxp->mlx_rx_groups = kmem_zalloc(mlxp->mlx_rx_groups_size, KM_SLEEP);

	mlxp->mlx_tx_ngroups = mlxp->mlx_props.mldp_tx_ngroups;
	mlxp->mlx_tx_groups_size = mlxp->mlx_tx_ngroups *
	    sizeof (mlxcx_ring_group_t);
	mlxp->mlx_tx_groups = kmem_zalloc(mlxp->mlx_tx_groups_size, KM_SLEEP);

	mlxp->mlx_attach |= MLXCX_ATTACH_GROUPS;

	/*
	 * Sets up the free/busy buffers list for keeping track of packet
	 * buffers.
	 */
	if (!mlxcx_setup_bufs(mlxp))
		goto err;
	mlxp->mlx_attach |= MLXCX_ATTACH_BUFS;

	/*
	 * Before we tell MAC about our rings/groups, we need to do enough
	 * setup on them to be sure about the numbers and configuration that
	 * we have. This will do basically everything short of allocating
	 * packet buffers and starting the rings up.
	 */
	for (i = 0; i < mlxp->mlx_tx_ngroups; ++i) {
		if (!mlxcx_tx_group_setup(mlxp, &mlxp->mlx_tx_groups[i]))
			goto err;
	}
	for (i = 0; i < mlxp->mlx_rx_ngroups; ++i) {
		if (!mlxcx_rx_group_setup(mlxp, &mlxp->mlx_rx_groups[i]))
			goto err;
	}

	/*
	 * Set up periodic fault check timers which check the queue states,
	 * set up should be after all the queues have been initialized and
	 * consequently the teardown of timers must happen before
	 * queue teardown.
	 */
	if (!mlxcx_setup_checktimers(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_CHKTIMERS;

	/*
	 * Some devices may not have a working temperature sensor; however,
	 * there isn't a great way for us to know. We shouldn't fail attach if
	 * this doesn't work.
	 */
	if (mlxcx_setup_sensors(mlxp)) {
		mlxp->mlx_attach |= MLXCX_ATTACH_SENSORS;
	}

	/*
	 * Finally, tell MAC that we exist!
	 */
	if (!mlxcx_register_mac(mlxp)) {
		goto err;
	}
	mlxp->mlx_attach |= MLXCX_ATTACH_MAC_HDL;

	/*
	 * This tells the interrupt handlers they can start processing events
	 * other than cmd completions and page requests.
	 */
	mlxcx_eq_set_attached(mlxp);

	return (DDI_SUCCESS);

err:
	mlxcx_teardown(mlxp);
	return (DDI_FAILURE);
}

static struct cb_ops mlxcx_cb_ops = {
	.cb_open = nulldev,
	.cb_close = nulldev,
	.cb_strategy = nodev,
	.cb_print = nodev,
	.cb_dump = nodev,
	.cb_read = nodev,
	.cb_write = nodev,
	.cb_ioctl = nodev,
	.cb_devmap = nodev,
	.cb_mmap = nodev,
	.cb_segmap = nodev,
	.cb_chpoll = nochpoll,
	.cb_prop_op = ddi_prop_op,
	.cb_flag = D_MP,
	.cb_rev = CB_REV,
	.cb_aread = nodev,
	.cb_awrite = nodev
};

static struct dev_ops mlxcx_dev_ops = {
	.devo_rev = DEVO_REV,
	.devo_refcnt = 0,
	.devo_getinfo = NULL,
	.devo_identify = nulldev,
	.devo_probe = nulldev,
	.devo_attach = mlxcx_attach,
	.devo_detach = mlxcx_detach,
	.devo_reset = nodev,
	.devo_quiesce = ddi_quiesce_not_supported,
	.devo_cb_ops = &mlxcx_cb_ops
};

static struct modldrv mlxcx_modldrv = {
	.drv_modops = &mod_driverops,
	.drv_linkinfo = "Mellanox Connect-X 4/5/6",
	.drv_dev_ops = &mlxcx_dev_ops
};

static struct modlinkage mlxcx_modlinkage = {
	.ml_rev = MODREV_1,
	.ml_linkage = { &mlxcx_modldrv, NULL }
};

int
_init(void)
{
	int ret;

	ret = ddi_soft_state_init(&mlxcx_softstate, sizeof (mlxcx_t), 0);
	if (ret != 0) {
		return (ret);
	}

	mac_init_ops(&mlxcx_dev_ops, MLXCX_MODULE_NAME);

	if ((ret = mod_install(&mlxcx_modlinkage)) != DDI_SUCCESS) {
		mac_fini_ops(&mlxcx_dev_ops);
		ddi_soft_state_fini(&mlxcx_softstate);
		return (ret);
	}

	return (DDI_SUCCESS);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&mlxcx_modlinkage, modinfop));
}

int
_fini(void)
{
	int ret;

	if ((ret = mod_remove(&mlxcx_modlinkage)) != DDI_SUCCESS) {
		return (ret);
	}

	mac_fini_ops(&mlxcx_dev_ops);

	ddi_soft_state_fini(&mlxcx_softstate);

	return (DDI_SUCCESS);
}