xref: /freebsd/sys/dev/cxgbe/adapter.h (revision 13de33a5dc2304b13d595d75d48c51793958474f)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  * $FreeBSD$
28  *
29  */
30 
31 #ifndef __T4_ADAPTER_H__
32 #define __T4_ADAPTER_H__
33 
34 #include <sys/kernel.h>
35 #include <sys/bus.h>
36 #include <sys/rman.h>
37 #include <sys/types.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/rwlock.h>
41 #include <sys/sx.h>
42 #include <vm/uma.h>
43 
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
46 #include <machine/bus.h>
47 #include <sys/socket.h>
48 #include <sys/sysctl.h>
49 #include <net/ethernet.h>
50 #include <net/if.h>
51 #include <net/if_var.h>
52 #include <net/if_media.h>
53 #include <netinet/in.h>
54 #include <netinet/tcp_lro.h>
55 
56 #include "offload.h"
57 #include "firmware/t4fw_interface.h"
58 
59 MALLOC_DECLARE(M_CXGBE);
60 #define CXGBE_UNIMPLEMENTED(s) \
61     panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
62 
63 #if defined(__i386__) || defined(__amd64__)
64 static __inline void
65 prefetch(void *x)
66 {
67 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
68 }
69 #else
70 #define prefetch(x)
71 #endif
72 
73 #ifndef SYSCTL_ADD_UQUAD
74 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
75 #define sysctl_handle_64 sysctl_handle_quad
76 #define CTLTYPE_U64 CTLTYPE_QUAD
77 #endif
78 
79 #if (__FreeBSD_version >= 900030) || \
80     ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000))
81 #define SBUF_DRAIN 1
82 #endif
83 
84 #ifdef __amd64__
85 /* XXX: need systemwide bus_space_read_8/bus_space_write_8 */
86 static __inline uint64_t
87 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
88     bus_size_t offset)
89 {
90 	KASSERT(tag == X86_BUS_SPACE_MEM,
91 	    ("%s: can only handle mem space", __func__));
92 
93 	return (*(volatile uint64_t *)(handle + offset));
94 }
95 
96 static __inline void
97 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
98     bus_size_t offset, uint64_t value)
99 {
100 	KASSERT(tag == X86_BUS_SPACE_MEM,
101 	    ("%s: can only handle mem space", __func__));
102 
103 	*(volatile uint64_t *)(bsh + offset) = value;
104 }
105 #else
106 static __inline uint64_t
107 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
108     bus_size_t offset)
109 {
110 	return (uint64_t)bus_space_read_4(tag, handle, offset) +
111 	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32);
112 }
113 
114 static __inline void
115 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
116     bus_size_t offset, uint64_t value)
117 {
118 	bus_space_write_4(tag, bsh, offset, value);
119 	bus_space_write_4(tag, bsh, offset + 4, value >> 32);
120 }
121 #endif
122 
123 struct adapter;
124 typedef struct adapter adapter_t;
125 
126 enum {
127 	FW_IQ_QSIZE = 256,
128 	FW_IQ_ESIZE = 64,	/* At least 64 mandated by the firmware spec */
129 
130 	RX_IQ_QSIZE = 1024,
131 	RX_IQ_ESIZE = 64,	/* At least 64 so CPL_RX_PKT will fit */
132 
133 	EQ_ESIZE = 64,		/* All egress queues use this entry size */
134 
135 	RX_FL_ESIZE = EQ_ESIZE,	/* 8 64bit addresses */
136 #if MJUMPAGESIZE != MCLBYTES
137 	FL_BUF_SIZES_MAX = 5,	/* cluster, jumbop, jumbo9k, jumbo16k, extra */
138 #else
139 	FL_BUF_SIZES_MAX = 4,	/* cluster, jumbo9k, jumbo16k, extra */
140 #endif
141 
142 	CTRL_EQ_QSIZE = 128,
143 
144 	TX_EQ_QSIZE = 1024,
145 	TX_SGL_SEGS = 36,
146 	TX_WR_FLITS = SGE_MAX_WR_LEN / 8
147 };
148 
149 enum {
150 	/* adapter intr_type */
151 	INTR_INTX	= (1 << 0),
152 	INTR_MSI 	= (1 << 1),
153 	INTR_MSIX	= (1 << 2)
154 };
155 
156 enum {
157 	/* flags understood by begin_synchronized_op */
158 	HOLD_LOCK	= (1 << 0),
159 	SLEEP_OK	= (1 << 1),
160 	INTR_OK		= (1 << 2),
161 
162 	/* flags understood by end_synchronized_op */
163 	LOCK_HELD	= HOLD_LOCK,
164 };
165 
166 enum {
167 	/* adapter flags */
168 	FULL_INIT_DONE	= (1 << 0),
169 	FW_OK		= (1 << 1),
170 	INTR_DIRECT	= (1 << 2),	/* direct interrupts for everything */
171 	MASTER_PF	= (1 << 3),
172 	ADAP_SYSCTL_CTX	= (1 << 4),
173 	TOM_INIT_DONE	= (1 << 5),
174 	BUF_PACKING_OK	= (1 << 6),
175 
176 	CXGBE_BUSY	= (1 << 9),
177 
178 	/* port flags */
179 	DOOMED		= (1 << 0),
180 	PORT_INIT_DONE	= (1 << 1),
181 	PORT_SYSCTL_CTX	= (1 << 2),
182 	HAS_TRACEQ	= (1 << 3),
183 };
184 
185 #define IS_DOOMED(pi)	((pi)->flags & DOOMED)
186 #define SET_DOOMED(pi)	do {(pi)->flags |= DOOMED;} while (0)
187 #define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
188 #define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
189 #define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
190 
191 struct port_info {
192 	device_t dev;
193 	struct adapter *adapter;
194 
195 	struct ifnet *ifp;
196 	struct ifmedia media;
197 
198 	struct mtx pi_lock;
199 	char lockname[16];
200 	unsigned long flags;
201 	int if_flags;
202 
203 	uint16_t *rss;
204 	uint16_t viid;
205 	int16_t  xact_addr_filt;/* index of exact MAC address filter */
206 	uint16_t rss_size;	/* size of VI's RSS table slice */
207 	uint8_t  lport;		/* associated offload logical port */
208 	int8_t   mdio_addr;
209 	uint8_t  port_type;
210 	uint8_t  mod_type;
211 	uint8_t  port_id;
212 	uint8_t  tx_chan;
213 
214 	/* These need to be int as they are used in sysctl */
215 	int ntxq;	/* # of tx queues */
216 	int first_txq;	/* index of first tx queue */
217 	int nrxq;	/* # of rx queues */
218 	int first_rxq;	/* index of first rx queue */
219 #ifdef TCP_OFFLOAD
220 	int nofldtxq;		/* # of offload tx queues */
221 	int first_ofld_txq;	/* index of first offload tx queue */
222 	int nofldrxq;		/* # of offload rx queues */
223 	int first_ofld_rxq;	/* index of first offload rx queue */
224 #endif
225 	int tmr_idx;
226 	int pktc_idx;
227 	int qsize_rxq;
228 	int qsize_txq;
229 
230 	int linkdnrc;
231 	struct link_config link_cfg;
232 	struct port_stats stats;
233 
234 	eventhandler_tag vlan_c;
235 
236 	struct callout tick;
237 	struct sysctl_ctx_list ctx;	/* from ifconfig up to driver detach */
238 
239 	uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */
240 };
241 
242 struct fl_sdesc {
243 	bus_dmamap_t map;
244 	caddr_t cl;
245 	uint8_t tag_idx;	/* the fl->tag entry this map comes from */
246 #ifdef INVARIANTS
247 	__be64 ba_hwtag;
248 #endif
249 };
250 
251 struct tx_desc {
252 	__be64 flit[8];
253 };
254 
255 struct tx_map {
256 	struct mbuf *m;
257 	bus_dmamap_t map;
258 };
259 
260 /* DMA maps used for tx */
261 struct tx_maps {
262 	struct tx_map *maps;
263 	uint32_t map_total;	/* # of DMA maps */
264 	uint32_t map_pidx;	/* next map to be used */
265 	uint32_t map_cidx;	/* reclaimed up to this index */
266 	uint32_t map_avail;	/* # of available maps */
267 };
268 
269 struct tx_sdesc {
270 	uint8_t desc_used;	/* # of hardware descriptors used by the WR */
271 	uint8_t credits;	/* NIC txq: # of frames sent out in the WR */
272 };
273 
274 enum {
275 	/* iq flags */
276 	IQ_ALLOCATED	= (1 << 0),	/* firmware resources allocated */
277 	IQ_HAS_FL	= (1 << 1),	/* iq associated with a freelist */
278 	IQ_INTR		= (1 << 2),	/* iq takes direct interrupt */
279 	IQ_LRO_ENABLED	= (1 << 3),	/* iq is an eth rxq with LRO enabled */
280 
281 	/* iq state */
282 	IQS_DISABLED	= 0,
283 	IQS_BUSY	= 1,
284 	IQS_IDLE	= 2,
285 };
286 
287 /*
288  * Ingress Queue: T4 is producer, driver is consumer.
289  */
290 struct sge_iq {
291 	bus_dma_tag_t desc_tag;
292 	bus_dmamap_t desc_map;
293 	bus_addr_t ba;		/* bus address of descriptor ring */
294 	uint32_t flags;
295 	uint16_t abs_id;	/* absolute SGE id for the iq */
296 	int8_t   intr_pktc_idx;	/* packet count threshold index */
297 	int8_t   pad0;
298 	__be64  *desc;		/* KVA of descriptor ring */
299 
300 	volatile int state;
301 	struct adapter *adapter;
302 	const __be64 *cdesc;	/* current descriptor */
303 	uint8_t  gen;		/* generation bit */
304 	uint8_t  intr_params;	/* interrupt holdoff parameters */
305 	uint8_t  intr_next;	/* XXX: holdoff for next interrupt */
306 	uint8_t  esize;		/* size (bytes) of each entry in the queue */
307 	uint16_t qsize;		/* size (# of entries) of the queue */
308 	uint16_t cidx;		/* consumer index */
309 	uint16_t cntxt_id;	/* SGE context id for the iq */
310 
311 	STAILQ_ENTRY(sge_iq) link;
312 };
313 
314 enum {
315 	EQ_CTRL		= 1,
316 	EQ_ETH		= 2,
317 #ifdef TCP_OFFLOAD
318 	EQ_OFLD		= 3,
319 #endif
320 
321 	/* eq flags */
322 	EQ_TYPEMASK	= 7,		/* 3 lsbits hold the type */
323 	EQ_ALLOCATED	= (1 << 3),	/* firmware resources allocated */
324 	EQ_DOOMED	= (1 << 4),	/* about to be destroyed */
325 	EQ_CRFLUSHED	= (1 << 5),	/* expecting an update from SGE */
326 	EQ_STALLED	= (1 << 6),	/* out of hw descriptors or dmamaps */
327 };
328 
329 /* Listed in order of preference.  Update t4_sysctls too if you change these */
330 enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB};
331 
332 /*
333  * Egress Queue: driver is producer, T4 is consumer.
334  *
335  * Note: A free list is an egress queue (driver produces the buffers and T4
336  * consumes them) but it's special enough to have its own struct (see sge_fl).
337  */
338 struct sge_eq {
339 	unsigned int flags;	/* MUST be first */
340 	unsigned int cntxt_id;	/* SGE context id for the eq */
341 	bus_dma_tag_t desc_tag;
342 	bus_dmamap_t desc_map;
343 	char lockname[16];
344 	struct mtx eq_lock;
345 
346 	struct tx_desc *desc;	/* KVA of descriptor ring */
347 	bus_addr_t ba;		/* bus address of descriptor ring */
348 	struct sge_qstat *spg;	/* status page, for convenience */
349 	int doorbells;
350 	volatile uint32_t *udb;	/* KVA of doorbell (lies within BAR2) */
351 	u_int udb_qid;		/* relative qid within the doorbell page */
352 	uint16_t cap;		/* max # of desc, for convenience */
353 	uint16_t avail;		/* available descriptors, for convenience */
354 	uint16_t qsize;		/* size (# of entries) of the queue */
355 	uint16_t cidx;		/* consumer idx (desc idx) */
356 	uint16_t pidx;		/* producer idx (desc idx) */
357 	uint16_t pending;	/* # of descriptors used since last doorbell */
358 	uint16_t iqid;		/* iq that gets egr_update for the eq */
359 	uint8_t tx_chan;	/* tx channel used by the eq */
360 	struct task tx_task;
361 	struct callout tx_callout;
362 
363 	/* stats */
364 
365 	uint32_t egr_update;	/* # of SGE_EGR_UPDATE notifications for eq */
366 	uint32_t unstalled;	/* recovered from stall */
367 };
368 
369 struct fl_buf_info {
370 	u_int size;
371 	int type;
372 	int hwtag:4;	/* tag in low 4 bits of the pa. */
373 	uma_zone_t zone;
374 };
375 #define FL_BUF_SIZES(sc)	(sc->sge.fl_buf_sizes)
376 #define FL_BUF_SIZE(sc, x)	(sc->sge.fl_buf_info[x].size)
377 #define FL_BUF_TYPE(sc, x)	(sc->sge.fl_buf_info[x].type)
378 #define FL_BUF_HWTAG(sc, x)	(sc->sge.fl_buf_info[x].hwtag)
379 #define FL_BUF_ZONE(sc, x)	(sc->sge.fl_buf_info[x].zone)
380 
381 enum {
382 	FL_STARVING	= (1 << 0), /* on the adapter's list of starving fl's */
383 	FL_DOOMED	= (1 << 1), /* about to be destroyed */
384 	FL_BUF_PACKING	= (1 << 2), /* buffer packing enabled */
385 };
386 
387 #define FL_RUNNING_LOW(fl)	(fl->cap - fl->needed <= fl->lowat)
388 #define FL_NOT_RUNNING_LOW(fl)	(fl->cap - fl->needed >= 2 * fl->lowat)
389 
390 struct sge_fl {
391 	bus_dma_tag_t desc_tag;
392 	bus_dmamap_t desc_map;
393 	bus_dma_tag_t tag[FL_BUF_SIZES_MAX]; /* only first FL_BUF_SIZES(sc) are
394 						valid */
395 	uint8_t tag_idx;
396 	struct mtx fl_lock;
397 	char lockname[16];
398 	int flags;
399 
400 	__be64 *desc;		/* KVA of descriptor ring, ptr to addresses */
401 	bus_addr_t ba;		/* bus address of descriptor ring */
402 	struct fl_sdesc *sdesc;	/* KVA of software descriptor ring */
403 	uint32_t cap;		/* max # of buffers, for convenience */
404 	uint16_t qsize;		/* size (# of entries) of the queue */
405 	uint16_t cntxt_id;	/* SGE context id for the freelist */
406 	uint32_t cidx;		/* consumer idx (buffer idx, NOT hw desc idx) */
407 	uint32_t rx_offset;	/* offset in fl buf (when buffer packing) */
408 	uint32_t pidx;		/* producer idx (buffer idx, NOT hw desc idx) */
409 	uint32_t needed;	/* # of buffers needed to fill up fl. */
410 	uint32_t lowat;		/* # of buffers <= this means fl needs help */
411 	uint32_t pending;	/* # of bufs allocated since last doorbell */
412 	u_int dmamap_failed;
413 	struct mbuf *mstash[8];
414 	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
415 };
416 
417 /* txq: SGE egress queue + what's needed for Ethernet NIC */
418 struct sge_txq {
419 	struct sge_eq eq;	/* MUST be first */
420 
421 	struct ifnet *ifp;	/* the interface this txq belongs to */
422 	bus_dma_tag_t tx_tag;	/* tag for transmit buffers */
423 	struct buf_ring *br;	/* tx buffer ring */
424 	struct tx_sdesc *sdesc;	/* KVA of software descriptor ring */
425 	struct mbuf *m;		/* held up due to temporary resource shortage */
426 
427 	struct tx_maps txmaps;
428 
429 	/* stats for common events first */
430 
431 	uint64_t txcsum;	/* # of times hardware assisted with checksum */
432 	uint64_t tso_wrs;	/* # of TSO work requests */
433 	uint64_t vlan_insertion;/* # of times VLAN tag was inserted */
434 	uint64_t imm_wrs;	/* # of work requests with immediate data */
435 	uint64_t sgl_wrs;	/* # of work requests with direct SGL */
436 	uint64_t txpkt_wrs;	/* # of txpkt work requests (not coalesced) */
437 	uint64_t txpkts_wrs;	/* # of coalesced tx work requests */
438 	uint64_t txpkts_pkts;	/* # of frames in coalesced tx work requests */
439 
440 	/* stats for not-that-common events */
441 
442 	uint32_t no_dmamap;	/* no DMA map to load the mbuf */
443 	uint32_t no_desc;	/* out of hardware descriptors */
444 } __aligned(CACHE_LINE_SIZE);
445 
446 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
447 struct sge_rxq {
448 	struct sge_iq iq;	/* MUST be first */
449 	struct sge_fl fl;	/* MUST follow iq */
450 
451 	struct ifnet *ifp;	/* the interface this rxq belongs to */
452 #if defined(INET) || defined(INET6)
453 	struct lro_ctrl lro;	/* LRO state */
454 #endif
455 
456 	/* stats for common events first */
457 
458 	uint64_t rxcsum;	/* # of times hardware assisted with checksum */
459 	uint64_t vlan_extraction;/* # of times VLAN tag was extracted */
460 
461 	/* stats for not-that-common events */
462 
463 } __aligned(CACHE_LINE_SIZE);
464 
465 static inline struct sge_rxq *
466 iq_to_rxq(struct sge_iq *iq)
467 {
468 
469 	return (__containerof(iq, struct sge_rxq, iq));
470 }
471 
472 
473 #ifdef TCP_OFFLOAD
474 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
475 struct sge_ofld_rxq {
476 	struct sge_iq iq;	/* MUST be first */
477 	struct sge_fl fl;	/* MUST follow iq */
478 } __aligned(CACHE_LINE_SIZE);
479 
480 static inline struct sge_ofld_rxq *
481 iq_to_ofld_rxq(struct sge_iq *iq)
482 {
483 
484 	return (__containerof(iq, struct sge_ofld_rxq, iq));
485 }
486 #endif
487 
488 struct wrqe {
489 	STAILQ_ENTRY(wrqe) link;
490 	struct sge_wrq *wrq;
491 	int wr_len;
492 	uint64_t wr[] __aligned(16);
493 };
494 
495 /*
496  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
497  * and offload tx queues are of this type.
498  */
499 struct sge_wrq {
500 	struct sge_eq eq;	/* MUST be first */
501 
502 	struct adapter *adapter;
503 
504 	/* List of WRs held up due to lack of tx descriptors */
505 	STAILQ_HEAD(, wrqe) wr_list;
506 
507 	/* stats for common events first */
508 
509 	uint64_t tx_wrs;	/* # of tx work requests */
510 
511 	/* stats for not-that-common events */
512 
513 	uint32_t no_desc;	/* out of hardware descriptors */
514 } __aligned(CACHE_LINE_SIZE);
515 
516 struct sge {
517 	int timer_val[SGE_NTIMERS];
518 	int counter_val[SGE_NCOUNTERS];
519 	int fl_starve_threshold;
520 	int eq_s_qpp;
521 	int iq_s_qpp;
522 
523 	int nrxq;	/* total # of Ethernet rx queues */
524 	int ntxq;	/* total # of Ethernet tx tx queues */
525 #ifdef TCP_OFFLOAD
526 	int nofldrxq;	/* total # of TOE rx queues */
527 	int nofldtxq;	/* total # of TOE tx queues */
528 #endif
529 	int niq;	/* total # of ingress queues */
530 	int neq;	/* total # of egress queues */
531 
532 	struct sge_iq fwq;	/* Firmware event queue */
533 	struct sge_wrq mgmtq;	/* Management queue (control queue) */
534 	struct sge_wrq *ctrlq;	/* Control queues */
535 	struct sge_txq *txq;	/* NIC tx queues */
536 	struct sge_rxq *rxq;	/* NIC rx queues */
537 #ifdef TCP_OFFLOAD
538 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
539 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
540 #endif
541 
542 	uint16_t iq_start;
543 	int eq_start;
544 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
545 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
546 
547 	u_int fl_buf_sizes __aligned(CACHE_LINE_SIZE);
548 	struct fl_buf_info fl_buf_info[FL_BUF_SIZES_MAX];
549 };
550 
551 struct rss_header;
552 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
553     struct mbuf *);
554 typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *);
555 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
556 
557 struct adapter {
558 	SLIST_ENTRY(adapter) link;
559 	device_t dev;
560 	struct cdev *cdev;
561 
562 	/* PCIe register resources */
563 	int regs_rid;
564 	struct resource *regs_res;
565 	int msix_rid;
566 	struct resource *msix_res;
567 	bus_space_handle_t bh;
568 	bus_space_tag_t bt;
569 	bus_size_t mmio_len;
570 	int udbs_rid;
571 	struct resource *udbs_res;
572 	volatile uint8_t *udbs_base;
573 
574 	unsigned int pf;
575 	unsigned int mbox;
576 
577 	/* Interrupt information */
578 	int intr_type;
579 	int intr_count;
580 	struct irq {
581 		struct resource *res;
582 		int rid;
583 		void *tag;
584 	} *irq;
585 
586 	bus_dma_tag_t dmat;	/* Parent DMA tag */
587 
588 	struct sge sge;
589 	int lro_timeout;
590 
591 	struct taskqueue *tq[NCHAN];	/* taskqueues that flush data out */
592 	struct port_info *port[MAX_NPORTS];
593 	uint8_t chan_map[NCHAN];
594 
595 #ifdef TCP_OFFLOAD
596 	void *tom_softc;	/* (struct tom_data *) */
597 	struct tom_tunables tt;
598 	void *iwarp_softc;	/* (struct c4iw_dev *) */
599 #endif
600 	struct l2t_data *l2t;	/* L2 table */
601 	struct tid_info tids;
602 
603 	int doorbells;
604 	int open_device_map;
605 #ifdef TCP_OFFLOAD
606 	int offload_map;
607 #endif
608 	int flags;
609 
610 	char ifp_lockname[16];
611 	struct mtx ifp_lock;
612 	struct ifnet *ifp;	/* tracer ifp */
613 	struct ifmedia media;
614 	int traceq;		/* iq used by all tracers, -1 if none */
615 	int tracer_valid;	/* bitmap of valid tracers */
616 	int tracer_enabled;	/* bitmap of enabled tracers */
617 
618 	char fw_version[32];
619 	char cfg_file[32];
620 	u_int cfcsum;
621 	struct adapter_params params;
622 	struct t4_virt_res vres;
623 
624 	uint16_t linkcaps;
625 	uint16_t niccaps;
626 	uint16_t toecaps;
627 	uint16_t rdmacaps;
628 	uint16_t iscsicaps;
629 	uint16_t fcoecaps;
630 
631 	struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */
632 
633 	struct mtx sc_lock;
634 	char lockname[16];
635 
636 	/* Starving free lists */
637 	struct mtx sfl_lock;	/* same cache-line as sc_lock? but that's ok */
638 	TAILQ_HEAD(, sge_fl) sfl;
639 	struct callout sfl_callout;
640 
641 	an_handler_t an_handler __aligned(CACHE_LINE_SIZE);
642 	fw_msg_handler_t fw_msg_handler[5];	/* NUM_FW6_TYPES */
643 	cpl_handler_t cpl_handler[0xef];	/* NUM_CPL_CMDS */
644 
645 #ifdef INVARIANTS
646 	const char *last_op;
647 	const void *last_op_thr;
648 #endif
649 };
650 
651 #define ADAPTER_LOCK(sc)		mtx_lock(&(sc)->sc_lock)
652 #define ADAPTER_UNLOCK(sc)		mtx_unlock(&(sc)->sc_lock)
653 #define ADAPTER_LOCK_ASSERT_OWNED(sc)	mtx_assert(&(sc)->sc_lock, MA_OWNED)
654 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
655 
656 /* XXX: not bulletproof, but much better than nothing */
657 #define ASSERT_SYNCHRONIZED_OP(sc)	\
658     KASSERT(IS_BUSY(sc) && \
659 	(mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
660 	("%s: operation not synchronized.", __func__))
661 
662 #define PORT_LOCK(pi)			mtx_lock(&(pi)->pi_lock)
663 #define PORT_UNLOCK(pi)			mtx_unlock(&(pi)->pi_lock)
664 #define PORT_LOCK_ASSERT_OWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_OWNED)
665 #define PORT_LOCK_ASSERT_NOTOWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_NOTOWNED)
666 
667 #define FL_LOCK(fl)			mtx_lock(&(fl)->fl_lock)
668 #define FL_TRYLOCK(fl)			mtx_trylock(&(fl)->fl_lock)
669 #define FL_UNLOCK(fl)			mtx_unlock(&(fl)->fl_lock)
670 #define FL_LOCK_ASSERT_OWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_OWNED)
671 #define FL_LOCK_ASSERT_NOTOWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_NOTOWNED)
672 
673 #define RXQ_FL_LOCK(rxq)		FL_LOCK(&(rxq)->fl)
674 #define RXQ_FL_UNLOCK(rxq)		FL_UNLOCK(&(rxq)->fl)
675 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq)	FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
676 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
677 
678 #define EQ_LOCK(eq)			mtx_lock(&(eq)->eq_lock)
679 #define EQ_TRYLOCK(eq)			mtx_trylock(&(eq)->eq_lock)
680 #define EQ_UNLOCK(eq)			mtx_unlock(&(eq)->eq_lock)
681 #define EQ_LOCK_ASSERT_OWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_OWNED)
682 #define EQ_LOCK_ASSERT_NOTOWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_NOTOWNED)
683 
684 #define TXQ_LOCK(txq)			EQ_LOCK(&(txq)->eq)
685 #define TXQ_TRYLOCK(txq)		EQ_TRYLOCK(&(txq)->eq)
686 #define TXQ_UNLOCK(txq)			EQ_UNLOCK(&(txq)->eq)
687 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
688 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
689 
690 #define for_each_txq(pi, iter, q) \
691 	for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \
692 	    iter < pi->ntxq; ++iter, ++q)
693 #define for_each_rxq(pi, iter, q) \
694 	for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \
695 	    iter < pi->nrxq; ++iter, ++q)
696 #define for_each_ofld_txq(pi, iter, q) \
697 	for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \
698 	    iter < pi->nofldtxq; ++iter, ++q)
699 #define for_each_ofld_rxq(pi, iter, q) \
700 	for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \
701 	    iter < pi->nofldrxq; ++iter, ++q)
702 
703 /* One for errors, one for firmware events */
704 #define T4_EXTRA_INTR 2
705 
706 static inline uint32_t
707 t4_read_reg(struct adapter *sc, uint32_t reg)
708 {
709 
710 	return bus_space_read_4(sc->bt, sc->bh, reg);
711 }
712 
713 static inline void
714 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
715 {
716 
717 	bus_space_write_4(sc->bt, sc->bh, reg, val);
718 }
719 
720 static inline uint64_t
721 t4_read_reg64(struct adapter *sc, uint32_t reg)
722 {
723 
724 	return t4_bus_space_read_8(sc->bt, sc->bh, reg);
725 }
726 
727 static inline void
728 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
729 {
730 
731 	t4_bus_space_write_8(sc->bt, sc->bh, reg, val);
732 }
733 
734 static inline void
735 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
736 {
737 
738 	*val = pci_read_config(sc->dev, reg, 1);
739 }
740 
741 static inline void
742 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
743 {
744 
745 	pci_write_config(sc->dev, reg, val, 1);
746 }
747 
748 static inline void
749 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
750 {
751 
752 	*val = pci_read_config(sc->dev, reg, 2);
753 }
754 
755 static inline void
756 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
757 {
758 
759 	pci_write_config(sc->dev, reg, val, 2);
760 }
761 
762 static inline void
763 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
764 {
765 
766 	*val = pci_read_config(sc->dev, reg, 4);
767 }
768 
769 static inline void
770 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
771 {
772 
773 	pci_write_config(sc->dev, reg, val, 4);
774 }
775 
776 static inline struct port_info *
777 adap2pinfo(struct adapter *sc, int idx)
778 {
779 
780 	return (sc->port[idx]);
781 }
782 
783 static inline void
784 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
785 {
786 
787 	bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN);
788 }
789 
790 static inline bool
791 is_10G_port(const struct port_info *pi)
792 {
793 
794 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0);
795 }
796 
797 static inline bool
798 is_40G_port(const struct port_info *pi)
799 {
800 
801 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0);
802 }
803 
804 static inline int
805 tx_resume_threshold(struct sge_eq *eq)
806 {
807 
808 	return (eq->qsize / 4);
809 }
810 
811 /* t4_main.c */
812 void t4_tx_task(void *, int);
813 void t4_tx_callout(void *);
814 int t4_os_find_pci_capability(struct adapter *, int);
815 int t4_os_pci_save_state(struct adapter *);
816 int t4_os_pci_restore_state(struct adapter *);
817 void t4_os_portmod_changed(const struct adapter *, int);
818 void t4_os_link_changed(struct adapter *, int, int, int);
819 void t4_iterate(void (*)(struct adapter *, void *), void *);
820 int t4_register_cpl_handler(struct adapter *, int, cpl_handler_t);
821 int t4_register_an_handler(struct adapter *, an_handler_t);
822 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
823 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
824 int begin_synchronized_op(struct adapter *, struct port_info *, int, char *);
825 void end_synchronized_op(struct adapter *, int);
826 
827 /* t4_sge.c */
828 void t4_sge_modload(void);
829 void t4_init_sge_cpl_handlers(struct adapter *);
830 void t4_tweak_chip_settings(struct adapter *);
831 int t4_read_chip_settings(struct adapter *);
832 int t4_create_dma_tag(struct adapter *);
833 void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *,
834     struct sysctl_oid_list *);
835 int t4_destroy_dma_tag(struct adapter *);
836 int t4_setup_adapter_queues(struct adapter *);
837 int t4_teardown_adapter_queues(struct adapter *);
838 int t4_setup_port_queues(struct port_info *);
839 int t4_teardown_port_queues(struct port_info *);
840 int t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int);
841 void t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t);
842 void t4_intr_all(void *);
843 void t4_intr(void *);
844 void t4_intr_err(void *);
845 void t4_intr_evt(void *);
846 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);
847 int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *);
848 void t4_update_fl_bufsize(struct ifnet *);
849 int can_resume_tx(struct sge_eq *);
850 
851 /* t4_tracer.c */
852 struct t4_tracer;
853 void t4_tracer_modload(void);
854 void t4_tracer_modunload(void);
855 void t4_tracer_port_detach(struct adapter *);
856 int t4_get_tracer(struct adapter *, struct t4_tracer *);
857 int t4_set_tracer(struct adapter *, struct t4_tracer *);
858 int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
859 int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
860 
861 static inline struct wrqe *
862 alloc_wrqe(int wr_len, struct sge_wrq *wrq)
863 {
864 	int len = offsetof(struct wrqe, wr) + wr_len;
865 	struct wrqe *wr;
866 
867 	wr = malloc(len, M_CXGBE, M_NOWAIT);
868 	if (__predict_false(wr == NULL))
869 		return (NULL);
870 	wr->wr_len = wr_len;
871 	wr->wrq = wrq;
872 	return (wr);
873 }
874 
875 static inline void *
876 wrtod(struct wrqe *wr)
877 {
878 	return (&wr->wr[0]);
879 }
880 
881 static inline void
882 free_wrqe(struct wrqe *wr)
883 {
884 	free(wr, M_CXGBE);
885 }
886 
887 static inline void
888 t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
889 {
890 	struct sge_wrq *wrq = wr->wrq;
891 
892 	TXQ_LOCK(wrq);
893 	t4_wrq_tx_locked(sc, wrq, wr);
894 	TXQ_UNLOCK(wrq);
895 }
896 
897 #endif
898