xref: /freebsd/sys/dev/cxgbe/adapter.h (revision 9268022b74279434ed6300244e3f977e56a8ceb5)
1 /*-
2  * Copyright (c) 2011 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: Navdeep Parhar <np@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  *
27  * $FreeBSD$
28  *
29  */
30 
31 #ifndef __T4_ADAPTER_H__
32 #define __T4_ADAPTER_H__
33 
34 #include <sys/kernel.h>
35 #include <sys/bus.h>
36 #include <sys/rman.h>
37 #include <sys/types.h>
38 #include <sys/lock.h>
39 #include <sys/malloc.h>
40 #include <sys/rwlock.h>
41 #include <sys/sx.h>
42 #include <vm/uma.h>
43 
44 #include <dev/pci/pcivar.h>
45 #include <dev/pci/pcireg.h>
46 #include <machine/bus.h>
47 #include <sys/socket.h>
48 #include <sys/sysctl.h>
49 #include <net/ethernet.h>
50 #include <net/if.h>
51 #include <net/if_var.h>
52 #include <net/if_media.h>
53 #include <netinet/in.h>
54 #include <netinet/tcp_lro.h>
55 
56 #include "offload.h"
57 #include "common/t4_msg.h"
58 #include "firmware/t4fw_interface.h"
59 
60 MALLOC_DECLARE(M_CXGBE);
61 #define CXGBE_UNIMPLEMENTED(s) \
62     panic("%s (%s, line %d) not implemented yet.", s, __FILE__, __LINE__)
63 
64 #if defined(__i386__) || defined(__amd64__)
65 static __inline void
66 prefetch(void *x)
67 {
68 	__asm volatile("prefetcht0 %0" :: "m" (*(unsigned long *)x));
69 }
70 #else
71 #define prefetch(x)
72 #endif
73 
74 #ifndef SYSCTL_ADD_UQUAD
75 #define SYSCTL_ADD_UQUAD SYSCTL_ADD_QUAD
76 #define sysctl_handle_64 sysctl_handle_quad
77 #define CTLTYPE_U64 CTLTYPE_QUAD
78 #endif
79 
80 #if (__FreeBSD_version >= 900030) || \
81     ((__FreeBSD_version >= 802507) && (__FreeBSD_version < 900000))
82 #define SBUF_DRAIN 1
83 #endif
84 
85 #ifdef __amd64__
86 /* XXX: need systemwide bus_space_read_8/bus_space_write_8 */
87 static __inline uint64_t
88 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
89     bus_size_t offset)
90 {
91 	KASSERT(tag == X86_BUS_SPACE_MEM,
92 	    ("%s: can only handle mem space", __func__));
93 
94 	return (*(volatile uint64_t *)(handle + offset));
95 }
96 
97 static __inline void
98 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
99     bus_size_t offset, uint64_t value)
100 {
101 	KASSERT(tag == X86_BUS_SPACE_MEM,
102 	    ("%s: can only handle mem space", __func__));
103 
104 	*(volatile uint64_t *)(bsh + offset) = value;
105 }
106 #else
107 static __inline uint64_t
108 t4_bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle,
109     bus_size_t offset)
110 {
111 	return (uint64_t)bus_space_read_4(tag, handle, offset) +
112 	    ((uint64_t)bus_space_read_4(tag, handle, offset + 4) << 32);
113 }
114 
115 static __inline void
116 t4_bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t bsh,
117     bus_size_t offset, uint64_t value)
118 {
119 	bus_space_write_4(tag, bsh, offset, value);
120 	bus_space_write_4(tag, bsh, offset + 4, value >> 32);
121 }
122 #endif
123 
124 struct adapter;
125 typedef struct adapter adapter_t;
126 
127 enum {
128 	/*
129 	 * All ingress queues use this entry size.  Note that the firmware event
130 	 * queue and any iq expecting CPL_RX_PKT in the descriptor needs this to
131 	 * be at least 64.
132 	 */
133 	IQ_ESIZE = 64,
134 
135 	/* Default queue sizes for all kinds of ingress queues */
136 	FW_IQ_QSIZE = 256,
137 	RX_IQ_QSIZE = 1024,
138 
139 	/* All egress queues use this entry size */
140 	EQ_ESIZE = 64,
141 
142 	/* Default queue sizes for all kinds of egress queues */
143 	CTRL_EQ_QSIZE = 128,
144 	TX_EQ_QSIZE = 1024,
145 
146 #if MJUMPAGESIZE != MCLBYTES
147 	SW_ZONE_SIZES = 4,	/* cluster, jumbop, jumbo9k, jumbo16k */
148 #else
149 	SW_ZONE_SIZES = 3,	/* cluster, jumbo9k, jumbo16k */
150 #endif
151 	CL_METADATA_SIZE = CACHE_LINE_SIZE,
152 
153 	SGE_MAX_WR_NDESC = SGE_MAX_WR_LEN / EQ_ESIZE, /* max WR size in desc */
154 	TX_SGL_SEGS = 36,
155 	TX_WR_FLITS = SGE_MAX_WR_LEN / 8
156 };
157 
158 enum {
159 	/* adapter intr_type */
160 	INTR_INTX	= (1 << 0),
161 	INTR_MSI 	= (1 << 1),
162 	INTR_MSIX	= (1 << 2)
163 };
164 
165 enum {
166 	XGMAC_MTU	= (1 << 0),
167 	XGMAC_PROMISC	= (1 << 1),
168 	XGMAC_ALLMULTI	= (1 << 2),
169 	XGMAC_VLANEX	= (1 << 3),
170 	XGMAC_UCADDR	= (1 << 4),
171 	XGMAC_MCADDRS	= (1 << 5),
172 
173 	XGMAC_ALL	= 0xffff
174 };
175 
176 enum {
177 	/* flags understood by begin_synchronized_op */
178 	HOLD_LOCK	= (1 << 0),
179 	SLEEP_OK	= (1 << 1),
180 	INTR_OK		= (1 << 2),
181 
182 	/* flags understood by end_synchronized_op */
183 	LOCK_HELD	= HOLD_LOCK,
184 };
185 
186 enum {
187 	/* adapter flags */
188 	FULL_INIT_DONE	= (1 << 0),
189 	FW_OK		= (1 << 1),
190 	/* INTR_DIRECT	= (1 << 2),	No longer used. */
191 	MASTER_PF	= (1 << 3),
192 	ADAP_SYSCTL_CTX	= (1 << 4),
193 	TOM_INIT_DONE	= (1 << 5),
194 	BUF_PACKING_OK	= (1 << 6),
195 
196 	CXGBE_BUSY	= (1 << 9),
197 
198 	/* port flags */
199 	DOOMED		= (1 << 0),
200 	PORT_INIT_DONE	= (1 << 1),
201 	PORT_SYSCTL_CTX	= (1 << 2),
202 	HAS_TRACEQ	= (1 << 3),
203 	INTR_RXQ	= (1 << 4),	/* All NIC rxq's take interrupts */
204 	INTR_OFLD_RXQ	= (1 << 5),	/* All TOE rxq's take interrupts */
205 	INTR_NM_RXQ	= (1 << 6),	/* All netmap rxq's take interrupts */
206 	INTR_ALL	= (INTR_RXQ | INTR_OFLD_RXQ | INTR_NM_RXQ),
207 };
208 
209 #define IS_DOOMED(pi)	((pi)->flags & DOOMED)
210 #define SET_DOOMED(pi)	do {(pi)->flags |= DOOMED;} while (0)
211 #define IS_BUSY(sc)	((sc)->flags & CXGBE_BUSY)
212 #define SET_BUSY(sc)	do {(sc)->flags |= CXGBE_BUSY;} while (0)
213 #define CLR_BUSY(sc)	do {(sc)->flags &= ~CXGBE_BUSY;} while (0)
214 
215 struct port_info {
216 	device_t dev;
217 	struct adapter *adapter;
218 
219 	struct ifnet *ifp;
220 	struct ifmedia media;
221 
222 	struct mtx pi_lock;
223 	char lockname[16];
224 	unsigned long flags;
225 	int if_flags;
226 
227 	uint16_t *rss;
228 	uint16_t viid;
229 	int16_t  xact_addr_filt;/* index of exact MAC address filter */
230 	uint16_t rss_size;	/* size of VI's RSS table slice */
231 	uint8_t  lport;		/* associated offload logical port */
232 	int8_t   mdio_addr;
233 	uint8_t  port_type;
234 	uint8_t  mod_type;
235 	uint8_t  port_id;
236 	uint8_t  tx_chan;
237 	uint8_t  rx_chan_map;	/* rx MPS channel bitmap */
238 
239 	/* These need to be int as they are used in sysctl */
240 	int ntxq;	/* # of tx queues */
241 	int first_txq;	/* index of first tx queue */
242 	int rsrv_noflowq; /* Reserve queue 0 for non-flowid packets */
243 	int nrxq;	/* # of rx queues */
244 	int first_rxq;	/* index of first rx queue */
245 #ifdef TCP_OFFLOAD
246 	int nofldtxq;		/* # of offload tx queues */
247 	int first_ofld_txq;	/* index of first offload tx queue */
248 	int nofldrxq;		/* # of offload rx queues */
249 	int first_ofld_rxq;	/* index of first offload rx queue */
250 #endif
251 #ifdef DEV_NETMAP
252 	int nnmtxq;		/* # of netmap tx queues */
253 	int first_nm_txq;	/* index of first netmap tx queue */
254 	int nnmrxq;		/* # of netmap rx queues */
255 	int first_nm_rxq;	/* index of first netmap rx queue */
256 
257 	struct ifnet *nm_ifp;
258 	struct ifmedia nm_media;
259 	int nmif_flags;
260 	uint16_t nm_viid;
261 	int16_t nm_xact_addr_filt;
262 	uint16_t nm_rss_size;	/* size of netmap VI's RSS table slice */
263 #endif
264 	int tmr_idx;
265 	int pktc_idx;
266 	int qsize_rxq;
267 	int qsize_txq;
268 
269 	int linkdnrc;
270 	struct link_config link_cfg;
271 
272 	struct timeval last_refreshed;
273  	struct port_stats stats;
274 	u_int tnl_cong_drops;
275 
276 	eventhandler_tag vlan_c;
277 
278 	struct callout tick;
279 	struct sysctl_ctx_list ctx;	/* from ifconfig up to driver detach */
280 
281 	uint8_t hw_addr[ETHER_ADDR_LEN]; /* factory MAC address, won't change */
282 };
283 
284 /* Where the cluster came from, how it has been carved up. */
285 struct cluster_layout {
286 	int8_t zidx;
287 	int8_t hwidx;
288 	uint16_t region1;	/* mbufs laid out within this region */
289 				/* region2 is the DMA region */
290 	uint16_t region3;	/* cluster_metadata within this region */
291 };
292 
293 struct cluster_metadata {
294 	u_int refcount;
295 #ifdef INVARIANTS
296 	struct fl_sdesc *sd;	/* For debug only.  Could easily be stale */
297 #endif
298 };
299 
300 struct fl_sdesc {
301 	caddr_t cl;
302 	uint16_t nmbuf;	/* # of driver originated mbufs with ref on cluster */
303 	struct cluster_layout cll;
304 };
305 
306 struct tx_desc {
307 	__be64 flit[8];
308 };
309 
310 struct tx_map {
311 	struct mbuf *m;
312 	bus_dmamap_t map;
313 };
314 
315 /* DMA maps used for tx */
316 struct tx_maps {
317 	struct tx_map *maps;
318 	uint32_t map_total;	/* # of DMA maps */
319 	uint32_t map_pidx;	/* next map to be used */
320 	uint32_t map_cidx;	/* reclaimed up to this index */
321 	uint32_t map_avail;	/* # of available maps */
322 };
323 
324 struct tx_sdesc {
325 	uint8_t desc_used;	/* # of hardware descriptors used by the WR */
326 	uint8_t credits;	/* NIC txq: # of frames sent out in the WR */
327 };
328 
329 
330 #define IQ_PAD (IQ_ESIZE - sizeof(struct rsp_ctrl) - sizeof(struct rss_header))
331 struct iq_desc {
332 	struct rss_header rss;
333 	uint8_t cpl[IQ_PAD];
334 	struct rsp_ctrl rsp;
335 };
336 #undef IQ_PAD
337 CTASSERT(sizeof(struct iq_desc) == IQ_ESIZE);
338 
339 enum {
340 	/* iq flags */
341 	IQ_ALLOCATED	= (1 << 0),	/* firmware resources allocated */
342 	IQ_HAS_FL	= (1 << 1),	/* iq associated with a freelist */
343 	IQ_INTR		= (1 << 2),	/* iq takes direct interrupt */
344 	IQ_LRO_ENABLED	= (1 << 3),	/* iq is an eth rxq with LRO enabled */
345 
346 	/* iq state */
347 	IQS_DISABLED	= 0,
348 	IQS_BUSY	= 1,
349 	IQS_IDLE	= 2,
350 };
351 
352 /*
353  * Ingress Queue: T4 is producer, driver is consumer.
354  */
355 struct sge_iq {
356 	uint32_t flags;
357 	volatile int state;
358 	struct adapter *adapter;
359 	struct iq_desc  *desc;	/* KVA of descriptor ring */
360 	int8_t   intr_pktc_idx;	/* packet count threshold index */
361 	uint8_t  gen;		/* generation bit */
362 	uint8_t  intr_params;	/* interrupt holdoff parameters */
363 	uint8_t  intr_next;	/* XXX: holdoff for next interrupt */
364 	uint16_t qsize;		/* size (# of entries) of the queue */
365 	uint16_t sidx;		/* index of the entry with the status page */
366 	uint16_t cidx;		/* consumer index */
367 	uint16_t cntxt_id;	/* SGE context id for the iq */
368 	uint16_t abs_id;	/* absolute SGE id for the iq */
369 
370 	STAILQ_ENTRY(sge_iq) link;
371 
372 	bus_dma_tag_t desc_tag;
373 	bus_dmamap_t desc_map;
374 	bus_addr_t ba;		/* bus address of descriptor ring */
375 };
376 
377 enum {
378 	EQ_CTRL		= 1,
379 	EQ_ETH		= 2,
380 #ifdef TCP_OFFLOAD
381 	EQ_OFLD		= 3,
382 #endif
383 
384 	/* eq flags */
385 	EQ_TYPEMASK	= 7,		/* 3 lsbits hold the type */
386 	EQ_ALLOCATED	= (1 << 3),	/* firmware resources allocated */
387 	EQ_DOOMED	= (1 << 4),	/* about to be destroyed */
388 	EQ_CRFLUSHED	= (1 << 5),	/* expecting an update from SGE */
389 	EQ_STALLED	= (1 << 6),	/* out of hw descriptors or dmamaps */
390 };
391 
392 /* Listed in order of preference.  Update t4_sysctls too if you change these */
393 enum {DOORBELL_UDB, DOORBELL_WCWR, DOORBELL_UDBWC, DOORBELL_KDB};
394 
395 /*
396  * Egress Queue: driver is producer, T4 is consumer.
397  *
398  * Note: A free list is an egress queue (driver produces the buffers and T4
399  * consumes them) but it's special enough to have its own struct (see sge_fl).
400  */
401 struct sge_eq {
402 	unsigned int flags;	/* MUST be first */
403 	unsigned int cntxt_id;	/* SGE context id for the eq */
404 	bus_dma_tag_t desc_tag;
405 	bus_dmamap_t desc_map;
406 	char lockname[16];
407 	struct mtx eq_lock;
408 
409 	struct tx_desc *desc;	/* KVA of descriptor ring */
410 	bus_addr_t ba;		/* bus address of descriptor ring */
411 	struct sge_qstat *spg;	/* status page, for convenience */
412 	uint16_t doorbells;
413 	volatile uint32_t *udb;	/* KVA of doorbell (lies within BAR2) */
414 	u_int udb_qid;		/* relative qid within the doorbell page */
415 	uint16_t cap;		/* max # of desc, for convenience */
416 	uint16_t avail;		/* available descriptors, for convenience */
417 	uint16_t qsize;		/* size (# of entries) of the queue */
418 	uint16_t cidx;		/* consumer idx (desc idx) */
419 	uint16_t pidx;		/* producer idx (desc idx) */
420 	uint16_t pending;	/* # of descriptors used since last doorbell */
421 	uint16_t iqid;		/* iq that gets egr_update for the eq */
422 	uint8_t tx_chan;	/* tx channel used by the eq */
423 	struct task tx_task;
424 	struct callout tx_callout;
425 
426 	/* stats */
427 
428 	uint32_t egr_update;	/* # of SGE_EGR_UPDATE notifications for eq */
429 	uint32_t unstalled;	/* recovered from stall */
430 };
431 
432 struct sw_zone_info {
433 	uma_zone_t zone;	/* zone that this cluster comes from */
434 	int size;		/* size of cluster: 2K, 4K, 9K, 16K, etc. */
435 	int type;		/* EXT_xxx type of the cluster */
436 	int8_t head_hwidx;
437 	int8_t tail_hwidx;
438 };
439 
440 struct hw_buf_info {
441 	int8_t zidx;		/* backpointer to zone; -ve means unused */
442 	int8_t next;		/* next hwidx for this zone; -1 means no more */
443 	int size;
444 };
445 
446 enum {
447 	FL_STARVING	= (1 << 0), /* on the adapter's list of starving fl's */
448 	FL_DOOMED	= (1 << 1), /* about to be destroyed */
449 	FL_BUF_PACKING	= (1 << 2), /* buffer packing enabled */
450 	FL_BUF_RESUME	= (1 << 3), /* resume from the middle of the frame */
451 };
452 
453 #define FL_RUNNING_LOW(fl) \
454     (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) <= fl->lowat)
455 #define FL_NOT_RUNNING_LOW(fl) \
456     (IDXDIFF(fl->dbidx * 8, fl->cidx, fl->sidx * 8) >= 2 * fl->lowat)
457 
458 struct sge_fl {
459 	struct mtx fl_lock;
460 	__be64 *desc;		/* KVA of descriptor ring, ptr to addresses */
461 	struct fl_sdesc *sdesc;	/* KVA of software descriptor ring */
462 	struct cluster_layout cll_def;	/* default refill zone, layout */
463 	uint16_t lowat;		/* # of buffers <= this means fl needs help */
464 	int flags;
465 	uint16_t buf_boundary;
466 
467 	/* The 16b idx all deal with hw descriptors */
468 	uint16_t dbidx;		/* hw pidx after last doorbell */
469 	uint16_t sidx;		/* index of status page */
470 	volatile uint16_t hw_cidx;
471 
472 	/* The 32b idx are all buffer idx, not hardware descriptor idx */
473 	uint32_t cidx;		/* consumer index */
474 	uint32_t pidx;		/* producer index */
475 
476 	uint32_t dbval;
477 	u_int rx_offset;	/* offset in fl buf (when buffer packing) */
478 	volatile uint32_t *udb;
479 
480 	uint64_t mbuf_allocated;/* # of mbuf allocated from zone_mbuf */
481 	uint64_t mbuf_inlined;	/* # of mbuf created within clusters */
482 	uint64_t cl_allocated;	/* # of clusters allocated */
483 	uint64_t cl_recycled;	/* # of clusters recycled */
484 	uint64_t cl_fast_recycled; /* # of clusters recycled (fast) */
485 
486 	/* These 3 are valid when FL_BUF_RESUME is set, stale otherwise. */
487 	struct mbuf *m0;
488 	struct mbuf **pnext;
489 	u_int remaining;
490 
491 	uint16_t qsize;		/* # of hw descriptors (status page included) */
492 	uint16_t cntxt_id;	/* SGE context id for the freelist */
493 	TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
494 	bus_dma_tag_t desc_tag;
495 	bus_dmamap_t desc_map;
496 	char lockname[16];
497 	bus_addr_t ba;		/* bus address of descriptor ring */
498 	struct cluster_layout cll_alt;	/* alternate refill zone, layout */
499 };
500 
501 /* txq: SGE egress queue + what's needed for Ethernet NIC */
502 struct sge_txq {
503 	struct sge_eq eq;	/* MUST be first */
504 
505 	struct ifnet *ifp;	/* the interface this txq belongs to */
506 	bus_dma_tag_t tx_tag;	/* tag for transmit buffers */
507 	struct buf_ring *br;	/* tx buffer ring */
508 	struct tx_sdesc *sdesc;	/* KVA of software descriptor ring */
509 	struct mbuf *m;		/* held up due to temporary resource shortage */
510 
511 	struct tx_maps txmaps;
512 
513 	/* stats for common events first */
514 
515 	uint64_t txcsum;	/* # of times hardware assisted with checksum */
516 	uint64_t tso_wrs;	/* # of TSO work requests */
517 	uint64_t vlan_insertion;/* # of times VLAN tag was inserted */
518 	uint64_t imm_wrs;	/* # of work requests with immediate data */
519 	uint64_t sgl_wrs;	/* # of work requests with direct SGL */
520 	uint64_t txpkt_wrs;	/* # of txpkt work requests (not coalesced) */
521 	uint64_t txpkts_wrs;	/* # of coalesced tx work requests */
522 	uint64_t txpkts_pkts;	/* # of frames in coalesced tx work requests */
523 
524 	/* stats for not-that-common events */
525 
526 	uint32_t no_dmamap;	/* no DMA map to load the mbuf */
527 	uint32_t no_desc;	/* out of hardware descriptors */
528 } __aligned(CACHE_LINE_SIZE);
529 
530 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
531 struct sge_rxq {
532 	struct sge_iq iq;	/* MUST be first */
533 	struct sge_fl fl;	/* MUST follow iq */
534 
535 	struct ifnet *ifp;	/* the interface this rxq belongs to */
536 #if defined(INET) || defined(INET6)
537 	struct lro_ctrl lro;	/* LRO state */
538 #endif
539 
540 	/* stats for common events first */
541 
542 	uint64_t rxcsum;	/* # of times hardware assisted with checksum */
543 	uint64_t vlan_extraction;/* # of times VLAN tag was extracted */
544 
545 	/* stats for not-that-common events */
546 
547 } __aligned(CACHE_LINE_SIZE);
548 
549 static inline struct sge_rxq *
550 iq_to_rxq(struct sge_iq *iq)
551 {
552 
553 	return (__containerof(iq, struct sge_rxq, iq));
554 }
555 
556 
557 #ifdef TCP_OFFLOAD
558 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
559 struct sge_ofld_rxq {
560 	struct sge_iq iq;	/* MUST be first */
561 	struct sge_fl fl;	/* MUST follow iq */
562 } __aligned(CACHE_LINE_SIZE);
563 
564 static inline struct sge_ofld_rxq *
565 iq_to_ofld_rxq(struct sge_iq *iq)
566 {
567 
568 	return (__containerof(iq, struct sge_ofld_rxq, iq));
569 }
570 #endif
571 
572 struct wrqe {
573 	STAILQ_ENTRY(wrqe) link;
574 	struct sge_wrq *wrq;
575 	int wr_len;
576 	uint64_t wr[] __aligned(16);
577 };
578 
579 /*
580  * wrq: SGE egress queue that is given prebuilt work requests.  Both the control
581  * and offload tx queues are of this type.
582  */
583 struct sge_wrq {
584 	struct sge_eq eq;	/* MUST be first */
585 
586 	struct adapter *adapter;
587 
588 	/* List of WRs held up due to lack of tx descriptors */
589 	STAILQ_HEAD(, wrqe) wr_list;
590 
591 	/* stats for common events first */
592 
593 	uint64_t tx_wrs;	/* # of tx work requests */
594 
595 	/* stats for not-that-common events */
596 
597 	uint32_t no_desc;	/* out of hardware descriptors */
598 } __aligned(CACHE_LINE_SIZE);
599 
600 
601 #ifdef DEV_NETMAP
602 struct sge_nm_rxq {
603 	struct port_info *pi;
604 
605 	struct iq_desc *iq_desc;
606 	uint16_t iq_abs_id;
607 	uint16_t iq_cntxt_id;
608 	uint16_t iq_cidx;
609 	uint16_t iq_sidx;
610 	uint8_t iq_gen;
611 
612 	__be64  *fl_desc;
613 	uint16_t fl_cntxt_id;
614 	uint32_t fl_cidx;
615 	uint32_t fl_pidx;
616 	uint32_t fl_sidx;
617 	uint32_t fl_db_val;
618 	u_int fl_hwidx:4;
619 
620 	u_int nid;		/* netmap ring # for this queue */
621 
622 	/* infrequently used items after this */
623 
624 	bus_dma_tag_t iq_desc_tag;
625 	bus_dmamap_t iq_desc_map;
626 	bus_addr_t iq_ba;
627 	int intr_idx;
628 
629 	bus_dma_tag_t fl_desc_tag;
630 	bus_dmamap_t fl_desc_map;
631 	bus_addr_t fl_ba;
632 } __aligned(CACHE_LINE_SIZE);
633 
634 struct sge_nm_txq {
635 	struct tx_desc *desc;
636 	uint16_t cidx;
637 	uint16_t pidx;
638 	uint16_t sidx;
639 	uint16_t equiqidx;	/* EQUIQ last requested at this pidx */
640 	uint16_t equeqidx;	/* EQUEQ last requested at this pidx */
641 	uint16_t dbidx;		/* pidx of the most recent doorbell */
642 	uint16_t doorbells;
643 	volatile uint32_t *udb;
644 	u_int udb_qid;
645 	u_int cntxt_id;
646 	__be32 cpl_ctrl0;	/* for convenience */
647 	u_int nid;		/* netmap ring # for this queue */
648 
649 	/* infrequently used items after this */
650 
651 	bus_dma_tag_t desc_tag;
652 	bus_dmamap_t desc_map;
653 	bus_addr_t ba;
654 	int iqidx;
655 } __aligned(CACHE_LINE_SIZE);
656 #endif
657 
658 struct sge {
659 	int timer_val[SGE_NTIMERS];
660 	int counter_val[SGE_NCOUNTERS];
661 	int fl_starve_threshold;
662 	int fl_starve_threshold2;
663 	int eq_s_qpp;
664 	int iq_s_qpp;
665 
666 	int nrxq;	/* total # of Ethernet rx queues */
667 	int ntxq;	/* total # of Ethernet tx tx queues */
668 #ifdef TCP_OFFLOAD
669 	int nofldrxq;	/* total # of TOE rx queues */
670 	int nofldtxq;	/* total # of TOE tx queues */
671 #endif
672 #ifdef DEV_NETMAP
673 	int nnmrxq;	/* total # of netmap rx queues */
674 	int nnmtxq;	/* total # of netmap tx queues */
675 #endif
676 	int niq;	/* total # of ingress queues */
677 	int neq;	/* total # of egress queues */
678 
679 	struct sge_iq fwq;	/* Firmware event queue */
680 	struct sge_wrq mgmtq;	/* Management queue (control queue) */
681 	struct sge_wrq *ctrlq;	/* Control queues */
682 	struct sge_txq *txq;	/* NIC tx queues */
683 	struct sge_rxq *rxq;	/* NIC rx queues */
684 #ifdef TCP_OFFLOAD
685 	struct sge_wrq *ofld_txq;	/* TOE tx queues */
686 	struct sge_ofld_rxq *ofld_rxq;	/* TOE rx queues */
687 #endif
688 #ifdef DEV_NETMAP
689 	struct sge_nm_txq *nm_txq;	/* netmap tx queues */
690 	struct sge_nm_rxq *nm_rxq;	/* netmap rx queues */
691 #endif
692 
693 	uint16_t iq_start;
694 	int eq_start;
695 	struct sge_iq **iqmap;	/* iq->cntxt_id to iq mapping */
696 	struct sge_eq **eqmap;	/* eq->cntxt_id to eq mapping */
697 
698 	int pack_boundary;
699 	int8_t safe_hwidx1;	/* may not have room for metadata */
700 	int8_t safe_hwidx2;	/* with room for metadata and maybe more */
701 	struct sw_zone_info sw_zone_info[SW_ZONE_SIZES];
702 	struct hw_buf_info hw_buf_info[SGE_FLBUF_SIZES];
703 };
704 
705 struct rss_header;
706 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
707     struct mbuf *);
708 typedef int (*an_handler_t)(struct sge_iq *, const struct rsp_ctrl *);
709 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
710 
711 struct adapter {
712 	SLIST_ENTRY(adapter) link;
713 	device_t dev;
714 	struct cdev *cdev;
715 
716 	/* PCIe register resources */
717 	int regs_rid;
718 	struct resource *regs_res;
719 	int msix_rid;
720 	struct resource *msix_res;
721 	bus_space_handle_t bh;
722 	bus_space_tag_t bt;
723 	bus_size_t mmio_len;
724 	int udbs_rid;
725 	struct resource *udbs_res;
726 	volatile uint8_t *udbs_base;
727 
728 	unsigned int pf;
729 	unsigned int mbox;
730 
731 	/* Interrupt information */
732 	int intr_type;
733 	int intr_count;
734 	struct irq {
735 		struct resource *res;
736 		int rid;
737 		void *tag;
738 	} *irq;
739 
740 	bus_dma_tag_t dmat;	/* Parent DMA tag */
741 
742 	struct sge sge;
743 	int lro_timeout;
744 
745 	struct taskqueue *tq[NCHAN];	/* taskqueues that flush data out */
746 	struct port_info *port[MAX_NPORTS];
747 	uint8_t chan_map[NCHAN];
748 
749 #ifdef TCP_OFFLOAD
750 	void *tom_softc;	/* (struct tom_data *) */
751 	struct tom_tunables tt;
752 	void *iwarp_softc;	/* (struct c4iw_dev *) */
753 	void *iscsi_softc;
754 #endif
755 	struct l2t_data *l2t;	/* L2 table */
756 	struct tid_info tids;
757 
758 	uint16_t doorbells;
759 	int open_device_map;
760 #ifdef TCP_OFFLOAD
761 	int offload_map;
762 #endif
763 	int flags;
764 
765 	char ifp_lockname[16];
766 	struct mtx ifp_lock;
767 	struct ifnet *ifp;	/* tracer ifp */
768 	struct ifmedia media;
769 	int traceq;		/* iq used by all tracers, -1 if none */
770 	int tracer_valid;	/* bitmap of valid tracers */
771 	int tracer_enabled;	/* bitmap of enabled tracers */
772 
773 	char fw_version[32];
774 	char cfg_file[32];
775 	u_int cfcsum;
776 	struct adapter_params params;
777 	struct t4_virt_res vres;
778 
779 	uint16_t linkcaps;
780 	uint16_t niccaps;
781 	uint16_t toecaps;
782 	uint16_t rdmacaps;
783 	uint16_t iscsicaps;
784 	uint16_t fcoecaps;
785 
786 	struct sysctl_ctx_list ctx; /* from adapter_full_init to full_uninit */
787 
788 	struct mtx sc_lock;
789 	char lockname[16];
790 
791 	/* Starving free lists */
792 	struct mtx sfl_lock;	/* same cache-line as sc_lock? but that's ok */
793 	TAILQ_HEAD(, sge_fl) sfl;
794 	struct callout sfl_callout;
795 
796 	struct mtx regwin_lock;	/* for indirect reads and memory windows */
797 
798 	an_handler_t an_handler __aligned(CACHE_LINE_SIZE);
799 	fw_msg_handler_t fw_msg_handler[5];	/* NUM_FW6_TYPES */
800 	cpl_handler_t cpl_handler[0xef];	/* NUM_CPL_CMDS */
801 
802 #ifdef INVARIANTS
803 	const char *last_op;
804 	const void *last_op_thr;
805 #endif
806 
807 	int sc_do_rxcopy;
808 };
809 
810 #define ADAPTER_LOCK(sc)		mtx_lock(&(sc)->sc_lock)
811 #define ADAPTER_UNLOCK(sc)		mtx_unlock(&(sc)->sc_lock)
812 #define ADAPTER_LOCK_ASSERT_OWNED(sc)	mtx_assert(&(sc)->sc_lock, MA_OWNED)
813 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) mtx_assert(&(sc)->sc_lock, MA_NOTOWNED)
814 
815 /* XXX: not bulletproof, but much better than nothing */
816 #define ASSERT_SYNCHRONIZED_OP(sc)	\
817     KASSERT(IS_BUSY(sc) && \
818 	(mtx_owned(&(sc)->sc_lock) || sc->last_op_thr == curthread), \
819 	("%s: operation not synchronized.", __func__))
820 
821 #define PORT_LOCK(pi)			mtx_lock(&(pi)->pi_lock)
822 #define PORT_UNLOCK(pi)			mtx_unlock(&(pi)->pi_lock)
823 #define PORT_LOCK_ASSERT_OWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_OWNED)
824 #define PORT_LOCK_ASSERT_NOTOWNED(pi)	mtx_assert(&(pi)->pi_lock, MA_NOTOWNED)
825 
826 #define FL_LOCK(fl)			mtx_lock(&(fl)->fl_lock)
827 #define FL_TRYLOCK(fl)			mtx_trylock(&(fl)->fl_lock)
828 #define FL_UNLOCK(fl)			mtx_unlock(&(fl)->fl_lock)
829 #define FL_LOCK_ASSERT_OWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_OWNED)
830 #define FL_LOCK_ASSERT_NOTOWNED(fl)	mtx_assert(&(fl)->fl_lock, MA_NOTOWNED)
831 
832 #define RXQ_FL_LOCK(rxq)		FL_LOCK(&(rxq)->fl)
833 #define RXQ_FL_UNLOCK(rxq)		FL_UNLOCK(&(rxq)->fl)
834 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq)	FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
835 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
836 
837 #define EQ_LOCK(eq)			mtx_lock(&(eq)->eq_lock)
838 #define EQ_TRYLOCK(eq)			mtx_trylock(&(eq)->eq_lock)
839 #define EQ_UNLOCK(eq)			mtx_unlock(&(eq)->eq_lock)
840 #define EQ_LOCK_ASSERT_OWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_OWNED)
841 #define EQ_LOCK_ASSERT_NOTOWNED(eq)	mtx_assert(&(eq)->eq_lock, MA_NOTOWNED)
842 
843 #define TXQ_LOCK(txq)			EQ_LOCK(&(txq)->eq)
844 #define TXQ_TRYLOCK(txq)		EQ_TRYLOCK(&(txq)->eq)
845 #define TXQ_UNLOCK(txq)			EQ_UNLOCK(&(txq)->eq)
846 #define TXQ_LOCK_ASSERT_OWNED(txq)	EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
847 #define TXQ_LOCK_ASSERT_NOTOWNED(txq)	EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
848 
849 #define for_each_txq(pi, iter, q) \
850 	for (q = &pi->adapter->sge.txq[pi->first_txq], iter = 0; \
851 	    iter < pi->ntxq; ++iter, ++q)
852 #define for_each_rxq(pi, iter, q) \
853 	for (q = &pi->adapter->sge.rxq[pi->first_rxq], iter = 0; \
854 	    iter < pi->nrxq; ++iter, ++q)
855 #define for_each_ofld_txq(pi, iter, q) \
856 	for (q = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq], iter = 0; \
857 	    iter < pi->nofldtxq; ++iter, ++q)
858 #define for_each_ofld_rxq(pi, iter, q) \
859 	for (q = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq], iter = 0; \
860 	    iter < pi->nofldrxq; ++iter, ++q)
861 #define for_each_nm_txq(pi, iter, q) \
862 	for (q = &pi->adapter->sge.nm_txq[pi->first_nm_txq], iter = 0; \
863 	    iter < pi->nnmtxq; ++iter, ++q)
864 #define for_each_nm_rxq(pi, iter, q) \
865 	for (q = &pi->adapter->sge.nm_rxq[pi->first_nm_rxq], iter = 0; \
866 	    iter < pi->nnmrxq; ++iter, ++q)
867 
868 #define IDXINCR(idx, incr, wrap) do { \
869 	idx = wrap - idx > incr ? idx + incr : incr - (wrap - idx); \
870 } while (0)
871 #define IDXDIFF(head, tail, wrap) \
872 	((head) >= (tail) ? (head) - (tail) : (wrap) - (tail) + (head))
873 
874 /* One for errors, one for firmware events */
875 #define T4_EXTRA_INTR 2
876 
877 static inline uint32_t
878 t4_read_reg(struct adapter *sc, uint32_t reg)
879 {
880 
881 	return bus_space_read_4(sc->bt, sc->bh, reg);
882 }
883 
884 static inline void
885 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
886 {
887 
888 	bus_space_write_4(sc->bt, sc->bh, reg, val);
889 }
890 
891 static inline uint64_t
892 t4_read_reg64(struct adapter *sc, uint32_t reg)
893 {
894 
895 	return t4_bus_space_read_8(sc->bt, sc->bh, reg);
896 }
897 
898 static inline void
899 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
900 {
901 
902 	t4_bus_space_write_8(sc->bt, sc->bh, reg, val);
903 }
904 
905 static inline void
906 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
907 {
908 
909 	*val = pci_read_config(sc->dev, reg, 1);
910 }
911 
912 static inline void
913 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
914 {
915 
916 	pci_write_config(sc->dev, reg, val, 1);
917 }
918 
919 static inline void
920 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
921 {
922 
923 	*val = pci_read_config(sc->dev, reg, 2);
924 }
925 
926 static inline void
927 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
928 {
929 
930 	pci_write_config(sc->dev, reg, val, 2);
931 }
932 
933 static inline void
934 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
935 {
936 
937 	*val = pci_read_config(sc->dev, reg, 4);
938 }
939 
940 static inline void
941 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
942 {
943 
944 	pci_write_config(sc->dev, reg, val, 4);
945 }
946 
947 static inline struct port_info *
948 adap2pinfo(struct adapter *sc, int idx)
949 {
950 
951 	return (sc->port[idx]);
952 }
953 
954 static inline void
955 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
956 {
957 
958 	bcopy(hw_addr, sc->port[idx]->hw_addr, ETHER_ADDR_LEN);
959 }
960 
961 static inline bool
962 is_10G_port(const struct port_info *pi)
963 {
964 
965 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) != 0);
966 }
967 
968 static inline bool
969 is_40G_port(const struct port_info *pi)
970 {
971 
972 	return ((pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) != 0);
973 }
974 
975 static inline int
976 tx_resume_threshold(struct sge_eq *eq)
977 {
978 
979 	return (eq->qsize / 4);
980 }
981 
982 /* t4_main.c */
983 void t4_tx_task(void *, int);
984 void t4_tx_callout(void *);
985 int t4_os_find_pci_capability(struct adapter *, int);
986 int t4_os_pci_save_state(struct adapter *);
987 int t4_os_pci_restore_state(struct adapter *);
988 void t4_os_portmod_changed(const struct adapter *, int);
989 void t4_os_link_changed(struct adapter *, int, int, int);
990 void t4_iterate(void (*)(struct adapter *, void *), void *);
991 int t4_register_cpl_handler(struct adapter *, int, cpl_handler_t);
992 int t4_register_an_handler(struct adapter *, an_handler_t);
993 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
994 int t4_filter_rpl(struct sge_iq *, const struct rss_header *, struct mbuf *);
995 int begin_synchronized_op(struct adapter *, struct port_info *, int, char *);
996 void end_synchronized_op(struct adapter *, int);
997 int update_mac_settings(struct ifnet *, int);
998 int adapter_full_init(struct adapter *);
999 int adapter_full_uninit(struct adapter *);
1000 int port_full_init(struct port_info *);
1001 int port_full_uninit(struct port_info *);
1002 
1003 #ifdef DEV_NETMAP
1004 /* t4_netmap.c */
1005 int create_netmap_ifnet(struct port_info *);
1006 int destroy_netmap_ifnet(struct port_info *);
1007 void t4_nm_intr(void *);
1008 #endif
1009 
1010 /* t4_sge.c */
1011 void t4_sge_modload(void);
1012 void t4_sge_modunload(void);
1013 uint64_t t4_sge_extfree_refs(void);
1014 void t4_init_sge_cpl_handlers(struct adapter *);
1015 void t4_tweak_chip_settings(struct adapter *);
1016 int t4_read_chip_settings(struct adapter *);
1017 int t4_create_dma_tag(struct adapter *);
1018 void t4_sge_sysctls(struct adapter *, struct sysctl_ctx_list *,
1019     struct sysctl_oid_list *);
1020 int t4_destroy_dma_tag(struct adapter *);
1021 int t4_setup_adapter_queues(struct adapter *);
1022 int t4_teardown_adapter_queues(struct adapter *);
1023 int t4_setup_port_queues(struct port_info *);
1024 int t4_teardown_port_queues(struct port_info *);
1025 int t4_alloc_tx_maps(struct tx_maps *, bus_dma_tag_t, int, int);
1026 void t4_free_tx_maps(struct tx_maps *, bus_dma_tag_t);
1027 void t4_intr_all(void *);
1028 void t4_intr(void *);
1029 void t4_intr_err(void *);
1030 void t4_intr_evt(void *);
1031 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);
1032 int t4_eth_tx(struct ifnet *, struct sge_txq *, struct mbuf *);
1033 void t4_update_fl_bufsize(struct ifnet *);
1034 int can_resume_tx(struct sge_eq *);
1035 
1036 /* t4_tracer.c */
1037 struct t4_tracer;
1038 void t4_tracer_modload(void);
1039 void t4_tracer_modunload(void);
1040 void t4_tracer_port_detach(struct adapter *);
1041 int t4_get_tracer(struct adapter *, struct t4_tracer *);
1042 int t4_set_tracer(struct adapter *, struct t4_tracer *);
1043 int t4_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
1044 int t5_trace_pkt(struct sge_iq *, const struct rss_header *, struct mbuf *);
1045 
1046 static inline struct wrqe *
1047 alloc_wrqe(int wr_len, struct sge_wrq *wrq)
1048 {
1049 	int len = offsetof(struct wrqe, wr) + wr_len;
1050 	struct wrqe *wr;
1051 
1052 	wr = malloc(len, M_CXGBE, M_NOWAIT);
1053 	if (__predict_false(wr == NULL))
1054 		return (NULL);
1055 	wr->wr_len = wr_len;
1056 	wr->wrq = wrq;
1057 	return (wr);
1058 }
1059 
1060 static inline void *
1061 wrtod(struct wrqe *wr)
1062 {
1063 	return (&wr->wr[0]);
1064 }
1065 
1066 static inline void
1067 free_wrqe(struct wrqe *wr)
1068 {
1069 	free(wr, M_CXGBE);
1070 }
1071 
1072 static inline void
1073 t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
1074 {
1075 	struct sge_wrq *wrq = wr->wrq;
1076 
1077 	TXQ_LOCK(wrq);
1078 	t4_wrq_tx_locked(sc, wrq, wr);
1079 	TXQ_UNLOCK(wrq);
1080 }
1081 
1082 #endif
1083