1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2011-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 #ifndef __CXGBE_ADAPTER_H
24 #define __CXGBE_ADAPTER_H
25
26 #include <sys/ddi.h>
27 #include <sys/mac_provider.h>
28 #include <sys/ethernet.h>
29 #include <sys/queue.h>
30 #include <sys/containerof.h>
31 #include <sys/ddi_ufm.h>
32
33 #include "offload.h"
34 #include "firmware/t4fw_interface.h"
35 #include "shared.h"
36
37 struct adapter;
38 typedef struct adapter adapter_t;
39
40 enum {
41 FW_IQ_QSIZE = 256,
42 FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */
43
44 RX_IQ_QSIZE = 1024,
45 RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */
46
47 EQ_ESIZE = 64, /* All egres queues use this entry size */
48
49 RX_FL_ESIZE = 64, /* 8 64bit addresses */
50
51 FL_BUF_SIZES = 4,
52
53 CTRL_EQ_QSIZE = 128,
54
55 TX_EQ_QSIZE = 1024,
56 TX_SGL_SEGS = 36,
57 TX_WR_FLITS = SGE_MAX_WR_LEN / 8
58 };
59
60 enum {
61 /* adapter flags */
62 FULL_INIT_DONE = (1 << 0),
63 FW_OK = (1 << 1),
64 INTR_FWD = (1 << 2),
65 INTR_ALLOCATED = (1 << 3),
66 MASTER_PF = (1 << 4),
67
68 CXGBE_BUSY = (1 << 9),
69
70 /* port flags */
71 DOOMED = (1 << 0),
72 PORT_INIT_DONE = (1 << 1),
73 };
74
75 enum {
76 /* Features */
77 CXGBE_HW_LSO = (1 << 0),
78 CXGBE_HW_CSUM = (1 << 1),
79 };
80
81 enum {
82 UDBS_SEG_SHIFT = 7, /* log2(UDBS_SEG_SIZE) */
83 UDBS_DB_OFFSET = 8, /* offset of the 4B doorbell in a segment */
84 UDBS_WR_OFFSET = 64, /* offset of the work request in a segment */
85 };
86
87 #define IS_DOOMED(pi) (pi->flags & DOOMED)
88 #define SET_DOOMED(pi) do { pi->flags |= DOOMED; } while (0)
89 #define IS_BUSY(sc) (sc->flags & CXGBE_BUSY)
90 #define SET_BUSY(sc) do { sc->flags |= CXGBE_BUSY; } while (0)
91 #define CLR_BUSY(sc) do { sc->flags &= ~CXGBE_BUSY; } while (0)
92
93 struct port_info {
94 PORT_INFO_HDR;
95
96 kmutex_t lock;
97 struct adapter *adapter;
98
99 #ifdef TCP_OFFLOAD_ENABLE
100 void *tdev;
101 #endif
102
103 unsigned int flags;
104
105 uint16_t viid;
106 int16_t xact_addr_filt; /* index of exact MAC address filter */
107 uint16_t rss_size; /* size of VI's RSS table slice */
108 uint16_t ntxq; /* # of tx queues */
109 uint16_t first_txq; /* index of first tx queue */
110 uint16_t nrxq; /* # of rx queues */
111 uint16_t first_rxq; /* index of first rx queue */
112 #ifdef TCP_OFFLOAD_ENABLE
113 uint16_t nofldtxq; /* # of offload tx queues */
114 uint16_t first_ofld_txq; /* index of first offload tx queue */
115 uint16_t nofldrxq; /* # of offload rx queues */
116 uint16_t first_ofld_rxq; /* index of first offload rx queue */
117 #endif
118 uint8_t lport; /* associated offload logical port */
119 int8_t mdio_addr;
120 uint8_t port_type;
121 uint8_t mod_type;
122 uint8_t port_id;
123 uint8_t tx_chan;
124 uint8_t rx_chan;
125 uint8_t rx_cchan;
126 uint8_t instance; /* Associated adapter instance */
127 uint8_t child_inst; /* Associated child instance */
128 uint8_t tmr_idx;
129 int8_t pktc_idx;
130 struct link_config link_cfg;
131 struct port_stats stats;
132 uint32_t features;
133 uint8_t macaddr_cnt;
134 u8 rss_mode;
135 u16 viid_mirror;
136 kstat_t *ksp_config;
137 kstat_t *ksp_info;
138 kstat_t *ksp_fec;
139
140 u8 vivld;
141 u8 vin;
142 u8 smt_idx;
143
144 u8 vivld_mirror;
145 u8 vin_mirror;
146 u8 smt_idx_mirror;
147 };
148
149 struct fl_sdesc {
150 struct rxbuf *rxb;
151 };
152
153 struct tx_desc {
154 __be64 flit[8];
155 };
156
157 /* DMA maps used for tx */
158 struct tx_maps {
159 ddi_dma_handle_t *map;
160 uint32_t map_total; /* # of DMA maps */
161 uint32_t map_pidx; /* next map to be used */
162 uint32_t map_cidx; /* reclaimed up to this index */
163 uint32_t map_avail; /* # of available maps */
164 };
165
166 struct tx_sdesc {
167 mblk_t *m;
168 uint32_t txb_used; /* # of bytes of tx copy buffer used */
169 uint16_t hdls_used; /* # of dma handles used */
170 uint16_t desc_used; /* # of hardware descriptors used */
171 };
172
173 enum {
174 /* iq flags */
175 IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */
176 IQ_INTR = (1 << 1), /* iq takes direct interrupt */
177 IQ_HAS_FL = (1 << 2), /* iq has fl */
178
179 /* iq state */
180 IQS_DISABLED = 0,
181 IQS_BUSY = 1,
182 IQS_IDLE = 2,
183 };
184
185 /*
186 * Ingress Queue: T4 is producer, driver is consumer.
187 */
188 struct sge_iq {
189 unsigned int flags;
190 ddi_dma_handle_t dhdl;
191 ddi_acc_handle_t ahdl;
192
193 volatile uint_t state;
194 __be64 *desc; /* KVA of descriptor ring */
195 uint64_t ba; /* bus address of descriptor ring */
196 const __be64 *cdesc; /* current descriptor */
197 struct adapter *adapter; /* associated adapter */
198 uint8_t gen; /* generation bit */
199 uint8_t intr_params; /* interrupt holdoff parameters */
200 int8_t intr_pktc_idx; /* packet count threshold index */
201 uint8_t intr_next; /* holdoff for next interrupt */
202 uint8_t esize; /* size (bytes) of each entry in the queue */
203 uint16_t qsize; /* size (# of entries) of the queue */
204 uint16_t cidx; /* consumer index */
205 uint16_t pending; /* # of descs processed since last doorbell */
206 uint16_t cntxt_id; /* SGE context id for the iq */
207 uint16_t abs_id; /* absolute SGE id for the iq */
208 kmutex_t lock; /* Rx access lock */
209 uint8_t polling;
210
211 STAILQ_ENTRY(sge_iq) link;
212 };
213
214 enum {
215 EQ_CTRL = 1,
216 EQ_ETH = 2,
217 #ifdef TCP_OFFLOAD_ENABLE
218 EQ_OFLD = 3,
219 #endif
220
221 /* eq flags */
222 EQ_TYPEMASK = 7, /* 3 lsbits hold the type */
223 EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */
224 EQ_DOOMED = (1 << 4), /* about to be destroyed */
225 EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */
226 EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */
227 EQ_MTX = (1 << 7), /* mutex has been initialized */
228 EQ_STARTED = (1 << 8), /* started */
229 };
230
231 /* Listed in order of preference. Update t4_sysctls too if you change these */
232 enum {DOORBELL_UDB=0x1 , DOORBELL_WCWR=0x2, DOORBELL_UDBWC=0x4, DOORBELL_KDB=0x8};
233
234 /*
235 * Egress Queue: driver is producer, T4 is consumer.
236 *
237 * Note: A free list is an egress queue (driver produces the buffers and T4
238 * consumes them) but it's special enough to have its own struct (see sge_fl).
239 */
240 struct sge_eq {
241 ddi_dma_handle_t desc_dhdl;
242 ddi_acc_handle_t desc_ahdl;
243 unsigned int flags;
244 kmutex_t lock;
245
246 struct tx_desc *desc; /* KVA of descriptor ring */
247 uint64_t ba; /* bus address of descriptor ring */
248 struct sge_qstat *spg; /* status page, for convenience */
249 int doorbells;
250 volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */
251 u_int udb_qid; /* relative qid within the doorbell page */
252 uint16_t cap; /* max # of desc, for convenience */
253 uint16_t avail; /* available descriptors, for convenience */
254 uint16_t qsize; /* size (# of entries) of the queue */
255 uint16_t cidx; /* consumer idx (desc idx) */
256 uint16_t pidx; /* producer idx (desc idx) */
257 uint16_t pending; /* # of descriptors used since last doorbell */
258 uint16_t iqid; /* iq that gets egr_update for the eq */
259 uint8_t tx_chan; /* tx channel used by the eq */
260 uint32_t cntxt_id; /* SGE context id for the eq */
261 };
262
263 enum {
264 /* fl flags */
265 FL_MTX = (1 << 0), /* mutex has been initialized */
266 FL_STARVING = (1 << 1), /* on the list of starving fl's */
267 FL_DOOMED = (1 << 2), /* about to be destroyed */
268 };
269
270 #define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat)
271 #define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat)
272
273 struct sge_fl {
274 unsigned int flags;
275 kmutex_t lock;
276 ddi_dma_handle_t dhdl;
277 ddi_acc_handle_t ahdl;
278
279 __be64 *desc; /* KVA of descriptor ring, ptr to addresses */
280 uint64_t ba; /* bus address of descriptor ring */
281 struct fl_sdesc *sdesc; /* KVA of software descriptor ring */
282 uint32_t cap; /* max # of buffers, for convenience */
283 uint16_t qsize; /* size (# of entries) of the queue */
284 uint16_t cntxt_id; /* SGE context id for the freelist */
285 uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */
286 uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */
287 uint32_t needed; /* # of buffers needed to fill up fl. */
288 uint32_t lowat; /* # of buffers <= this means fl needs help */
289 uint32_t pending; /* # of bufs allocated since last doorbell */
290 uint32_t offset; /* current packet within the larger buffer */
291 uint16_t copy_threshold; /* anything this size or less is copied up */
292
293 uint64_t copied_up; /* # of frames copied into mblk and handed up */
294 uint64_t passed_up; /* # of frames wrapped in mblk and handed up */
295 uint64_t allocb_fail; /* # of mblk allocation failures */
296
297 TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
298 };
299
300 /* txq: SGE egress queue + miscellaneous items */
301 struct sge_txq {
302 struct sge_eq eq; /* MUST be first */
303
304 struct port_info *port; /* the port this txq belongs to */
305 struct tx_sdesc *sdesc; /* KVA of software descriptor ring */
306 mac_ring_handle_t ring_handle;
307
308 /* DMA handles used for tx */
309 ddi_dma_handle_t *tx_dhdl;
310 uint32_t tx_dhdl_total; /* Total # of handles */
311 uint32_t tx_dhdl_pidx; /* next handle to be used */
312 uint32_t tx_dhdl_cidx; /* reclaimed up to this index */
313 uint32_t tx_dhdl_avail; /* # of available handles */
314
315 /* Copy buffers for tx */
316 ddi_dma_handle_t txb_dhdl;
317 ddi_acc_handle_t txb_ahdl;
318 caddr_t txb_va; /* KVA of copy buffers area */
319 uint64_t txb_ba; /* bus address of copy buffers area */
320 uint32_t txb_size; /* total size */
321 uint32_t txb_next; /* offset of next useable area in the buffer */
322 uint32_t txb_avail; /* # of bytes available */
323 uint16_t copy_threshold; /* anything this size or less is copied up */
324
325 uint64_t txpkts; /* # of ethernet packets */
326 uint64_t txbytes; /* # of ethernet bytes */
327 kstat_t *ksp;
328
329 /* stats for common events first */
330
331 uint64_t txcsum; /* # of times hardware assisted with checksum */
332 uint64_t tso_wrs; /* # of IPv4 TSO work requests */
333 uint64_t imm_wrs; /* # of work requests with immediate data */
334 uint64_t sgl_wrs; /* # of work requests with direct SGL */
335 uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */
336 uint64_t txpkts_wrs; /* # of coalesced tx work requests */
337 uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */
338 uint64_t txb_used; /* # of tx copy buffers used (64 byte each) */
339 uint64_t hdl_used; /* # of DMA handles used */
340
341 /* stats for not-that-common events */
342
343 uint32_t txb_full; /* txb ran out of space */
344 uint32_t dma_hdl_failed; /* couldn't obtain DMA handle */
345 uint32_t dma_map_failed; /* couldn't obtain DMA mapping */
346 uint32_t qfull; /* out of hardware descriptors */
347 uint32_t qflush; /* # of SGE_EGR_UPDATE notifications for txq */
348 uint32_t pullup_early; /* # of pullups before starting frame's SGL */
349 uint32_t pullup_late; /* # of pullups while building frame's SGL */
350 uint32_t pullup_failed; /* # of failed pullups */
351 uint32_t csum_failed; /* # of csum reqs we failed to fulfill */
352 };
353
354 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
355 struct sge_rxq {
356 struct sge_iq iq; /* MUST be first */
357 struct sge_fl fl;
358
359 struct port_info *port; /* the port this rxq belongs to */
360 kstat_t *ksp;
361
362 mac_ring_handle_t ring_handle;
363 uint64_t ring_gen_num;
364
365 /* stats for common events first */
366
367 uint64_t rxcsum; /* # of times hardware assisted with checksum */
368 uint64_t rxpkts; /* # of ethernet packets */
369 uint64_t rxbytes; /* # of ethernet bytes */
370
371 /* stats for not-that-common events */
372
373 uint32_t nomem; /* mblk allocation during rx failed */
374 };
375
376 #ifdef TCP_OFFLOAD_ENABLE
377 /* ofld_rxq: SGE ingress queue + SGE free list + miscellaneous items */
378 struct sge_ofld_rxq {
379 struct sge_iq iq; /* MUST be first */
380 struct sge_fl fl;
381 };
382
383 /*
384 * wrq: SGE egress queue that is given prebuilt work requests. Both the control
385 * and offload tx queues are of this type.
386 */
387 struct sge_wrq {
388 struct sge_eq eq; /* MUST be first */
389
390 struct adapter *adapter;
391
392 /* List of WRs held up due to lack of tx descriptors */
393 struct mblk_pair wr_list;
394
395 /* stats for common events first */
396
397 uint64_t tx_wrs; /* # of tx work requests */
398
399 /* stats for not-that-common events */
400
401 uint32_t no_desc; /* out of hardware descriptors */
402 };
403 #endif
404
405 struct sge {
406 int fl_starve_threshold;
407 int s_qpp;
408
409 int nrxq; /* total rx queues (all ports and the rest) */
410 int ntxq; /* total tx queues (all ports and the rest) */
411 #ifdef TCP_OFFLOAD_ENABLE
412 int nofldrxq; /* total # of TOE rx queues */
413 int nofldtxq; /* total # of TOE tx queues */
414 #endif
415 int niq; /* total ingress queues */
416 int neq; /* total egress queues */
417 int stat_len; /* length of status page at ring end */
418 int pktshift; /* padding between CPL & packet data */
419 int fl_align; /* response queue message alignment */
420
421 struct sge_iq fwq; /* Firmware event queue */
422 #ifdef TCP_OFFLOAD_ENABLE
423 struct sge_wrq mgmtq; /* Management queue (Control queue) */
424 #endif
425 struct sge_txq *txq; /* NIC tx queues */
426 struct sge_rxq *rxq; /* NIC rx queues */
427 #ifdef TCP_OFFLOAD_ENABLE
428 struct sge_wrq *ctrlq; /* Control queues */
429 struct sge_wrq *ofld_txq; /* TOE tx queues */
430 struct sge_ofld_rxq *ofld_rxq; /* TOE rx queues */
431 #endif
432
433 int iq_start; /* iq context id map start index */
434 int eq_start; /* eq context id map start index */
435 int iqmap_sz; /* size of iq context id map */
436 int eqmap_sz; /* size of eq context id map */
437 struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */
438 struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */
439
440 /* Device access and DMA attributes for all the descriptor rings */
441 ddi_device_acc_attr_t acc_attr_desc;
442 ddi_dma_attr_t dma_attr_desc;
443
444 /* Device access and DMA attributes for tx buffers */
445 ddi_device_acc_attr_t acc_attr_tx;
446 ddi_dma_attr_t dma_attr_tx;
447
448 /* Device access and DMA attributes for rx buffers are in rxb_params */
449 kmem_cache_t *rxbuf_cache;
450 struct rxbuf_cache_params rxb_params;
451 };
452
453 struct driver_properties {
454 /* There is a driver.conf variable for each of these */
455 int max_ntxq_10g;
456 int max_nrxq_10g;
457 int max_ntxq_1g;
458 int max_nrxq_1g;
459 #ifdef TCP_OFFLOAD_ENABLE
460 int max_nofldtxq_10g;
461 int max_nofldrxq_10g;
462 int max_nofldtxq_1g;
463 int max_nofldrxq_1g;
464 #endif
465 int intr_types;
466 int tmr_idx_10g;
467 int pktc_idx_10g;
468 int tmr_idx_1g;
469 int pktc_idx_1g;
470 int qsize_txq;
471 int qsize_rxq;
472
473 int timer_val[SGE_NTIMERS];
474 int counter_val[SGE_NCOUNTERS];
475
476 int wc;
477
478 int multi_rings;
479 int t4_fw_install;
480 };
481
482 struct rss_header;
483 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
484 mblk_t *);
485 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
486
487 struct t4_mbox_list {
488 STAILQ_ENTRY(t4_mbox_list) link;
489 };
490
491 struct adapter {
492 SLIST_ENTRY(adapter) link;
493 dev_info_t *dip;
494 dev_t dev;
495
496 unsigned int pf;
497 unsigned int mbox;
498
499 unsigned int vpd_busy;
500 unsigned int vpd_flag;
501
502 u32 t4_bar0;
503
504 uint_t open; /* character device is open */
505
506 /* PCI config space access handle */
507 ddi_acc_handle_t pci_regh;
508
509 /* MMIO register access handle */
510 ddi_acc_handle_t regh;
511 caddr_t regp;
512 /* BAR1 register access handle */
513 ddi_acc_handle_t reg1h;
514 caddr_t reg1p;
515
516 /* Interrupt information */
517 int intr_type;
518 int intr_count;
519 int intr_cap;
520 uint_t intr_pri;
521 ddi_intr_handle_t *intr_handle;
522
523 struct driver_properties props;
524 kstat_t *ksp;
525 kstat_t *ksp_stat;
526
527 struct sge sge;
528
529 struct port_info *port[MAX_NPORTS];
530 ddi_taskq_t *tq[NCHAN];
531 uint8_t chan_map[NCHAN];
532 uint32_t filter_mode;
533
534 struct l2t_data *l2t; /* L2 table */
535 struct tid_info tids;
536
537 int doorbells;
538 int registered_device_map;
539 int open_device_map;
540 int flags;
541
542 unsigned int cfcsum;
543 struct adapter_params params;
544 struct t4_virt_res vres;
545
546 #ifdef TCP_OFFLOAD_ENABLE
547 struct uld_softc tom;
548 struct tom_tunables tt;
549 #endif
550
551 #ifdef TCP_OFFLOAD_ENABLE
552 int offload_map;
553 #endif
554 uint16_t linkcaps;
555 uint16_t niccaps;
556 uint16_t toecaps;
557 uint16_t rdmacaps;
558 uint16_t iscsicaps;
559 uint16_t fcoecaps;
560
561 fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */
562 cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */
563
564 kmutex_t lock;
565 kcondvar_t cv;
566
567 /* Starving free lists */
568 kmutex_t sfl_lock; /* same cache-line as sc_lock? but that's ok */
569 TAILQ_HEAD(, sge_fl) sfl;
570 timeout_id_t sfl_timer;
571
572 /* Sensors */
573 id_t temp_sensor;
574 id_t volt_sensor;
575
576 ddi_ufm_handle_t *ufm_hdl;
577
578 /* support for single-threading access to adapter mailbox registers */
579 kmutex_t mbox_lock;
580 STAILQ_HEAD(, t4_mbox_list) mbox_list;
581 };
582
583 enum {
584 NIC_H = 0,
585 TOM_H,
586 IW_H,
587 ISCSI_H
588 };
589
590 struct memwin {
591 uint32_t base;
592 uint32_t aperture;
593 };
594
595 #define ADAPTER_LOCK(sc) mutex_enter(&(sc)->lock)
596 #define ADAPTER_UNLOCK(sc) mutex_exit(&(sc)->lock)
597 #define ADAPTER_LOCK_ASSERT_OWNED(sc) ASSERT(mutex_owned(&(sc)->lock))
598 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) ASSERT(!mutex_owned(&(sc)->lock))
599
600 #define PORT_LOCK(pi) mutex_enter(&(pi)->lock)
601 #define PORT_UNLOCK(pi) mutex_exit(&(pi)->lock)
602 #define PORT_LOCK_ASSERT_OWNED(pi) ASSERT(mutex_owned(&(pi)->lock))
603 #define PORT_LOCK_ASSERT_NOTOWNED(pi) ASSERT(!mutex_owned(&(pi)->lock))
604
605 #define IQ_LOCK(iq) mutex_enter(&(iq)->lock)
606 #define IQ_UNLOCK(iq) mutex_exit(&(iq)->lock)
607 #define IQ_LOCK_ASSERT_OWNED(iq) ASSERT(mutex_owned(&(iq)->lock))
608 #define IQ_LOCK_ASSERT_NOTOWNED(iq) ASSERT(!mutex_owned(&(iq)->lock))
609
610 #define FL_LOCK(fl) mutex_enter(&(fl)->lock)
611 #define FL_UNLOCK(fl) mutex_exit(&(fl)->lock)
612 #define FL_LOCK_ASSERT_OWNED(fl) ASSERT(mutex_owned(&(fl)->lock))
613 #define FL_LOCK_ASSERT_NOTOWNED(fl) ASSERT(!mutex_owned(&(fl)->lock))
614
615 #define RXQ_LOCK(rxq) IQ_LOCK(&(rxq)->iq)
616 #define RXQ_UNLOCK(rxq) IQ_UNLOCK(&(rxq)->iq)
617 #define RXQ_LOCK_ASSERT_OWNED(rxq) IQ_LOCK_ASSERT_OWNED(&(rxq)->iq)
618 #define RXQ_LOCK_ASSERT_NOTOWNED(rxq) IQ_LOCK_ASSERT_NOTOWNED(&(rxq)->iq)
619
620 #define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl)
621 #define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl)
622 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
623 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
624
625 #define EQ_LOCK(eq) mutex_enter(&(eq)->lock)
626 #define EQ_UNLOCK(eq) mutex_exit(&(eq)->lock)
627 #define EQ_LOCK_ASSERT_OWNED(eq) ASSERT(mutex_owned(&(eq)->lock))
628 #define EQ_LOCK_ASSERT_NOTOWNED(eq) ASSERT(!mutex_owned(&(eq)->lock))
629
630 #define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq)
631 #define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq)
632 #define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
633 #define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
634
635 #define for_each_txq(pi, iter, txq) \
636 txq = &pi->adapter->sge.txq[pi->first_txq]; \
637 for (iter = 0; iter < pi->ntxq; ++iter, ++txq)
638 #define for_each_rxq(pi, iter, rxq) \
639 rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
640 for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
641 #define for_each_ofld_txq(pi, iter, ofld_txq) \
642 ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
643 for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
644 #define for_each_ofld_rxq(pi, iter, ofld_rxq) \
645 ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
646 for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
647
648 #define NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1)
649
650 /* One for errors, one for firmware events */
651 #define T4_EXTRA_INTR 2
652
653 typedef kmutex_t t4_os_lock_t;
654
t4_os_lock(t4_os_lock_t * lock)655 static inline void t4_os_lock(t4_os_lock_t *lock)
656 {
657 mutex_enter(lock);
658 }
659
t4_os_unlock(t4_os_lock_t * lock)660 static inline void t4_os_unlock(t4_os_lock_t *lock)
661 {
662 mutex_exit(lock);
663 }
664
t4_mbox_list_add(struct adapter * adap,struct t4_mbox_list * entry)665 static inline void t4_mbox_list_add(struct adapter *adap,
666 struct t4_mbox_list *entry)
667 {
668 t4_os_lock(&adap->mbox_lock);
669 STAILQ_INSERT_TAIL(&adap->mbox_list, entry, link);
670 t4_os_unlock(&adap->mbox_lock);
671 }
672
t4_mbox_list_del(struct adapter * adap,struct t4_mbox_list * entry)673 static inline void t4_mbox_list_del(struct adapter *adap,
674 struct t4_mbox_list *entry)
675 {
676 t4_os_lock(&adap->mbox_lock);
677 STAILQ_REMOVE(&adap->mbox_list, entry, t4_mbox_list, link);
678 t4_os_unlock(&adap->mbox_lock);
679 }
680
681 static inline struct t4_mbox_list *
t4_mbox_list_first_entry(struct adapter * adap)682 t4_mbox_list_first_entry(struct adapter *adap)
683 {
684 return STAILQ_FIRST(&adap->mbox_list);
685 }
686
687 static inline uint32_t
t4_read_reg(struct adapter * sc,uint32_t reg)688 t4_read_reg(struct adapter *sc, uint32_t reg)
689 {
690 /* LINTED: E_BAD_PTR_CAST_ALIGN */
691 return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
692 }
693
694 static inline void
t4_write_reg(struct adapter * sc,uint32_t reg,uint32_t val)695 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
696 {
697 /* LINTED: E_BAD_PTR_CAST_ALIGN */
698 ddi_put32(sc->regh, (uint32_t *)(sc->regp + reg), val);
699 }
700
701 static inline void
t4_os_pci_read_cfg1(struct adapter * sc,int reg,uint8_t * val)702 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
703 {
704 *val = pci_config_get8(sc->pci_regh, reg);
705 }
706
707 static inline void
t4_os_pci_write_cfg1(struct adapter * sc,int reg,uint8_t val)708 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
709 {
710 pci_config_put8(sc->pci_regh, reg, val);
711 }
712
713 static inline void
t4_os_pci_read_cfg2(struct adapter * sc,int reg,uint16_t * val)714 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
715 {
716 *val = pci_config_get16(sc->pci_regh, reg);
717 }
718
719 static inline void
t4_os_pci_write_cfg2(struct adapter * sc,int reg,uint16_t val)720 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
721 {
722 pci_config_put16(sc->pci_regh, reg, val);
723 }
724
725 static inline void
t4_os_pci_read_cfg4(struct adapter * sc,int reg,uint32_t * val)726 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
727 {
728 *val = pci_config_get32(sc->pci_regh, reg);
729 }
730
731 static inline void
t4_os_pci_write_cfg4(struct adapter * sc,int reg,uint32_t val)732 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
733 {
734 pci_config_put32(sc->pci_regh, reg, val);
735 }
736
737 static inline uint32_t
t4_read_reg32(struct adapter * sc,uint32_t reg)738 t4_read_reg32(struct adapter *sc, uint32_t reg)
739 {
740 return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
741 }
742
743 static inline uint64_t
t4_read_reg64(struct adapter * sc,uint32_t reg)744 t4_read_reg64(struct adapter *sc, uint32_t reg)
745 {
746 return (ddi_get64(sc->regh, (uint64_t *)(sc->regp + reg)));
747 }
748
749 static inline void
t4_write_reg64(struct adapter * sc,uint32_t reg,uint64_t val)750 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
751 {
752 ddi_put64(sc->regh, (uint64_t *)(sc->regp + reg), val);
753 }
754
755 static inline struct port_info *
adap2pinfo(struct adapter * sc,int idx)756 adap2pinfo(struct adapter *sc, int idx)
757 {
758 return (sc->port[idx]);
759 }
760
761 static inline void
t4_os_set_hw_addr(struct adapter * sc,int idx,uint8_t hw_addr[])762 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
763 {
764 bcopy(hw_addr, sc->port[idx]->hw_addr, ETHERADDRL);
765 }
766
767 static inline bool
is_10G_port(const struct port_info * pi)768 is_10G_port(const struct port_info *pi)
769 {
770 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) != 0);
771 }
772
773 static inline struct sge_rxq *
iq_to_rxq(struct sge_iq * iq)774 iq_to_rxq(struct sge_iq *iq)
775 {
776 return (__containerof(iq, struct sge_rxq, iq));
777 }
778
779 static inline bool
is_25G_port(const struct port_info * pi)780 is_25G_port(const struct port_info *pi)
781 {
782 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) != 0);
783 }
784
785 static inline bool
is_40G_port(const struct port_info * pi)786 is_40G_port(const struct port_info *pi)
787 {
788 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) != 0);
789 }
790
791 static inline bool
is_50G_port(const struct port_info * pi)792 is_50G_port(const struct port_info *pi)
793 {
794 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) != 0);
795 }
796
797 static inline bool
is_100G_port(const struct port_info * pi)798 is_100G_port(const struct port_info *pi)
799 {
800 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) != 0);
801 }
802
803 static inline bool
is_10XG_port(const struct port_info * pi)804 is_10XG_port(const struct port_info *pi)
805 {
806 return (is_10G_port(pi) || is_40G_port(pi) ||
807 is_25G_port(pi) || is_50G_port(pi) ||
808 is_100G_port(pi));
809 }
810
811 #ifdef TCP_OFFLOAD_ENABLE
812 int t4_wrq_tx_locked(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m0);
813
814 static inline int
t4_wrq_tx(struct adapter * sc,struct sge_wrq * wrq,mblk_t * m)815 t4_wrq_tx(struct adapter *sc, struct sge_wrq *wrq, mblk_t *m)
816 {
817 int rc;
818
819 TXQ_LOCK(wrq);
820 rc = t4_wrq_tx_locked(sc, wrq, m);
821 TXQ_UNLOCK(wrq);
822 return (rc);
823 }
824 #endif
825
826 /**
827 * t4_os_pci_read_seeprom - read four bytes of SEEPROM/VPD contents
828 * @adapter: the adapter
829 * @addr: SEEPROM/VPD Address to read
830 * @valp: where to store the value read
831 *
832 * Read a 32-bit value from the given address in the SEEPROM/VPD. The address
833 * must be four-byte aligned. Returns 0 on success, a negative erro number
834 * on failure.
835 */
t4_os_pci_read_seeprom(adapter_t * adapter,int addr,u32 * valp)836 static inline int t4_os_pci_read_seeprom(adapter_t *adapter,
837 int addr, u32 *valp)
838 {
839 int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
840 int ret;
841
842 ret = t4_seeprom_read(adapter, addr, valp);
843
844 return ret >= 0 ? 0 : ret;
845 }
846
847 /**
848 * t4_os_pci_write_seeprom - write four bytes of SEEPROM/VPD contents
849 * @adapter: the adapter
850 * @addr: SEEPROM/VPD Address to write
851 * @val: the value write
852 *
853 * Write a 32-bit value to the given address in the SEEPROM/VPD. The address
854 * must be four-byte aligned. Returns 0 on success, a negative erro number
855 * on failure.
856 */
t4_os_pci_write_seeprom(adapter_t * adapter,int addr,u32 val)857 static inline int t4_os_pci_write_seeprom(adapter_t *adapter,
858 int addr, u32 val)
859 {
860 int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
861 int ret;
862
863 ret = t4_seeprom_write(adapter, addr, val);
864
865 return ret >= 0 ? 0 : ret;
866 }
867
t4_os_pci_set_vpd_size(struct adapter * adapter,size_t len)868 static inline int t4_os_pci_set_vpd_size(struct adapter *adapter, size_t len)
869 {
870 return 0;
871 }
872
t4_use_ldst(struct adapter * adap)873 static inline unsigned int t4_use_ldst(struct adapter *adap)
874 {
875 return (adap->flags & FW_OK);
876 }
877 #define t4_os_alloc(_size) kmem_alloc(_size, KM_SLEEP)
878
t4_db_full(struct adapter * adap)879 static inline void t4_db_full(struct adapter *adap) {}
t4_db_dropped(struct adapter * adap)880 static inline void t4_db_dropped(struct adapter *adap) {}
881
882 /* t4_nexus.c */
883 int t4_os_find_pci_capability(struct adapter *sc, int cap);
884 void t4_os_portmod_changed(struct adapter *sc, int idx);
885 int adapter_full_init(struct adapter *sc);
886 int adapter_full_uninit(struct adapter *sc);
887 int port_full_init(struct port_info *pi);
888 int port_full_uninit(struct port_info *pi);
889 void enable_port_queues(struct port_info *pi);
890 void disable_port_queues(struct port_info *pi);
891 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
892 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
893 void t4_iterate(void (*func)(int, void *), void *arg);
894
895 /* t4_sge.c */
896 void t4_sge_init(struct adapter *sc);
897 int t4_setup_adapter_queues(struct adapter *sc);
898 int t4_teardown_adapter_queues(struct adapter *sc);
899 int t4_setup_port_queues(struct port_info *pi);
900 int t4_teardown_port_queues(struct port_info *pi);
901 uint_t t4_intr_all(caddr_t arg1, caddr_t arg2);
902 uint_t t4_intr(caddr_t arg1, caddr_t arg2);
903 uint_t t4_intr_err(caddr_t arg1, caddr_t arg2);
904 int t4_mgmt_tx(struct adapter *sc, mblk_t *m);
905 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
906 uint32_t position_memwin(struct adapter *, int, uint32_t);
907
908 mblk_t *t4_eth_tx(void *, mblk_t *);
909 mblk_t *t4_mc_tx(void *arg, mblk_t *m);
910 mblk_t *t4_ring_rx(struct sge_rxq *rxq, int poll_bytes);
911 int t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
912 int flags);
913
914 /* t4_mac.c */
915 void t4_mc_init(struct port_info *pi);
916 void t4_mc_cb_init(struct port_info *);
917 void t4_os_link_changed(struct adapter *sc, int idx, int link_stat);
918 void t4_mac_rx(struct port_info *pi, struct sge_rxq *rxq, mblk_t *m);
919 void t4_mac_tx_update(struct port_info *pi, struct sge_txq *txq);
920 int t4_addmac(void *arg, const uint8_t *ucaddr);
921
922 /* t4_ioctl.c */
923 int t4_ioctl(struct adapter *sc, int cmd, void *data, int mode);
924
925 struct l2t_data *t4_init_l2t(struct adapter *sc);
926 int begin_synchronized_op(struct port_info *pi, int hold, int waitok);
927 void end_synchronized_op(struct port_info *pi, int held);
928 #endif /* __CXGBE_ADAPTER_H */
929