1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2011-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 /*
24 * Copyright 2024 Oxide Computer Company
25 */
26
27 #ifndef __CXGBE_ADAPTER_H
28 #define __CXGBE_ADAPTER_H
29
30 #include <sys/ddi.h>
31 #include <sys/mac_provider.h>
32 #include <sys/ethernet.h>
33 #include <sys/queue.h>
34 #include <sys/containerof.h>
35 #include <sys/ddi_ufm.h>
36
37 #include "offload.h"
38 #include "firmware/t4fw_interface.h"
39 #include "shared.h"
40
41 struct adapter;
42 typedef struct adapter adapter_t;
43
44 enum {
45 FW_IQ_QSIZE = 256,
46 FW_IQ_ESIZE = 64, /* At least 64 mandated by the firmware spec */
47
48 RX_IQ_QSIZE = 1024,
49 RX_IQ_ESIZE = 64, /* At least 64 so CPL_RX_PKT will fit */
50
51 EQ_ESIZE = 64, /* All egres queues use this entry size */
52
53 RX_FL_ESIZE = 64, /* 8 64bit addresses */
54
55 FL_BUF_SIZES = 4,
56
57 CTRL_EQ_QSIZE = 128,
58
59 TX_EQ_QSIZE = 1024,
60 TX_SGL_SEGS = 36,
61 TX_WR_FLITS = SGE_MAX_WR_LEN / 8
62 };
63
64 enum {
65 /* adapter flags */
66 FULL_INIT_DONE = (1 << 0),
67 FW_OK = (1 << 1),
68 INTR_FWD = (1 << 2),
69 INTR_ALLOCATED = (1 << 3),
70 MASTER_PF = (1 << 4),
71
72 CXGBE_BUSY = (1 << 9),
73
74 /* port flags */
75 DOOMED = (1 << 0),
76 PORT_INIT_DONE = (1 << 1),
77 };
78
79 enum {
80 /* Features */
81 CXGBE_HW_LSO = (1 << 0),
82 CXGBE_HW_CSUM = (1 << 1),
83 };
84
85 enum {
86 UDBS_SEG_SHIFT = 7, /* log2(UDBS_SEG_SIZE) */
87 UDBS_DB_OFFSET = 8, /* offset of the 4B doorbell in a segment */
88 UDBS_WR_OFFSET = 64, /* offset of the work request in a segment */
89 };
90
91 #define IS_DOOMED(pi) (pi->flags & DOOMED)
92 #define SET_DOOMED(pi) do { pi->flags |= DOOMED; } while (0)
93 #define IS_BUSY(sc) (sc->flags & CXGBE_BUSY)
94 #define SET_BUSY(sc) do { sc->flags |= CXGBE_BUSY; } while (0)
95 #define CLR_BUSY(sc) do { sc->flags &= ~CXGBE_BUSY; } while (0)
96
97 struct port_info {
98 dev_info_t *dip;
99 mac_handle_t mh;
100 mac_callbacks_t *mc;
101 void *props;
102 int mtu;
103 uint8_t hw_addr[ETHERADDRL];
104
105 kmutex_t lock;
106 struct adapter *adapter;
107
108 unsigned int flags;
109
110 uint16_t viid;
111 int16_t xact_addr_filt; /* index of exact MAC address filter */
112 uint16_t rss_size; /* size of VI's RSS table slice */
113 uint16_t ntxq; /* # of tx queues */
114 uint16_t first_txq; /* index of first tx queue */
115 uint16_t nrxq; /* # of rx queues */
116 uint16_t first_rxq; /* index of first rx queue */
117 uint8_t lport; /* associated offload logical port */
118 int8_t mdio_addr;
119 uint8_t port_type;
120 uint8_t mod_type;
121 uint8_t port_id;
122 uint8_t tx_chan;
123 uint8_t rx_chan;
124 uint8_t rx_cchan;
125 uint8_t instance; /* Associated adapter instance */
126 uint8_t child_inst; /* Associated child instance */
127 uint8_t tmr_idx;
128 int8_t pktc_idx;
129 struct link_config link_cfg;
130 struct port_stats stats;
131 uint32_t features;
132 uint8_t macaddr_cnt;
133 u8 rss_mode;
134 u16 viid_mirror;
135 kstat_t *ksp_config;
136 kstat_t *ksp_info;
137 kstat_t *ksp_fec;
138
139 u8 vivld;
140 u8 vin;
141 u8 smt_idx;
142
143 u8 vivld_mirror;
144 u8 vin_mirror;
145 u8 smt_idx_mirror;
146 };
147
148 struct fl_sdesc {
149 struct rxbuf *rxb;
150 };
151
152 struct tx_desc {
153 __be64 flit[8];
154 };
155
156 /* DMA maps used for tx */
157 struct tx_maps {
158 ddi_dma_handle_t *map;
159 uint32_t map_total; /* # of DMA maps */
160 uint32_t map_pidx; /* next map to be used */
161 uint32_t map_cidx; /* reclaimed up to this index */
162 uint32_t map_avail; /* # of available maps */
163 };
164
165 struct tx_sdesc {
166 mblk_t *m;
167 uint32_t txb_used; /* # of bytes of tx copy buffer used */
168 uint16_t hdls_used; /* # of dma handles used */
169 uint16_t desc_used; /* # of hardware descriptors used */
170 };
171
172 enum {
173 /* iq flags */
174 IQ_ALLOCATED = (1 << 0), /* firmware resources allocated */
175 IQ_INTR = (1 << 1), /* iq takes direct interrupt */
176 IQ_HAS_FL = (1 << 2), /* iq has fl */
177
178 /* iq state */
179 IQS_DISABLED = 0,
180 IQS_BUSY = 1,
181 IQS_IDLE = 2,
182 };
183
184 struct rxbuf_cache_params {
185 dev_info_t *dip;
186 ddi_dma_attr_t dma_attr_rx;
187 ddi_device_acc_attr_t acc_attr_rx;
188 size_t buf_size;
189 };
190
191 /*
192 * Ingress Queue: T4 is producer, driver is consumer.
193 */
194 struct sge_iq {
195 unsigned int flags;
196 ddi_dma_handle_t dhdl;
197 ddi_acc_handle_t ahdl;
198
199 volatile uint_t state;
200 __be64 *desc; /* KVA of descriptor ring */
201 uint64_t ba; /* bus address of descriptor ring */
202 const __be64 *cdesc; /* current descriptor */
203 struct adapter *adapter; /* associated adapter */
204 uint8_t gen; /* generation bit */
205 uint8_t intr_params; /* interrupt holdoff parameters */
206 int8_t intr_pktc_idx; /* packet count threshold index */
207 uint8_t intr_next; /* holdoff for next interrupt */
208 uint8_t esize; /* size (bytes) of each entry in the queue */
209 uint16_t qsize; /* size (# of entries) of the queue */
210 uint16_t cidx; /* consumer index */
211 uint16_t pending; /* # of descs processed since last doorbell */
212 uint16_t cntxt_id; /* SGE context id for the iq */
213 uint16_t abs_id; /* absolute SGE id for the iq */
214 kmutex_t lock; /* Rx access lock */
215 uint8_t polling;
216
217 STAILQ_ENTRY(sge_iq) link;
218 };
219
220 enum {
221 EQ_CTRL = 1,
222 EQ_ETH = 2,
223
224 /* eq flags */
225 EQ_TYPEMASK = 7, /* 3 lsbits hold the type */
226 EQ_ALLOCATED = (1 << 3), /* firmware resources allocated */
227 EQ_DOOMED = (1 << 4), /* about to be destroyed */
228 EQ_CRFLUSHED = (1 << 5), /* expecting an update from SGE */
229 EQ_STALLED = (1 << 6), /* out of hw descriptors or dmamaps */
230 EQ_MTX = (1 << 7), /* mutex has been initialized */
231 EQ_STARTED = (1 << 8), /* started */
232 };
233
234 /* Listed in order of preference. Update t4_sysctls too if you change these */
235 enum {
236 DOORBELL_UDB = 0x1,
237 DOORBELL_WCWR = 0x2,
238 DOORBELL_UDBWC = 0x4,
239 DOORBELL_KDB = 0x8
240 };
241
242 /*
243 * Egress Queue: driver is producer, T4 is consumer.
244 *
245 * Note: A free list is an egress queue (driver produces the buffers and T4
246 * consumes them) but it's special enough to have its own struct (see sge_fl).
247 */
248 struct sge_eq {
249 ddi_dma_handle_t desc_dhdl;
250 ddi_acc_handle_t desc_ahdl;
251 unsigned int flags;
252 kmutex_t lock;
253
254 struct tx_desc *desc; /* KVA of descriptor ring */
255 uint64_t ba; /* bus address of descriptor ring */
256 struct sge_qstat *spg; /* status page, for convenience */
257 int doorbells;
258 volatile uint32_t *udb; /* KVA of doorbell (lies within BAR2) */
259 uint_t udb_qid; /* relative qid within the doorbell page */
260 uint16_t cap; /* max # of desc, for convenience */
261 uint16_t avail; /* available descriptors, for convenience */
262 uint16_t qsize; /* size (# of entries) of the queue */
263 uint16_t cidx; /* consumer idx (desc idx) */
264 uint16_t pidx; /* producer idx (desc idx) */
265 uint16_t pending; /* # of descriptors used since last doorbell */
266 uint16_t iqid; /* iq that gets egr_update for the eq */
267 uint8_t tx_chan; /* tx channel used by the eq */
268 uint32_t cntxt_id; /* SGE context id for the eq */
269 };
270
271 enum {
272 /* fl flags */
273 FL_MTX = (1 << 0), /* mutex has been initialized */
274 FL_STARVING = (1 << 1), /* on the list of starving fl's */
275 FL_DOOMED = (1 << 2), /* about to be destroyed */
276 };
277
278 #define FL_RUNNING_LOW(fl) (fl->cap - fl->needed <= fl->lowat)
279 #define FL_NOT_RUNNING_LOW(fl) (fl->cap - fl->needed >= 2 * fl->lowat)
280
281 struct sge_fl {
282 unsigned int flags;
283 kmutex_t lock;
284 ddi_dma_handle_t dhdl;
285 ddi_acc_handle_t ahdl;
286
287 __be64 *desc; /* KVA of descriptor ring, ptr to addresses */
288 uint64_t ba; /* bus address of descriptor ring */
289 struct fl_sdesc *sdesc; /* KVA of software descriptor ring */
290 uint32_t cap; /* max # of buffers, for convenience */
291 uint16_t qsize; /* size (# of entries) of the queue */
292 uint16_t cntxt_id; /* SGE context id for the freelist */
293 uint32_t cidx; /* consumer idx (buffer idx, NOT hw desc idx) */
294 uint32_t pidx; /* producer idx (buffer idx, NOT hw desc idx) */
295 uint32_t needed; /* # of buffers needed to fill up fl. */
296 uint32_t lowat; /* # of buffers <= this means fl needs help */
297 uint32_t pending; /* # of bufs allocated since last doorbell */
298 uint32_t offset; /* current packet within the larger buffer */
299 uint16_t copy_threshold; /* anything this size or less is copied up */
300
301 uint64_t copied_up; /* # of frames copied into mblk and handed up */
302 uint64_t passed_up; /* # of frames wrapped in mblk and handed up */
303 uint64_t allocb_fail; /* # of mblk allocation failures */
304
305 TAILQ_ENTRY(sge_fl) link; /* All starving freelists */
306 };
307
308 /* txq: SGE egress queue + miscellaneous items */
309 struct sge_txq {
310 struct sge_eq eq; /* MUST be first */
311
312 struct port_info *port; /* the port this txq belongs to */
313 struct tx_sdesc *sdesc; /* KVA of software descriptor ring */
314 mac_ring_handle_t ring_handle;
315
316 /* DMA handles used for tx */
317 ddi_dma_handle_t *tx_dhdl;
318 uint32_t tx_dhdl_total; /* Total # of handles */
319 uint32_t tx_dhdl_pidx; /* next handle to be used */
320 uint32_t tx_dhdl_cidx; /* reclaimed up to this index */
321 uint32_t tx_dhdl_avail; /* # of available handles */
322
323 /* Copy buffers for tx */
324 ddi_dma_handle_t txb_dhdl;
325 ddi_acc_handle_t txb_ahdl;
326 caddr_t txb_va; /* KVA of copy buffers area */
327 uint64_t txb_ba; /* bus address of copy buffers area */
328 uint32_t txb_size; /* total size */
329 uint32_t txb_next; /* offset of next useable area in the buffer */
330 uint32_t txb_avail; /* # of bytes available */
331 uint16_t copy_threshold; /* anything this size or less is copied up */
332
333 uint64_t txpkts; /* # of ethernet packets */
334 uint64_t txbytes; /* # of ethernet bytes */
335 kstat_t *ksp;
336
337 /* stats for common events first */
338
339 uint64_t txcsum; /* # of times hardware assisted with checksum */
340 uint64_t tso_wrs; /* # of IPv4 TSO work requests */
341 uint64_t imm_wrs; /* # of work requests with immediate data */
342 uint64_t sgl_wrs; /* # of work requests with direct SGL */
343 uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */
344 uint64_t txpkts_wrs; /* # of coalesced tx work requests */
345 uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */
346 uint64_t txb_used; /* # of tx copy buffers used (64 byte each) */
347 uint64_t hdl_used; /* # of DMA handles used */
348
349 /* stats for not-that-common events */
350
351 uint32_t txb_full; /* txb ran out of space */
352 uint32_t dma_hdl_failed; /* couldn't obtain DMA handle */
353 uint32_t dma_map_failed; /* couldn't obtain DMA mapping */
354 uint32_t qfull; /* out of hardware descriptors */
355 uint32_t qflush; /* # of SGE_EGR_UPDATE notifications for txq */
356 uint32_t pullup_early; /* # of pullups before starting frame's SGL */
357 uint32_t pullup_late; /* # of pullups while building frame's SGL */
358 uint32_t pullup_failed; /* # of failed pullups */
359 uint32_t csum_failed; /* # of csum reqs we failed to fulfill */
360 };
361
362 /* rxq: SGE ingress queue + SGE free list + miscellaneous items */
363 struct sge_rxq {
364 struct sge_iq iq; /* MUST be first */
365 struct sge_fl fl;
366
367 struct port_info *port; /* the port this rxq belongs to */
368 kstat_t *ksp;
369
370 mac_ring_handle_t ring_handle;
371 uint64_t ring_gen_num;
372
373 /* stats for common events first */
374
375 uint64_t rxcsum; /* # of times hardware assisted with checksum */
376 uint64_t rxpkts; /* # of ethernet packets */
377 uint64_t rxbytes; /* # of ethernet bytes */
378
379 /* stats for not-that-common events */
380
381 uint32_t nomem; /* mblk allocation during rx failed */
382 };
383
384 struct sge {
385 int fl_starve_threshold;
386 int s_qpp;
387
388 int nrxq; /* total rx queues (all ports and the rest) */
389 int ntxq; /* total tx queues (all ports and the rest) */
390 int niq; /* total ingress queues */
391 int neq; /* total egress queues */
392 int stat_len; /* length of status page at ring end */
393 int pktshift; /* padding between CPL & packet data */
394 int fl_align; /* response queue message alignment */
395
396 struct sge_iq fwq; /* Firmware event queue */
397 struct sge_txq *txq; /* NIC tx queues */
398 struct sge_rxq *rxq; /* NIC rx queues */
399
400 int iq_start; /* iq context id map start index */
401 int eq_start; /* eq context id map start index */
402 int iqmap_sz; /* size of iq context id map */
403 int eqmap_sz; /* size of eq context id map */
404 struct sge_iq **iqmap; /* iq->cntxt_id to iq mapping */
405 struct sge_eq **eqmap; /* eq->cntxt_id to eq mapping */
406
407 /* Device access and DMA attributes for all the descriptor rings */
408 ddi_device_acc_attr_t acc_attr_desc;
409 ddi_dma_attr_t dma_attr_desc;
410
411 /* Device access and DMA attributes for tx buffers */
412 ddi_device_acc_attr_t acc_attr_tx;
413 ddi_dma_attr_t dma_attr_tx;
414
415 /* Device access and DMA attributes for rx buffers are in rxb_params */
416 kmem_cache_t *rxbuf_cache;
417 struct rxbuf_cache_params rxb_params;
418 };
419
420 struct driver_properties {
421 /* There is a driver.conf variable for each of these */
422 int max_ntxq_10g;
423 int max_nrxq_10g;
424 int max_ntxq_1g;
425 int max_nrxq_1g;
426 int intr_types;
427 int tmr_idx_10g;
428 int pktc_idx_10g;
429 int tmr_idx_1g;
430 int pktc_idx_1g;
431 int qsize_txq;
432 int qsize_rxq;
433
434 int timer_val[SGE_NTIMERS];
435 int counter_val[SGE_NCOUNTERS];
436
437 int wc;
438
439 int multi_rings;
440 int t4_fw_install;
441 };
442
443 struct rss_header;
444 typedef int (*cpl_handler_t)(struct sge_iq *, const struct rss_header *,
445 mblk_t *);
446 typedef int (*fw_msg_handler_t)(struct adapter *, const __be64 *);
447
448 struct t4_mbox_list {
449 STAILQ_ENTRY(t4_mbox_list) link;
450 };
451
452 struct adapter {
453 SLIST_ENTRY(adapter) link;
454 dev_info_t *dip;
455 dev_t dev;
456
457 unsigned int pf;
458 unsigned int mbox;
459
460 unsigned int vpd_busy;
461 unsigned int vpd_flag;
462
463 u32 t4_bar0;
464
465 uint_t open; /* character device is open */
466
467 /* PCI config space access handle */
468 ddi_acc_handle_t pci_regh;
469
470 /* MMIO register access handle */
471 ddi_acc_handle_t regh;
472 caddr_t regp;
473 /* BAR1 register access handle */
474 ddi_acc_handle_t reg1h;
475 caddr_t reg1p;
476
477 /* Interrupt information */
478 int intr_type;
479 int intr_count;
480 int intr_cap;
481 uint_t intr_pri;
482 ddi_intr_handle_t *intr_handle;
483
484 struct driver_properties props;
485 kstat_t *ksp;
486 kstat_t *ksp_stat;
487
488 struct sge sge;
489
490 struct port_info *port[MAX_NPORTS];
491 ddi_taskq_t *tq[NCHAN];
492 uint8_t chan_map[NCHAN];
493 uint32_t filter_mode;
494
495 struct l2t_data *l2t; /* L2 table */
496 struct tid_info tids;
497
498 int doorbells;
499 int registered_device_map;
500 int open_device_map;
501 int flags;
502
503 unsigned int cfcsum;
504 struct adapter_params params;
505 struct t4_virt_res vres;
506
507 uint16_t linkcaps;
508 uint16_t niccaps;
509 uint16_t toecaps;
510 uint16_t rdmacaps;
511 uint16_t iscsicaps;
512 uint16_t fcoecaps;
513
514 fw_msg_handler_t fw_msg_handler[5]; /* NUM_FW6_TYPES */
515 cpl_handler_t cpl_handler[0xef]; /* NUM_CPL_CMDS */
516
517 kmutex_t lock;
518 kcondvar_t cv;
519
520 /* Starving free lists */
521 kmutex_t sfl_lock; /* same cache-line as sc_lock? but that's ok */
522 TAILQ_HEAD(, sge_fl) sfl;
523 timeout_id_t sfl_timer;
524
525 /* Sensors */
526 id_t temp_sensor;
527 id_t volt_sensor;
528
529 ddi_ufm_handle_t *ufm_hdl;
530
531 /* support for single-threading access to adapter mailbox registers */
532 kmutex_t mbox_lock;
533 STAILQ_HEAD(, t4_mbox_list) mbox_list;
534 };
535
536 enum {
537 NIC_H = 0,
538 TOM_H,
539 IW_H,
540 ISCSI_H
541 };
542
543 struct memwin {
544 uint32_t base;
545 uint32_t aperture;
546 };
547
548 #define ADAPTER_LOCK(sc) mutex_enter(&(sc)->lock)
549 #define ADAPTER_UNLOCK(sc) mutex_exit(&(sc)->lock)
550 #define ADAPTER_LOCK_ASSERT_OWNED(sc) ASSERT(mutex_owned(&(sc)->lock))
551 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) ASSERT(!mutex_owned(&(sc)->lock))
552
553 #define PORT_LOCK(pi) mutex_enter(&(pi)->lock)
554 #define PORT_UNLOCK(pi) mutex_exit(&(pi)->lock)
555 #define PORT_LOCK_ASSERT_OWNED(pi) ASSERT(mutex_owned(&(pi)->lock))
556 #define PORT_LOCK_ASSERT_NOTOWNED(pi) ASSERT(!mutex_owned(&(pi)->lock))
557
558 #define IQ_LOCK(iq) mutex_enter(&(iq)->lock)
559 #define IQ_UNLOCK(iq) mutex_exit(&(iq)->lock)
560 #define IQ_LOCK_ASSERT_OWNED(iq) ASSERT(mutex_owned(&(iq)->lock))
561 #define IQ_LOCK_ASSERT_NOTOWNED(iq) ASSERT(!mutex_owned(&(iq)->lock))
562
563 #define FL_LOCK(fl) mutex_enter(&(fl)->lock)
564 #define FL_UNLOCK(fl) mutex_exit(&(fl)->lock)
565 #define FL_LOCK_ASSERT_OWNED(fl) ASSERT(mutex_owned(&(fl)->lock))
566 #define FL_LOCK_ASSERT_NOTOWNED(fl) ASSERT(!mutex_owned(&(fl)->lock))
567
568 #define RXQ_LOCK(rxq) IQ_LOCK(&(rxq)->iq)
569 #define RXQ_UNLOCK(rxq) IQ_UNLOCK(&(rxq)->iq)
570 #define RXQ_LOCK_ASSERT_OWNED(rxq) IQ_LOCK_ASSERT_OWNED(&(rxq)->iq)
571 #define RXQ_LOCK_ASSERT_NOTOWNED(rxq) IQ_LOCK_ASSERT_NOTOWNED(&(rxq)->iq)
572
573 #define RXQ_FL_LOCK(rxq) FL_LOCK(&(rxq)->fl)
574 #define RXQ_FL_UNLOCK(rxq) FL_UNLOCK(&(rxq)->fl)
575 #define RXQ_FL_LOCK_ASSERT_OWNED(rxq) FL_LOCK_ASSERT_OWNED(&(rxq)->fl)
576 #define RXQ_FL_LOCK_ASSERT_NOTOWNED(rxq) FL_LOCK_ASSERT_NOTOWNED(&(rxq)->fl)
577
578 #define EQ_LOCK(eq) mutex_enter(&(eq)->lock)
579 #define EQ_UNLOCK(eq) mutex_exit(&(eq)->lock)
580 #define EQ_LOCK_ASSERT_OWNED(eq) ASSERT(mutex_owned(&(eq)->lock))
581 #define EQ_LOCK_ASSERT_NOTOWNED(eq) ASSERT(!mutex_owned(&(eq)->lock))
582
583 #define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq)
584 #define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq)
585 #define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
586 #define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
587
588 #define for_each_txq(pi, iter, txq) \
589 txq = &pi->adapter->sge.txq[pi->first_txq]; \
590 for (iter = 0; iter < pi->ntxq; ++iter, ++txq)
591 #define for_each_rxq(pi, iter, rxq) \
592 rxq = &pi->adapter->sge.rxq[pi->first_rxq]; \
593 for (iter = 0; iter < pi->nrxq; ++iter, ++rxq)
594 #define for_each_ofld_txq(pi, iter, ofld_txq) \
595 ofld_txq = &pi->adapter->sge.ofld_txq[pi->first_ofld_txq]; \
596 for (iter = 0; iter < pi->nofldtxq; ++iter, ++ofld_txq)
597 #define for_each_ofld_rxq(pi, iter, ofld_rxq) \
598 ofld_rxq = &pi->adapter->sge.ofld_rxq[pi->first_ofld_rxq]; \
599 for (iter = 0; iter < pi->nofldrxq; ++iter, ++ofld_rxq)
600
601 #define NFIQ(sc) ((sc)->intr_count > 1 ? (sc)->intr_count - 1 : 1)
602
603 /* One for errors, one for firmware events */
604 #define T4_EXTRA_INTR 2
605
t4_mbox_list_add(struct adapter * adap,struct t4_mbox_list * entry)606 static inline void t4_mbox_list_add(struct adapter *adap,
607 struct t4_mbox_list *entry)
608 {
609 mutex_enter(&adap->mbox_lock);
610 STAILQ_INSERT_TAIL(&adap->mbox_list, entry, link);
611 mutex_exit(&adap->mbox_lock);
612 }
613
t4_mbox_list_del(struct adapter * adap,struct t4_mbox_list * entry)614 static inline void t4_mbox_list_del(struct adapter *adap,
615 struct t4_mbox_list *entry)
616 {
617 mutex_enter(&adap->mbox_lock);
618 STAILQ_REMOVE(&adap->mbox_list, entry, t4_mbox_list, link);
619 mutex_exit(&adap->mbox_lock);
620 }
621
622 static inline struct t4_mbox_list *
t4_mbox_list_first_entry(struct adapter * adap)623 t4_mbox_list_first_entry(struct adapter *adap)
624 {
625 return (STAILQ_FIRST(&adap->mbox_list));
626 }
627
628 static inline uint32_t
t4_read_reg(struct adapter * sc,uint32_t reg)629 t4_read_reg(struct adapter *sc, uint32_t reg)
630 {
631 /* LINTED: E_BAD_PTR_CAST_ALIGN */
632 return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
633 }
634
635 static inline void
t4_write_reg(struct adapter * sc,uint32_t reg,uint32_t val)636 t4_write_reg(struct adapter *sc, uint32_t reg, uint32_t val)
637 {
638 /* LINTED: E_BAD_PTR_CAST_ALIGN */
639 ddi_put32(sc->regh, (uint32_t *)(sc->regp + reg), val);
640 }
641
642 static inline void
t4_os_pci_read_cfg1(struct adapter * sc,int reg,uint8_t * val)643 t4_os_pci_read_cfg1(struct adapter *sc, int reg, uint8_t *val)
644 {
645 *val = pci_config_get8(sc->pci_regh, reg);
646 }
647
648 static inline void
t4_os_pci_write_cfg1(struct adapter * sc,int reg,uint8_t val)649 t4_os_pci_write_cfg1(struct adapter *sc, int reg, uint8_t val)
650 {
651 pci_config_put8(sc->pci_regh, reg, val);
652 }
653
654 static inline void
t4_os_pci_read_cfg2(struct adapter * sc,int reg,uint16_t * val)655 t4_os_pci_read_cfg2(struct adapter *sc, int reg, uint16_t *val)
656 {
657 *val = pci_config_get16(sc->pci_regh, reg);
658 }
659
660 static inline void
t4_os_pci_write_cfg2(struct adapter * sc,int reg,uint16_t val)661 t4_os_pci_write_cfg2(struct adapter *sc, int reg, uint16_t val)
662 {
663 pci_config_put16(sc->pci_regh, reg, val);
664 }
665
666 static inline void
t4_os_pci_read_cfg4(struct adapter * sc,int reg,uint32_t * val)667 t4_os_pci_read_cfg4(struct adapter *sc, int reg, uint32_t *val)
668 {
669 *val = pci_config_get32(sc->pci_regh, reg);
670 }
671
672 static inline void
t4_os_pci_write_cfg4(struct adapter * sc,int reg,uint32_t val)673 t4_os_pci_write_cfg4(struct adapter *sc, int reg, uint32_t val)
674 {
675 pci_config_put32(sc->pci_regh, reg, val);
676 }
677
678 static inline uint32_t
t4_read_reg32(struct adapter * sc,uint32_t reg)679 t4_read_reg32(struct adapter *sc, uint32_t reg)
680 {
681 return (ddi_get32(sc->regh, (uint32_t *)(sc->regp + reg)));
682 }
683
684 static inline uint64_t
t4_read_reg64(struct adapter * sc,uint32_t reg)685 t4_read_reg64(struct adapter *sc, uint32_t reg)
686 {
687 return (ddi_get64(sc->regh, (uint64_t *)(sc->regp + reg)));
688 }
689
690 static inline void
t4_write_reg64(struct adapter * sc,uint32_t reg,uint64_t val)691 t4_write_reg64(struct adapter *sc, uint32_t reg, uint64_t val)
692 {
693 ddi_put64(sc->regh, (uint64_t *)(sc->regp + reg), val);
694 }
695
696 static inline struct port_info *
adap2pinfo(struct adapter * sc,int idx)697 adap2pinfo(struct adapter *sc, int idx)
698 {
699 return (sc->port[idx]);
700 }
701
702 static inline void
t4_os_set_hw_addr(struct adapter * sc,int idx,uint8_t hw_addr[])703 t4_os_set_hw_addr(struct adapter *sc, int idx, uint8_t hw_addr[])
704 {
705 bcopy(hw_addr, sc->port[idx]->hw_addr, ETHERADDRL);
706 }
707
708 static inline bool
is_10G_port(const struct port_info * pi)709 is_10G_port(const struct port_info *pi)
710 {
711 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) != 0);
712 }
713
714 static inline struct sge_rxq *
iq_to_rxq(struct sge_iq * iq)715 iq_to_rxq(struct sge_iq *iq)
716 {
717 return (__containerof(iq, struct sge_rxq, iq));
718 }
719
720 static inline bool
is_25G_port(const struct port_info * pi)721 is_25G_port(const struct port_info *pi)
722 {
723 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) != 0);
724 }
725
726 static inline bool
is_40G_port(const struct port_info * pi)727 is_40G_port(const struct port_info *pi)
728 {
729 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) != 0);
730 }
731
732 static inline bool
is_50G_port(const struct port_info * pi)733 is_50G_port(const struct port_info *pi)
734 {
735 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) != 0);
736 }
737
738 static inline bool
is_100G_port(const struct port_info * pi)739 is_100G_port(const struct port_info *pi)
740 {
741 return ((pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) != 0);
742 }
743
744 static inline bool
is_10XG_port(const struct port_info * pi)745 is_10XG_port(const struct port_info *pi)
746 {
747 return (is_10G_port(pi) || is_40G_port(pi) ||
748 is_25G_port(pi) || is_50G_port(pi) ||
749 is_100G_port(pi));
750 }
751
752 /*
753 * t4_os_pci_read_seeprom - read four bytes of SEEPROM/VPD contents
754 * @adapter: the adapter
755 * @addr: SEEPROM/VPD Address to read
756 * @valp: where to store the value read
757 *
758 * Read a 32-bit value from the given address in the SEEPROM/VPD. The address
759 * must be four-byte aligned. Returns 0 on success, a negative erro number
760 * on failure.
761 */
t4_os_pci_read_seeprom(adapter_t * adapter,int addr,u32 * valp)762 static inline int t4_os_pci_read_seeprom(adapter_t *adapter, int addr,
763 u32 *valp)
764 {
765 int t4_seeprom_read(struct adapter *adapter, u32 addr, u32 *data);
766 int ret;
767
768 ret = t4_seeprom_read(adapter, addr, valp);
769
770 return (ret >= 0 ? 0 : ret);
771 }
772
773 /*
774 * t4_os_pci_write_seeprom - write four bytes of SEEPROM/VPD contents
775 * @adapter: the adapter
776 * @addr: SEEPROM/VPD Address to write
777 * @val: the value write
778 *
779 * Write a 32-bit value to the given address in the SEEPROM/VPD. The address
780 * must be four-byte aligned. Returns 0 on success, a negative erro number
781 * on failure.
782 */
t4_os_pci_write_seeprom(adapter_t * adapter,int addr,u32 val)783 static inline int t4_os_pci_write_seeprom(adapter_t *adapter, int addr, u32 val)
784 {
785 int t4_seeprom_write(struct adapter *adapter, u32 addr, u32 data);
786 int ret;
787
788 ret = t4_seeprom_write(adapter, addr, val);
789
790 return (ret >= 0 ? 0 : ret);
791 }
792
t4_os_pci_set_vpd_size(struct adapter * adapter,size_t len)793 static inline int t4_os_pci_set_vpd_size(struct adapter *adapter, size_t len)
794 {
795 return (0);
796 }
797
t4_use_ldst(struct adapter * adap)798 static inline unsigned int t4_use_ldst(struct adapter *adap)
799 {
800 return (adap->flags & FW_OK);
801 }
802
t4_db_full(struct adapter * adap)803 static inline void t4_db_full(struct adapter *adap) {}
t4_db_dropped(struct adapter * adap)804 static inline void t4_db_dropped(struct adapter *adap) {}
805
806 /* t4_nexus.c */
807 int t4_os_find_pci_capability(struct adapter *sc, int cap);
808 void t4_os_portmod_changed(struct adapter *sc, int idx);
809 int adapter_full_init(struct adapter *sc);
810 int adapter_full_uninit(struct adapter *sc);
811 int port_full_init(struct port_info *pi);
812 int port_full_uninit(struct port_info *pi);
813 void enable_port_queues(struct port_info *pi);
814 void disable_port_queues(struct port_info *pi);
815 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h);
816 int t4_register_fw_msg_handler(struct adapter *, int, fw_msg_handler_t);
817 void t4_iterate(void (*func)(int, void *), void *arg);
818
819 /* t4_debug.c */
820 void t4_debug_init(void);
821 void t4_debug_fini(void);
822
823 /* t4_sge.c */
824 void t4_sge_init(struct adapter *sc);
825 int t4_setup_adapter_queues(struct adapter *sc);
826 int t4_teardown_adapter_queues(struct adapter *sc);
827 int t4_setup_port_queues(struct port_info *pi);
828 int t4_teardown_port_queues(struct port_info *pi);
829 uint_t t4_intr_all(caddr_t arg1, caddr_t arg2);
830 uint_t t4_intr(caddr_t arg1, caddr_t arg2);
831 uint_t t4_intr_err(caddr_t arg1, caddr_t arg2);
832 int t4_mgmt_tx(struct adapter *sc, mblk_t *m);
833 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *);
834 uint32_t position_memwin(struct adapter *, int, uint32_t);
835
836 mblk_t *t4_eth_tx(void *, mblk_t *);
837 mblk_t *t4_mc_tx(void *arg, mblk_t *m);
838 mblk_t *t4_ring_rx(struct sge_rxq *rxq, int poll_bytes);
839 int t4_alloc_tx_maps(struct adapter *sc, struct tx_maps *txmaps, int count,
840 int flags);
841
842 /* t4_mac.c */
843 void t4_mc_init(struct port_info *pi);
844 void t4_mc_cb_init(struct port_info *);
845 void t4_os_link_changed(struct adapter *sc, int idx, int link_stat);
846 void t4_mac_rx(struct port_info *pi, struct sge_rxq *rxq, mblk_t *m);
847 void t4_mac_tx_update(struct port_info *pi, struct sge_txq *txq);
848 int t4_addmac(void *arg, const uint8_t *ucaddr);
849
850 /* t4_ioctl.c */
851 int t4_ioctl(struct adapter *sc, int cmd, void *data, int mode);
852
853 struct l2t_data *t4_init_l2t(struct adapter *sc);
854 int begin_synchronized_op(struct port_info *pi, int hold, int waitok);
855 void end_synchronized_op(struct port_info *pi, int held);
856
857 #define setbit(a, i) ((a)[(i)/NBBY] |= 1<<((i)%NBBY))
858 #define clrbit(a, i) ((a)[(i)/NBBY] &= ~(1<<((i)%NBBY)))
859 #define isset(a, i) ((a)[(i)/NBBY] & (1<<((i)%NBBY)))
860
861 #endif /* __CXGBE_ADAPTER_H */
862