1 /*
2 * This file and its contents are supplied under the terms of the
3 * Common Development and Distribution License ("CDDL"), version 1.0.
4 * You may only use this file in accordance with the terms of version
5 * 1.0 of the CDDL.
6 *
7 * A full copy of the text of the CDDL should have accompanied this
8 * source. A copy of the CDDL is also available via the Internet at
9 * http://www.illumos.org/license/CDDL.
10 */
11
12 /*
13 * This file is part of the Chelsio T4 support code.
14 *
15 * Copyright (C) 2011-2013 Chelsio Communications. All rights reserved.
16 *
17 * This program is distributed in the hope that it will be useful, but WITHOUT
18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
20 * release for licensing terms and conditions.
21 */
22
23 /*
24 * Copyright 2025 Oxide Computer Company
25 */
26
27 #ifndef __CXGBE_ADAPTER_H
28 #define __CXGBE_ADAPTER_H
29
30 #include <sys/ddi.h>
31 #include <sys/mac_provider.h>
32 #include <sys/ethernet.h>
33 #include <sys/list.h>
34 #include <sys/containerof.h>
35 #include <sys/ddi_ufm.h>
36 #include <sys/mac_provider.h>
37
38 #include "firmware/t4fw_interface.h"
39 #include "shared.h"
40
41 struct adapter;
42 struct port_info;
43 typedef struct adapter adapter_t;
44 struct sge_fl;
45
46 /* See the _Ingress Context Contents_ section of the T4 Programmers Guide. */
47 typedef enum t4_iq_esize {
48 T4_IQ_ESIZE_16B = 0,
49 T4_IQ_ESIZE_32B = 1,
50 T4_IQ_ESIZE_64B = 2,
51 T4_IQ_ESIZE_128B = 3,
52 } t4_iq_esize_t;
53
54 #define FW_IQ_QSIZE 256
55 /* At least 64 bytes mandated by the firmware spec */
56 #define FW_IQ_ESIZE T4_IQ_ESIZE_64B
57
58 #define T4_RX_DEF_QSIZE 1024
59 /* At least 64 so CPL_RX_PKT will fit */
60 #define RX_IQ_ESIZE T4_IQ_ESIZE_64B
61
62 /* A flit is an 8 byte quantity. */
63 #define FLIT_NUM_BYTES 8
64 #define FLITS_TO_BYTES(nflits) ((nflits) * FLIT_NUM_BYTES)
65
66 /*
67 * Egress Queues (EQ) are made up of units called "host credits". Each credit is
68 * always 8 flits (64 bytes) in size. The number of entries in the queue as well
69 * as the producer and consumer indexes (pidx/cidx) are phrased in units of
70 * credits.
71 *
72 * A freelist (FL) is a type of EQ. It consists of 16-byte aligned, 8-byte
73 * pointers to data buffers meant to hold the data of incoming packets. Since an
74 * EQ host credit is always 8 flits, and an FL buffer pointer is a single flit,
75 * each credit holds 8 FL buffer pointers.
76 *
77 */
78 #define FLITS_PER_EQ_HC 8
79 #define EQ_HC_SIZE FLITS_PER_EQ_HC * FLIT_NUM_BYTES
80 #define FL_BUF_PTR_PER_HC FLITS_PER_EQ_HC
81
82 /*
83 * Given a number of host credits, calculate the total number of flits
84 * contained in those credits.
85 */
86 #define EQ_HC_TO_FLITS(num_credits) ((num_credits) * FLITS_PER_EQ_HC)
87
88 /*
89 * Given a number of flits, calculate how many host credits are needed to hold
90 * them.
91 */
92 #define EQ_FLITS_TO_HC(num_flits) (howmany(num_flits, FLITS_PER_EQ_HC))
93
94 /*
95 * We constrain the max "usable" EQ size so that there is always room for the
96 * status page, which may require 1-2 host credits.
97 */
98 #define T4_MAX_EQ_SIZE (UINT16_MAX - 2)
99 #define T4_TX_DEF_QSIZE 1024
100 #define TX_SGL_SEGS 36
101
102 /* The maximum number of flits/credits a single WR may consume. */
103 #define TX_WR_MAX_FLITS (SGE_MAX_WR_LEN / FLIT_NUM_BYTES)
104 #define TX_WR_MAX_CREDITS (TX_WR_MAX_FLITS / FLITS_PER_EQ_HC)
105
106 CTASSERT(TX_WR_MAX_FLITS == 64);
107 CTASSERT(TX_WR_MAX_CREDITS == 8);
108
109 #define UDBS_SEG_SHIFT 7 /* log2(UDBS_SEG_SIZE) */
110 #define UDBS_DB_OFFSET 8 /* offset of the 4B doorbell in a segment */
111 #define UDBS_WR_OFFSET 64 /* offset of the work request in a segment */
112
113 /*
114 * A sentinel to mark when the interrupts for an IQ are being forwarded from
115 * another IQ which is receiving the actual interrupt.
116 */
117 #define INTR_FORWARDED UINT_MAX
118
119 struct fl_desc {
120 uint64_t dptr[FL_BUF_PTR_PER_HC];
121 };
122
123 struct fl_sdesc {
124 struct rxbuf *rxb;
125 };
126
127 typedef struct t4_eq_host_credit {
128 uint64_t flit[8];
129 } t4_eq_host_credit_t;
130
131 struct tx_sdesc {
132 mblk_t *mp_head;
133 mblk_t *mp_tail;
134 uint32_t txb_used; /* # of bytes of tx copy buffer used */
135 uint16_t hdls_used; /* # of dma handles used */
136 uint16_t credits_used; /* # of EQ host credits used */
137 uint64_t _pad;
138 };
139
140 typedef enum t4_iq_flags {
141 IQ_ALLOC_HOST = (1 << 0), /* host-side resources allocated */
142 IQ_ALLOC_DEV = (1 << 1), /* device-side resource allocated */
143 IQ_INTR = (1 << 2), /* iq takes direct interrupt */
144
145 /* Runtime state flags: */
146 IQ_ENABLED = (1 << 3),
147 IQ_POLLING = (1 << 4),
148 } t4_iq_flags_t;
149
150 struct rxbuf_cache_params {
151 dev_info_t *dip;
152 ddi_dma_attr_t dma_attr_rx;
153 ddi_device_acc_attr_t acc_attr_rx;
154 size_t buf_size;
155 };
156
157 struct sge_iq_stats {
158 uint64_t sis_processed; /* # entries processed from IQ */
159 uint64_t sis_overflow; /* # entries bearing overflow flag */
160 };
161
162 /*
163 * These values are designed to match up with what is posted to GTS registers
164 * when processing an ingress queue.
165 *
166 * See: t4_iq_update_intr_cfg() and t4_iq_gts_update().
167 */
168 typedef enum t4_gts_config {
169 TGC_SE_INTR_ARM = 1,
170 TGC_TIMER0 = (0 << 1),
171 TGC_TIMER1 = (1 << 1),
172 TGC_TIMER2 = (2 << 1),
173 TGC_TIMER3 = (3 << 1),
174 TGC_TIMER4 = (4 << 1),
175 TGC_TIMER5 = (5 << 1),
176 TGC_START_COUNTER = (6 << 1),
177 } t4_gts_config_t;
178
179 /*
180 * Event IQs are used for firmware events, Tx EGR updates, and IQ forwarded
181 * interrupts.
182 *
183 * Ethernet Rx IQs are used for receiving incoming packets.
184 */
185 typedef enum t4_iq_type {
186 TIQT_UNINIT,
187 TIQT_EVENT,
188 TIQT_ETH_RX,
189 } t4_iq_type_t;
190
191 /* Ingress Queue: T4 is producer, driver is consumer. */
192 typedef struct t4_sge_iq {
193 kmutex_t tsi_lock;
194
195 t4_iq_type_t tsi_iqtype; /* Write Once */
196 t4_iq_flags_t tsi_flags; /* tsi_lock */
197
198 /*
199 * This field is non-NULL only for Rx queues. It points to the event
200 * queue which receives interrupts on its behalf. The event queue
201 * processes these "forwarded interrupts" in t4_process_event_iq() and
202 * calls into t4_process_rx_iq() for each Rx queue with an interrupt
203 * notification message.
204 */
205 struct t4_sge_iq *tsi_intr_evtq; /* Write Once */
206 /*
207 * This field is only used by the event queues.
208 *
209 * As the event queue processes forwarded interrupt notification
210 * messages it adds the destination rx queue receving the notification
211 * to this list. After the event queue finishes processing its own
212 * messages, it then uses this list to process the rx queues which have
213 * pending notifications.
214 */
215 list_node_t tsi_intr_fwd_node; /* tsi_lock */
216 /*
217 * This field is used by both event queues and rx queues.
218 *
219 * For event queues this field holds the interrupt vector assigned to
220 * this queue.
221 *
222 * For rx queues it holds the sentinel value INTR_FORWARDED to indicate
223 * it has its interrupts forwarded by the event queue. In the current
224 * iteration of this driver all rx queues will always have their
225 * interrupts forwarded.
226 */
227 uint_t tsi_intr_idx; /* Write Once */
228
229 ddi_dma_handle_t tsi_desc_dhdl; /* Write Once */
230 ddi_acc_handle_t tsi_desc_ahdl; /* Write Once */
231
232 /* KVA of descriptor ring */
233 void *tsi_desc; /* Write Once */
234 /* bus address of descriptor ring */
235 uint64_t tsi_desc_ba; /* Write Once */
236 /* current descriptor (at CIDX) */
237 const void *tsi_cdesc; /* tsi_lock */
238
239 /* Sizing and status */
240 /* size of each entry in the queue */
241 t4_iq_esize_t tsi_esize; /* Write Once */
242 /* entry size in bytes */
243 uint16_t tsi_esize_bytes; /* Write Once */
244 /* number of entries in the queue */
245 uint16_t tsi_qsize; /* Write Once */
246 /* number of usable entries in the queue */
247 uint16_t tsi_cap; /* Write Once */
248 /* consumer index */
249 uint16_t tsi_cidx; /* tsi_lock */
250 /* generation bit */
251 uint8_t tsi_gen; /* tsi_lock */
252
253 /* GTS config to re-arm queue notification */
254 t4_gts_config_t tsi_gts_rearm; /* tsi_lock */
255 /* packet count threshold index */
256 int8_t tsi_intr_pktc_idx; /* tsi_lock */
257
258 /* SGE context ID for IQ */
259 uint16_t tsi_cntxt_id; /* Write Once */
260 /* absolute SGE ID for IQ */
261 uint16_t tsi_abs_id; /* Write Once */
262
263 /* associated adapter */
264 struct adapter *tsi_adapter; /* Write Once */
265 /* associated freelist (if any) */
266 struct sge_fl *tsi_fl; /* Write Once */
267
268 struct sge_iq_stats tsi_stats; /* tsi_lock */
269 } t4_sge_iq_t;
270
271 /* Result of servicing IQ in t4_iq_service() call */
272 typedef enum t4_iq_result {
273 TIR_SUCCESS, /* All available entries processed successfully */
274 TIR_DISABLED, /* IQ is disabled */
275 TIR_POLLING, /* non-polling service req'd on polling-cfg'd IQ */
276 TIR_ALLOC_FAIL, /* could not allocate packet buffer(s) */
277 TIR_BUDGET_MAX, /* hit budget limit while processing entries */
278 } t4_iq_result_t;
279
280 /*
281 * Details used when servicing an IQ as part of polling.
282 */
283 struct t4_poll_req {
284 mblk_t *tpr_mp;
285 uint_t tpr_byte_budget;
286 };
287
288 typedef enum t4_eq_flags {
289 /* Initialization state flags: */
290 EQ_ALLOC_HOST = (1 << 0), /* host-side resources allocated */
291 EQ_ALLOC_DEV = (1 << 1), /* EQ allocated in device firmware */
292 EQ_ALLOC_DESC = (1 << 2), /* descriptor inputs allocated */
293
294 /* Runtime state flags: */
295
296 EQ_ENABLED = (1 << 3), /* ready for submitted work requests */
297 /*
298 * Short on resources (memory and/or descriptors) while attempting to
299 * enqueue work in EQ
300 */
301 EQ_CORKED = (1 << 4),
302 } t4_eq_flags_t;
303
304 /*
305 * These are the Egress Queue doorbell methods. They are listed in order of
306 * preference (WCWR most preferred, KDB least). This ordering is important as
307 * the datapath uses ffs (find first set) to pick the preferred method.
308 *
309 * The first three are "user space" doorbells. They are mapped in BAR2 and are
310 * provided to allow kernel-bypass network stacks. However, they can also be
311 * used in the kernel and provide benefits such as write combining and per-queue
312 * registers (versus KDB which is a single register).
313 *
314 * The WCWR, Write Combining Work Request, is the preferred method. It allows
315 * the driver to push a WR directly to the device without the need for it to
316 * perform a DMA read of the hardware ring (to read the EQ host credit). Instead
317 * it comes in via the BAR2/UDB memory space and the device increments the pidx
318 * accordingly. However, the WCWR is limited to a single WR. It's use is
319 * intended for low latency situations or low rate of work, not for throughput.
320 *
321 * The maximum WCWR for T4 is 256 bytes. For T5/T6 it is 64-128 bytes, depending
322 * on the write-combining size of the platform.
323 *
324 * T4 Firmware Interface Specification, §9.2 Egress Queues and Work Requests.
325 */
326 typedef enum t4_doorbells {
327 DOORBELL_WCWR = (1 << 0),
328 DOORBELL_UDBWC = (1 << 1),
329 DOORBELL_UDB = (1 << 2),
330 DOORBELL_KDB = (1 << 3),
331 } t4_doorbells_t;
332
333 /* Egress Queue: driver is producer, T4 is consumer. */
334 typedef struct t4_sge_eq {
335 kmutex_t tse_lock;
336
337 t4_eq_flags_t tse_flags; /* tse_lock */
338
339 ddi_dma_handle_t tse_ring_dhdl; /* Write Once */
340 ddi_acc_handle_t tse_ring_ahdl; /* Write Once */
341
342 /*
343 * The ring type is pointer to void because the ring does not consist of
344 * descriptors but rather host credits. These host credits carry
345 * variable length work requests (WR) as well as the status page (SP) at
346 * the end of the ring. We use void* to facilitate the type punning
347 * required to work with these various types of EQ entries.
348 *
349 * In order to access credits and their individual flits we make use to
350 * the t4_eq_host_credit_t type.
351 */
352 void *tse_ring; /* KVA of ring - Write Once */
353 uint64_t tse_ring_ba; /* bus address of ring - Write Once */
354
355 /*
356 * tse_qsize: The number of host credits that may be used for data. This
357 * value is static for the lifetime of the queue.
358 *
359 * tse_qsize_spg: The total number of host credits in the queue. This is
360 * 1-2 more credits than tse_qsize to account for the status page at the
361 * end of the queue. The status page credits cannot be used for sending
362 * data, rather the beginning of the status page is considered the end
363 * of the queue as far as the datapath is concerned.
364 *
365 * tse_avail: The number of host credits that are currently available
366 * for use by the host. This is never more than 'tse_qsize - 1' in order
367 * to avoid 'tse_pidx==tse_cidx' which we use to indicate an empty
368 * queue. This number is updated as credits are used/recycled.
369 *
370 * tse_pending: The number of credits that have been written by the host
371 * but still require a doorbell before the device can consume them. Said
372 * another way, it's the number of credits the host's pidx is ahead of
373 * the device's cidx.
374 */
375 uint16_t tse_qsize; /* Write Once */
376 uint16_t tse_qsize_spg; /* Write Once */
377 uint16_t tse_avail; /* tse_lock */
378 uint16_t tse_pending; /* tse_lock */
379
380 /*
381 * The pidx is the driver's position in the queue, pointing to the next
382 * credit to consume. The cidx is the device's position in the queue,
383 * pointing to the last credit it has consumed as of the last status
384 * update.
385 */
386 uint16_t tse_cidx; /* tse_lock */
387 uint16_t tse_pidx; /* tse_lock */
388
389 /* Doorbell bits */
390 t4_doorbells_t tse_doorbells; /* Write Once */
391 /* KVA of doorbell (lies within BAR2) */
392 caddr_t tse_udb; /* Write Once */
393 /* relative qid within the doorbell page */
394 uint_t tse_udb_qid; /* Write Once */
395
396 struct sge_qstat *tse_spg; /* status page - Write Once */
397 /* IQ that gets egr_update msg for EQ */
398 uint16_t tse_iqid; /* Write Once */
399 /* tx channel used by the EQ */
400 uint8_t tse_tx_chan; /* Write Once */
401 /* SGE context id for the EQ */
402 uint32_t tse_cntxt_id; /* Write Once */
403 } t4_sge_eq_t;
404
405 typedef enum t4_sfl_flags {
406 SFL_STARVING = (1 << 0), /* on the list of starving fl's */
407 SFL_DOOMED = (1 << 1), /* about to be destroyed */
408 } t4_sfl_flags_t;
409
410 struct sge_fl_stats {
411 /* These stats describe the receiving of data. */
412 uint64_t copy; /* # of frames copied (allocb) */
413 uint64_t copy_fail; /* # of allocb failures */
414 uint64_t wrap; /* # of frames wrapped (desballoc) */
415 uint64_t wrap_fail; /* # of deballoc failures */
416
417 /* These stats describe the refilling of rx (FL) buffers. */
418 uint64_t rxb_recycle; /* # of rx buffers recycled */
419 uint64_t rxb_alloc; /* # of rx buffers allocated */
420 uint64_t rxb_alloc_fail; /* # of rx buffers that failed to allocb */
421 };
422
423 struct sge_fl {
424 /*
425 * EQ for passing freelist entries to adapter.
426 * Must be first field in struct
427 */
428 t4_sge_eq_t eq; /* Write Once */
429
430 /*
431 * Index at which new buffers are to be placed in the FL descriptor
432 * which is currently being produced for the device.
433 */
434 uint8_t cidx_sdesc; /* FL_LOCK */
435 uint8_t pidx_sdesc; /* FL_LOCK */
436
437 /* KVA of the software descriptor ring. */
438 struct fl_sdesc *sdesc; /* Write Once */
439 /* Total number of buffers in the FL. */
440 uint32_t bufs_cap; /* Write Once */
441 /*
442 * Number of buffers available to receive data, buffers owned by the
443 * device.
444 */
445 uint32_t bufs_avail; /* FL_LOCK */
446 /* Number of buffers at which the FL is considered "starving". */
447 uint32_t bufs_lowat; /* Write Once */
448 /* The byte offset in the current FL buffer. */
449 uint32_t offset; /* FL_LOCK */
450 /* Any packet smaller or equal to this is copied (allocb). */
451 uint16_t copy_threshold; /* Write Once */
452
453 /* Starvation-related state for this freelist. */
454 t4_sfl_flags_t sfl_flags; /* adapter->sfl_lock */
455 list_node_t sfl_node; /* adapter->sfl_lock */
456
457 struct sge_fl_stats stats; /* FL_LOCK */
458 };
459
460 struct sge_txq_stats {
461 /* stats for common events first */
462 uint64_t txpkts; /* # of ethernet packets */
463 uint64_t txbytes; /* # of ethernet bytes */
464 uint64_t txcsum; /* # of times hardware assisted with checksum */
465 uint64_t tso_wrs; /* # of IPv4 TSO work requests */
466 uint64_t imm_wrs; /* # of work requests with immediate data */
467 uint64_t sgl_wrs; /* # of work requests with direct SGL */
468 uint64_t txpkt_wrs; /* # of txpkt work requests (not coalesced) */
469 uint64_t txpkts_wrs; /* # of coalesced tx work requests */
470 uint64_t txpkts_pkts; /* # of frames in coalesced tx work requests */
471 uint64_t txb_used; /* # of tx copy buffers used (64 byte each) */
472 uint64_t hdl_used; /* # of DMA handles used */
473
474 /* stats for not-that-common events */
475 uint32_t txb_full; /* txb ran out of space */
476 uint32_t dma_hdl_failed; /* couldn't obtain DMA handle */
477 uint32_t dma_map_failed; /* couldn't obtain DMA mapping */
478 uint32_t qfull; /* out of hardware descriptors */
479 uint32_t pullup_early; /* # of pullups before starting frame's SGL */
480 uint32_t pullup_late; /* # of pullups while building frame's SGL */
481 uint32_t pullup_failed; /* # of failed pullups */
482 uint32_t csum_failed; /* # of csum reqs we failed to fulfill */
483 };
484
485 /* Ethernet packet transmission queue */
486 struct sge_txq {
487 t4_sge_eq_t eq;
488
489 struct port_info *port;
490 struct tx_sdesc *sdesc; /* KVA of software descriptor ring */
491
492 mac_ring_handle_t ring_handle;
493
494 /* DMA handles used for tx */
495 ddi_dma_handle_t *tx_dhdl;
496 uint32_t tx_dhdl_total; /* Total # of handles */
497 uint32_t tx_dhdl_pidx; /* next handle to be used */
498 uint32_t tx_dhdl_cidx; /* reclaimed up to this index */
499 uint32_t tx_dhdl_avail; /* # of available handles */
500
501 /* Copy buffers for tx */
502 ddi_dma_handle_t txb_dhdl;
503 ddi_acc_handle_t txb_ahdl;
504 caddr_t txb_va; /* KVA of copy buffers area */
505 uint64_t txb_ba; /* bus address of copy buffers area */
506 uint32_t txb_size; /* total size */
507 uint32_t txb_next; /* offset of next useable area in the buffer */
508 uint32_t txb_avail; /* # of bytes available */
509 uint16_t copy_threshold; /* anything this size or less is copied up */
510
511 kstat_t *ksp;
512 struct sge_txq_stats stats;
513 };
514
515 struct sge_rxq_stats {
516 /* stats for common events first */
517 uint64_t rxcsum; /* # of times hardware assisted with checksum */
518 uint64_t rxpkts; /* # of ethernet packets */
519 uint64_t rxbytes; /* # of ethernet bytes */
520 };
521
522 /* Ethernet packet receive queue */
523 struct sge_rxq {
524 t4_sge_iq_t iq;
525 struct sge_fl fl; /* Freelist for packet receive buffers */
526
527 struct port_info *port;
528
529 mac_ring_handle_t ring_handle;
530 uint64_t ring_gen_num;
531
532 kstat_t *ksp;
533 struct sge_rxq_stats stats;
534 };
535
536 typedef enum t4_port_flags {
537 TPF_INIT_DONE = (1 << 0),
538 TPF_OPEN = (1 << 1),
539 TPF_VI_ENABLED = (1 << 2),
540 } t4_port_flags_t;
541
542 typedef enum t4_port_feat {
543 CXGBE_HW_LSO = (1 << 0),
544 CXGBE_HW_CSUM = (1 << 1),
545 } t4_port_feat_t;
546
547
548 struct port_info {
549 kmutex_t lock;
550 dev_info_t *dip;
551 struct adapter *adapter;
552 uint8_t port_id;
553
554 t4_port_flags_t flags;
555 t4_port_feat_t features;
556
557 mac_handle_t mh;
558 int mtu;
559 uint8_t hw_addr[ETHERADDRL];
560 int16_t xact_addr_filt; /* index of exact MAC address filter */
561
562 uint16_t rxq_count; /* # of RX queues */
563 uint16_t rxq_start; /* index of first RX queue */
564 uint16_t txq_count; /* # of TX queues */
565 uint16_t txq_start; /* index of first TX queue */
566
567 /*
568 * Array of IQs for queue events, such as interrupt forward events
569 * for Rx queue processing and completion events for Tx queues.
570 * Only available when TIP_PER_PORT is selected. The size is based
571 * on adapter.intr_queue_cfg.intr_per_port.
572 */
573 t4_sge_iq_t *intr_iqs;
574
575 kstat_t *ksp_config;
576 kstat_t *ksp_info;
577 kstat_t *ksp_fec;
578
579 /* Port attributes/data set by common code: */
580 uint16_t viid;
581 uint16_t rss_size; /* size of VI's RSS table slice */
582
583 uint8_t port_type;
584 int8_t mdio_addr;
585 uint8_t mod_type;
586
587 uint8_t lport;
588 uint8_t tx_chan;
589 uint8_t rx_chan;
590 uint8_t rx_cchan;
591
592 uint8_t rss_mode;
593
594 uint8_t tmr_idx;
595 int8_t pktc_idx;
596 uint8_t dbq_timer_idx;
597
598 struct link_config link_cfg;
599 uint8_t macaddr_cnt;
600
601 u8 vivld;
602 u8 vin;
603 u8 smt_idx;
604
605 /* Mirroring bits utilized by common code (unused by our driver) */
606 u16 viid_mirror;
607 u8 vivld_mirror;
608 u8 vin_mirror;
609 };
610
611 struct sge_info {
612 uint_t fl_starve_threshold;
613 uint64_t dbq_timer_tick;
614 uint16_t dbq_timers[SGE_NDBQTIMERS];
615
616 uint_t eq_spg_len; /* EQ status page length in host credits */
617 uint_t pktshift; /* padding between CPL & packet data */
618 uint_t fl_align; /* response queue message alignment */
619 uint8_t fwq_tmr_idx; /* Intr. coalesce timer for FWQ */
620 int8_t fwq_pktc_idx; /* Intr. coalesce count for FWQ */
621
622 t4_sge_iq_t fwq; /* Firmware event queue */
623
624 uint_t rxq_count; /* total RX queues (all ports and the rest) */
625 uint_t txq_count; /* total TX queues (all ports and the rest) */
626 struct sge_txq *txq; /* NIC TX queues */
627 struct sge_rxq *rxq; /* NIC RX queues */
628
629 /*
630 * Adapters uses 16-bit "context IDs" to uniquely identify queues.
631 *
632 * References to the queues, indexed by said context IDs are maintained
633 * here, using the start/end values queried from the adapter.
634 */
635 uint_t iqmap_start; /* IQ context id map start index */
636 uint_t rxqmap_start; /* IQ context id map start index */
637 uint_t eqmap_start; /* EQ context id map start index */
638 uint_t iqmap_sz; /* size of IQ context id map */
639 uint_t eqmap_sz; /* size of EQ context id map */
640 t4_sge_iq_t **iqmap; /* iq->cntxt_id to IQ mapping */
641 t4_sge_eq_t **eqmap; /* eq->cntxt_id to EQ mapping */
642
643 /* Device access and DMA attributes for all the descriptor rings */
644 ddi_device_acc_attr_t acc_attr_desc;
645 ddi_dma_attr_t dma_attr_desc;
646
647 /* Device access and DMA attributes for TX buffers */
648 ddi_device_acc_attr_t acc_attr_tx;
649 ddi_dma_attr_t dma_attr_tx;
650
651 /* Device access and DMA attributes for RX buffers are in rxb_params */
652 kmem_cache_t *rxbuf_cache;
653 struct rxbuf_cache_params rxb_params;
654 };
655
656 struct driver_properties {
657 uint8_t ethq_tmr_idx;
658 int8_t ethq_pktc_idx;
659 uint8_t dbq_timer_idx;
660 uint8_t fwq_tmr_idx;
661 int8_t fwq_pktc_idx;
662 uint16_t qsize_txq;
663 uint16_t qsize_rxq;
664
665 uint_t holdoff_timer_us[SGE_NTIMERS];
666 uint_t holdoff_pktcnt[SGE_NCOUNTERS];
667
668 bool write_combine;
669 int t4_fw_install;
670 };
671
672 typedef struct t4_mbox_waiter {
673 list_node_t node;
674 kthread_t *thread;
675 } t4_mbox_waiter_t;
676
677 typedef enum t4_adapter_flags {
678 /* Initialization progress status bits */
679 TAF_INIT_DONE = (1 << 0),
680 TAF_FW_OK = (1 << 1),
681 TAF_INTR_ALLOC = (1 << 2),
682
683 /* State & capability bits */
684 TAF_MASTER_PF = (1 << 8),
685 TAF_DBQ_TIMER = (1 << 9),
686 } t4_adapter_flags_t;
687
688 /* Plan for interrupt allocation */
689 typedef enum t4_intr_plan {
690 /* Everything on a single interrupt */
691 TIP_SINGLE,
692 /* One for device errors, one FWQ (including forwarded intrs) */
693 TIP_ERR_QUEUES,
694 /* 1 + 1 for errors and FWQ, with rest divided evenly between ports */
695 TIP_PER_PORT,
696 } t4_intr_plan_t;
697
698 struct t4_intrs_queues {
699 /* The DDI_INTR_TYPE_* value negotiated. */
700 int intr_type;
701
702 /*
703 * The plan for interrupt allocation, based on the interrupt type
704 * and number of interrupts available. See the block comment in
705 * t4_nexus.c for more information.
706 */
707 t4_intr_plan_t intr_plan;
708
709 /*
710 * The number of interrupts available (intr_avail) for use vs. the
711 * number of interrupts the driver has decided to make use of
712 * (intr_count). These values may be different depending on the
713 * number available and the port count of the attached part.
714 */
715 int intr_avail;
716 int intr_count;
717
718 /*
719 * The number of interrupts per port for use with event queues.
720 * These interrupts are used to take delivery of Tx recycling
721 * messages and Rx packet delivery.
722 */
723 uint_t intr_per_port;
724
725 /*
726 * Track the number of IQs allocated for use with interrupts. We track
727 * this to know how many IQs we have leftover for Rx queue usage.
728 */
729 uint_t num_iqs;
730
731 /* The maximum number of RX/TX queues per port. */
732 uint_t port_max_rxq;
733 uint_t port_max_txq;
734 };
735
736 /*
737 * WO - Write Once at initialization time.
738 */
739 struct adapter {
740 list_node_t node;
741 dev_info_t *dip;
742 dev_t dev;
743
744 unsigned int pf;
745 unsigned int mbox;
746
747 unsigned int vpd_busy;
748 unsigned int vpd_flag;
749
750 u32 t4_bar0;
751
752 uint_t open; /* character device is open */
753
754 /* PCI config space access handle */
755 ddi_acc_handle_t pci_regh;
756
757 /* MMIO register access handle */
758 ddi_acc_handle_t regh;
759 caddr_t regp;
760 /* BAR2 register access handle */
761 ddi_acc_handle_t bar2_hdl;
762 caddr_t bar2_ptr;
763
764 /* Interrupt information */
765 ddi_intr_handle_t *intr_handle;
766 int intr_cap;
767 uint_t intr_pri;
768
769 struct driver_properties props;
770 kstat_t *ksp;
771 kstat_t *ksp_stat;
772
773 struct sge_info sge;
774
775 struct port_info *port[MAX_NPORTS];
776 uint8_t chan_map[NCHAN];
777 uint32_t filter_mode;
778
779 t4_adapter_flags_t flags;
780 t4_doorbells_t doorbells;
781
782 unsigned int cfcsum;
783 struct adapter_params params;
784 struct t4_intrs_queues intr_queue_cfg;
785
786 kmutex_t lock;
787 kcondvar_t cv;
788
789 /*
790 * Starving freelist state
791 *
792 * sfl_lock protects the `sfl_flags` and `sfl_node` fields in all sge_fl
793 * structs owned by this adapter.
794 */
795 kmutex_t sfl_lock;
796 list_t sfl_list;
797 timeout_id_t sfl_timer;
798
799 /* Sensors */
800 id_t temp_sensor;
801 id_t volt_sensor;
802
803 ddi_ufm_handle_t *ufm_hdl;
804
805 /* support for single-threading access to adapter mailbox registers */
806 kmutex_t mbox_lock;
807 kcondvar_t mbox_cv;
808 list_t mbox_list;
809 };
810
811 #define ADAPTER_LOCK(sc) mutex_enter(&(sc)->lock)
812 #define ADAPTER_UNLOCK(sc) mutex_exit(&(sc)->lock)
813 #define ADAPTER_LOCK_ASSERT_OWNED(sc) ASSERT(mutex_owned(&(sc)->lock))
814 #define ADAPTER_LOCK_ASSERT_NOTOWNED(sc) ASSERT(!mutex_owned(&(sc)->lock))
815
816 #define PORT_LOCK(pi) mutex_enter(&(pi)->lock)
817 #define PORT_UNLOCK(pi) mutex_exit(&(pi)->lock)
818 #define PORT_LOCK_ASSERT_OWNED(pi) ASSERT(mutex_owned(&(pi)->lock))
819 #define PORT_LOCK_ASSERT_NOTOWNED(pi) ASSERT(!mutex_owned(&(pi)->lock))
820
821 #define IQ_LOCK(iq) mutex_enter(&(iq)->tsi_lock)
822 #define IQ_UNLOCK(iq) mutex_exit(&(iq)->tsi_lock)
823 #define IQ_LOCK_ASSERT_OWNED(iq) ASSERT(mutex_owned(&(iq)->tsi_lock))
824 #define IQ_LOCK_ASSERT_NOTOWNED(iq) ASSERT(!mutex_owned(&(iq)->tsi_lock))
825
826 #define EQ_LOCK(eq) mutex_enter(&(eq)->tse_lock)
827 #define EQ_UNLOCK(eq) mutex_exit(&(eq)->tse_lock)
828 #define EQ_LOCK_ASSERT_OWNED(eq) ASSERT(mutex_owned(&(eq)->tse_lock))
829 #define EQ_LOCK_ASSERT_NOTOWNED(eq) ASSERT(!mutex_owned(&(eq)->tse_lock))
830
831 /* Freelist state is protected by its EQ lock */
832 #define FL_LOCK(fl) EQ_LOCK(&(fl)->eq)
833 #define FL_UNLOCK(fl) EQ_UNLOCK(&(fl)->eq)
834 #define FL_LOCK_ASSERT_OWNED(fl) EQ_LOCK_ASSERT_OWNED(&(fl)->eq)
835 #define FL_LOCK_ASSERT_NOTOWNED(fl) EQ_LOCK_ASSERT_NOTOWNED(&(fl)->eq)
836
837 #define TXQ_LOCK(txq) EQ_LOCK(&(txq)->eq)
838 #define TXQ_UNLOCK(txq) EQ_UNLOCK(&(txq)->eq)
839 #define TXQ_LOCK_ASSERT_OWNED(txq) EQ_LOCK_ASSERT_OWNED(&(txq)->eq)
840 #define TXQ_LOCK_ASSERT_NOTOWNED(txq) EQ_LOCK_ASSERT_NOTOWNED(&(txq)->eq)
841
842 #define for_each_txq(pi, iter, txq) \
843 txq = &pi->adapter->sge.txq[pi->txq_start]; \
844 for (iter = 0; iter < pi->txq_count; ++iter, ++txq)
845 #define for_each_rxq(pi, iter, rxq) \
846 rxq = &pi->adapter->sge.rxq[pi->rxq_start]; \
847 for (iter = 0; iter < pi->rxq_count; ++iter, ++rxq)
848
849 static inline struct port_info *
adap2pinfo(struct adapter * sc,int idx)850 adap2pinfo(struct adapter *sc, int idx)
851 {
852 return (sc->port[idx]);
853 }
854
t4_use_ldst(struct adapter * adap)855 static inline unsigned int t4_use_ldst(struct adapter *adap)
856 {
857 return (adap->flags & TAF_FW_OK);
858 }
859
t4_db_full(struct adapter * adap)860 static inline void t4_db_full(struct adapter *adap) {}
t4_db_dropped(struct adapter * adap)861 static inline void t4_db_dropped(struct adapter *adap) {}
862
863 /* Is chip version equal to specified value? */
864 static inline bool
t4_cver_eq(const adapter_t * adap,uint8_t ver)865 t4_cver_eq(const adapter_t *adap, uint8_t ver)
866 {
867 return (CHELSIO_CHIP_VERSION(adap->params.chip) == ver);
868 }
869
870 /* Is chip version greater than or equal to specified value? */
871 static inline bool
t4_cver_ge(const adapter_t * adap,uint8_t ver)872 t4_cver_ge(const adapter_t *adap, uint8_t ver)
873 {
874 return (CHELSIO_CHIP_VERSION(adap->params.chip) >= ver);
875 }
876
877 /* t4_nexus.c */
878 int t4_port_full_init(struct port_info *);
879
880 uint32_t t4_read_reg(struct adapter *, uint32_t);
881 void t4_write_reg(struct adapter *, uint32_t, uint32_t);
882 uint64_t t4_read_reg64(struct adapter *, uint32_t);
883 void t4_write_reg64(struct adapter *, uint32_t, uint64_t);
884
885 void t4_mbox_waiter_add(struct adapter *, t4_mbox_waiter_t *);
886 void t4_mbox_waiter_remove(struct adapter *, t4_mbox_waiter_t *);
887 bool t4_mbox_wait_owner(struct adapter *, uint_t, bool);
888
889 /* t4_debug.c */
890 void t4_debug_init(void);
891 void t4_debug_fini(void);
892
893 /* t4_sge.c */
894 void t4_sge_init(struct adapter *);
895 int t4_alloc_evt_iqs(struct adapter *);
896 void t4_free_evt_iqs(struct adapter *);
897 void t4_port_kstats_init(struct port_info *);
898 void t4_port_kstats_fini(struct port_info *);
899 int t4_port_queues_init(struct port_info *);
900 void t4_port_queues_fini(struct port_info *);
901 void t4_port_queues_enable(struct port_info *pi);
902 void t4_port_queues_disable(struct port_info *pi);
903 uint_t t4_intr_all(caddr_t, caddr_t);
904 uint_t t4_intr_err(caddr_t, caddr_t);
905 uint_t t4_intr_fwq(caddr_t, caddr_t);
906 uint_t t4_intr_port_queue(caddr_t, caddr_t);
907 void t4_iq_gts_update(t4_sge_iq_t *, t4_gts_config_t, uint16_t);
908 void t4_iq_update_intr_cfg(t4_sge_iq_t *, uint8_t, int8_t);
909 void t4_eq_update_dbq_timer(t4_sge_eq_t *, struct port_info *);
910
911 mblk_t *t4_eth_tx(void *, mblk_t *);
912 t4_iq_result_t t4_process_rx_iq(t4_sge_iq_t *, uint_t, struct t4_poll_req *);
913
914 /* t4_mac.c */
915 void t4_os_link_changed(struct adapter *sc, int idx, int link_stat);
916 void t4_mac_tx_update(struct port_info *pi, struct sge_txq *txq);
917 int t4_addmac(void *arg, const uint8_t *ucaddr);
918 const char **t4_get_priv_props(struct port_info *, size_t *);
919 uint8_t t4_choose_holdoff_timer(struct adapter *, uint_t);
920 int8_t t4_choose_holdoff_pktcnt(struct adapter *, int);
921 uint_t t4_choose_dbq_timer(struct adapter *, uint_t);
922 extern mac_callbacks_t t4_mac_callbacks;
923
924 /* t4_ioctl.c */
925 int t4_ioctl(struct adapter *sc, int cmd, void *data, int mode);
926
927 #endif /* __CXGBE_ADAPTER_H */
928