xref: /freebsd/sys/dev/cxgb/cxgb_main.c (revision 77c1fcec914bae1232762ca0b0462abd6957ff26)
1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 
4 Copyright (c) 2007-2009, Chelsio Inc.
5 All rights reserved.
6 
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9 
10  1. Redistributions of source code must retain the above copyright notice,
11     this list of conditions and the following disclaimer.
12 
13  2. Neither the name of the Chelsio Corporation nor the names of its
14     contributors may be used to endorse or promote products derived from
15     this software without specific prior written permission.
16 
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
28 
29 ***************************************************************************/
30 
31 #include <sys/cdefs.h>
32 __FBSDID("$FreeBSD$");
33 
34 #include "opt_inet.h"
35 
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/kernel.h>
39 #include <sys/bus.h>
40 #include <sys/module.h>
41 #include <sys/pciio.h>
42 #include <sys/conf.h>
43 #include <machine/bus.h>
44 #include <machine/resource.h>
45 #include <sys/ktr.h>
46 #include <sys/rman.h>
47 #include <sys/ioccom.h>
48 #include <sys/mbuf.h>
49 #include <sys/linker.h>
50 #include <sys/firmware.h>
51 #include <sys/socket.h>
52 #include <sys/sockio.h>
53 #include <sys/smp.h>
54 #include <sys/sysctl.h>
55 #include <sys/syslog.h>
56 #include <sys/queue.h>
57 #include <sys/taskqueue.h>
58 #include <sys/proc.h>
59 
60 #include <net/bpf.h>
61 #include <net/ethernet.h>
62 #include <net/if.h>
63 #include <net/if_var.h>
64 #include <net/if_arp.h>
65 #include <net/if_dl.h>
66 #include <net/if_media.h>
67 #include <net/if_types.h>
68 #include <net/if_vlan_var.h>
69 
70 #include <netinet/in_systm.h>
71 #include <netinet/in.h>
72 #include <netinet/if_ether.h>
73 #include <netinet/ip.h>
74 #include <netinet/ip.h>
75 #include <netinet/tcp.h>
76 #include <netinet/udp.h>
77 #include <netinet/netdump/netdump.h>
78 
79 #include <dev/pci/pcireg.h>
80 #include <dev/pci/pcivar.h>
81 #include <dev/pci/pci_private.h>
82 
83 #include <cxgb_include.h>
84 
85 #ifdef PRIV_SUPPORTED
86 #include <sys/priv.h>
87 #endif
88 
89 static int cxgb_setup_interrupts(adapter_t *);
90 static void cxgb_teardown_interrupts(adapter_t *);
91 static void cxgb_init(void *);
92 static int cxgb_init_locked(struct port_info *);
93 static int cxgb_uninit_locked(struct port_info *);
94 static int cxgb_uninit_synchronized(struct port_info *);
95 static int cxgb_ioctl(struct ifnet *, unsigned long, caddr_t);
96 static int cxgb_media_change(struct ifnet *);
97 static int cxgb_ifm_type(int);
98 static void cxgb_build_medialist(struct port_info *);
99 static void cxgb_media_status(struct ifnet *, struct ifmediareq *);
100 static uint64_t cxgb_get_counter(struct ifnet *, ift_counter);
101 static int setup_sge_qsets(adapter_t *);
102 static void cxgb_async_intr(void *);
103 static void cxgb_tick_handler(void *, int);
104 static void cxgb_tick(void *);
105 static void link_check_callout(void *);
106 static void check_link_status(void *, int);
107 static void setup_rss(adapter_t *sc);
108 static int alloc_filters(struct adapter *);
109 static int setup_hw_filters(struct adapter *);
110 static int set_filter(struct adapter *, int, const struct filter_info *);
111 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
112     unsigned int, u64, u64);
113 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
114     unsigned int, u64, u64);
115 #ifdef TCP_OFFLOAD
116 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
117 #endif
118 
119 /* Attachment glue for the PCI controller end of the device.  Each port of
120  * the device is attached separately, as defined later.
121  */
122 static int cxgb_controller_probe(device_t);
123 static int cxgb_controller_attach(device_t);
124 static int cxgb_controller_detach(device_t);
125 static void cxgb_free(struct adapter *);
126 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
127     unsigned int end);
128 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
129 static int cxgb_get_regs_len(void);
130 static void touch_bars(device_t dev);
131 static void cxgb_update_mac_settings(struct port_info *p);
132 #ifdef TCP_OFFLOAD
133 static int toe_capability(struct port_info *, int);
134 #endif
135 
136 /* Table for probing the cards.  The desc field isn't actually used */
137 struct cxgb_ident {
138 	uint16_t	vendor;
139 	uint16_t	device;
140 	int		index;
141 	char		*desc;
142 } cxgb_identifiers[] = {
143 	{PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
144 	{PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
145 	{PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
146 	{PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
147 	{PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
148 	{PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
149 	{PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
150 	{PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
151 	{PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
152 	{PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
153 	{PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
154 	{PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
155 	{PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
156 	{PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
157 	{0, 0, 0, NULL}
158 };
159 
160 static device_method_t cxgb_controller_methods[] = {
161 	DEVMETHOD(device_probe,		cxgb_controller_probe),
162 	DEVMETHOD(device_attach,	cxgb_controller_attach),
163 	DEVMETHOD(device_detach,	cxgb_controller_detach),
164 
165 	DEVMETHOD_END
166 };
167 
168 static driver_t cxgb_controller_driver = {
169 	"cxgbc",
170 	cxgb_controller_methods,
171 	sizeof(struct adapter)
172 };
173 
174 static int cxgbc_mod_event(module_t, int, void *);
175 static devclass_t	cxgb_controller_devclass;
176 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgb_controller_devclass,
177     cxgbc_mod_event, 0);
178 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
179     nitems(cxgb_identifiers) - 1);
180 MODULE_VERSION(cxgbc, 1);
181 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
182 
183 /*
184  * Attachment glue for the ports.  Attachment is done directly to the
185  * controller device.
186  */
187 static int cxgb_port_probe(device_t);
188 static int cxgb_port_attach(device_t);
189 static int cxgb_port_detach(device_t);
190 
191 static device_method_t cxgb_port_methods[] = {
192 	DEVMETHOD(device_probe,		cxgb_port_probe),
193 	DEVMETHOD(device_attach,	cxgb_port_attach),
194 	DEVMETHOD(device_detach,	cxgb_port_detach),
195 	{ 0, 0 }
196 };
197 
198 static driver_t cxgb_port_driver = {
199 	"cxgb",
200 	cxgb_port_methods,
201 	0
202 };
203 
204 static d_ioctl_t cxgb_extension_ioctl;
205 static d_open_t cxgb_extension_open;
206 static d_close_t cxgb_extension_close;
207 
208 static struct cdevsw cxgb_cdevsw = {
209        .d_version =    D_VERSION,
210        .d_flags =      0,
211        .d_open =       cxgb_extension_open,
212        .d_close =      cxgb_extension_close,
213        .d_ioctl =      cxgb_extension_ioctl,
214        .d_name =       "cxgb",
215 };
216 
217 static devclass_t	cxgb_port_devclass;
218 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, cxgb_port_devclass, 0, 0);
219 MODULE_VERSION(cxgb, 1);
220 
221 NETDUMP_DEFINE(cxgb);
222 
223 static struct mtx t3_list_lock;
224 static SLIST_HEAD(, adapter) t3_list;
225 #ifdef TCP_OFFLOAD
226 static struct mtx t3_uld_list_lock;
227 static SLIST_HEAD(, uld_info) t3_uld_list;
228 #endif
229 
230 /*
231  * The driver uses the best interrupt scheme available on a platform in the
232  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
233  * of these schemes the driver may consider as follows:
234  *
235  * msi = 2: choose from among all three options
236  * msi = 1 : only consider MSI and pin interrupts
237  * msi = 0: force pin interrupts
238  */
239 static int msi_allowed = 2;
240 
241 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD, 0, "CXGB driver parameters");
242 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
243     "MSI-X, MSI, INTx selector");
244 
245 /*
246  * The driver uses an auto-queue algorithm by default.
247  * To disable it and force a single queue-set per port, use multiq = 0
248  */
249 static int multiq = 1;
250 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
251     "use min(ncpus/ports, 8) queue-sets per port");
252 
253 /*
254  * By default the driver will not update the firmware unless
255  * it was compiled against a newer version
256  *
257  */
258 static int force_fw_update = 0;
259 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
260     "update firmware even if up to date");
261 
262 int cxgb_use_16k_clusters = -1;
263 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
264     &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
265 
266 static int nfilters = -1;
267 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
268     &nfilters, 0, "max number of entries in the filter table");
269 
270 enum {
271 	MAX_TXQ_ENTRIES      = 16384,
272 	MAX_CTRL_TXQ_ENTRIES = 1024,
273 	MAX_RSPQ_ENTRIES     = 16384,
274 	MAX_RX_BUFFERS       = 16384,
275 	MAX_RX_JUMBO_BUFFERS = 16384,
276 	MIN_TXQ_ENTRIES      = 4,
277 	MIN_CTRL_TXQ_ENTRIES = 4,
278 	MIN_RSPQ_ENTRIES     = 32,
279 	MIN_FL_ENTRIES       = 32,
280 	MIN_FL_JUMBO_ENTRIES = 32
281 };
282 
283 struct filter_info {
284 	u32 sip;
285 	u32 sip_mask;
286 	u32 dip;
287 	u16 sport;
288 	u16 dport;
289 	u32 vlan:12;
290 	u32 vlan_prio:3;
291 	u32 mac_hit:1;
292 	u32 mac_idx:4;
293 	u32 mac_vld:1;
294 	u32 pkt_type:2;
295 	u32 report_filter_id:1;
296 	u32 pass:1;
297 	u32 rss:1;
298 	u32 qset:3;
299 	u32 locked:1;
300 	u32 valid:1;
301 };
302 
303 enum { FILTER_NO_VLAN_PRI = 7 };
304 
305 #define EEPROM_MAGIC 0x38E2F10C
306 
307 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
308 
309 
310 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
311 
312 
313 static __inline char
314 t3rev2char(struct adapter *adapter)
315 {
316 	char rev = 'z';
317 
318 	switch(adapter->params.rev) {
319 	case T3_REV_A:
320 		rev = 'a';
321 		break;
322 	case T3_REV_B:
323 	case T3_REV_B2:
324 		rev = 'b';
325 		break;
326 	case T3_REV_C:
327 		rev = 'c';
328 		break;
329 	}
330 	return rev;
331 }
332 
333 static struct cxgb_ident *
334 cxgb_get_ident(device_t dev)
335 {
336 	struct cxgb_ident *id;
337 
338 	for (id = cxgb_identifiers; id->desc != NULL; id++) {
339 		if ((id->vendor == pci_get_vendor(dev)) &&
340 		    (id->device == pci_get_device(dev))) {
341 			return (id);
342 		}
343 	}
344 	return (NULL);
345 }
346 
347 static const struct adapter_info *
348 cxgb_get_adapter_info(device_t dev)
349 {
350 	struct cxgb_ident *id;
351 	const struct adapter_info *ai;
352 
353 	id = cxgb_get_ident(dev);
354 	if (id == NULL)
355 		return (NULL);
356 
357 	ai = t3_get_adapter_info(id->index);
358 
359 	return (ai);
360 }
361 
362 static int
363 cxgb_controller_probe(device_t dev)
364 {
365 	const struct adapter_info *ai;
366 	char *ports, buf[80];
367 	int nports;
368 
369 	ai = cxgb_get_adapter_info(dev);
370 	if (ai == NULL)
371 		return (ENXIO);
372 
373 	nports = ai->nports0 + ai->nports1;
374 	if (nports == 1)
375 		ports = "port";
376 	else
377 		ports = "ports";
378 
379 	snprintf(buf, sizeof(buf), "%s, %d %s", ai->desc, nports, ports);
380 	device_set_desc_copy(dev, buf);
381 	return (BUS_PROBE_DEFAULT);
382 }
383 
384 #define FW_FNAME "cxgb_t3fw"
385 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
386 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
387 
388 static int
389 upgrade_fw(adapter_t *sc)
390 {
391 	const struct firmware *fw;
392 	int status;
393 	u32 vers;
394 
395 	if ((fw = firmware_get(FW_FNAME)) == NULL)  {
396 		device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
397 		return (ENOENT);
398 	} else
399 		device_printf(sc->dev, "installing firmware on card\n");
400 	status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
401 
402 	if (status != 0) {
403 		device_printf(sc->dev, "failed to install firmware: %d\n",
404 		    status);
405 	} else {
406 		t3_get_fw_version(sc, &vers);
407 		snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
408 		    G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
409 		    G_FW_VERSION_MICRO(vers));
410 	}
411 
412 	firmware_put(fw, FIRMWARE_UNLOAD);
413 
414 	return (status);
415 }
416 
417 /*
418  * The cxgb_controller_attach function is responsible for the initial
419  * bringup of the device.  Its responsibilities include:
420  *
421  *  1. Determine if the device supports MSI or MSI-X.
422  *  2. Allocate bus resources so that we can access the Base Address Register
423  *  3. Create and initialize mutexes for the controller and its control
424  *     logic such as SGE and MDIO.
425  *  4. Call hardware specific setup routine for the adapter as a whole.
426  *  5. Allocate the BAR for doing MSI-X.
427  *  6. Setup the line interrupt iff MSI-X is not supported.
428  *  7. Create the driver's taskq.
429  *  8. Start one task queue service thread.
430  *  9. Check if the firmware and SRAM are up-to-date.  They will be
431  *     auto-updated later (before FULL_INIT_DONE), if required.
432  * 10. Create a child device for each MAC (port)
433  * 11. Initialize T3 private state.
434  * 12. Trigger the LED
435  * 13. Setup offload iff supported.
436  * 14. Reset/restart the tick callout.
437  * 15. Attach sysctls
438  *
439  * NOTE: Any modification or deviation from this list MUST be reflected in
440  * the above comment.  Failure to do so will result in problems on various
441  * error conditions including link flapping.
442  */
443 static int
444 cxgb_controller_attach(device_t dev)
445 {
446 	device_t child;
447 	const struct adapter_info *ai;
448 	struct adapter *sc;
449 	int i, error = 0;
450 	uint32_t vers;
451 	int port_qsets = 1;
452 	int msi_needed, reg;
453 	char buf[80];
454 
455 	sc = device_get_softc(dev);
456 	sc->dev = dev;
457 	sc->msi_count = 0;
458 	ai = cxgb_get_adapter_info(dev);
459 
460 	snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
461 	    device_get_unit(dev));
462 	ADAPTER_LOCK_INIT(sc, sc->lockbuf);
463 
464 	snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
465 	    device_get_unit(dev));
466 	snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
467 	    device_get_unit(dev));
468 	snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
469 	    device_get_unit(dev));
470 
471 	MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
472 	MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
473 	MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
474 
475 	mtx_lock(&t3_list_lock);
476 	SLIST_INSERT_HEAD(&t3_list, sc, link);
477 	mtx_unlock(&t3_list_lock);
478 
479 	/* find the PCIe link width and set max read request to 4KB*/
480 	if (pci_find_cap(dev, PCIY_EXPRESS, &reg) == 0) {
481 		uint16_t lnk;
482 
483 		lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
484 		sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
485 		if (sc->link_width < 8 &&
486 		    (ai->caps & SUPPORTED_10000baseT_Full)) {
487 			device_printf(sc->dev,
488 			    "PCIe x%d Link, expect reduced performance\n",
489 			    sc->link_width);
490 		}
491 
492 		pci_set_max_read_req(dev, 4096);
493 	}
494 
495 	touch_bars(dev);
496 	pci_enable_busmaster(dev);
497 	/*
498 	 * Allocate the registers and make them available to the driver.
499 	 * The registers that we care about for NIC mode are in BAR 0
500 	 */
501 	sc->regs_rid = PCIR_BAR(0);
502 	if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
503 	    &sc->regs_rid, RF_ACTIVE)) == NULL) {
504 		device_printf(dev, "Cannot allocate BAR region 0\n");
505 		error = ENXIO;
506 		goto out;
507 	}
508 
509 	sc->bt = rman_get_bustag(sc->regs_res);
510 	sc->bh = rman_get_bushandle(sc->regs_res);
511 	sc->mmio_len = rman_get_size(sc->regs_res);
512 
513 	for (i = 0; i < MAX_NPORTS; i++)
514 		sc->port[i].adapter = sc;
515 
516 	if (t3_prep_adapter(sc, ai, 1) < 0) {
517 		printf("prep adapter failed\n");
518 		error = ENODEV;
519 		goto out;
520 	}
521 
522 	sc->udbs_rid = PCIR_BAR(2);
523 	sc->udbs_res = NULL;
524 	if (is_offload(sc) &&
525 	    ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
526 		   &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
527 		device_printf(dev, "Cannot allocate BAR region 1\n");
528 		error = ENXIO;
529 		goto out;
530 	}
531 
532         /* Allocate the BAR for doing MSI-X.  If it succeeds, try to allocate
533 	 * enough messages for the queue sets.  If that fails, try falling
534 	 * back to MSI.  If that fails, then try falling back to the legacy
535 	 * interrupt pin model.
536 	 */
537 	sc->msix_regs_rid = 0x20;
538 	if ((msi_allowed >= 2) &&
539 	    (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
540 	    &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
541 
542 		if (multiq)
543 			port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
544 		msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
545 
546 		if (pci_msix_count(dev) == 0 ||
547 		    (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
548 		    sc->msi_count != msi_needed) {
549 			device_printf(dev, "alloc msix failed - "
550 				      "msi_count=%d, msi_needed=%d, err=%d; "
551 				      "will try MSI\n", sc->msi_count,
552 				      msi_needed, error);
553 			sc->msi_count = 0;
554 			port_qsets = 1;
555 			pci_release_msi(dev);
556 			bus_release_resource(dev, SYS_RES_MEMORY,
557 			    sc->msix_regs_rid, sc->msix_regs_res);
558 			sc->msix_regs_res = NULL;
559 		} else {
560 			sc->flags |= USING_MSIX;
561 			sc->cxgb_intr = cxgb_async_intr;
562 			device_printf(dev,
563 				      "using MSI-X interrupts (%u vectors)\n",
564 				      sc->msi_count);
565 		}
566 	}
567 
568 	if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
569 		sc->msi_count = 1;
570 		if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
571 			device_printf(dev, "alloc msi failed - "
572 				      "err=%d; will try INTx\n", error);
573 			sc->msi_count = 0;
574 			port_qsets = 1;
575 			pci_release_msi(dev);
576 		} else {
577 			sc->flags |= USING_MSI;
578 			sc->cxgb_intr = t3_intr_msi;
579 			device_printf(dev, "using MSI interrupts\n");
580 		}
581 	}
582 	if (sc->msi_count == 0) {
583 		device_printf(dev, "using line interrupts\n");
584 		sc->cxgb_intr = t3b_intr;
585 	}
586 
587 	/* Create a private taskqueue thread for handling driver events */
588 	sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
589 	    taskqueue_thread_enqueue, &sc->tq);
590 	if (sc->tq == NULL) {
591 		device_printf(dev, "failed to allocate controller task queue\n");
592 		goto out;
593 	}
594 
595 	taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
596 	    device_get_nameunit(dev));
597 	TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
598 
599 
600 	/* Create a periodic callout for checking adapter status */
601 	callout_init(&sc->cxgb_tick_ch, 1);
602 
603 	if (t3_check_fw_version(sc) < 0 || force_fw_update) {
604 		/*
605 		 * Warn user that a firmware update will be attempted in init.
606 		 */
607 		device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
608 		    FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
609 		sc->flags &= ~FW_UPTODATE;
610 	} else {
611 		sc->flags |= FW_UPTODATE;
612 	}
613 
614 	if (t3_check_tpsram_version(sc) < 0) {
615 		/*
616 		 * Warn user that a firmware update will be attempted in init.
617 		 */
618 		device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
619 		    t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
620 		sc->flags &= ~TPS_UPTODATE;
621 	} else {
622 		sc->flags |= TPS_UPTODATE;
623 	}
624 
625 	/*
626 	 * Create a child device for each MAC.  The ethernet attachment
627 	 * will be done in these children.
628 	 */
629 	for (i = 0; i < (sc)->params.nports; i++) {
630 		struct port_info *pi;
631 
632 		if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
633 			device_printf(dev, "failed to add child port\n");
634 			error = EINVAL;
635 			goto out;
636 		}
637 		pi = &sc->port[i];
638 		pi->adapter = sc;
639 		pi->nqsets = port_qsets;
640 		pi->first_qset = i*port_qsets;
641 		pi->port_id = i;
642 		pi->tx_chan = i >= ai->nports0;
643 		pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
644 		sc->rxpkt_map[pi->txpkt_intf] = i;
645 		sc->port[i].tx_chan = i >= ai->nports0;
646 		sc->portdev[i] = child;
647 		device_set_softc(child, pi);
648 	}
649 	if ((error = bus_generic_attach(dev)) != 0)
650 		goto out;
651 
652 	/* initialize sge private state */
653 	t3_sge_init_adapter(sc);
654 
655 	t3_led_ready(sc);
656 
657 	error = t3_get_fw_version(sc, &vers);
658 	if (error)
659 		goto out;
660 
661 	snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
662 	    G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
663 	    G_FW_VERSION_MICRO(vers));
664 
665 	snprintf(buf, sizeof(buf), "%s %sNIC\t E/C: %s S/N: %s",
666 		 ai->desc, is_offload(sc) ? "R" : "",
667 		 sc->params.vpd.ec, sc->params.vpd.sn);
668 	device_set_desc_copy(dev, buf);
669 
670 	snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
671 		 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
672 		 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
673 
674 	device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
675 	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
676 	t3_add_attach_sysctls(sc);
677 
678 #ifdef TCP_OFFLOAD
679 	for (i = 0; i < NUM_CPL_HANDLERS; i++)
680 		sc->cpl_handler[i] = cpl_not_handled;
681 #endif
682 
683 	t3_intr_clear(sc);
684 	error = cxgb_setup_interrupts(sc);
685 out:
686 	if (error)
687 		cxgb_free(sc);
688 
689 	return (error);
690 }
691 
692 /*
693  * The cxgb_controller_detach routine is called with the device is
694  * unloaded from the system.
695  */
696 
697 static int
698 cxgb_controller_detach(device_t dev)
699 {
700 	struct adapter *sc;
701 
702 	sc = device_get_softc(dev);
703 
704 	cxgb_free(sc);
705 
706 	return (0);
707 }
708 
709 /*
710  * The cxgb_free() is called by the cxgb_controller_detach() routine
711  * to tear down the structures that were built up in
712  * cxgb_controller_attach(), and should be the final piece of work
713  * done when fully unloading the driver.
714  *
715  *
716  *  1. Shutting down the threads started by the cxgb_controller_attach()
717  *     routine.
718  *  2. Stopping the lower level device and all callouts (cxgb_down_locked()).
719  *  3. Detaching all of the port devices created during the
720  *     cxgb_controller_attach() routine.
721  *  4. Removing the device children created via cxgb_controller_attach().
722  *  5. Releasing PCI resources associated with the device.
723  *  6. Turning off the offload support, iff it was turned on.
724  *  7. Destroying the mutexes created in cxgb_controller_attach().
725  *
726  */
727 static void
728 cxgb_free(struct adapter *sc)
729 {
730 	int i, nqsets = 0;
731 
732 	ADAPTER_LOCK(sc);
733 	sc->flags |= CXGB_SHUTDOWN;
734 	ADAPTER_UNLOCK(sc);
735 
736 	/*
737 	 * Make sure all child devices are gone.
738 	 */
739 	bus_generic_detach(sc->dev);
740 	for (i = 0; i < (sc)->params.nports; i++) {
741 		if (sc->portdev[i] &&
742 		    device_delete_child(sc->dev, sc->portdev[i]) != 0)
743 			device_printf(sc->dev, "failed to delete child port\n");
744 		nqsets += sc->port[i].nqsets;
745 	}
746 
747 	/*
748 	 * At this point, it is as if cxgb_port_detach has run on all ports, and
749 	 * cxgb_down has run on the adapter.  All interrupts have been silenced,
750 	 * all open devices have been closed.
751 	 */
752 	KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
753 					   __func__, sc->open_device_map));
754 	for (i = 0; i < sc->params.nports; i++) {
755 		KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
756 						  __func__, i));
757 	}
758 
759 	/*
760 	 * Finish off the adapter's callouts.
761 	 */
762 	callout_drain(&sc->cxgb_tick_ch);
763 	callout_drain(&sc->sge_timer_ch);
764 
765 	/*
766 	 * Release resources grabbed under FULL_INIT_DONE by cxgb_up.  The
767 	 * sysctls are cleaned up by the kernel linker.
768 	 */
769 	if (sc->flags & FULL_INIT_DONE) {
770  		t3_free_sge_resources(sc, nqsets);
771  		sc->flags &= ~FULL_INIT_DONE;
772  	}
773 
774 	/*
775 	 * Release all interrupt resources.
776 	 */
777 	cxgb_teardown_interrupts(sc);
778 	if (sc->flags & (USING_MSI | USING_MSIX)) {
779 		device_printf(sc->dev, "releasing msi message(s)\n");
780 		pci_release_msi(sc->dev);
781 	} else {
782 		device_printf(sc->dev, "no msi message to release\n");
783 	}
784 
785 	if (sc->msix_regs_res != NULL) {
786 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
787 		    sc->msix_regs_res);
788 	}
789 
790 	/*
791 	 * Free the adapter's taskqueue.
792 	 */
793 	if (sc->tq != NULL) {
794 		taskqueue_free(sc->tq);
795 		sc->tq = NULL;
796 	}
797 
798 	free(sc->filters, M_DEVBUF);
799 	t3_sge_free(sc);
800 
801 	if (sc->udbs_res != NULL)
802 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
803 		    sc->udbs_res);
804 
805 	if (sc->regs_res != NULL)
806 		bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
807 		    sc->regs_res);
808 
809 	MTX_DESTROY(&sc->mdio_lock);
810 	MTX_DESTROY(&sc->sge.reg_lock);
811 	MTX_DESTROY(&sc->elmer_lock);
812 	mtx_lock(&t3_list_lock);
813 	SLIST_REMOVE(&t3_list, sc, adapter, link);
814 	mtx_unlock(&t3_list_lock);
815 	ADAPTER_LOCK_DEINIT(sc);
816 }
817 
818 /**
819  *	setup_sge_qsets - configure SGE Tx/Rx/response queues
820  *	@sc: the controller softc
821  *
822  *	Determines how many sets of SGE queues to use and initializes them.
823  *	We support multiple queue sets per port if we have MSI-X, otherwise
824  *	just one queue set per port.
825  */
826 static int
827 setup_sge_qsets(adapter_t *sc)
828 {
829 	int i, j, err, irq_idx = 0, qset_idx = 0;
830 	u_int ntxq = SGE_TXQ_PER_SET;
831 
832 	if ((err = t3_sge_alloc(sc)) != 0) {
833 		device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
834 		return (err);
835 	}
836 
837 	if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
838 		irq_idx = -1;
839 
840 	for (i = 0; i < (sc)->params.nports; i++) {
841 		struct port_info *pi = &sc->port[i];
842 
843 		for (j = 0; j < pi->nqsets; j++, qset_idx++) {
844 			err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
845 			    (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
846 			    &sc->params.sge.qset[qset_idx], ntxq, pi);
847 			if (err) {
848 				t3_free_sge_resources(sc, qset_idx);
849 				device_printf(sc->dev,
850 				    "t3_sge_alloc_qset failed with %d\n", err);
851 				return (err);
852 			}
853 		}
854 	}
855 
856 	return (0);
857 }
858 
859 static void
860 cxgb_teardown_interrupts(adapter_t *sc)
861 {
862 	int i;
863 
864 	for (i = 0; i < SGE_QSETS; i++) {
865 		if (sc->msix_intr_tag[i] == NULL) {
866 
867 			/* Should have been setup fully or not at all */
868 			KASSERT(sc->msix_irq_res[i] == NULL &&
869 				sc->msix_irq_rid[i] == 0,
870 				("%s: half-done interrupt (%d).", __func__, i));
871 
872 			continue;
873 		}
874 
875 		bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
876 				  sc->msix_intr_tag[i]);
877 		bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
878 				     sc->msix_irq_res[i]);
879 
880 		sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
881 		sc->msix_irq_rid[i] = 0;
882 	}
883 
884 	if (sc->intr_tag) {
885 		KASSERT(sc->irq_res != NULL,
886 			("%s: half-done interrupt.", __func__));
887 
888 		bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
889 		bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
890 				     sc->irq_res);
891 
892 		sc->irq_res = sc->intr_tag = NULL;
893 		sc->irq_rid = 0;
894 	}
895 }
896 
897 static int
898 cxgb_setup_interrupts(adapter_t *sc)
899 {
900 	struct resource *res;
901 	void *tag;
902 	int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
903 
904 	sc->irq_rid = intr_flag ? 1 : 0;
905 	sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
906 					     RF_SHAREABLE | RF_ACTIVE);
907 	if (sc->irq_res == NULL) {
908 		device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
909 			      intr_flag, sc->irq_rid);
910 		err = EINVAL;
911 		sc->irq_rid = 0;
912 	} else {
913 		err = bus_setup_intr(sc->dev, sc->irq_res,
914 		    INTR_MPSAFE | INTR_TYPE_NET, NULL,
915 		    sc->cxgb_intr, sc, &sc->intr_tag);
916 
917 		if (err) {
918 			device_printf(sc->dev,
919 				      "Cannot set up interrupt (%x, %u, %d)\n",
920 				      intr_flag, sc->irq_rid, err);
921 			bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
922 					     sc->irq_res);
923 			sc->irq_res = sc->intr_tag = NULL;
924 			sc->irq_rid = 0;
925 		}
926 	}
927 
928 	/* That's all for INTx or MSI */
929 	if (!(intr_flag & USING_MSIX) || err)
930 		return (err);
931 
932 	bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
933 	for (i = 0; i < sc->msi_count - 1; i++) {
934 		rid = i + 2;
935 		res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
936 					     RF_SHAREABLE | RF_ACTIVE);
937 		if (res == NULL) {
938 			device_printf(sc->dev, "Cannot allocate interrupt "
939 				      "for message %d\n", rid);
940 			err = EINVAL;
941 			break;
942 		}
943 
944 		err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
945 				     NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
946 		if (err) {
947 			device_printf(sc->dev, "Cannot set up interrupt "
948 				      "for message %d (%d)\n", rid, err);
949 			bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
950 			break;
951 		}
952 
953 		sc->msix_irq_rid[i] = rid;
954 		sc->msix_irq_res[i] = res;
955 		sc->msix_intr_tag[i] = tag;
956 		bus_describe_intr(sc->dev, res, tag, "qs%d", i);
957 	}
958 
959 	if (err)
960 		cxgb_teardown_interrupts(sc);
961 
962 	return (err);
963 }
964 
965 
966 static int
967 cxgb_port_probe(device_t dev)
968 {
969 	struct port_info *p;
970 	char buf[80];
971 	const char *desc;
972 
973 	p = device_get_softc(dev);
974 	desc = p->phy.desc;
975 	snprintf(buf, sizeof(buf), "Port %d %s", p->port_id, desc);
976 	device_set_desc_copy(dev, buf);
977 	return (0);
978 }
979 
980 
981 static int
982 cxgb_makedev(struct port_info *pi)
983 {
984 
985 	pi->port_cdev = make_dev(&cxgb_cdevsw, pi->ifp->if_dunit,
986 	    UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
987 
988 	if (pi->port_cdev == NULL)
989 		return (ENOMEM);
990 
991 	pi->port_cdev->si_drv1 = (void *)pi;
992 
993 	return (0);
994 }
995 
996 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
997     IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
998     IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
999 #define CXGB_CAP_ENABLE CXGB_CAP
1000 
1001 static int
1002 cxgb_port_attach(device_t dev)
1003 {
1004 	struct port_info *p;
1005 	struct ifnet *ifp;
1006 	int err;
1007 	struct adapter *sc;
1008 
1009 	p = device_get_softc(dev);
1010 	sc = p->adapter;
1011 	snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1012 	    device_get_unit(device_get_parent(dev)), p->port_id);
1013 	PORT_LOCK_INIT(p, p->lockbuf);
1014 
1015 	callout_init(&p->link_check_ch, 1);
1016 	TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1017 
1018 	/* Allocate an ifnet object and set it up */
1019 	ifp = p->ifp = if_alloc(IFT_ETHER);
1020 	if (ifp == NULL) {
1021 		device_printf(dev, "Cannot allocate ifnet\n");
1022 		return (ENOMEM);
1023 	}
1024 
1025 	if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1026 	ifp->if_init = cxgb_init;
1027 	ifp->if_softc = p;
1028 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
1029 	ifp->if_ioctl = cxgb_ioctl;
1030 	ifp->if_transmit = cxgb_transmit;
1031 	ifp->if_qflush = cxgb_qflush;
1032 	ifp->if_get_counter = cxgb_get_counter;
1033 
1034 	ifp->if_capabilities = CXGB_CAP;
1035 #ifdef TCP_OFFLOAD
1036 	if (is_offload(sc))
1037 		ifp->if_capabilities |= IFCAP_TOE4;
1038 #endif
1039 	ifp->if_capenable = CXGB_CAP_ENABLE;
1040 	ifp->if_hwassist = CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1041 	    CSUM_UDP_IPV6 | CSUM_TCP_IPV6;
1042 
1043 	/*
1044 	 * Disable TSO on 4-port - it isn't supported by the firmware.
1045 	 */
1046 	if (sc->params.nports > 2) {
1047 		ifp->if_capabilities &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1048 		ifp->if_capenable &= ~(IFCAP_TSO | IFCAP_VLAN_HWTSO);
1049 		ifp->if_hwassist &= ~CSUM_TSO;
1050 	}
1051 
1052 	ether_ifattach(ifp, p->hw_addr);
1053 
1054 	/* Attach driver netdump methods. */
1055 	NETDUMP_SET(ifp, cxgb);
1056 
1057 #ifdef DEFAULT_JUMBO
1058 	if (sc->params.nports <= 2)
1059 		ifp->if_mtu = ETHERMTU_JUMBO;
1060 #endif
1061 	if ((err = cxgb_makedev(p)) != 0) {
1062 		printf("makedev failed %d\n", err);
1063 		return (err);
1064 	}
1065 
1066 	/* Create a list of media supported by this port */
1067 	ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1068 	    cxgb_media_status);
1069 	cxgb_build_medialist(p);
1070 
1071 	t3_sge_init_port(p);
1072 
1073 	return (err);
1074 }
1075 
1076 /*
1077  * cxgb_port_detach() is called via the device_detach methods when
1078  * cxgb_free() calls the bus_generic_detach.  It is responsible for
1079  * removing the device from the view of the kernel, i.e. from all
1080  * interfaces lists etc.  This routine is only called when the driver is
1081  * being unloaded, not when the link goes down.
1082  */
1083 static int
1084 cxgb_port_detach(device_t dev)
1085 {
1086 	struct port_info *p;
1087 	struct adapter *sc;
1088 	int i;
1089 
1090 	p = device_get_softc(dev);
1091 	sc = p->adapter;
1092 
1093 	/* Tell cxgb_ioctl and if_init that the port is going away */
1094 	ADAPTER_LOCK(sc);
1095 	SET_DOOMED(p);
1096 	wakeup(&sc->flags);
1097 	while (IS_BUSY(sc))
1098 		mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1099 	SET_BUSY(sc);
1100 	ADAPTER_UNLOCK(sc);
1101 
1102 	if (p->port_cdev != NULL)
1103 		destroy_dev(p->port_cdev);
1104 
1105 	cxgb_uninit_synchronized(p);
1106 	ether_ifdetach(p->ifp);
1107 
1108 	for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1109 		struct sge_qset *qs = &sc->sge.qs[i];
1110 		struct sge_txq *txq = &qs->txq[TXQ_ETH];
1111 
1112 		callout_drain(&txq->txq_watchdog);
1113 		callout_drain(&txq->txq_timer);
1114 	}
1115 
1116 	PORT_LOCK_DEINIT(p);
1117 	if_free(p->ifp);
1118 	p->ifp = NULL;
1119 
1120 	ADAPTER_LOCK(sc);
1121 	CLR_BUSY(sc);
1122 	wakeup_one(&sc->flags);
1123 	ADAPTER_UNLOCK(sc);
1124 	return (0);
1125 }
1126 
1127 void
1128 t3_fatal_err(struct adapter *sc)
1129 {
1130 	u_int fw_status[4];
1131 
1132 	if (sc->flags & FULL_INIT_DONE) {
1133 		t3_sge_stop(sc);
1134 		t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1135 		t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1136 		t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1137 		t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1138 		t3_intr_disable(sc);
1139 	}
1140 	device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1141 	if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1142 		device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1143 		    fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1144 }
1145 
1146 int
1147 t3_os_find_pci_capability(adapter_t *sc, int cap)
1148 {
1149 	device_t dev;
1150 	struct pci_devinfo *dinfo;
1151 	pcicfgregs *cfg;
1152 	uint32_t status;
1153 	uint8_t ptr;
1154 
1155 	dev = sc->dev;
1156 	dinfo = device_get_ivars(dev);
1157 	cfg = &dinfo->cfg;
1158 
1159 	status = pci_read_config(dev, PCIR_STATUS, 2);
1160 	if (!(status & PCIM_STATUS_CAPPRESENT))
1161 		return (0);
1162 
1163 	switch (cfg->hdrtype & PCIM_HDRTYPE) {
1164 	case 0:
1165 	case 1:
1166 		ptr = PCIR_CAP_PTR;
1167 		break;
1168 	case 2:
1169 		ptr = PCIR_CAP_PTR_2;
1170 		break;
1171 	default:
1172 		return (0);
1173 		break;
1174 	}
1175 	ptr = pci_read_config(dev, ptr, 1);
1176 
1177 	while (ptr != 0) {
1178 		if (pci_read_config(dev, ptr + PCICAP_ID, 1) == cap)
1179 			return (ptr);
1180 		ptr = pci_read_config(dev, ptr + PCICAP_NEXTPTR, 1);
1181 	}
1182 
1183 	return (0);
1184 }
1185 
1186 int
1187 t3_os_pci_save_state(struct adapter *sc)
1188 {
1189 	device_t dev;
1190 	struct pci_devinfo *dinfo;
1191 
1192 	dev = sc->dev;
1193 	dinfo = device_get_ivars(dev);
1194 
1195 	pci_cfg_save(dev, dinfo, 0);
1196 	return (0);
1197 }
1198 
1199 int
1200 t3_os_pci_restore_state(struct adapter *sc)
1201 {
1202 	device_t dev;
1203 	struct pci_devinfo *dinfo;
1204 
1205 	dev = sc->dev;
1206 	dinfo = device_get_ivars(dev);
1207 
1208 	pci_cfg_restore(dev, dinfo);
1209 	return (0);
1210 }
1211 
1212 /**
1213  *	t3_os_link_changed - handle link status changes
1214  *	@sc: the adapter associated with the link change
1215  *	@port_id: the port index whose link status has changed
1216  *	@link_status: the new status of the link
1217  *	@speed: the new speed setting
1218  *	@duplex: the new duplex setting
1219  *	@fc: the new flow-control setting
1220  *
1221  *	This is the OS-dependent handler for link status changes.  The OS
1222  *	neutral handler takes care of most of the processing for these events,
1223  *	then calls this handler for any OS-specific processing.
1224  */
1225 void
1226 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1227      int duplex, int fc, int mac_was_reset)
1228 {
1229 	struct port_info *pi = &adapter->port[port_id];
1230 	struct ifnet *ifp = pi->ifp;
1231 
1232 	/* no race with detach, so ifp should always be good */
1233 	KASSERT(ifp, ("%s: if detached.", __func__));
1234 
1235 	/* Reapply mac settings if they were lost due to a reset */
1236 	if (mac_was_reset) {
1237 		PORT_LOCK(pi);
1238 		cxgb_update_mac_settings(pi);
1239 		PORT_UNLOCK(pi);
1240 	}
1241 
1242 	if (link_status) {
1243 		ifp->if_baudrate = IF_Mbps(speed);
1244 		if_link_state_change(ifp, LINK_STATE_UP);
1245 	} else
1246 		if_link_state_change(ifp, LINK_STATE_DOWN);
1247 }
1248 
1249 /**
1250  *	t3_os_phymod_changed - handle PHY module changes
1251  *	@phy: the PHY reporting the module change
1252  *	@mod_type: new module type
1253  *
1254  *	This is the OS-dependent handler for PHY module changes.  It is
1255  *	invoked when a PHY module is removed or inserted for any OS-specific
1256  *	processing.
1257  */
1258 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1259 {
1260 	static const char *mod_str[] = {
1261 		NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1262 	};
1263 	struct port_info *pi = &adap->port[port_id];
1264 	int mod = pi->phy.modtype;
1265 
1266 	if (mod != pi->media.ifm_cur->ifm_data)
1267 		cxgb_build_medialist(pi);
1268 
1269 	if (mod == phy_modtype_none)
1270 		if_printf(pi->ifp, "PHY module unplugged\n");
1271 	else {
1272 		KASSERT(mod < ARRAY_SIZE(mod_str),
1273 			("invalid PHY module type %d", mod));
1274 		if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1275 	}
1276 }
1277 
1278 void
1279 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1280 {
1281 
1282 	/*
1283 	 * The ifnet might not be allocated before this gets called,
1284 	 * as this is called early on in attach by t3_prep_adapter
1285 	 * save the address off in the port structure
1286 	 */
1287 	if (cxgb_debug)
1288 		printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1289 	bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1290 }
1291 
1292 /*
1293  * Programs the XGMAC based on the settings in the ifnet.  These settings
1294  * include MTU, MAC address, mcast addresses, etc.
1295  */
1296 static void
1297 cxgb_update_mac_settings(struct port_info *p)
1298 {
1299 	struct ifnet *ifp = p->ifp;
1300 	struct t3_rx_mode rm;
1301 	struct cmac *mac = &p->mac;
1302 	int mtu, hwtagging;
1303 
1304 	PORT_LOCK_ASSERT_OWNED(p);
1305 
1306 	bcopy(IF_LLADDR(ifp), p->hw_addr, ETHER_ADDR_LEN);
1307 
1308 	mtu = ifp->if_mtu;
1309 	if (ifp->if_capenable & IFCAP_VLAN_MTU)
1310 		mtu += ETHER_VLAN_ENCAP_LEN;
1311 
1312 	hwtagging = (ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0;
1313 
1314 	t3_mac_set_mtu(mac, mtu);
1315 	t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1316 	t3_mac_set_address(mac, 0, p->hw_addr);
1317 	t3_init_rx_mode(&rm, p);
1318 	t3_mac_set_rx_mode(mac, &rm);
1319 }
1320 
1321 
1322 static int
1323 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1324 			      unsigned long n)
1325 {
1326 	int attempts = 5;
1327 
1328 	while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1329 		if (!--attempts)
1330 			return (ETIMEDOUT);
1331 		t3_os_sleep(10);
1332 	}
1333 	return 0;
1334 }
1335 
1336 static int
1337 init_tp_parity(struct adapter *adap)
1338 {
1339 	int i;
1340 	struct mbuf *m;
1341 	struct cpl_set_tcb_field *greq;
1342 	unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1343 
1344 	t3_tp_set_offload_mode(adap, 1);
1345 
1346 	for (i = 0; i < 16; i++) {
1347 		struct cpl_smt_write_req *req;
1348 
1349 		m = m_gethdr(M_WAITOK, MT_DATA);
1350 		req = mtod(m, struct cpl_smt_write_req *);
1351 		m->m_len = m->m_pkthdr.len = sizeof(*req);
1352 		memset(req, 0, sizeof(*req));
1353 		req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1354 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1355 		req->iff = i;
1356 		t3_mgmt_tx(adap, m);
1357 	}
1358 
1359 	for (i = 0; i < 2048; i++) {
1360 		struct cpl_l2t_write_req *req;
1361 
1362 		m = m_gethdr(M_WAITOK, MT_DATA);
1363 		req = mtod(m, struct cpl_l2t_write_req *);
1364 		m->m_len = m->m_pkthdr.len = sizeof(*req);
1365 		memset(req, 0, sizeof(*req));
1366 		req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1367 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1368 		req->params = htonl(V_L2T_W_IDX(i));
1369 		t3_mgmt_tx(adap, m);
1370 	}
1371 
1372 	for (i = 0; i < 2048; i++) {
1373 		struct cpl_rte_write_req *req;
1374 
1375 		m = m_gethdr(M_WAITOK, MT_DATA);
1376 		req = mtod(m, struct cpl_rte_write_req *);
1377 		m->m_len = m->m_pkthdr.len = sizeof(*req);
1378 		memset(req, 0, sizeof(*req));
1379 		req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1380 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1381 		req->l2t_idx = htonl(V_L2T_W_IDX(i));
1382 		t3_mgmt_tx(adap, m);
1383 	}
1384 
1385 	m = m_gethdr(M_WAITOK, MT_DATA);
1386 	greq = mtod(m, struct cpl_set_tcb_field *);
1387 	m->m_len = m->m_pkthdr.len = sizeof(*greq);
1388 	memset(greq, 0, sizeof(*greq));
1389 	greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1390 	OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1391 	greq->mask = htobe64(1);
1392 	t3_mgmt_tx(adap, m);
1393 
1394 	i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1395 	t3_tp_set_offload_mode(adap, 0);
1396 	return (i);
1397 }
1398 
1399 /**
1400  *	setup_rss - configure Receive Side Steering (per-queue connection demux)
1401  *	@adap: the adapter
1402  *
1403  *	Sets up RSS to distribute packets to multiple receive queues.  We
1404  *	configure the RSS CPU lookup table to distribute to the number of HW
1405  *	receive queues, and the response queue lookup table to narrow that
1406  *	down to the response queues actually configured for each port.
1407  *	We always configure the RSS mapping for two ports since the mapping
1408  *	table has plenty of entries.
1409  */
1410 static void
1411 setup_rss(adapter_t *adap)
1412 {
1413 	int i;
1414 	u_int nq[2];
1415 	uint8_t cpus[SGE_QSETS + 1];
1416 	uint16_t rspq_map[RSS_TABLE_SIZE];
1417 
1418 	for (i = 0; i < SGE_QSETS; ++i)
1419 		cpus[i] = i;
1420 	cpus[SGE_QSETS] = 0xff;
1421 
1422 	nq[0] = nq[1] = 0;
1423 	for_each_port(adap, i) {
1424 		const struct port_info *pi = adap2pinfo(adap, i);
1425 
1426 		nq[pi->tx_chan] += pi->nqsets;
1427 	}
1428 	for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1429 		rspq_map[i] = nq[0] ? i % nq[0] : 0;
1430 		rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1431 	}
1432 
1433 	/* Calculate the reverse RSS map table */
1434 	for (i = 0; i < SGE_QSETS; ++i)
1435 		adap->rrss_map[i] = 0xff;
1436 	for (i = 0; i < RSS_TABLE_SIZE; ++i)
1437 		if (adap->rrss_map[rspq_map[i]] == 0xff)
1438 			adap->rrss_map[rspq_map[i]] = i;
1439 
1440 	t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1441 		      F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1442 	              F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1443 	              cpus, rspq_map);
1444 
1445 }
1446 static void
1447 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1448 			      int hi, int port)
1449 {
1450 	struct mbuf *m;
1451 	struct mngt_pktsched_wr *req;
1452 
1453 	m = m_gethdr(M_NOWAIT, MT_DATA);
1454 	if (m) {
1455 		req = mtod(m, struct mngt_pktsched_wr *);
1456 		req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1457 		req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1458 		req->sched = sched;
1459 		req->idx = qidx;
1460 		req->min = lo;
1461 		req->max = hi;
1462 		req->binding = port;
1463 		m->m_len = m->m_pkthdr.len = sizeof(*req);
1464 		t3_mgmt_tx(adap, m);
1465 	}
1466 }
1467 
1468 static void
1469 bind_qsets(adapter_t *sc)
1470 {
1471 	int i, j;
1472 
1473 	for (i = 0; i < (sc)->params.nports; ++i) {
1474 		const struct port_info *pi = adap2pinfo(sc, i);
1475 
1476 		for (j = 0; j < pi->nqsets; ++j) {
1477 			send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1478 					  -1, pi->tx_chan);
1479 
1480 		}
1481 	}
1482 }
1483 
1484 static void
1485 update_tpeeprom(struct adapter *adap)
1486 {
1487 	const struct firmware *tpeeprom;
1488 
1489 	uint32_t version;
1490 	unsigned int major, minor;
1491 	int ret, len;
1492 	char rev, name[32];
1493 
1494 	t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1495 
1496 	major = G_TP_VERSION_MAJOR(version);
1497 	minor = G_TP_VERSION_MINOR(version);
1498 	if (major == TP_VERSION_MAJOR  && minor == TP_VERSION_MINOR)
1499 		return;
1500 
1501 	rev = t3rev2char(adap);
1502 	snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1503 
1504 	tpeeprom = firmware_get(name);
1505 	if (tpeeprom == NULL) {
1506 		device_printf(adap->dev,
1507 			      "could not load TP EEPROM: unable to load %s\n",
1508 			      name);
1509 		return;
1510 	}
1511 
1512 	len = tpeeprom->datasize - 4;
1513 
1514 	ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1515 	if (ret)
1516 		goto release_tpeeprom;
1517 
1518 	if (len != TP_SRAM_LEN) {
1519 		device_printf(adap->dev,
1520 			      "%s length is wrong len=%d expected=%d\n", name,
1521 			      len, TP_SRAM_LEN);
1522 		return;
1523 	}
1524 
1525 	ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1526 	    TP_SRAM_OFFSET);
1527 
1528 	if (!ret) {
1529 		device_printf(adap->dev,
1530 			"Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1531 			 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1532 	} else
1533 		device_printf(adap->dev,
1534 			      "Protocol SRAM image update in EEPROM failed\n");
1535 
1536 release_tpeeprom:
1537 	firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1538 
1539 	return;
1540 }
1541 
1542 static int
1543 update_tpsram(struct adapter *adap)
1544 {
1545 	const struct firmware *tpsram;
1546 	int ret;
1547 	char rev, name[32];
1548 
1549 	rev = t3rev2char(adap);
1550 	snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1551 
1552 	update_tpeeprom(adap);
1553 
1554 	tpsram = firmware_get(name);
1555 	if (tpsram == NULL){
1556 		device_printf(adap->dev, "could not load TP SRAM\n");
1557 		return (EINVAL);
1558 	} else
1559 		device_printf(adap->dev, "updating TP SRAM\n");
1560 
1561 	ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1562 	if (ret)
1563 		goto release_tpsram;
1564 
1565 	ret = t3_set_proto_sram(adap, tpsram->data);
1566 	if (ret)
1567 		device_printf(adap->dev, "loading protocol SRAM failed\n");
1568 
1569 release_tpsram:
1570 	firmware_put(tpsram, FIRMWARE_UNLOAD);
1571 
1572 	return ret;
1573 }
1574 
1575 /**
1576  *	cxgb_up - enable the adapter
1577  *	@adap: adapter being enabled
1578  *
1579  *	Called when the first port is enabled, this function performs the
1580  *	actions necessary to make an adapter operational, such as completing
1581  *	the initialization of HW modules, and enabling interrupts.
1582  */
1583 static int
1584 cxgb_up(struct adapter *sc)
1585 {
1586 	int err = 0;
1587 	unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1588 
1589 	KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1590 					   __func__, sc->open_device_map));
1591 
1592 	if ((sc->flags & FULL_INIT_DONE) == 0) {
1593 
1594 		ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1595 
1596 		if ((sc->flags & FW_UPTODATE) == 0)
1597 			if ((err = upgrade_fw(sc)))
1598 				goto out;
1599 
1600 		if ((sc->flags & TPS_UPTODATE) == 0)
1601 			if ((err = update_tpsram(sc)))
1602 				goto out;
1603 
1604 		if (is_offload(sc) && nfilters != 0) {
1605 			sc->params.mc5.nservers = 0;
1606 
1607 			if (nfilters < 0)
1608 				sc->params.mc5.nfilters = mxf;
1609 			else
1610 				sc->params.mc5.nfilters = min(nfilters, mxf);
1611 		}
1612 
1613 		err = t3_init_hw(sc, 0);
1614 		if (err)
1615 			goto out;
1616 
1617 		t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1618 		t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1619 
1620 		err = setup_sge_qsets(sc);
1621 		if (err)
1622 			goto out;
1623 
1624 		alloc_filters(sc);
1625 		setup_rss(sc);
1626 
1627 		t3_add_configured_sysctls(sc);
1628 		sc->flags |= FULL_INIT_DONE;
1629 	}
1630 
1631 	t3_intr_clear(sc);
1632 	t3_sge_start(sc);
1633 	t3_intr_enable(sc);
1634 
1635 	if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1636 	    is_offload(sc) && init_tp_parity(sc) == 0)
1637 		sc->flags |= TP_PARITY_INIT;
1638 
1639 	if (sc->flags & TP_PARITY_INIT) {
1640 		t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1641 		t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1642 	}
1643 
1644 	if (!(sc->flags & QUEUES_BOUND)) {
1645 		bind_qsets(sc);
1646 		setup_hw_filters(sc);
1647 		sc->flags |= QUEUES_BOUND;
1648 	}
1649 
1650 	t3_sge_reset_adapter(sc);
1651 out:
1652 	return (err);
1653 }
1654 
1655 /*
1656  * Called when the last open device is closed.  Does NOT undo all of cxgb_up's
1657  * work.  Specifically, the resources grabbed under FULL_INIT_DONE are released
1658  * during controller_detach, not here.
1659  */
1660 static void
1661 cxgb_down(struct adapter *sc)
1662 {
1663 	t3_sge_stop(sc);
1664 	t3_intr_disable(sc);
1665 }
1666 
1667 /*
1668  * if_init for cxgb ports.
1669  */
1670 static void
1671 cxgb_init(void *arg)
1672 {
1673 	struct port_info *p = arg;
1674 	struct adapter *sc = p->adapter;
1675 
1676 	ADAPTER_LOCK(sc);
1677 	cxgb_init_locked(p); /* releases adapter lock */
1678 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1679 }
1680 
1681 static int
1682 cxgb_init_locked(struct port_info *p)
1683 {
1684 	struct adapter *sc = p->adapter;
1685 	struct ifnet *ifp = p->ifp;
1686 	struct cmac *mac = &p->mac;
1687 	int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1688 
1689 	ADAPTER_LOCK_ASSERT_OWNED(sc);
1690 
1691 	while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1692 		gave_up_lock = 1;
1693 		if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1694 			rc = EINTR;
1695 			goto done;
1696 		}
1697 	}
1698 	if (IS_DOOMED(p)) {
1699 		rc = ENXIO;
1700 		goto done;
1701 	}
1702 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1703 
1704 	/*
1705 	 * The code that runs during one-time adapter initialization can sleep
1706 	 * so it's important not to hold any locks across it.
1707 	 */
1708 	may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1709 
1710 	if (may_sleep) {
1711 		SET_BUSY(sc);
1712 		gave_up_lock = 1;
1713 		ADAPTER_UNLOCK(sc);
1714 	}
1715 
1716 	if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1717 			goto done;
1718 
1719 	PORT_LOCK(p);
1720 	if (isset(&sc->open_device_map, p->port_id) &&
1721 	    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
1722 		PORT_UNLOCK(p);
1723 		goto done;
1724 	}
1725 	t3_port_intr_enable(sc, p->port_id);
1726 	if (!mac->multiport)
1727 		t3_mac_init(mac);
1728 	cxgb_update_mac_settings(p);
1729 	t3_link_start(&p->phy, mac, &p->link_config);
1730 	t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1731 	ifp->if_drv_flags |= IFF_DRV_RUNNING;
1732 	ifp->if_drv_flags &= ~IFF_DRV_OACTIVE;
1733 	PORT_UNLOCK(p);
1734 
1735 	for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1736 		struct sge_qset *qs = &sc->sge.qs[i];
1737 		struct sge_txq *txq = &qs->txq[TXQ_ETH];
1738 
1739 		callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1740 				 txq->txq_watchdog.c_cpu);
1741 	}
1742 
1743 	/* all ok */
1744 	setbit(&sc->open_device_map, p->port_id);
1745 	callout_reset(&p->link_check_ch,
1746 	    p->phy.caps & SUPPORTED_LINK_IRQ ?  hz * 3 : hz / 4,
1747 	    link_check_callout, p);
1748 
1749 done:
1750 	if (may_sleep) {
1751 		ADAPTER_LOCK(sc);
1752 		KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1753 		CLR_BUSY(sc);
1754 	}
1755 	if (gave_up_lock)
1756 		wakeup_one(&sc->flags);
1757 	ADAPTER_UNLOCK(sc);
1758 	return (rc);
1759 }
1760 
1761 static int
1762 cxgb_uninit_locked(struct port_info *p)
1763 {
1764 	struct adapter *sc = p->adapter;
1765 	int rc;
1766 
1767 	ADAPTER_LOCK_ASSERT_OWNED(sc);
1768 
1769 	while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1770 		if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1771 			rc = EINTR;
1772 			goto done;
1773 		}
1774 	}
1775 	if (IS_DOOMED(p)) {
1776 		rc = ENXIO;
1777 		goto done;
1778 	}
1779 	KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1780 	SET_BUSY(sc);
1781 	ADAPTER_UNLOCK(sc);
1782 
1783 	rc = cxgb_uninit_synchronized(p);
1784 
1785 	ADAPTER_LOCK(sc);
1786 	KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1787 	CLR_BUSY(sc);
1788 	wakeup_one(&sc->flags);
1789 done:
1790 	ADAPTER_UNLOCK(sc);
1791 	return (rc);
1792 }
1793 
1794 /*
1795  * Called on "ifconfig down", and from port_detach
1796  */
1797 static int
1798 cxgb_uninit_synchronized(struct port_info *pi)
1799 {
1800 	struct adapter *sc = pi->adapter;
1801 	struct ifnet *ifp = pi->ifp;
1802 
1803 	/*
1804 	 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1805 	 */
1806 	ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1807 
1808 	/*
1809 	 * Clear this port's bit from the open device map, and then drain all
1810 	 * the tasks that can access/manipulate this port's port_info or ifp.
1811 	 * We disable this port's interrupts here and so the slow/ext
1812 	 * interrupt tasks won't be enqueued.  The tick task will continue to
1813 	 * be enqueued every second but the runs after this drain will not see
1814 	 * this port in the open device map.
1815 	 *
1816 	 * A well behaved task must take open_device_map into account and ignore
1817 	 * ports that are not open.
1818 	 */
1819 	clrbit(&sc->open_device_map, pi->port_id);
1820 	t3_port_intr_disable(sc, pi->port_id);
1821 	taskqueue_drain(sc->tq, &sc->slow_intr_task);
1822 	taskqueue_drain(sc->tq, &sc->tick_task);
1823 
1824 	callout_drain(&pi->link_check_ch);
1825 	taskqueue_drain(sc->tq, &pi->link_check_task);
1826 
1827 	PORT_LOCK(pi);
1828 	ifp->if_drv_flags &= ~(IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1829 
1830 	/* disable pause frames */
1831 	t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1832 
1833 	/* Reset RX FIFO HWM */
1834 	t3_set_reg_field(sc, A_XGM_RXFIFO_CFG +  pi->mac.offset,
1835 			 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1836 
1837 	DELAY(100 * 1000);
1838 
1839 	/* Wait for TXFIFO empty */
1840 	t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1841 			F_TXFIFO_EMPTY, 1, 20, 5);
1842 
1843 	DELAY(100 * 1000);
1844 	t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1845 
1846 	pi->phy.ops->power_down(&pi->phy, 1);
1847 
1848 	PORT_UNLOCK(pi);
1849 
1850 	pi->link_config.link_ok = 0;
1851 	t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1852 
1853 	if (sc->open_device_map == 0)
1854 		cxgb_down(pi->adapter);
1855 
1856 	return (0);
1857 }
1858 
1859 /*
1860  * Mark lro enabled or disabled in all qsets for this port
1861  */
1862 static int
1863 cxgb_set_lro(struct port_info *p, int enabled)
1864 {
1865 	int i;
1866 	struct adapter *adp = p->adapter;
1867 	struct sge_qset *q;
1868 
1869 	for (i = 0; i < p->nqsets; i++) {
1870 		q = &adp->sge.qs[p->first_qset + i];
1871 		q->lro.enabled = (enabled != 0);
1872 	}
1873 	return (0);
1874 }
1875 
1876 static int
1877 cxgb_ioctl(struct ifnet *ifp, unsigned long command, caddr_t data)
1878 {
1879 	struct port_info *p = ifp->if_softc;
1880 	struct adapter *sc = p->adapter;
1881 	struct ifreq *ifr = (struct ifreq *)data;
1882 	int flags, error = 0, mtu;
1883 	uint32_t mask;
1884 
1885 	switch (command) {
1886 	case SIOCSIFMTU:
1887 		ADAPTER_LOCK(sc);
1888 		error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1889 		if (error) {
1890 fail:
1891 			ADAPTER_UNLOCK(sc);
1892 			return (error);
1893 		}
1894 
1895 		mtu = ifr->ifr_mtu;
1896 		if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1897 			error = EINVAL;
1898 		} else {
1899 			ifp->if_mtu = mtu;
1900 			PORT_LOCK(p);
1901 			cxgb_update_mac_settings(p);
1902 			PORT_UNLOCK(p);
1903 		}
1904 		ADAPTER_UNLOCK(sc);
1905 		break;
1906 	case SIOCSIFFLAGS:
1907 		ADAPTER_LOCK(sc);
1908 		if (IS_DOOMED(p)) {
1909 			error = ENXIO;
1910 			goto fail;
1911 		}
1912 		if (ifp->if_flags & IFF_UP) {
1913 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1914 				flags = p->if_flags;
1915 				if (((ifp->if_flags ^ flags) & IFF_PROMISC) ||
1916 				    ((ifp->if_flags ^ flags) & IFF_ALLMULTI)) {
1917 					if (IS_BUSY(sc)) {
1918 						error = EBUSY;
1919 						goto fail;
1920 					}
1921 					PORT_LOCK(p);
1922 					cxgb_update_mac_settings(p);
1923 					PORT_UNLOCK(p);
1924 				}
1925 				ADAPTER_UNLOCK(sc);
1926 			} else
1927 				error = cxgb_init_locked(p);
1928 			p->if_flags = ifp->if_flags;
1929 		} else if (ifp->if_drv_flags & IFF_DRV_RUNNING)
1930 			error = cxgb_uninit_locked(p);
1931 		else
1932 			ADAPTER_UNLOCK(sc);
1933 
1934 		ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1935 		break;
1936 	case SIOCADDMULTI:
1937 	case SIOCDELMULTI:
1938 		ADAPTER_LOCK(sc);
1939 		error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1940 		if (error)
1941 			goto fail;
1942 
1943 		if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
1944 			PORT_LOCK(p);
1945 			cxgb_update_mac_settings(p);
1946 			PORT_UNLOCK(p);
1947 		}
1948 		ADAPTER_UNLOCK(sc);
1949 
1950 		break;
1951 	case SIOCSIFCAP:
1952 		ADAPTER_LOCK(sc);
1953 		error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1954 		if (error)
1955 			goto fail;
1956 
1957 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
1958 		if (mask & IFCAP_TXCSUM) {
1959 			ifp->if_capenable ^= IFCAP_TXCSUM;
1960 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
1961 
1962 			if (IFCAP_TSO4 & ifp->if_capenable &&
1963 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1964 				ifp->if_capenable &= ~IFCAP_TSO4;
1965 				if_printf(ifp,
1966 				    "tso4 disabled due to -txcsum.\n");
1967 			}
1968 		}
1969 		if (mask & IFCAP_TXCSUM_IPV6) {
1970 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
1971 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1972 
1973 			if (IFCAP_TSO6 & ifp->if_capenable &&
1974 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
1975 				ifp->if_capenable &= ~IFCAP_TSO6;
1976 				if_printf(ifp,
1977 				    "tso6 disabled due to -txcsum6.\n");
1978 			}
1979 		}
1980 		if (mask & IFCAP_RXCSUM)
1981 			ifp->if_capenable ^= IFCAP_RXCSUM;
1982 		if (mask & IFCAP_RXCSUM_IPV6)
1983 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
1984 
1985 		/*
1986 		 * Note that we leave CSUM_TSO alone (it is always set).  The
1987 		 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1988 		 * sending a TSO request our way, so it's sufficient to toggle
1989 		 * IFCAP_TSOx only.
1990 		 */
1991 		if (mask & IFCAP_TSO4) {
1992 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
1993 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
1994 				if_printf(ifp, "enable txcsum first.\n");
1995 				error = EAGAIN;
1996 				goto fail;
1997 			}
1998 			ifp->if_capenable ^= IFCAP_TSO4;
1999 		}
2000 		if (mask & IFCAP_TSO6) {
2001 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
2002 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
2003 				if_printf(ifp, "enable txcsum6 first.\n");
2004 				error = EAGAIN;
2005 				goto fail;
2006 			}
2007 			ifp->if_capenable ^= IFCAP_TSO6;
2008 		}
2009 		if (mask & IFCAP_LRO) {
2010 			ifp->if_capenable ^= IFCAP_LRO;
2011 
2012 			/* Safe to do this even if cxgb_up not called yet */
2013 			cxgb_set_lro(p, ifp->if_capenable & IFCAP_LRO);
2014 		}
2015 #ifdef TCP_OFFLOAD
2016 		if (mask & IFCAP_TOE4) {
2017 			int enable = (ifp->if_capenable ^ mask) & IFCAP_TOE4;
2018 
2019 			error = toe_capability(p, enable);
2020 			if (error == 0)
2021 				ifp->if_capenable ^= mask;
2022 		}
2023 #endif
2024 		if (mask & IFCAP_VLAN_HWTAGGING) {
2025 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
2026 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2027 				PORT_LOCK(p);
2028 				cxgb_update_mac_settings(p);
2029 				PORT_UNLOCK(p);
2030 			}
2031 		}
2032 		if (mask & IFCAP_VLAN_MTU) {
2033 			ifp->if_capenable ^= IFCAP_VLAN_MTU;
2034 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
2035 				PORT_LOCK(p);
2036 				cxgb_update_mac_settings(p);
2037 				PORT_UNLOCK(p);
2038 			}
2039 		}
2040 		if (mask & IFCAP_VLAN_HWTSO)
2041 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
2042 		if (mask & IFCAP_VLAN_HWCSUM)
2043 			ifp->if_capenable ^= IFCAP_VLAN_HWCSUM;
2044 
2045 #ifdef VLAN_CAPABILITIES
2046 		VLAN_CAPABILITIES(ifp);
2047 #endif
2048 		ADAPTER_UNLOCK(sc);
2049 		break;
2050 	case SIOCSIFMEDIA:
2051 	case SIOCGIFMEDIA:
2052 		error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2053 		break;
2054 	default:
2055 		error = ether_ioctl(ifp, command, data);
2056 	}
2057 
2058 	return (error);
2059 }
2060 
2061 static int
2062 cxgb_media_change(struct ifnet *ifp)
2063 {
2064 	return (EOPNOTSUPP);
2065 }
2066 
2067 /*
2068  * Translates phy->modtype to the correct Ethernet media subtype.
2069  */
2070 static int
2071 cxgb_ifm_type(int mod)
2072 {
2073 	switch (mod) {
2074 	case phy_modtype_sr:
2075 		return (IFM_10G_SR);
2076 	case phy_modtype_lr:
2077 		return (IFM_10G_LR);
2078 	case phy_modtype_lrm:
2079 		return (IFM_10G_LRM);
2080 	case phy_modtype_twinax:
2081 		return (IFM_10G_TWINAX);
2082 	case phy_modtype_twinax_long:
2083 		return (IFM_10G_TWINAX_LONG);
2084 	case phy_modtype_none:
2085 		return (IFM_NONE);
2086 	case phy_modtype_unknown:
2087 		return (IFM_UNKNOWN);
2088 	}
2089 
2090 	KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2091 	return (IFM_UNKNOWN);
2092 }
2093 
2094 /*
2095  * Rebuilds the ifmedia list for this port, and sets the current media.
2096  */
2097 static void
2098 cxgb_build_medialist(struct port_info *p)
2099 {
2100 	struct cphy *phy = &p->phy;
2101 	struct ifmedia *media = &p->media;
2102 	int mod = phy->modtype;
2103 	int m = IFM_ETHER | IFM_FDX;
2104 
2105 	PORT_LOCK(p);
2106 
2107 	ifmedia_removeall(media);
2108 	if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2109 		/* Copper (RJ45) */
2110 
2111 		if (phy->caps & SUPPORTED_10000baseT_Full)
2112 			ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2113 
2114 		if (phy->caps & SUPPORTED_1000baseT_Full)
2115 			ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2116 
2117 		if (phy->caps & SUPPORTED_100baseT_Full)
2118 			ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2119 
2120 		if (phy->caps & SUPPORTED_10baseT_Full)
2121 			ifmedia_add(media, m | IFM_10_T, mod, NULL);
2122 
2123 		ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2124 		ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2125 
2126 	} else if (phy->caps & SUPPORTED_TP) {
2127 		/* Copper (CX4) */
2128 
2129 		KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2130 			("%s: unexpected cap 0x%x", __func__, phy->caps));
2131 
2132 		ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2133 		ifmedia_set(media, m | IFM_10G_CX4);
2134 
2135 	} else if (phy->caps & SUPPORTED_FIBRE &&
2136 		   phy->caps & SUPPORTED_10000baseT_Full) {
2137 		/* 10G optical (but includes SFP+ twinax) */
2138 
2139 		m |= cxgb_ifm_type(mod);
2140 		if (IFM_SUBTYPE(m) == IFM_NONE)
2141 			m &= ~IFM_FDX;
2142 
2143 		ifmedia_add(media, m, mod, NULL);
2144 		ifmedia_set(media, m);
2145 
2146 	} else if (phy->caps & SUPPORTED_FIBRE &&
2147 		   phy->caps & SUPPORTED_1000baseT_Full) {
2148 		/* 1G optical */
2149 
2150 		/* XXX: Lie and claim to be SX, could actually be any 1G-X */
2151 		ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2152 		ifmedia_set(media, m | IFM_1000_SX);
2153 
2154 	} else {
2155 		KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2156 			    phy->caps));
2157 	}
2158 
2159 	PORT_UNLOCK(p);
2160 }
2161 
2162 static void
2163 cxgb_media_status(struct ifnet *ifp, struct ifmediareq *ifmr)
2164 {
2165 	struct port_info *p = ifp->if_softc;
2166 	struct ifmedia_entry *cur = p->media.ifm_cur;
2167 	int speed = p->link_config.speed;
2168 
2169 	if (cur->ifm_data != p->phy.modtype) {
2170 		cxgb_build_medialist(p);
2171 		cur = p->media.ifm_cur;
2172 	}
2173 
2174 	ifmr->ifm_status = IFM_AVALID;
2175 	if (!p->link_config.link_ok)
2176 		return;
2177 
2178 	ifmr->ifm_status |= IFM_ACTIVE;
2179 
2180 	/*
2181 	 * active and current will differ iff current media is autoselect.  That
2182 	 * can happen only for copper RJ45.
2183 	 */
2184 	if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2185 		return;
2186 	KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2187 		("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2188 
2189 	ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2190 	if (speed == SPEED_10000)
2191 		ifmr->ifm_active |= IFM_10G_T;
2192 	else if (speed == SPEED_1000)
2193 		ifmr->ifm_active |= IFM_1000_T;
2194 	else if (speed == SPEED_100)
2195 		ifmr->ifm_active |= IFM_100_TX;
2196 	else if (speed == SPEED_10)
2197 		ifmr->ifm_active |= IFM_10_T;
2198 	else
2199 		KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2200 			    speed));
2201 }
2202 
2203 static uint64_t
2204 cxgb_get_counter(struct ifnet *ifp, ift_counter c)
2205 {
2206 	struct port_info *pi = ifp->if_softc;
2207 	struct adapter *sc = pi->adapter;
2208 	struct cmac *mac = &pi->mac;
2209 	struct mac_stats *mstats = &mac->stats;
2210 
2211 	cxgb_refresh_stats(pi);
2212 
2213 	switch (c) {
2214 	case IFCOUNTER_IPACKETS:
2215 		return (mstats->rx_frames);
2216 
2217 	case IFCOUNTER_IERRORS:
2218 		return (mstats->rx_jabber + mstats->rx_data_errs +
2219 		    mstats->rx_sequence_errs + mstats->rx_runt +
2220 		    mstats->rx_too_long + mstats->rx_mac_internal_errs +
2221 		    mstats->rx_short + mstats->rx_fcs_errs);
2222 
2223 	case IFCOUNTER_OPACKETS:
2224 		return (mstats->tx_frames);
2225 
2226 	case IFCOUNTER_OERRORS:
2227 		return (mstats->tx_excess_collisions + mstats->tx_underrun +
2228 		    mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2229 		    mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2230 
2231 	case IFCOUNTER_COLLISIONS:
2232 		return (mstats->tx_total_collisions);
2233 
2234 	case IFCOUNTER_IBYTES:
2235 		return (mstats->rx_octets);
2236 
2237 	case IFCOUNTER_OBYTES:
2238 		return (mstats->tx_octets);
2239 
2240 	case IFCOUNTER_IMCASTS:
2241 		return (mstats->rx_mcast_frames);
2242 
2243 	case IFCOUNTER_OMCASTS:
2244 		return (mstats->tx_mcast_frames);
2245 
2246 	case IFCOUNTER_IQDROPS:
2247 		return (mstats->rx_cong_drops);
2248 
2249 	case IFCOUNTER_OQDROPS: {
2250 		int i;
2251 		uint64_t drops;
2252 
2253 		drops = 0;
2254 		if (sc->flags & FULL_INIT_DONE) {
2255 			for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2256 				drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2257 		}
2258 
2259 		return (drops);
2260 
2261 	}
2262 
2263 	default:
2264 		return (if_get_counter_default(ifp, c));
2265 	}
2266 }
2267 
2268 static void
2269 cxgb_async_intr(void *data)
2270 {
2271 	adapter_t *sc = data;
2272 
2273 	t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2274 	(void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2275 	taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2276 }
2277 
2278 static void
2279 link_check_callout(void *arg)
2280 {
2281 	struct port_info *pi = arg;
2282 	struct adapter *sc = pi->adapter;
2283 
2284 	if (!isset(&sc->open_device_map, pi->port_id))
2285 		return;
2286 
2287 	taskqueue_enqueue(sc->tq, &pi->link_check_task);
2288 }
2289 
2290 static void
2291 check_link_status(void *arg, int pending)
2292 {
2293 	struct port_info *pi = arg;
2294 	struct adapter *sc = pi->adapter;
2295 
2296 	if (!isset(&sc->open_device_map, pi->port_id))
2297 		return;
2298 
2299 	t3_link_changed(sc, pi->port_id);
2300 
2301 	if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2302 	    pi->link_config.link_ok == 0)
2303 		callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2304 }
2305 
2306 void
2307 t3_os_link_intr(struct port_info *pi)
2308 {
2309 	/*
2310 	 * Schedule a link check in the near future.  If the link is flapping
2311 	 * rapidly we'll keep resetting the callout and delaying the check until
2312 	 * things stabilize a bit.
2313 	 */
2314 	callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2315 }
2316 
2317 static void
2318 check_t3b2_mac(struct adapter *sc)
2319 {
2320 	int i;
2321 
2322 	if (sc->flags & CXGB_SHUTDOWN)
2323 		return;
2324 
2325 	for_each_port(sc, i) {
2326 		struct port_info *p = &sc->port[i];
2327 		int status;
2328 #ifdef INVARIANTS
2329 		struct ifnet *ifp = p->ifp;
2330 #endif
2331 
2332 		if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2333 		    !p->link_config.link_ok)
2334 			continue;
2335 
2336 		KASSERT(ifp->if_drv_flags & IFF_DRV_RUNNING,
2337 			("%s: state mismatch (drv_flags %x, device_map %x)",
2338 			 __func__, ifp->if_drv_flags, sc->open_device_map));
2339 
2340 		PORT_LOCK(p);
2341 		status = t3b2_mac_watchdog_task(&p->mac);
2342 		if (status == 1)
2343 			p->mac.stats.num_toggled++;
2344 		else if (status == 2) {
2345 			struct cmac *mac = &p->mac;
2346 
2347 			cxgb_update_mac_settings(p);
2348 			t3_link_start(&p->phy, mac, &p->link_config);
2349 			t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2350 			t3_port_intr_enable(sc, p->port_id);
2351 			p->mac.stats.num_resets++;
2352 		}
2353 		PORT_UNLOCK(p);
2354 	}
2355 }
2356 
2357 static void
2358 cxgb_tick(void *arg)
2359 {
2360 	adapter_t *sc = (adapter_t *)arg;
2361 
2362 	if (sc->flags & CXGB_SHUTDOWN)
2363 		return;
2364 
2365 	taskqueue_enqueue(sc->tq, &sc->tick_task);
2366 	callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2367 }
2368 
2369 void
2370 cxgb_refresh_stats(struct port_info *pi)
2371 {
2372 	struct timeval tv;
2373 	const struct timeval interval = {0, 250000};    /* 250ms */
2374 
2375 	getmicrotime(&tv);
2376 	timevalsub(&tv, &interval);
2377 	if (timevalcmp(&tv, &pi->last_refreshed, <))
2378 		return;
2379 
2380 	PORT_LOCK(pi);
2381 	t3_mac_update_stats(&pi->mac);
2382 	PORT_UNLOCK(pi);
2383 	getmicrotime(&pi->last_refreshed);
2384 }
2385 
2386 static void
2387 cxgb_tick_handler(void *arg, int count)
2388 {
2389 	adapter_t *sc = (adapter_t *)arg;
2390 	const struct adapter_params *p = &sc->params;
2391 	int i;
2392 	uint32_t cause, reset;
2393 
2394 	if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2395 		return;
2396 
2397 	if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2398 		check_t3b2_mac(sc);
2399 
2400 	cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2401 	if (cause) {
2402 		struct sge_qset *qs = &sc->sge.qs[0];
2403 		uint32_t mask, v;
2404 
2405 		v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2406 
2407 		mask = 1;
2408 		for (i = 0; i < SGE_QSETS; i++) {
2409 			if (v & mask)
2410 				qs[i].rspq.starved++;
2411 			mask <<= 1;
2412 		}
2413 
2414 		mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2415 
2416 		for (i = 0; i < SGE_QSETS * 2; i++) {
2417 			if (v & mask) {
2418 				qs[i / 2].fl[i % 2].empty++;
2419 			}
2420 			mask <<= 1;
2421 		}
2422 
2423 		/* clear */
2424 		t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2425 		t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2426 	}
2427 
2428 	for (i = 0; i < sc->params.nports; i++) {
2429 		struct port_info *pi = &sc->port[i];
2430 		struct cmac *mac = &pi->mac;
2431 
2432 		if (!isset(&sc->open_device_map, pi->port_id))
2433 			continue;
2434 
2435 		cxgb_refresh_stats(pi);
2436 
2437 		if (mac->multiport)
2438 			continue;
2439 
2440 		/* Count rx fifo overflows, once per second */
2441 		cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2442 		reset = 0;
2443 		if (cause & F_RXFIFO_OVERFLOW) {
2444 			mac->stats.rx_fifo_ovfl++;
2445 			reset |= F_RXFIFO_OVERFLOW;
2446 		}
2447 		t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2448 	}
2449 }
2450 
2451 static void
2452 touch_bars(device_t dev)
2453 {
2454 	/*
2455 	 * Don't enable yet
2456 	 */
2457 #if !defined(__LP64__) && 0
2458 	u32 v;
2459 
2460 	pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2461 	pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2462 	pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2463 	pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2464 	pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2465 	pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2466 #endif
2467 }
2468 
2469 static int
2470 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2471 {
2472 	uint8_t *buf;
2473 	int err = 0;
2474 	u32 aligned_offset, aligned_len, *p;
2475 	struct adapter *adapter = pi->adapter;
2476 
2477 
2478 	aligned_offset = offset & ~3;
2479 	aligned_len = (len + (offset & 3) + 3) & ~3;
2480 
2481 	if (aligned_offset != offset || aligned_len != len) {
2482 		buf = malloc(aligned_len, M_DEVBUF, M_WAITOK|M_ZERO);
2483 		if (!buf)
2484 			return (ENOMEM);
2485 		err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2486 		if (!err && aligned_len > 4)
2487 			err = t3_seeprom_read(adapter,
2488 					      aligned_offset + aligned_len - 4,
2489 					      (u32 *)&buf[aligned_len - 4]);
2490 		if (err)
2491 			goto out;
2492 		memcpy(buf + (offset & 3), data, len);
2493 	} else
2494 		buf = (uint8_t *)(uintptr_t)data;
2495 
2496 	err = t3_seeprom_wp(adapter, 0);
2497 	if (err)
2498 		goto out;
2499 
2500 	for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2501 		err = t3_seeprom_write(adapter, aligned_offset, *p);
2502 		aligned_offset += 4;
2503 	}
2504 
2505 	if (!err)
2506 		err = t3_seeprom_wp(adapter, 1);
2507 out:
2508 	if (buf != data)
2509 		free(buf, M_DEVBUF);
2510 	return err;
2511 }
2512 
2513 
2514 static int
2515 in_range(int val, int lo, int hi)
2516 {
2517 	return val < 0 || (val <= hi && val >= lo);
2518 }
2519 
2520 static int
2521 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2522 {
2523        return (0);
2524 }
2525 
2526 static int
2527 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2528 {
2529        return (0);
2530 }
2531 
2532 static int
2533 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2534     int fflag, struct thread *td)
2535 {
2536 	int mmd, error = 0;
2537 	struct port_info *pi = dev->si_drv1;
2538 	adapter_t *sc = pi->adapter;
2539 
2540 #ifdef PRIV_SUPPORTED
2541 	if (priv_check(td, PRIV_DRIVER)) {
2542 		if (cxgb_debug)
2543 			printf("user does not have access to privileged ioctls\n");
2544 		return (EPERM);
2545 	}
2546 #else
2547 	if (suser(td)) {
2548 		if (cxgb_debug)
2549 			printf("user does not have access to privileged ioctls\n");
2550 		return (EPERM);
2551 	}
2552 #endif
2553 
2554 	switch (cmd) {
2555 	case CHELSIO_GET_MIIREG: {
2556 		uint32_t val;
2557 		struct cphy *phy = &pi->phy;
2558 		struct ch_mii_data *mid = (struct ch_mii_data *)data;
2559 
2560 		if (!phy->mdio_read)
2561 			return (EOPNOTSUPP);
2562 		if (is_10G(sc)) {
2563 			mmd = mid->phy_id >> 8;
2564 			if (!mmd)
2565 				mmd = MDIO_DEV_PCS;
2566 			else if (mmd > MDIO_DEV_VEND2)
2567 				return (EINVAL);
2568 
2569 			error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2570 					     mid->reg_num, &val);
2571 		} else
2572 		        error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2573 					     mid->reg_num & 0x1f, &val);
2574 		if (error == 0)
2575 			mid->val_out = val;
2576 		break;
2577 	}
2578 	case CHELSIO_SET_MIIREG: {
2579 		struct cphy *phy = &pi->phy;
2580 		struct ch_mii_data *mid = (struct ch_mii_data *)data;
2581 
2582 		if (!phy->mdio_write)
2583 			return (EOPNOTSUPP);
2584 		if (is_10G(sc)) {
2585 			mmd = mid->phy_id >> 8;
2586 			if (!mmd)
2587 				mmd = MDIO_DEV_PCS;
2588 			else if (mmd > MDIO_DEV_VEND2)
2589 				return (EINVAL);
2590 
2591 			error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2592 					      mmd, mid->reg_num, mid->val_in);
2593 		} else
2594 			error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2595 					      mid->reg_num & 0x1f,
2596 					      mid->val_in);
2597 		break;
2598 	}
2599 	case CHELSIO_SETREG: {
2600 		struct ch_reg *edata = (struct ch_reg *)data;
2601 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2602 			return (EFAULT);
2603 		t3_write_reg(sc, edata->addr, edata->val);
2604 		break;
2605 	}
2606 	case CHELSIO_GETREG: {
2607 		struct ch_reg *edata = (struct ch_reg *)data;
2608 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2609 			return (EFAULT);
2610 		edata->val = t3_read_reg(sc, edata->addr);
2611 		break;
2612 	}
2613 	case CHELSIO_GET_SGE_CONTEXT: {
2614 		struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2615 		mtx_lock_spin(&sc->sge.reg_lock);
2616 		switch (ecntxt->cntxt_type) {
2617 		case CNTXT_TYPE_EGRESS:
2618 			error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2619 			    ecntxt->data);
2620 			break;
2621 		case CNTXT_TYPE_FL:
2622 			error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2623 			    ecntxt->data);
2624 			break;
2625 		case CNTXT_TYPE_RSP:
2626 			error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2627 			    ecntxt->data);
2628 			break;
2629 		case CNTXT_TYPE_CQ:
2630 			error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2631 			    ecntxt->data);
2632 			break;
2633 		default:
2634 			error = EINVAL;
2635 			break;
2636 		}
2637 		mtx_unlock_spin(&sc->sge.reg_lock);
2638 		break;
2639 	}
2640 	case CHELSIO_GET_SGE_DESC: {
2641 		struct ch_desc *edesc = (struct ch_desc *)data;
2642 		int ret;
2643 		if (edesc->queue_num >= SGE_QSETS * 6)
2644 			return (EINVAL);
2645 		ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2646 		    edesc->queue_num % 6, edesc->idx, edesc->data);
2647 		if (ret < 0)
2648 			return (EINVAL);
2649 		edesc->size = ret;
2650 		break;
2651 	}
2652 	case CHELSIO_GET_QSET_PARAMS: {
2653 		struct qset_params *q;
2654 		struct ch_qset_params *t = (struct ch_qset_params *)data;
2655 		int q1 = pi->first_qset;
2656 		int nqsets = pi->nqsets;
2657 		int i;
2658 
2659 		if (t->qset_idx >= nqsets)
2660 			return EINVAL;
2661 
2662 		i = q1 + t->qset_idx;
2663 		q = &sc->params.sge.qset[i];
2664 		t->rspq_size   = q->rspq_size;
2665 		t->txq_size[0] = q->txq_size[0];
2666 		t->txq_size[1] = q->txq_size[1];
2667 		t->txq_size[2] = q->txq_size[2];
2668 		t->fl_size[0]  = q->fl_size;
2669 		t->fl_size[1]  = q->jumbo_size;
2670 		t->polling     = q->polling;
2671 		t->lro         = q->lro;
2672 		t->intr_lat    = q->coalesce_usecs;
2673 		t->cong_thres  = q->cong_thres;
2674 		t->qnum        = i;
2675 
2676 		if ((sc->flags & FULL_INIT_DONE) == 0)
2677 			t->vector = 0;
2678 		else if (sc->flags & USING_MSIX)
2679 			t->vector = rman_get_start(sc->msix_irq_res[i]);
2680 		else
2681 			t->vector = rman_get_start(sc->irq_res);
2682 
2683 		break;
2684 	}
2685 	case CHELSIO_GET_QSET_NUM: {
2686 		struct ch_reg *edata = (struct ch_reg *)data;
2687 		edata->val = pi->nqsets;
2688 		break;
2689 	}
2690 	case CHELSIO_LOAD_FW: {
2691 		uint8_t *fw_data;
2692 		uint32_t vers;
2693 		struct ch_mem_range *t = (struct ch_mem_range *)data;
2694 
2695 		/*
2696 		 * You're allowed to load a firmware only before FULL_INIT_DONE
2697 		 *
2698 		 * FW_UPTODATE is also set so the rest of the initialization
2699 		 * will not overwrite what was loaded here.  This gives you the
2700 		 * flexibility to load any firmware (and maybe shoot yourself in
2701 		 * the foot).
2702 		 */
2703 
2704 		ADAPTER_LOCK(sc);
2705 		if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2706 			ADAPTER_UNLOCK(sc);
2707 			return (EBUSY);
2708 		}
2709 
2710 		fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2711 		if (!fw_data)
2712 			error = ENOMEM;
2713 		else
2714 			error = copyin(t->buf, fw_data, t->len);
2715 
2716 		if (!error)
2717 			error = -t3_load_fw(sc, fw_data, t->len);
2718 
2719 		if (t3_get_fw_version(sc, &vers) == 0) {
2720 			snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2721 			    "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2722 			    G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2723 		}
2724 
2725 		if (!error)
2726 			sc->flags |= FW_UPTODATE;
2727 
2728 		free(fw_data, M_DEVBUF);
2729 		ADAPTER_UNLOCK(sc);
2730 		break;
2731 	}
2732 	case CHELSIO_LOAD_BOOT: {
2733 		uint8_t *boot_data;
2734 		struct ch_mem_range *t = (struct ch_mem_range *)data;
2735 
2736 		boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2737 		if (!boot_data)
2738 			return ENOMEM;
2739 
2740 		error = copyin(t->buf, boot_data, t->len);
2741 		if (!error)
2742 			error = -t3_load_boot(sc, boot_data, t->len);
2743 
2744 		free(boot_data, M_DEVBUF);
2745 		break;
2746 	}
2747 	case CHELSIO_GET_PM: {
2748 		struct ch_pm *m = (struct ch_pm *)data;
2749 		struct tp_params *p = &sc->params.tp;
2750 
2751 		if (!is_offload(sc))
2752 			return (EOPNOTSUPP);
2753 
2754 		m->tx_pg_sz = p->tx_pg_size;
2755 		m->tx_num_pg = p->tx_num_pgs;
2756 		m->rx_pg_sz  = p->rx_pg_size;
2757 		m->rx_num_pg = p->rx_num_pgs;
2758 		m->pm_total  = p->pmtx_size + p->chan_rx_size * p->nchan;
2759 
2760 		break;
2761 	}
2762 	case CHELSIO_SET_PM: {
2763 		struct ch_pm *m = (struct ch_pm *)data;
2764 		struct tp_params *p = &sc->params.tp;
2765 
2766 		if (!is_offload(sc))
2767 			return (EOPNOTSUPP);
2768 		if (sc->flags & FULL_INIT_DONE)
2769 			return (EBUSY);
2770 
2771 		if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2772 		    !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2773 			return (EINVAL);	/* not power of 2 */
2774 		if (!(m->rx_pg_sz & 0x14000))
2775 			return (EINVAL);	/* not 16KB or 64KB */
2776 		if (!(m->tx_pg_sz & 0x1554000))
2777 			return (EINVAL);
2778 		if (m->tx_num_pg == -1)
2779 			m->tx_num_pg = p->tx_num_pgs;
2780 		if (m->rx_num_pg == -1)
2781 			m->rx_num_pg = p->rx_num_pgs;
2782 		if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2783 			return (EINVAL);
2784 		if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2785 		    m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2786 			return (EINVAL);
2787 
2788 		p->rx_pg_size = m->rx_pg_sz;
2789 		p->tx_pg_size = m->tx_pg_sz;
2790 		p->rx_num_pgs = m->rx_num_pg;
2791 		p->tx_num_pgs = m->tx_num_pg;
2792 		break;
2793 	}
2794 	case CHELSIO_SETMTUTAB: {
2795 		struct ch_mtus *m = (struct ch_mtus *)data;
2796 		int i;
2797 
2798 		if (!is_offload(sc))
2799 			return (EOPNOTSUPP);
2800 		if (offload_running(sc))
2801 			return (EBUSY);
2802 		if (m->nmtus != NMTUS)
2803 			return (EINVAL);
2804 		if (m->mtus[0] < 81)         /* accommodate SACK */
2805 			return (EINVAL);
2806 
2807 		/*
2808 		 * MTUs must be in ascending order
2809 		 */
2810 		for (i = 1; i < NMTUS; ++i)
2811 			if (m->mtus[i] < m->mtus[i - 1])
2812 				return (EINVAL);
2813 
2814 		memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2815 		break;
2816 	}
2817 	case CHELSIO_GETMTUTAB: {
2818 		struct ch_mtus *m = (struct ch_mtus *)data;
2819 
2820 		if (!is_offload(sc))
2821 			return (EOPNOTSUPP);
2822 
2823 		memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2824 		m->nmtus = NMTUS;
2825 		break;
2826 	}
2827 	case CHELSIO_GET_MEM: {
2828 		struct ch_mem_range *t = (struct ch_mem_range *)data;
2829 		struct mc7 *mem;
2830 		uint8_t *useraddr;
2831 		u64 buf[32];
2832 
2833 		/*
2834 		 * Use these to avoid modifying len/addr in the return
2835 		 * struct
2836 		 */
2837 		uint32_t len = t->len, addr = t->addr;
2838 
2839 		if (!is_offload(sc))
2840 			return (EOPNOTSUPP);
2841 		if (!(sc->flags & FULL_INIT_DONE))
2842 			return (EIO);         /* need the memory controllers */
2843 		if ((addr & 0x7) || (len & 0x7))
2844 			return (EINVAL);
2845 		if (t->mem_id == MEM_CM)
2846 			mem = &sc->cm;
2847 		else if (t->mem_id == MEM_PMRX)
2848 			mem = &sc->pmrx;
2849 		else if (t->mem_id == MEM_PMTX)
2850 			mem = &sc->pmtx;
2851 		else
2852 			return (EINVAL);
2853 
2854 		/*
2855 		 * Version scheme:
2856 		 * bits 0..9: chip version
2857 		 * bits 10..15: chip revision
2858 		 */
2859 		t->version = 3 | (sc->params.rev << 10);
2860 
2861 		/*
2862 		 * Read 256 bytes at a time as len can be large and we don't
2863 		 * want to use huge intermediate buffers.
2864 		 */
2865 		useraddr = (uint8_t *)t->buf;
2866 		while (len) {
2867 			unsigned int chunk = min(len, sizeof(buf));
2868 
2869 			error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2870 			if (error)
2871 				return (-error);
2872 			if (copyout(buf, useraddr, chunk))
2873 				return (EFAULT);
2874 			useraddr += chunk;
2875 			addr += chunk;
2876 			len -= chunk;
2877 		}
2878 		break;
2879 	}
2880 	case CHELSIO_READ_TCAM_WORD: {
2881 		struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2882 
2883 		if (!is_offload(sc))
2884 			return (EOPNOTSUPP);
2885 		if (!(sc->flags & FULL_INIT_DONE))
2886 			return (EIO);         /* need MC5 */
2887 		return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2888 		break;
2889 	}
2890 	case CHELSIO_SET_TRACE_FILTER: {
2891 		struct ch_trace *t = (struct ch_trace *)data;
2892 		const struct trace_params *tp;
2893 
2894 		tp = (const struct trace_params *)&t->sip;
2895 		if (t->config_tx)
2896 			t3_config_trace_filter(sc, tp, 0, t->invert_match,
2897 					       t->trace_tx);
2898 		if (t->config_rx)
2899 			t3_config_trace_filter(sc, tp, 1, t->invert_match,
2900 					       t->trace_rx);
2901 		break;
2902 	}
2903 	case CHELSIO_SET_PKTSCHED: {
2904 		struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2905 		if (sc->open_device_map == 0)
2906 			return (EAGAIN);
2907 		send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2908 		    p->binding);
2909 		break;
2910 	}
2911 	case CHELSIO_IFCONF_GETREGS: {
2912 		struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2913 		int reglen = cxgb_get_regs_len();
2914 		uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2915 		if (buf == NULL) {
2916 			return (ENOMEM);
2917 		}
2918 		if (regs->len > reglen)
2919 			regs->len = reglen;
2920 		else if (regs->len < reglen)
2921 			error = ENOBUFS;
2922 
2923 		if (!error) {
2924 			cxgb_get_regs(sc, regs, buf);
2925 			error = copyout(buf, regs->data, reglen);
2926 		}
2927 		free(buf, M_DEVBUF);
2928 
2929 		break;
2930 	}
2931 	case CHELSIO_SET_HW_SCHED: {
2932 		struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2933 		unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2934 
2935 		if ((sc->flags & FULL_INIT_DONE) == 0)
2936 			return (EAGAIN);       /* need TP to be initialized */
2937 		if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2938 		    !in_range(t->channel, 0, 1) ||
2939 		    !in_range(t->kbps, 0, 10000000) ||
2940 		    !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2941 		    !in_range(t->flow_ipg, 0,
2942 			      dack_ticks_to_usec(sc, 0x7ff)))
2943 			return (EINVAL);
2944 
2945 		if (t->kbps >= 0) {
2946 			error = t3_config_sched(sc, t->kbps, t->sched);
2947 			if (error < 0)
2948 				return (-error);
2949 		}
2950 		if (t->class_ipg >= 0)
2951 			t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2952 		if (t->flow_ipg >= 0) {
2953 			t->flow_ipg *= 1000;     /* us -> ns */
2954 			t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2955 		}
2956 		if (t->mode >= 0) {
2957 			int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2958 
2959 			t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2960 					 bit, t->mode ? bit : 0);
2961 		}
2962 		if (t->channel >= 0)
2963 			t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2964 					 1 << t->sched, t->channel << t->sched);
2965 		break;
2966 	}
2967 	case CHELSIO_GET_EEPROM: {
2968 		int i;
2969 		struct ch_eeprom *e = (struct ch_eeprom *)data;
2970 		uint8_t *buf;
2971 
2972 		if (e->offset & 3 || e->offset >= EEPROMSIZE ||
2973 		    e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
2974 			return (EINVAL);
2975 		}
2976 
2977 		buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2978 		if (buf == NULL) {
2979 			return (ENOMEM);
2980 		}
2981 		e->magic = EEPROM_MAGIC;
2982 		for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2983 			error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2984 
2985 		if (!error)
2986 			error = copyout(buf + e->offset, e->data, e->len);
2987 
2988 		free(buf, M_DEVBUF);
2989 		break;
2990 	}
2991 	case CHELSIO_CLEAR_STATS: {
2992 		if (!(sc->flags & FULL_INIT_DONE))
2993 			return EAGAIN;
2994 
2995 		PORT_LOCK(pi);
2996 		t3_mac_update_stats(&pi->mac);
2997 		memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2998 		PORT_UNLOCK(pi);
2999 		break;
3000 	}
3001 	case CHELSIO_GET_UP_LA: {
3002 		struct ch_up_la *la = (struct ch_up_la *)data;
3003 		uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
3004 		if (buf == NULL) {
3005 			return (ENOMEM);
3006 		}
3007 		if (la->bufsize < LA_BUFSIZE)
3008 			error = ENOBUFS;
3009 
3010 		if (!error)
3011 			error = -t3_get_up_la(sc, &la->stopped, &la->idx,
3012 					      &la->bufsize, buf);
3013 		if (!error)
3014 			error = copyout(buf, la->data, la->bufsize);
3015 
3016 		free(buf, M_DEVBUF);
3017 		break;
3018 	}
3019 	case CHELSIO_GET_UP_IOQS: {
3020 		struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
3021 		uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
3022 		uint32_t *v;
3023 
3024 		if (buf == NULL) {
3025 			return (ENOMEM);
3026 		}
3027 		if (ioqs->bufsize < IOQS_BUFSIZE)
3028 			error = ENOBUFS;
3029 
3030 		if (!error)
3031 			error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
3032 
3033 		if (!error) {
3034 			v = (uint32_t *)buf;
3035 
3036 			ioqs->ioq_rx_enable = *v++;
3037 			ioqs->ioq_tx_enable = *v++;
3038 			ioqs->ioq_rx_status = *v++;
3039 			ioqs->ioq_tx_status = *v++;
3040 
3041 			error = copyout(v, ioqs->data, ioqs->bufsize);
3042 		}
3043 
3044 		free(buf, M_DEVBUF);
3045 		break;
3046 	}
3047 	case CHELSIO_SET_FILTER: {
3048 		struct ch_filter *f = (struct ch_filter *)data;
3049 		struct filter_info *p;
3050 		unsigned int nfilters = sc->params.mc5.nfilters;
3051 
3052 		if (!is_offload(sc))
3053 			return (EOPNOTSUPP);	/* No TCAM */
3054 		if (!(sc->flags & FULL_INIT_DONE))
3055 			return (EAGAIN);	/* mc5 not setup yet */
3056 		if (nfilters == 0)
3057 			return (EBUSY);		/* TOE will use TCAM */
3058 
3059 		/* sanity checks */
3060 		if (f->filter_id >= nfilters ||
3061 		    (f->val.dip && f->mask.dip != 0xffffffff) ||
3062 		    (f->val.sport && f->mask.sport != 0xffff) ||
3063 		    (f->val.dport && f->mask.dport != 0xffff) ||
3064 		    (f->val.vlan && f->mask.vlan != 0xfff) ||
3065 		    (f->val.vlan_prio &&
3066 			f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3067 		    (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3068 		    f->qset >= SGE_QSETS ||
3069 		    sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3070 			return (EINVAL);
3071 
3072 		/* Was allocated with M_WAITOK */
3073 		KASSERT(sc->filters, ("filter table NULL\n"));
3074 
3075 		p = &sc->filters[f->filter_id];
3076 		if (p->locked)
3077 			return (EPERM);
3078 
3079 		bzero(p, sizeof(*p));
3080 		p->sip = f->val.sip;
3081 		p->sip_mask = f->mask.sip;
3082 		p->dip = f->val.dip;
3083 		p->sport = f->val.sport;
3084 		p->dport = f->val.dport;
3085 		p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3086 		p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3087 		    FILTER_NO_VLAN_PRI;
3088 		p->mac_hit = f->mac_hit;
3089 		p->mac_vld = f->mac_addr_idx != 0xffff;
3090 		p->mac_idx = f->mac_addr_idx;
3091 		p->pkt_type = f->proto;
3092 		p->report_filter_id = f->want_filter_id;
3093 		p->pass = f->pass;
3094 		p->rss = f->rss;
3095 		p->qset = f->qset;
3096 
3097 		error = set_filter(sc, f->filter_id, p);
3098 		if (error == 0)
3099 			p->valid = 1;
3100 		break;
3101 	}
3102 	case CHELSIO_DEL_FILTER: {
3103 		struct ch_filter *f = (struct ch_filter *)data;
3104 		struct filter_info *p;
3105 		unsigned int nfilters = sc->params.mc5.nfilters;
3106 
3107 		if (!is_offload(sc))
3108 			return (EOPNOTSUPP);
3109 		if (!(sc->flags & FULL_INIT_DONE))
3110 			return (EAGAIN);
3111 		if (nfilters == 0 || sc->filters == NULL)
3112 			return (EINVAL);
3113 		if (f->filter_id >= nfilters)
3114 		       return (EINVAL);
3115 
3116 		p = &sc->filters[f->filter_id];
3117 		if (p->locked)
3118 			return (EPERM);
3119 		if (!p->valid)
3120 			return (EFAULT); /* Read "Bad address" as "Bad index" */
3121 
3122 		bzero(p, sizeof(*p));
3123 		p->sip = p->sip_mask = 0xffffffff;
3124 		p->vlan = 0xfff;
3125 		p->vlan_prio = FILTER_NO_VLAN_PRI;
3126 		p->pkt_type = 1;
3127 		error = set_filter(sc, f->filter_id, p);
3128 		break;
3129 	}
3130 	case CHELSIO_GET_FILTER: {
3131 		struct ch_filter *f = (struct ch_filter *)data;
3132 		struct filter_info *p;
3133 		unsigned int i, nfilters = sc->params.mc5.nfilters;
3134 
3135 		if (!is_offload(sc))
3136 			return (EOPNOTSUPP);
3137 		if (!(sc->flags & FULL_INIT_DONE))
3138 			return (EAGAIN);
3139 		if (nfilters == 0 || sc->filters == NULL)
3140 			return (EINVAL);
3141 
3142 		i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3143 		for (; i < nfilters; i++) {
3144 			p = &sc->filters[i];
3145 			if (!p->valid)
3146 				continue;
3147 
3148 			bzero(f, sizeof(*f));
3149 
3150 			f->filter_id = i;
3151 			f->val.sip = p->sip;
3152 			f->mask.sip = p->sip_mask;
3153 			f->val.dip = p->dip;
3154 			f->mask.dip = p->dip ? 0xffffffff : 0;
3155 			f->val.sport = p->sport;
3156 			f->mask.sport = p->sport ? 0xffff : 0;
3157 			f->val.dport = p->dport;
3158 			f->mask.dport = p->dport ? 0xffff : 0;
3159 			f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3160 			f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3161 			f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3162 			    0 : p->vlan_prio;
3163 			f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3164 			    0 : FILTER_NO_VLAN_PRI;
3165 			f->mac_hit = p->mac_hit;
3166 			f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3167 			f->proto = p->pkt_type;
3168 			f->want_filter_id = p->report_filter_id;
3169 			f->pass = p->pass;
3170 			f->rss = p->rss;
3171 			f->qset = p->qset;
3172 
3173 			break;
3174 		}
3175 
3176 		if (i == nfilters)
3177 			f->filter_id = 0xffffffff;
3178 		break;
3179 	}
3180 	default:
3181 		return (EOPNOTSUPP);
3182 		break;
3183 	}
3184 
3185 	return (error);
3186 }
3187 
3188 static __inline void
3189 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3190     unsigned int end)
3191 {
3192 	uint32_t *p = (uint32_t *)(buf + start);
3193 
3194 	for ( ; start <= end; start += sizeof(uint32_t))
3195 		*p++ = t3_read_reg(ap, start);
3196 }
3197 
3198 #define T3_REGMAP_SIZE (3 * 1024)
3199 static int
3200 cxgb_get_regs_len(void)
3201 {
3202 	return T3_REGMAP_SIZE;
3203 }
3204 
3205 static void
3206 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3207 {
3208 
3209 	/*
3210 	 * Version scheme:
3211 	 * bits 0..9: chip version
3212 	 * bits 10..15: chip revision
3213 	 * bit 31: set for PCIe cards
3214 	 */
3215 	regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3216 
3217 	/*
3218 	 * We skip the MAC statistics registers because they are clear-on-read.
3219 	 * Also reading multi-register stats would need to synchronize with the
3220 	 * periodic mac stats accumulation.  Hard to justify the complexity.
3221 	 */
3222 	memset(buf, 0, cxgb_get_regs_len());
3223 	reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3224 	reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3225 	reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3226 	reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3227 	reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3228 	reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3229 		       XGM_REG(A_XGM_SERDES_STAT3, 1));
3230 	reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3231 		       XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3232 }
3233 
3234 static int
3235 alloc_filters(struct adapter *sc)
3236 {
3237 	struct filter_info *p;
3238 	unsigned int nfilters = sc->params.mc5.nfilters;
3239 
3240 	if (nfilters == 0)
3241 		return (0);
3242 
3243 	p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3244 	sc->filters = p;
3245 
3246 	p = &sc->filters[nfilters - 1];
3247 	p->vlan = 0xfff;
3248 	p->vlan_prio = FILTER_NO_VLAN_PRI;
3249 	p->pass = p->rss = p->valid = p->locked = 1;
3250 
3251 	return (0);
3252 }
3253 
3254 static int
3255 setup_hw_filters(struct adapter *sc)
3256 {
3257 	int i, rc;
3258 	unsigned int nfilters = sc->params.mc5.nfilters;
3259 
3260 	if (!sc->filters)
3261 		return (0);
3262 
3263 	t3_enable_filters(sc);
3264 
3265 	for (i = rc = 0; i < nfilters && !rc; i++) {
3266 		if (sc->filters[i].locked)
3267 			rc = set_filter(sc, i, &sc->filters[i]);
3268 	}
3269 
3270 	return (rc);
3271 }
3272 
3273 static int
3274 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3275 {
3276 	int len;
3277 	struct mbuf *m;
3278 	struct ulp_txpkt *txpkt;
3279 	struct work_request_hdr *wr;
3280 	struct cpl_pass_open_req *oreq;
3281 	struct cpl_set_tcb_field *sreq;
3282 
3283 	len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3284 	KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3285 
3286 	id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3287 	      sc->params.mc5.nfilters;
3288 
3289 	m = m_gethdr(M_WAITOK, MT_DATA);
3290 	m->m_len = m->m_pkthdr.len = len;
3291 	bzero(mtod(m, char *), len);
3292 
3293 	wr = mtod(m, struct work_request_hdr *);
3294 	wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3295 
3296 	oreq = (struct cpl_pass_open_req *)(wr + 1);
3297 	txpkt = (struct ulp_txpkt *)oreq;
3298 	txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3299 	txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3300 	OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3301 	oreq->local_port = htons(f->dport);
3302 	oreq->peer_port = htons(f->sport);
3303 	oreq->local_ip = htonl(f->dip);
3304 	oreq->peer_ip = htonl(f->sip);
3305 	oreq->peer_netmask = htonl(f->sip_mask);
3306 	oreq->opt0h = 0;
3307 	oreq->opt0l = htonl(F_NO_OFFLOAD);
3308 	oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3309 			 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3310 			 V_VLAN_PRI(f->vlan_prio >> 1) |
3311 			 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3312 			 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3313 			 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3314 
3315 	sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3316 	set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3317 			  (f->report_filter_id << 15) | (1 << 23) |
3318 			  ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3319 	set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3320 	t3_mgmt_tx(sc, m);
3321 
3322 	if (f->pass && !f->rss) {
3323 		len = sizeof(*sreq);
3324 		m = m_gethdr(M_WAITOK, MT_DATA);
3325 		m->m_len = m->m_pkthdr.len = len;
3326 		bzero(mtod(m, char *), len);
3327 		sreq = mtod(m, struct cpl_set_tcb_field *);
3328 		sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3329 		mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3330 				 (u64)sc->rrss_map[f->qset] << 19);
3331 		t3_mgmt_tx(sc, m);
3332 	}
3333 	return 0;
3334 }
3335 
3336 static inline void
3337 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3338     unsigned int word, u64 mask, u64 val)
3339 {
3340 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3341 	req->reply = V_NO_REPLY(1);
3342 	req->cpu_idx = 0;
3343 	req->word = htons(word);
3344 	req->mask = htobe64(mask);
3345 	req->val = htobe64(val);
3346 }
3347 
3348 static inline void
3349 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3350     unsigned int word, u64 mask, u64 val)
3351 {
3352 	struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3353 
3354 	txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3355 	txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3356 	mk_set_tcb_field(req, tid, word, mask, val);
3357 }
3358 
3359 void
3360 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3361 {
3362 	struct adapter *sc;
3363 
3364 	mtx_lock(&t3_list_lock);
3365 	SLIST_FOREACH(sc, &t3_list, link) {
3366 		/*
3367 		 * func should not make any assumptions about what state sc is
3368 		 * in - the only guarantee is that sc->sc_lock is a valid lock.
3369 		 */
3370 		func(sc, arg);
3371 	}
3372 	mtx_unlock(&t3_list_lock);
3373 }
3374 
3375 #ifdef TCP_OFFLOAD
3376 static int
3377 toe_capability(struct port_info *pi, int enable)
3378 {
3379 	int rc;
3380 	struct adapter *sc = pi->adapter;
3381 
3382 	ADAPTER_LOCK_ASSERT_OWNED(sc);
3383 
3384 	if (!is_offload(sc))
3385 		return (ENODEV);
3386 
3387 	if (enable) {
3388 		if (!(sc->flags & FULL_INIT_DONE)) {
3389 			log(LOG_WARNING,
3390 			    "You must enable a cxgb interface first\n");
3391 			return (EAGAIN);
3392 		}
3393 
3394 		if (isset(&sc->offload_map, pi->port_id))
3395 			return (0);
3396 
3397 		if (!(sc->flags & TOM_INIT_DONE)) {
3398 			rc = t3_activate_uld(sc, ULD_TOM);
3399 			if (rc == EAGAIN) {
3400 				log(LOG_WARNING,
3401 				    "You must kldload t3_tom.ko before trying "
3402 				    "to enable TOE on a cxgb interface.\n");
3403 			}
3404 			if (rc != 0)
3405 				return (rc);
3406 			KASSERT(sc->tom_softc != NULL,
3407 			    ("%s: TOM activated but softc NULL", __func__));
3408 			KASSERT(sc->flags & TOM_INIT_DONE,
3409 			    ("%s: TOM activated but flag not set", __func__));
3410 		}
3411 
3412 		setbit(&sc->offload_map, pi->port_id);
3413 
3414 		/*
3415 		 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3416 		 * enabled on any port.  Need to figure out how to enable,
3417 		 * disable, load, and unload iWARP cleanly.
3418 		 */
3419 		if (!isset(&sc->offload_map, MAX_NPORTS) &&
3420 		    t3_activate_uld(sc, ULD_IWARP) == 0)
3421 			setbit(&sc->offload_map, MAX_NPORTS);
3422 	} else {
3423 		if (!isset(&sc->offload_map, pi->port_id))
3424 			return (0);
3425 
3426 		KASSERT(sc->flags & TOM_INIT_DONE,
3427 		    ("%s: TOM never initialized?", __func__));
3428 		clrbit(&sc->offload_map, pi->port_id);
3429 	}
3430 
3431 	return (0);
3432 }
3433 
3434 /*
3435  * Add an upper layer driver to the global list.
3436  */
3437 int
3438 t3_register_uld(struct uld_info *ui)
3439 {
3440 	int rc = 0;
3441 	struct uld_info *u;
3442 
3443 	mtx_lock(&t3_uld_list_lock);
3444 	SLIST_FOREACH(u, &t3_uld_list, link) {
3445 	    if (u->uld_id == ui->uld_id) {
3446 		    rc = EEXIST;
3447 		    goto done;
3448 	    }
3449 	}
3450 
3451 	SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3452 	ui->refcount = 0;
3453 done:
3454 	mtx_unlock(&t3_uld_list_lock);
3455 	return (rc);
3456 }
3457 
3458 int
3459 t3_unregister_uld(struct uld_info *ui)
3460 {
3461 	int rc = EINVAL;
3462 	struct uld_info *u;
3463 
3464 	mtx_lock(&t3_uld_list_lock);
3465 
3466 	SLIST_FOREACH(u, &t3_uld_list, link) {
3467 	    if (u == ui) {
3468 		    if (ui->refcount > 0) {
3469 			    rc = EBUSY;
3470 			    goto done;
3471 		    }
3472 
3473 		    SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3474 		    rc = 0;
3475 		    goto done;
3476 	    }
3477 	}
3478 done:
3479 	mtx_unlock(&t3_uld_list_lock);
3480 	return (rc);
3481 }
3482 
3483 int
3484 t3_activate_uld(struct adapter *sc, int id)
3485 {
3486 	int rc = EAGAIN;
3487 	struct uld_info *ui;
3488 
3489 	mtx_lock(&t3_uld_list_lock);
3490 
3491 	SLIST_FOREACH(ui, &t3_uld_list, link) {
3492 		if (ui->uld_id == id) {
3493 			rc = ui->activate(sc);
3494 			if (rc == 0)
3495 				ui->refcount++;
3496 			goto done;
3497 		}
3498 	}
3499 done:
3500 	mtx_unlock(&t3_uld_list_lock);
3501 
3502 	return (rc);
3503 }
3504 
3505 int
3506 t3_deactivate_uld(struct adapter *sc, int id)
3507 {
3508 	int rc = EINVAL;
3509 	struct uld_info *ui;
3510 
3511 	mtx_lock(&t3_uld_list_lock);
3512 
3513 	SLIST_FOREACH(ui, &t3_uld_list, link) {
3514 		if (ui->uld_id == id) {
3515 			rc = ui->deactivate(sc);
3516 			if (rc == 0)
3517 				ui->refcount--;
3518 			goto done;
3519 		}
3520 	}
3521 done:
3522 	mtx_unlock(&t3_uld_list_lock);
3523 
3524 	return (rc);
3525 }
3526 
3527 static int
3528 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3529     struct mbuf *m)
3530 {
3531 	m_freem(m);
3532 	return (EDOOFUS);
3533 }
3534 
3535 int
3536 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3537 {
3538 	uintptr_t *loc, new;
3539 
3540 	if (opcode >= NUM_CPL_HANDLERS)
3541 		return (EINVAL);
3542 
3543 	new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3544 	loc = (uintptr_t *) &sc->cpl_handler[opcode];
3545 	atomic_store_rel_ptr(loc, new);
3546 
3547 	return (0);
3548 }
3549 #endif
3550 
3551 static int
3552 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3553 {
3554 	int rc = 0;
3555 
3556 	switch (cmd) {
3557 	case MOD_LOAD:
3558 		mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3559 		SLIST_INIT(&t3_list);
3560 #ifdef TCP_OFFLOAD
3561 		mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3562 		SLIST_INIT(&t3_uld_list);
3563 #endif
3564 		break;
3565 
3566 	case MOD_UNLOAD:
3567 #ifdef TCP_OFFLOAD
3568 		mtx_lock(&t3_uld_list_lock);
3569 		if (!SLIST_EMPTY(&t3_uld_list)) {
3570 			rc = EBUSY;
3571 			mtx_unlock(&t3_uld_list_lock);
3572 			break;
3573 		}
3574 		mtx_unlock(&t3_uld_list_lock);
3575 		mtx_destroy(&t3_uld_list_lock);
3576 #endif
3577 		mtx_lock(&t3_list_lock);
3578 		if (!SLIST_EMPTY(&t3_list)) {
3579 			rc = EBUSY;
3580 			mtx_unlock(&t3_list_lock);
3581 			break;
3582 		}
3583 		mtx_unlock(&t3_list_lock);
3584 		mtx_destroy(&t3_list_lock);
3585 		break;
3586 	}
3587 
3588 	return (rc);
3589 }
3590 
3591 #ifdef NETDUMP
3592 static void
3593 cxgb_netdump_init(struct ifnet *ifp, int *nrxr, int *ncl, int *clsize)
3594 {
3595 	struct port_info *pi;
3596 	adapter_t *adap;
3597 
3598 	pi = if_getsoftc(ifp);
3599 	adap = pi->adapter;
3600 	ADAPTER_LOCK(adap);
3601 	*nrxr = SGE_QSETS;
3602 	*ncl = adap->sge.qs[0].fl[1].size;
3603 	*clsize = adap->sge.qs[0].fl[1].buf_size;
3604 	ADAPTER_UNLOCK(adap);
3605 }
3606 
3607 static void
3608 cxgb_netdump_event(struct ifnet *ifp, enum netdump_ev event)
3609 {
3610 	struct port_info *pi;
3611 	struct sge_qset *qs;
3612 	int i;
3613 
3614 	pi = if_getsoftc(ifp);
3615 	if (event == NETDUMP_START)
3616 		for (i = 0; i < SGE_QSETS; i++) {
3617 			qs = &pi->adapter->sge.qs[i];
3618 
3619 			/* Need to reinit after netdump_mbuf_dump(). */
3620 			qs->fl[0].zone = zone_pack;
3621 			qs->fl[1].zone = zone_clust;
3622 			qs->lro.enabled = 0;
3623 		}
3624 }
3625 
3626 static int
3627 cxgb_netdump_transmit(struct ifnet *ifp, struct mbuf *m)
3628 {
3629 	struct port_info *pi;
3630 	struct sge_qset *qs;
3631 
3632 	pi = if_getsoftc(ifp);
3633 	if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
3634 	    IFF_DRV_RUNNING)
3635 		return (ENOENT);
3636 
3637 	qs = &pi->adapter->sge.qs[pi->first_qset];
3638 	return (cxgb_netdump_encap(qs, &m));
3639 }
3640 
3641 static int
3642 cxgb_netdump_poll(struct ifnet *ifp, int count)
3643 {
3644 	struct port_info *pi;
3645 	adapter_t *adap;
3646 	int i;
3647 
3648 	pi = if_getsoftc(ifp);
3649 	if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
3650 		return (ENOENT);
3651 
3652 	adap = pi->adapter;
3653 	for (i = 0; i < SGE_QSETS; i++)
3654 		(void)cxgb_netdump_poll_rx(adap, &adap->sge.qs[i]);
3655 	(void)cxgb_netdump_poll_tx(&adap->sge.qs[pi->first_qset]);
3656 	return (0);
3657 }
3658 #endif /* NETDUMP */
3659