1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause
3
4 Copyright (c) 2007-2009, Chelsio Inc.
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
12
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
28
29 ***************************************************************************/
30
31 #include <sys/cdefs.h>
32 #include "opt_inet.h"
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bus.h>
38 #include <sys/module.h>
39 #include <sys/conf.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/ktr.h>
43 #include <sys/rman.h>
44 #include <sys/ioccom.h>
45 #include <sys/mbuf.h>
46 #include <sys/linker.h>
47 #include <sys/firmware.h>
48 #include <sys/socket.h>
49 #include <sys/sockio.h>
50 #include <sys/smp.h>
51 #include <sys/sysctl.h>
52 #include <sys/syslog.h>
53 #include <sys/queue.h>
54 #include <sys/taskqueue.h>
55 #include <sys/proc.h>
56
57 #include <net/bpf.h>
58 #include <net/debugnet.h>
59 #include <net/ethernet.h>
60 #include <net/if.h>
61 #include <net/if_var.h>
62 #include <net/if_arp.h>
63 #include <net/if_dl.h>
64 #include <net/if_media.h>
65 #include <net/if_types.h>
66 #include <net/if_vlan_var.h>
67
68 #include <netinet/in_systm.h>
69 #include <netinet/in.h>
70 #include <netinet/if_ether.h>
71 #include <netinet/ip.h>
72 #include <netinet/ip.h>
73 #include <netinet/tcp.h>
74 #include <netinet/udp.h>
75
76 #include <dev/pci/pcireg.h>
77 #include <dev/pci/pcivar.h>
78
79 #include <cxgb_include.h>
80
81 #ifdef PRIV_SUPPORTED
82 #include <sys/priv.h>
83 #endif
84
85 static int cxgb_setup_interrupts(adapter_t *);
86 static void cxgb_teardown_interrupts(adapter_t *);
87 static void cxgb_init(void *);
88 static int cxgb_init_locked(struct port_info *);
89 static int cxgb_uninit_locked(struct port_info *);
90 static int cxgb_uninit_synchronized(struct port_info *);
91 static int cxgb_ioctl(if_t, unsigned long, caddr_t);
92 static int cxgb_media_change(if_t);
93 static int cxgb_ifm_type(int);
94 static void cxgb_build_medialist(struct port_info *);
95 static void cxgb_media_status(if_t, struct ifmediareq *);
96 static uint64_t cxgb_get_counter(if_t, ift_counter);
97 static int setup_sge_qsets(adapter_t *);
98 static void cxgb_async_intr(void *);
99 static void cxgb_tick_handler(void *, int);
100 static void cxgb_tick(void *);
101 static void link_check_callout(void *);
102 static void check_link_status(void *, int);
103 static void setup_rss(adapter_t *sc);
104 static int alloc_filters(struct adapter *);
105 static int setup_hw_filters(struct adapter *);
106 static int set_filter(struct adapter *, int, const struct filter_info *);
107 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
108 unsigned int, u64, u64);
109 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
110 unsigned int, u64, u64);
111 #ifdef TCP_OFFLOAD
112 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
113 #endif
114
115 /* Attachment glue for the PCI controller end of the device. Each port of
116 * the device is attached separately, as defined later.
117 */
118 static int cxgb_controller_probe(device_t);
119 static int cxgb_controller_attach(device_t);
120 static int cxgb_controller_detach(device_t);
121 static void cxgb_free(struct adapter *);
122 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
123 unsigned int end);
124 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
125 static int cxgb_get_regs_len(void);
126 static void touch_bars(device_t dev);
127 static void cxgb_update_mac_settings(struct port_info *p);
128 #ifdef TCP_OFFLOAD
129 static int toe_capability(struct port_info *, int);
130 #endif
131
132 /* Table for probing the cards. The desc field isn't actually used */
133 struct cxgb_ident {
134 uint16_t vendor;
135 uint16_t device;
136 int index;
137 char *desc;
138 } cxgb_identifiers[] = {
139 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
140 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
141 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
142 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
143 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
144 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
145 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
146 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
147 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
148 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
149 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
150 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
151 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
152 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
153 {0, 0, 0, NULL}
154 };
155
156 static device_method_t cxgb_controller_methods[] = {
157 DEVMETHOD(device_probe, cxgb_controller_probe),
158 DEVMETHOD(device_attach, cxgb_controller_attach),
159 DEVMETHOD(device_detach, cxgb_controller_detach),
160
161 DEVMETHOD_END
162 };
163
164 static driver_t cxgb_controller_driver = {
165 "cxgbc",
166 cxgb_controller_methods,
167 sizeof(struct adapter)
168 };
169
170 static int cxgbc_mod_event(module_t, int, void *);
171
172 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgbc_mod_event, NULL);
173 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
174 nitems(cxgb_identifiers) - 1);
175 MODULE_VERSION(cxgbc, 1);
176 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
177
178 /*
179 * Attachment glue for the ports. Attachment is done directly to the
180 * controller device.
181 */
182 static int cxgb_port_probe(device_t);
183 static int cxgb_port_attach(device_t);
184 static int cxgb_port_detach(device_t);
185
186 static device_method_t cxgb_port_methods[] = {
187 DEVMETHOD(device_probe, cxgb_port_probe),
188 DEVMETHOD(device_attach, cxgb_port_attach),
189 DEVMETHOD(device_detach, cxgb_port_detach),
190 { 0, 0 }
191 };
192
193 static driver_t cxgb_port_driver = {
194 "cxgb",
195 cxgb_port_methods,
196 0
197 };
198
199 static d_ioctl_t cxgb_extension_ioctl;
200 static d_open_t cxgb_extension_open;
201 static d_close_t cxgb_extension_close;
202
203 static struct cdevsw cxgb_cdevsw = {
204 .d_version = D_VERSION,
205 .d_flags = 0,
206 .d_open = cxgb_extension_open,
207 .d_close = cxgb_extension_close,
208 .d_ioctl = cxgb_extension_ioctl,
209 .d_name = "cxgb",
210 };
211
212 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, 0, 0);
213 MODULE_VERSION(cxgb, 1);
214
215 DEBUGNET_DEFINE(cxgb);
216
217 static struct mtx t3_list_lock;
218 static SLIST_HEAD(, adapter) t3_list;
219 #ifdef TCP_OFFLOAD
220 static struct mtx t3_uld_list_lock;
221 static SLIST_HEAD(, uld_info) t3_uld_list;
222 #endif
223
224 /*
225 * The driver uses the best interrupt scheme available on a platform in the
226 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
227 * of these schemes the driver may consider as follows:
228 *
229 * msi = 2: choose from among all three options
230 * msi = 1 : only consider MSI and pin interrupts
231 * msi = 0: force pin interrupts
232 */
233 static int msi_allowed = 2;
234
235 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
236 "CXGB driver parameters");
237 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
238 "MSI-X, MSI, INTx selector");
239
240 /*
241 * The driver uses an auto-queue algorithm by default.
242 * To disable it and force a single queue-set per port, use multiq = 0
243 */
244 static int multiq = 1;
245 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
246 "use min(ncpus/ports, 8) queue-sets per port");
247
248 /*
249 * By default the driver will not update the firmware unless
250 * it was compiled against a newer version
251 *
252 */
253 static int force_fw_update = 0;
254 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
255 "update firmware even if up to date");
256
257 int cxgb_use_16k_clusters = -1;
258 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
259 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
260
261 static int nfilters = -1;
262 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
263 &nfilters, 0, "max number of entries in the filter table");
264
265 enum {
266 MAX_TXQ_ENTRIES = 16384,
267 MAX_CTRL_TXQ_ENTRIES = 1024,
268 MAX_RSPQ_ENTRIES = 16384,
269 MAX_RX_BUFFERS = 16384,
270 MAX_RX_JUMBO_BUFFERS = 16384,
271 MIN_TXQ_ENTRIES = 4,
272 MIN_CTRL_TXQ_ENTRIES = 4,
273 MIN_RSPQ_ENTRIES = 32,
274 MIN_FL_ENTRIES = 32,
275 MIN_FL_JUMBO_ENTRIES = 32
276 };
277
278 struct filter_info {
279 u32 sip;
280 u32 sip_mask;
281 u32 dip;
282 u16 sport;
283 u16 dport;
284 u32 vlan:12;
285 u32 vlan_prio:3;
286 u32 mac_hit:1;
287 u32 mac_idx:4;
288 u32 mac_vld:1;
289 u32 pkt_type:2;
290 u32 report_filter_id:1;
291 u32 pass:1;
292 u32 rss:1;
293 u32 qset:3;
294 u32 locked:1;
295 u32 valid:1;
296 };
297
298 enum { FILTER_NO_VLAN_PRI = 7 };
299
300 #define EEPROM_MAGIC 0x38E2F10C
301
302 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
303
304
305 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
306
307
308 static __inline char
t3rev2char(struct adapter * adapter)309 t3rev2char(struct adapter *adapter)
310 {
311 char rev = 'z';
312
313 switch(adapter->params.rev) {
314 case T3_REV_A:
315 rev = 'a';
316 break;
317 case T3_REV_B:
318 case T3_REV_B2:
319 rev = 'b';
320 break;
321 case T3_REV_C:
322 rev = 'c';
323 break;
324 }
325 return rev;
326 }
327
328 static struct cxgb_ident *
cxgb_get_ident(device_t dev)329 cxgb_get_ident(device_t dev)
330 {
331 struct cxgb_ident *id;
332
333 for (id = cxgb_identifiers; id->desc != NULL; id++) {
334 if ((id->vendor == pci_get_vendor(dev)) &&
335 (id->device == pci_get_device(dev))) {
336 return (id);
337 }
338 }
339 return (NULL);
340 }
341
342 static const struct adapter_info *
cxgb_get_adapter_info(device_t dev)343 cxgb_get_adapter_info(device_t dev)
344 {
345 struct cxgb_ident *id;
346 const struct adapter_info *ai;
347
348 id = cxgb_get_ident(dev);
349 if (id == NULL)
350 return (NULL);
351
352 ai = t3_get_adapter_info(id->index);
353
354 return (ai);
355 }
356
357 static int
cxgb_controller_probe(device_t dev)358 cxgb_controller_probe(device_t dev)
359 {
360 const struct adapter_info *ai;
361 const char *ports;
362 int nports;
363
364 ai = cxgb_get_adapter_info(dev);
365 if (ai == NULL)
366 return (ENXIO);
367
368 nports = ai->nports0 + ai->nports1;
369 if (nports == 1)
370 ports = "port";
371 else
372 ports = "ports";
373
374 device_set_descf(dev, "%s, %d %s", ai->desc, nports, ports);
375 return (BUS_PROBE_DEFAULT);
376 }
377
378 #define FW_FNAME "cxgb_t3fw"
379 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
380 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
381
382 static int
upgrade_fw(adapter_t * sc)383 upgrade_fw(adapter_t *sc)
384 {
385 const struct firmware *fw;
386 int status;
387 u32 vers;
388
389 if ((fw = firmware_get(FW_FNAME)) == NULL) {
390 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
391 return (ENOENT);
392 } else
393 device_printf(sc->dev, "installing firmware on card\n");
394 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
395
396 if (status != 0) {
397 device_printf(sc->dev, "failed to install firmware: %d\n",
398 status);
399 } else {
400 t3_get_fw_version(sc, &vers);
401 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
402 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
403 G_FW_VERSION_MICRO(vers));
404 }
405
406 firmware_put(fw, FIRMWARE_UNLOAD);
407
408 return (status);
409 }
410
411 /*
412 * The cxgb_controller_attach function is responsible for the initial
413 * bringup of the device. Its responsibilities include:
414 *
415 * 1. Determine if the device supports MSI or MSI-X.
416 * 2. Allocate bus resources so that we can access the Base Address Register
417 * 3. Create and initialize mutexes for the controller and its control
418 * logic such as SGE and MDIO.
419 * 4. Call hardware specific setup routine for the adapter as a whole.
420 * 5. Allocate the BAR for doing MSI-X.
421 * 6. Setup the line interrupt iff MSI-X is not supported.
422 * 7. Create the driver's taskq.
423 * 8. Start one task queue service thread.
424 * 9. Check if the firmware and SRAM are up-to-date. They will be
425 * auto-updated later (before FULL_INIT_DONE), if required.
426 * 10. Create a child device for each MAC (port)
427 * 11. Initialize T3 private state.
428 * 12. Trigger the LED
429 * 13. Setup offload iff supported.
430 * 14. Reset/restart the tick callout.
431 * 15. Attach sysctls
432 *
433 * NOTE: Any modification or deviation from this list MUST be reflected in
434 * the above comment. Failure to do so will result in problems on various
435 * error conditions including link flapping.
436 */
437 static int
cxgb_controller_attach(device_t dev)438 cxgb_controller_attach(device_t dev)
439 {
440 device_t child;
441 const struct adapter_info *ai;
442 struct adapter *sc;
443 int i, error = 0;
444 uint32_t vers;
445 int port_qsets = 1;
446 int msi_needed, reg;
447
448 sc = device_get_softc(dev);
449 sc->dev = dev;
450 sc->msi_count = 0;
451 ai = cxgb_get_adapter_info(dev);
452
453 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
454 device_get_unit(dev));
455 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
456
457 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
458 device_get_unit(dev));
459 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
460 device_get_unit(dev));
461 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
462 device_get_unit(dev));
463
464 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
465 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
466 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
467
468 mtx_lock(&t3_list_lock);
469 SLIST_INSERT_HEAD(&t3_list, sc, link);
470 mtx_unlock(&t3_list_lock);
471
472 /* find the PCIe link width and set max read request to 4KB*/
473 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
474 uint16_t lnk;
475
476 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
477 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
478 if (sc->link_width < 8 &&
479 (ai->caps & SUPPORTED_10000baseT_Full)) {
480 device_printf(sc->dev,
481 "PCIe x%d Link, expect reduced performance\n",
482 sc->link_width);
483 }
484
485 pci_set_max_read_req(dev, 4096);
486 }
487
488 touch_bars(dev);
489 pci_enable_busmaster(dev);
490 /*
491 * Allocate the registers and make them available to the driver.
492 * The registers that we care about for NIC mode are in BAR 0
493 */
494 sc->regs_rid = PCIR_BAR(0);
495 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
496 &sc->regs_rid, RF_ACTIVE)) == NULL) {
497 device_printf(dev, "Cannot allocate BAR region 0\n");
498 error = ENXIO;
499 goto out;
500 }
501
502 sc->bt = rman_get_bustag(sc->regs_res);
503 sc->bh = rman_get_bushandle(sc->regs_res);
504 sc->mmio_len = rman_get_size(sc->regs_res);
505
506 for (i = 0; i < MAX_NPORTS; i++)
507 sc->port[i].adapter = sc;
508
509 if (t3_prep_adapter(sc, ai, 1) < 0) {
510 printf("prep adapter failed\n");
511 error = ENODEV;
512 goto out;
513 }
514
515 sc->udbs_rid = PCIR_BAR(2);
516 sc->udbs_res = NULL;
517 if (is_offload(sc) &&
518 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
519 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
520 device_printf(dev, "Cannot allocate BAR region 1\n");
521 error = ENXIO;
522 goto out;
523 }
524
525 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
526 * enough messages for the queue sets. If that fails, try falling
527 * back to MSI. If that fails, then try falling back to the legacy
528 * interrupt pin model.
529 */
530 sc->msix_regs_rid = 0x20;
531 if ((msi_allowed >= 2) &&
532 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
533 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
534
535 if (multiq)
536 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
537 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
538
539 if (pci_msix_count(dev) == 0 ||
540 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
541 sc->msi_count != msi_needed) {
542 device_printf(dev, "alloc msix failed - "
543 "msi_count=%d, msi_needed=%d, err=%d; "
544 "will try MSI\n", sc->msi_count,
545 msi_needed, error);
546 sc->msi_count = 0;
547 port_qsets = 1;
548 pci_release_msi(dev);
549 bus_release_resource(dev, SYS_RES_MEMORY,
550 sc->msix_regs_rid, sc->msix_regs_res);
551 sc->msix_regs_res = NULL;
552 } else {
553 sc->flags |= USING_MSIX;
554 sc->cxgb_intr = cxgb_async_intr;
555 device_printf(dev,
556 "using MSI-X interrupts (%u vectors)\n",
557 sc->msi_count);
558 }
559 }
560
561 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
562 sc->msi_count = 1;
563 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
564 device_printf(dev, "alloc msi failed - "
565 "err=%d; will try INTx\n", error);
566 sc->msi_count = 0;
567 port_qsets = 1;
568 pci_release_msi(dev);
569 } else {
570 sc->flags |= USING_MSI;
571 sc->cxgb_intr = t3_intr_msi;
572 device_printf(dev, "using MSI interrupts\n");
573 }
574 }
575 if (sc->msi_count == 0) {
576 device_printf(dev, "using line interrupts\n");
577 sc->cxgb_intr = t3b_intr;
578 }
579
580 /* Create a private taskqueue thread for handling driver events */
581 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
582 taskqueue_thread_enqueue, &sc->tq);
583 if (sc->tq == NULL) {
584 device_printf(dev, "failed to allocate controller task queue\n");
585 goto out;
586 }
587
588 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
589 device_get_nameunit(dev));
590 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
591
592
593 /* Create a periodic callout for checking adapter status */
594 callout_init(&sc->cxgb_tick_ch, 1);
595
596 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
597 /*
598 * Warn user that a firmware update will be attempted in init.
599 */
600 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
601 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
602 sc->flags &= ~FW_UPTODATE;
603 } else {
604 sc->flags |= FW_UPTODATE;
605 }
606
607 if (t3_check_tpsram_version(sc) < 0) {
608 /*
609 * Warn user that a firmware update will be attempted in init.
610 */
611 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
612 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
613 sc->flags &= ~TPS_UPTODATE;
614 } else {
615 sc->flags |= TPS_UPTODATE;
616 }
617
618 /*
619 * Create a child device for each MAC. The ethernet attachment
620 * will be done in these children.
621 */
622 for (i = 0; i < (sc)->params.nports; i++) {
623 struct port_info *pi;
624
625 if ((child = device_add_child(dev, "cxgb",
626 DEVICE_UNIT_ANY)) == NULL) {
627 device_printf(dev, "failed to add child port\n");
628 error = EINVAL;
629 goto out;
630 }
631 pi = &sc->port[i];
632 pi->adapter = sc;
633 pi->nqsets = port_qsets;
634 pi->first_qset = i*port_qsets;
635 pi->port_id = i;
636 pi->tx_chan = i >= ai->nports0;
637 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
638 sc->rxpkt_map[pi->txpkt_intf] = i;
639 sc->port[i].tx_chan = i >= ai->nports0;
640 sc->portdev[i] = child;
641 device_set_softc(child, pi);
642 }
643 bus_attach_children(dev);
644
645 /* initialize sge private state */
646 t3_sge_init_adapter(sc);
647
648 t3_led_ready(sc);
649
650 error = t3_get_fw_version(sc, &vers);
651 if (error)
652 goto out;
653
654 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
655 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
656 G_FW_VERSION_MICRO(vers));
657
658 device_set_descf(dev, "%s %sNIC\t E/C: %s S/N: %s",
659 ai->desc, is_offload(sc) ? "R" : "",
660 sc->params.vpd.ec, sc->params.vpd.sn);
661
662 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
663 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
664 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
665
666 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
667 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
668 t3_add_attach_sysctls(sc);
669
670 #ifdef TCP_OFFLOAD
671 for (i = 0; i < NUM_CPL_HANDLERS; i++)
672 sc->cpl_handler[i] = cpl_not_handled;
673 #endif
674
675 t3_intr_clear(sc);
676 error = cxgb_setup_interrupts(sc);
677 out:
678 if (error)
679 cxgb_free(sc);
680
681 return (error);
682 }
683
684 /*
685 * The cxgb_controller_detach routine is called with the device is
686 * unloaded from the system.
687 */
688
689 static int
cxgb_controller_detach(device_t dev)690 cxgb_controller_detach(device_t dev)
691 {
692 struct adapter *sc;
693
694 sc = device_get_softc(dev);
695
696 cxgb_free(sc);
697
698 return (0);
699 }
700
701 /*
702 * The cxgb_free() is called by the cxgb_controller_detach() routine
703 * to tear down the structures that were built up in
704 * cxgb_controller_attach(), and should be the final piece of work
705 * done when fully unloading the driver.
706 *
707 *
708 * 1. Shutting down the threads started by the cxgb_controller_attach()
709 * routine.
710 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
711 * 3. Detaching all of the port devices created during the
712 * cxgb_controller_attach() routine.
713 * 4. Removing the device children created via cxgb_controller_attach().
714 * 5. Releasing PCI resources associated with the device.
715 * 6. Turning off the offload support, iff it was turned on.
716 * 7. Destroying the mutexes created in cxgb_controller_attach().
717 *
718 */
719 static void
cxgb_free(struct adapter * sc)720 cxgb_free(struct adapter *sc)
721 {
722 int i, nqsets = 0;
723
724 ADAPTER_LOCK(sc);
725 sc->flags |= CXGB_SHUTDOWN;
726 ADAPTER_UNLOCK(sc);
727
728 /*
729 * Make sure all child devices are gone.
730 */
731 bus_detach_children(sc->dev);
732 for (i = 0; i < (sc)->params.nports; i++) {
733 if (sc->portdev[i] &&
734 device_delete_child(sc->dev, sc->portdev[i]) != 0)
735 device_printf(sc->dev, "failed to delete child port\n");
736 nqsets += sc->port[i].nqsets;
737 }
738
739 /*
740 * At this point, it is as if cxgb_port_detach has run on all ports, and
741 * cxgb_down has run on the adapter. All interrupts have been silenced,
742 * all open devices have been closed.
743 */
744 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
745 __func__, sc->open_device_map));
746 for (i = 0; i < sc->params.nports; i++) {
747 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
748 __func__, i));
749 }
750
751 /*
752 * Finish off the adapter's callouts.
753 */
754 callout_drain(&sc->cxgb_tick_ch);
755 callout_drain(&sc->sge_timer_ch);
756
757 /*
758 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
759 * sysctls are cleaned up by the kernel linker.
760 */
761 if (sc->flags & FULL_INIT_DONE) {
762 t3_free_sge_resources(sc, nqsets);
763 sc->flags &= ~FULL_INIT_DONE;
764 }
765
766 /*
767 * Release all interrupt resources.
768 */
769 cxgb_teardown_interrupts(sc);
770 if (sc->flags & (USING_MSI | USING_MSIX)) {
771 device_printf(sc->dev, "releasing msi message(s)\n");
772 pci_release_msi(sc->dev);
773 } else {
774 device_printf(sc->dev, "no msi message to release\n");
775 }
776
777 if (sc->msix_regs_res != NULL) {
778 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
779 sc->msix_regs_res);
780 }
781
782 /*
783 * Free the adapter's taskqueue.
784 */
785 if (sc->tq != NULL) {
786 taskqueue_free(sc->tq);
787 sc->tq = NULL;
788 }
789
790 free(sc->filters, M_DEVBUF);
791 t3_sge_free(sc);
792
793 if (sc->udbs_res != NULL)
794 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
795 sc->udbs_res);
796
797 if (sc->regs_res != NULL)
798 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
799 sc->regs_res);
800
801 MTX_DESTROY(&sc->mdio_lock);
802 MTX_DESTROY(&sc->sge.reg_lock);
803 MTX_DESTROY(&sc->elmer_lock);
804 mtx_lock(&t3_list_lock);
805 SLIST_REMOVE(&t3_list, sc, adapter, link);
806 mtx_unlock(&t3_list_lock);
807 ADAPTER_LOCK_DEINIT(sc);
808 }
809
810 /**
811 * setup_sge_qsets - configure SGE Tx/Rx/response queues
812 * @sc: the controller softc
813 *
814 * Determines how many sets of SGE queues to use and initializes them.
815 * We support multiple queue sets per port if we have MSI-X, otherwise
816 * just one queue set per port.
817 */
818 static int
setup_sge_qsets(adapter_t * sc)819 setup_sge_qsets(adapter_t *sc)
820 {
821 int i, j, err, irq_idx = 0, qset_idx = 0;
822 u_int ntxq = SGE_TXQ_PER_SET;
823
824 if ((err = t3_sge_alloc(sc)) != 0) {
825 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
826 return (err);
827 }
828
829 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
830 irq_idx = -1;
831
832 for (i = 0; i < (sc)->params.nports; i++) {
833 struct port_info *pi = &sc->port[i];
834
835 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
836 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
837 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
838 &sc->params.sge.qset[qset_idx], ntxq, pi);
839 if (err) {
840 t3_free_sge_resources(sc, qset_idx);
841 device_printf(sc->dev,
842 "t3_sge_alloc_qset failed with %d\n", err);
843 return (err);
844 }
845 }
846 }
847
848 sc->nqsets = qset_idx;
849
850 return (0);
851 }
852
853 static void
cxgb_teardown_interrupts(adapter_t * sc)854 cxgb_teardown_interrupts(adapter_t *sc)
855 {
856 int i;
857
858 for (i = 0; i < SGE_QSETS; i++) {
859 if (sc->msix_intr_tag[i] == NULL) {
860
861 /* Should have been setup fully or not at all */
862 KASSERT(sc->msix_irq_res[i] == NULL &&
863 sc->msix_irq_rid[i] == 0,
864 ("%s: half-done interrupt (%d).", __func__, i));
865
866 continue;
867 }
868
869 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
870 sc->msix_intr_tag[i]);
871 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
872 sc->msix_irq_res[i]);
873
874 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
875 sc->msix_irq_rid[i] = 0;
876 }
877
878 if (sc->intr_tag) {
879 KASSERT(sc->irq_res != NULL,
880 ("%s: half-done interrupt.", __func__));
881
882 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
883 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
884 sc->irq_res);
885
886 sc->irq_res = sc->intr_tag = NULL;
887 sc->irq_rid = 0;
888 }
889 }
890
891 static int
cxgb_setup_interrupts(adapter_t * sc)892 cxgb_setup_interrupts(adapter_t *sc)
893 {
894 struct resource *res;
895 void *tag;
896 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
897
898 sc->irq_rid = intr_flag ? 1 : 0;
899 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
900 RF_SHAREABLE | RF_ACTIVE);
901 if (sc->irq_res == NULL) {
902 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
903 intr_flag, sc->irq_rid);
904 err = EINVAL;
905 sc->irq_rid = 0;
906 } else {
907 err = bus_setup_intr(sc->dev, sc->irq_res,
908 INTR_MPSAFE | INTR_TYPE_NET, NULL,
909 sc->cxgb_intr, sc, &sc->intr_tag);
910
911 if (err) {
912 device_printf(sc->dev,
913 "Cannot set up interrupt (%x, %u, %d)\n",
914 intr_flag, sc->irq_rid, err);
915 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
916 sc->irq_res);
917 sc->irq_res = sc->intr_tag = NULL;
918 sc->irq_rid = 0;
919 }
920 }
921
922 /* That's all for INTx or MSI */
923 if (!(intr_flag & USING_MSIX) || err)
924 return (err);
925
926 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
927 for (i = 0; i < sc->msi_count - 1; i++) {
928 rid = i + 2;
929 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
930 RF_SHAREABLE | RF_ACTIVE);
931 if (res == NULL) {
932 device_printf(sc->dev, "Cannot allocate interrupt "
933 "for message %d\n", rid);
934 err = EINVAL;
935 break;
936 }
937
938 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
939 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
940 if (err) {
941 device_printf(sc->dev, "Cannot set up interrupt "
942 "for message %d (%d)\n", rid, err);
943 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
944 break;
945 }
946
947 sc->msix_irq_rid[i] = rid;
948 sc->msix_irq_res[i] = res;
949 sc->msix_intr_tag[i] = tag;
950 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
951 }
952
953 if (err)
954 cxgb_teardown_interrupts(sc);
955
956 return (err);
957 }
958
959
960 static int
cxgb_port_probe(device_t dev)961 cxgb_port_probe(device_t dev)
962 {
963 struct port_info *p;
964 const char *desc;
965
966 p = device_get_softc(dev);
967 desc = p->phy.desc;
968 device_set_descf(dev, "Port %d %s", p->port_id, desc);
969 return (0);
970 }
971
972
973 static int
cxgb_makedev(struct port_info * pi)974 cxgb_makedev(struct port_info *pi)
975 {
976
977 pi->port_cdev = make_dev(&cxgb_cdevsw, if_getdunit(pi->ifp),
978 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
979
980 if (pi->port_cdev == NULL)
981 return (ENOMEM);
982
983 pi->port_cdev->si_drv1 = (void *)pi;
984
985 return (0);
986 }
987
988 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
989 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
990 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
991 #define CXGB_CAP_ENABLE CXGB_CAP
992
993 static int
cxgb_port_attach(device_t dev)994 cxgb_port_attach(device_t dev)
995 {
996 struct port_info *p;
997 if_t ifp;
998 int err;
999 struct adapter *sc;
1000
1001 p = device_get_softc(dev);
1002 sc = p->adapter;
1003 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1004 device_get_unit(device_get_parent(dev)), p->port_id);
1005 PORT_LOCK_INIT(p, p->lockbuf);
1006
1007 callout_init(&p->link_check_ch, 1);
1008 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1009
1010 /* Allocate an ifnet object and set it up */
1011 ifp = p->ifp = if_alloc(IFT_ETHER);
1012 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1013 if_setinitfn(ifp, cxgb_init);
1014 if_setsoftc(ifp, p);
1015 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1016 if_setioctlfn(ifp, cxgb_ioctl);
1017 if_settransmitfn(ifp, cxgb_transmit);
1018 if_setqflushfn(ifp, cxgb_qflush);
1019 if_setgetcounterfn(ifp, cxgb_get_counter);
1020
1021 if_setcapabilities(ifp, CXGB_CAP);
1022 #ifdef TCP_OFFLOAD
1023 if (is_offload(sc))
1024 if_setcapabilitiesbit(ifp, IFCAP_TOE4, 0);
1025 #endif
1026 if_setcapenable(ifp, CXGB_CAP_ENABLE);
1027 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1028 CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1029 if_sethwtsomax(ifp, IP_MAXPACKET);
1030 if_sethwtsomaxsegcount(ifp, 36);
1031 if_sethwtsomaxsegsize(ifp, 65536);
1032
1033 /*
1034 * Disable TSO on 4-port - it isn't supported by the firmware.
1035 */
1036 if (sc->params.nports > 2) {
1037 if_setcapabilitiesbit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO);
1038 if_setcapenablebit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO);
1039 if_sethwassistbits(ifp, 0, CSUM_TSO);
1040 }
1041
1042 /* Create a list of media supported by this port */
1043 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1044 cxgb_media_status);
1045 cxgb_build_medialist(p);
1046
1047 ether_ifattach(ifp, p->hw_addr);
1048
1049 /* Attach driver debugnet methods. */
1050 DEBUGNET_SET(ifp, cxgb);
1051
1052 #ifdef DEFAULT_JUMBO
1053 if (sc->params.nports <= 2)
1054 if_setmtu(ifp, ETHERMTU_JUMBO);
1055 #endif
1056 if ((err = cxgb_makedev(p)) != 0) {
1057 printf("makedev failed %d\n", err);
1058 return (err);
1059 }
1060
1061 t3_sge_init_port(p);
1062
1063 return (err);
1064 }
1065
1066 /*
1067 * cxgb_port_detach() is called via the device_detach methods when
1068 * cxgb_free() calls the bus_detach_children. It is responsible for
1069 * removing the device from the view of the kernel, i.e. from all
1070 * interfaces lists etc. This routine is only called when the driver is
1071 * being unloaded, not when the link goes down.
1072 */
1073 static int
cxgb_port_detach(device_t dev)1074 cxgb_port_detach(device_t dev)
1075 {
1076 struct port_info *p;
1077 struct adapter *sc;
1078 int i;
1079
1080 p = device_get_softc(dev);
1081 sc = p->adapter;
1082
1083 /* Tell cxgb_ioctl and if_init that the port is going away */
1084 ADAPTER_LOCK(sc);
1085 SET_DOOMED(p);
1086 wakeup(&sc->flags);
1087 while (IS_BUSY(sc))
1088 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1089 SET_BUSY(sc);
1090 ADAPTER_UNLOCK(sc);
1091
1092 if (p->port_cdev != NULL)
1093 destroy_dev(p->port_cdev);
1094
1095 cxgb_uninit_synchronized(p);
1096 ether_ifdetach(p->ifp);
1097
1098 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1099 struct sge_qset *qs = &sc->sge.qs[i];
1100 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1101
1102 callout_drain(&txq->txq_watchdog);
1103 callout_drain(&txq->txq_timer);
1104 }
1105
1106 PORT_LOCK_DEINIT(p);
1107 if_free(p->ifp);
1108 p->ifp = NULL;
1109
1110 ADAPTER_LOCK(sc);
1111 CLR_BUSY(sc);
1112 wakeup_one(&sc->flags);
1113 ADAPTER_UNLOCK(sc);
1114 return (0);
1115 }
1116
1117 void
t3_fatal_err(struct adapter * sc)1118 t3_fatal_err(struct adapter *sc)
1119 {
1120 u_int fw_status[4];
1121
1122 if (sc->flags & FULL_INIT_DONE) {
1123 t3_sge_stop(sc);
1124 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1125 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1126 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1127 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1128 t3_intr_disable(sc);
1129 }
1130 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1131 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1132 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1133 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1134 }
1135
1136 int
t3_os_find_pci_capability(adapter_t * sc,int cap)1137 t3_os_find_pci_capability(adapter_t *sc, int cap)
1138 {
1139 int rc, reg = 0;
1140
1141 rc = pci_find_cap(sc->dev, cap, ®);
1142 return (rc == 0 ? reg : 0);
1143 }
1144
1145 int
t3_os_pci_save_state(struct adapter * sc)1146 t3_os_pci_save_state(struct adapter *sc)
1147 {
1148 pci_save_state(sc->dev);
1149 return (0);
1150 }
1151
1152 int
t3_os_pci_restore_state(struct adapter * sc)1153 t3_os_pci_restore_state(struct adapter *sc)
1154 {
1155 pci_restore_state(sc->dev);
1156 return (0);
1157 }
1158
1159 /**
1160 * t3_os_link_changed - handle link status changes
1161 * @sc: the adapter associated with the link change
1162 * @port_id: the port index whose link status has changed
1163 * @link_status: the new status of the link
1164 * @speed: the new speed setting
1165 * @duplex: the new duplex setting
1166 * @fc: the new flow-control setting
1167 *
1168 * This is the OS-dependent handler for link status changes. The OS
1169 * neutral handler takes care of most of the processing for these events,
1170 * then calls this handler for any OS-specific processing.
1171 */
1172 void
t3_os_link_changed(adapter_t * adapter,int port_id,int link_status,int speed,int duplex,int fc,int mac_was_reset)1173 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1174 int duplex, int fc, int mac_was_reset)
1175 {
1176 struct port_info *pi = &adapter->port[port_id];
1177 if_t ifp = pi->ifp;
1178
1179 /* no race with detach, so ifp should always be good */
1180 KASSERT(ifp, ("%s: if detached.", __func__));
1181
1182 /* Reapply mac settings if they were lost due to a reset */
1183 if (mac_was_reset) {
1184 PORT_LOCK(pi);
1185 cxgb_update_mac_settings(pi);
1186 PORT_UNLOCK(pi);
1187 }
1188
1189 if (link_status) {
1190 if_setbaudrate(ifp, IF_Mbps(speed));
1191 if_link_state_change(ifp, LINK_STATE_UP);
1192 } else
1193 if_link_state_change(ifp, LINK_STATE_DOWN);
1194 }
1195
1196 /**
1197 * t3_os_phymod_changed - handle PHY module changes
1198 * @phy: the PHY reporting the module change
1199 * @mod_type: new module type
1200 *
1201 * This is the OS-dependent handler for PHY module changes. It is
1202 * invoked when a PHY module is removed or inserted for any OS-specific
1203 * processing.
1204 */
t3_os_phymod_changed(struct adapter * adap,int port_id)1205 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1206 {
1207 static const char *mod_str[] = {
1208 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1209 };
1210 struct port_info *pi = &adap->port[port_id];
1211 int mod = pi->phy.modtype;
1212
1213 if (mod != pi->media.ifm_cur->ifm_data)
1214 cxgb_build_medialist(pi);
1215
1216 if (mod == phy_modtype_none)
1217 if_printf(pi->ifp, "PHY module unplugged\n");
1218 else {
1219 KASSERT(mod < ARRAY_SIZE(mod_str),
1220 ("invalid PHY module type %d", mod));
1221 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1222 }
1223 }
1224
1225 void
t3_os_set_hw_addr(adapter_t * adapter,int port_idx,u8 hw_addr[])1226 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1227 {
1228
1229 /*
1230 * The ifnet might not be allocated before this gets called,
1231 * as this is called early on in attach by t3_prep_adapter
1232 * save the address off in the port structure
1233 */
1234 if (cxgb_debug)
1235 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1236 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1237 }
1238
1239 /*
1240 * Programs the XGMAC based on the settings in the ifnet. These settings
1241 * include MTU, MAC address, mcast addresses, etc.
1242 */
1243 static void
cxgb_update_mac_settings(struct port_info * p)1244 cxgb_update_mac_settings(struct port_info *p)
1245 {
1246 if_t ifp = p->ifp;
1247 struct t3_rx_mode rm;
1248 struct cmac *mac = &p->mac;
1249 int mtu, hwtagging;
1250
1251 PORT_LOCK_ASSERT_OWNED(p);
1252
1253 bcopy(if_getlladdr(ifp), p->hw_addr, ETHER_ADDR_LEN);
1254
1255 mtu = if_getmtu(ifp);
1256 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU)
1257 mtu += ETHER_VLAN_ENCAP_LEN;
1258
1259 hwtagging = (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0;
1260
1261 t3_mac_set_mtu(mac, mtu);
1262 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1263 t3_mac_set_address(mac, 0, p->hw_addr);
1264 t3_init_rx_mode(&rm, p);
1265 t3_mac_set_rx_mode(mac, &rm);
1266 }
1267
1268
1269 static int
await_mgmt_replies(struct adapter * adap,unsigned long init_cnt,unsigned long n)1270 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1271 unsigned long n)
1272 {
1273 int attempts = 5;
1274
1275 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1276 if (!--attempts)
1277 return (ETIMEDOUT);
1278 t3_os_sleep(10);
1279 }
1280 return 0;
1281 }
1282
1283 static int
init_tp_parity(struct adapter * adap)1284 init_tp_parity(struct adapter *adap)
1285 {
1286 int i;
1287 struct mbuf *m;
1288 struct cpl_set_tcb_field *greq;
1289 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1290
1291 t3_tp_set_offload_mode(adap, 1);
1292
1293 for (i = 0; i < 16; i++) {
1294 struct cpl_smt_write_req *req;
1295
1296 m = m_gethdr(M_WAITOK, MT_DATA);
1297 req = mtod(m, struct cpl_smt_write_req *);
1298 m->m_len = m->m_pkthdr.len = sizeof(*req);
1299 memset(req, 0, sizeof(*req));
1300 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1301 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1302 req->iff = i;
1303 t3_mgmt_tx(adap, m);
1304 }
1305
1306 for (i = 0; i < 2048; i++) {
1307 struct cpl_l2t_write_req *req;
1308
1309 m = m_gethdr(M_WAITOK, MT_DATA);
1310 req = mtod(m, struct cpl_l2t_write_req *);
1311 m->m_len = m->m_pkthdr.len = sizeof(*req);
1312 memset(req, 0, sizeof(*req));
1313 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1314 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1315 req->params = htonl(V_L2T_W_IDX(i));
1316 t3_mgmt_tx(adap, m);
1317 }
1318
1319 for (i = 0; i < 2048; i++) {
1320 struct cpl_rte_write_req *req;
1321
1322 m = m_gethdr(M_WAITOK, MT_DATA);
1323 req = mtod(m, struct cpl_rte_write_req *);
1324 m->m_len = m->m_pkthdr.len = sizeof(*req);
1325 memset(req, 0, sizeof(*req));
1326 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1327 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1328 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1329 t3_mgmt_tx(adap, m);
1330 }
1331
1332 m = m_gethdr(M_WAITOK, MT_DATA);
1333 greq = mtod(m, struct cpl_set_tcb_field *);
1334 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1335 memset(greq, 0, sizeof(*greq));
1336 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1337 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1338 greq->mask = htobe64(1);
1339 t3_mgmt_tx(adap, m);
1340
1341 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1342 t3_tp_set_offload_mode(adap, 0);
1343 return (i);
1344 }
1345
1346 /**
1347 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1348 * @adap: the adapter
1349 *
1350 * Sets up RSS to distribute packets to multiple receive queues. We
1351 * configure the RSS CPU lookup table to distribute to the number of HW
1352 * receive queues, and the response queue lookup table to narrow that
1353 * down to the response queues actually configured for each port.
1354 * We always configure the RSS mapping for two ports since the mapping
1355 * table has plenty of entries.
1356 */
1357 static void
setup_rss(adapter_t * adap)1358 setup_rss(adapter_t *adap)
1359 {
1360 int i;
1361 u_int nq[2];
1362 uint8_t cpus[SGE_QSETS + 1];
1363 uint16_t rspq_map[RSS_TABLE_SIZE];
1364
1365 for (i = 0; i < SGE_QSETS; ++i)
1366 cpus[i] = i;
1367 cpus[SGE_QSETS] = 0xff;
1368
1369 nq[0] = nq[1] = 0;
1370 for_each_port(adap, i) {
1371 const struct port_info *pi = adap2pinfo(adap, i);
1372
1373 nq[pi->tx_chan] += pi->nqsets;
1374 }
1375 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1376 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1377 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1378 }
1379
1380 /* Calculate the reverse RSS map table */
1381 for (i = 0; i < SGE_QSETS; ++i)
1382 adap->rrss_map[i] = 0xff;
1383 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1384 if (adap->rrss_map[rspq_map[i]] == 0xff)
1385 adap->rrss_map[rspq_map[i]] = i;
1386
1387 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1388 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1389 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1390 cpus, rspq_map);
1391
1392 }
1393 static void
send_pktsched_cmd(struct adapter * adap,int sched,int qidx,int lo,int hi,int port)1394 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1395 int hi, int port)
1396 {
1397 struct mbuf *m;
1398 struct mngt_pktsched_wr *req;
1399
1400 m = m_gethdr(M_NOWAIT, MT_DATA);
1401 if (m) {
1402 req = mtod(m, struct mngt_pktsched_wr *);
1403 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1404 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1405 req->sched = sched;
1406 req->idx = qidx;
1407 req->min = lo;
1408 req->max = hi;
1409 req->binding = port;
1410 m->m_len = m->m_pkthdr.len = sizeof(*req);
1411 t3_mgmt_tx(adap, m);
1412 }
1413 }
1414
1415 static void
bind_qsets(adapter_t * sc)1416 bind_qsets(adapter_t *sc)
1417 {
1418 int i, j;
1419
1420 for (i = 0; i < (sc)->params.nports; ++i) {
1421 const struct port_info *pi = adap2pinfo(sc, i);
1422
1423 for (j = 0; j < pi->nqsets; ++j) {
1424 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1425 -1, pi->tx_chan);
1426
1427 }
1428 }
1429 }
1430
1431 static void
update_tpeeprom(struct adapter * adap)1432 update_tpeeprom(struct adapter *adap)
1433 {
1434 const struct firmware *tpeeprom;
1435
1436 uint32_t version;
1437 unsigned int major, minor;
1438 int ret, len;
1439 char rev, name[32];
1440
1441 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1442
1443 major = G_TP_VERSION_MAJOR(version);
1444 minor = G_TP_VERSION_MINOR(version);
1445 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1446 return;
1447
1448 rev = t3rev2char(adap);
1449 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1450
1451 tpeeprom = firmware_get(name);
1452 if (tpeeprom == NULL) {
1453 device_printf(adap->dev,
1454 "could not load TP EEPROM: unable to load %s\n",
1455 name);
1456 return;
1457 }
1458
1459 len = tpeeprom->datasize - 4;
1460
1461 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1462 if (ret)
1463 goto release_tpeeprom;
1464
1465 if (len != TP_SRAM_LEN) {
1466 device_printf(adap->dev,
1467 "%s length is wrong len=%d expected=%d\n", name,
1468 len, TP_SRAM_LEN);
1469 return;
1470 }
1471
1472 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1473 TP_SRAM_OFFSET);
1474
1475 if (!ret) {
1476 device_printf(adap->dev,
1477 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1478 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1479 } else
1480 device_printf(adap->dev,
1481 "Protocol SRAM image update in EEPROM failed\n");
1482
1483 release_tpeeprom:
1484 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1485
1486 return;
1487 }
1488
1489 static int
update_tpsram(struct adapter * adap)1490 update_tpsram(struct adapter *adap)
1491 {
1492 const struct firmware *tpsram;
1493 int ret;
1494 char rev, name[32];
1495
1496 rev = t3rev2char(adap);
1497 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1498
1499 update_tpeeprom(adap);
1500
1501 tpsram = firmware_get(name);
1502 if (tpsram == NULL){
1503 device_printf(adap->dev, "could not load TP SRAM\n");
1504 return (EINVAL);
1505 } else
1506 device_printf(adap->dev, "updating TP SRAM\n");
1507
1508 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1509 if (ret)
1510 goto release_tpsram;
1511
1512 ret = t3_set_proto_sram(adap, tpsram->data);
1513 if (ret)
1514 device_printf(adap->dev, "loading protocol SRAM failed\n");
1515
1516 release_tpsram:
1517 firmware_put(tpsram, FIRMWARE_UNLOAD);
1518
1519 return ret;
1520 }
1521
1522 /**
1523 * cxgb_up - enable the adapter
1524 * @adap: adapter being enabled
1525 *
1526 * Called when the first port is enabled, this function performs the
1527 * actions necessary to make an adapter operational, such as completing
1528 * the initialization of HW modules, and enabling interrupts.
1529 */
1530 static int
cxgb_up(struct adapter * sc)1531 cxgb_up(struct adapter *sc)
1532 {
1533 int err = 0;
1534 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1535
1536 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1537 __func__, sc->open_device_map));
1538
1539 if ((sc->flags & FULL_INIT_DONE) == 0) {
1540
1541 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1542
1543 if ((sc->flags & FW_UPTODATE) == 0)
1544 if ((err = upgrade_fw(sc)))
1545 goto out;
1546
1547 if ((sc->flags & TPS_UPTODATE) == 0)
1548 if ((err = update_tpsram(sc)))
1549 goto out;
1550
1551 if (is_offload(sc) && nfilters != 0) {
1552 sc->params.mc5.nservers = 0;
1553
1554 if (nfilters < 0)
1555 sc->params.mc5.nfilters = mxf;
1556 else
1557 sc->params.mc5.nfilters = min(nfilters, mxf);
1558 }
1559
1560 err = t3_init_hw(sc, 0);
1561 if (err)
1562 goto out;
1563
1564 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1565 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1566
1567 err = setup_sge_qsets(sc);
1568 if (err)
1569 goto out;
1570
1571 alloc_filters(sc);
1572 setup_rss(sc);
1573
1574 t3_add_configured_sysctls(sc);
1575 sc->flags |= FULL_INIT_DONE;
1576 }
1577
1578 t3_intr_clear(sc);
1579 t3_sge_start(sc);
1580 t3_intr_enable(sc);
1581
1582 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1583 is_offload(sc) && init_tp_parity(sc) == 0)
1584 sc->flags |= TP_PARITY_INIT;
1585
1586 if (sc->flags & TP_PARITY_INIT) {
1587 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1588 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1589 }
1590
1591 if (!(sc->flags & QUEUES_BOUND)) {
1592 bind_qsets(sc);
1593 setup_hw_filters(sc);
1594 sc->flags |= QUEUES_BOUND;
1595 }
1596
1597 t3_sge_reset_adapter(sc);
1598 out:
1599 return (err);
1600 }
1601
1602 /*
1603 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1604 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1605 * during controller_detach, not here.
1606 */
1607 static void
cxgb_down(struct adapter * sc)1608 cxgb_down(struct adapter *sc)
1609 {
1610 t3_sge_stop(sc);
1611 t3_intr_disable(sc);
1612 }
1613
1614 /*
1615 * if_init for cxgb ports.
1616 */
1617 static void
cxgb_init(void * arg)1618 cxgb_init(void *arg)
1619 {
1620 struct port_info *p = arg;
1621 struct adapter *sc = p->adapter;
1622
1623 ADAPTER_LOCK(sc);
1624 cxgb_init_locked(p); /* releases adapter lock */
1625 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1626 }
1627
1628 static int
cxgb_init_locked(struct port_info * p)1629 cxgb_init_locked(struct port_info *p)
1630 {
1631 struct adapter *sc = p->adapter;
1632 if_t ifp = p->ifp;
1633 struct cmac *mac = &p->mac;
1634 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1635
1636 ADAPTER_LOCK_ASSERT_OWNED(sc);
1637
1638 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1639 gave_up_lock = 1;
1640 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1641 rc = EINTR;
1642 goto done;
1643 }
1644 }
1645 if (IS_DOOMED(p)) {
1646 rc = ENXIO;
1647 goto done;
1648 }
1649 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1650
1651 /*
1652 * The code that runs during one-time adapter initialization can sleep
1653 * so it's important not to hold any locks across it.
1654 */
1655 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1656
1657 if (may_sleep) {
1658 SET_BUSY(sc);
1659 gave_up_lock = 1;
1660 ADAPTER_UNLOCK(sc);
1661 }
1662
1663 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1664 goto done;
1665
1666 PORT_LOCK(p);
1667 if (isset(&sc->open_device_map, p->port_id) &&
1668 (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
1669 PORT_UNLOCK(p);
1670 goto done;
1671 }
1672 t3_port_intr_enable(sc, p->port_id);
1673 if (!mac->multiport)
1674 t3_mac_init(mac);
1675 cxgb_update_mac_settings(p);
1676 t3_link_start(&p->phy, mac, &p->link_config);
1677 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1678 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1679 PORT_UNLOCK(p);
1680
1681 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1682 struct sge_qset *qs = &sc->sge.qs[i];
1683 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1684
1685 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1686 txq->txq_watchdog.c_cpu);
1687 }
1688
1689 /* all ok */
1690 setbit(&sc->open_device_map, p->port_id);
1691 callout_reset(&p->link_check_ch,
1692 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1693 link_check_callout, p);
1694
1695 done:
1696 if (may_sleep) {
1697 ADAPTER_LOCK(sc);
1698 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1699 CLR_BUSY(sc);
1700 }
1701 if (gave_up_lock)
1702 wakeup_one(&sc->flags);
1703 ADAPTER_UNLOCK(sc);
1704 return (rc);
1705 }
1706
1707 static int
cxgb_uninit_locked(struct port_info * p)1708 cxgb_uninit_locked(struct port_info *p)
1709 {
1710 struct adapter *sc = p->adapter;
1711 int rc;
1712
1713 ADAPTER_LOCK_ASSERT_OWNED(sc);
1714
1715 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1716 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1717 rc = EINTR;
1718 goto done;
1719 }
1720 }
1721 if (IS_DOOMED(p)) {
1722 rc = ENXIO;
1723 goto done;
1724 }
1725 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1726 SET_BUSY(sc);
1727 ADAPTER_UNLOCK(sc);
1728
1729 rc = cxgb_uninit_synchronized(p);
1730
1731 ADAPTER_LOCK(sc);
1732 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1733 CLR_BUSY(sc);
1734 wakeup_one(&sc->flags);
1735 done:
1736 ADAPTER_UNLOCK(sc);
1737 return (rc);
1738 }
1739
1740 /*
1741 * Called on "ifconfig down", and from port_detach
1742 */
1743 static int
cxgb_uninit_synchronized(struct port_info * pi)1744 cxgb_uninit_synchronized(struct port_info *pi)
1745 {
1746 struct adapter *sc = pi->adapter;
1747 if_t ifp = pi->ifp;
1748
1749 /*
1750 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1751 */
1752 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1753
1754 /*
1755 * Clear this port's bit from the open device map, and then drain all
1756 * the tasks that can access/manipulate this port's port_info or ifp.
1757 * We disable this port's interrupts here and so the slow/ext
1758 * interrupt tasks won't be enqueued. The tick task will continue to
1759 * be enqueued every second but the runs after this drain will not see
1760 * this port in the open device map.
1761 *
1762 * A well behaved task must take open_device_map into account and ignore
1763 * ports that are not open.
1764 */
1765 clrbit(&sc->open_device_map, pi->port_id);
1766 t3_port_intr_disable(sc, pi->port_id);
1767 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1768 taskqueue_drain(sc->tq, &sc->tick_task);
1769
1770 callout_drain(&pi->link_check_ch);
1771 taskqueue_drain(sc->tq, &pi->link_check_task);
1772
1773 PORT_LOCK(pi);
1774 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1775
1776 /* disable pause frames */
1777 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1778
1779 /* Reset RX FIFO HWM */
1780 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1781 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1782
1783 DELAY(100 * 1000);
1784
1785 /* Wait for TXFIFO empty */
1786 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1787 F_TXFIFO_EMPTY, 1, 20, 5);
1788
1789 DELAY(100 * 1000);
1790 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1791
1792 pi->phy.ops->power_down(&pi->phy, 1);
1793
1794 PORT_UNLOCK(pi);
1795
1796 pi->link_config.link_ok = 0;
1797 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1798
1799 if (sc->open_device_map == 0)
1800 cxgb_down(pi->adapter);
1801
1802 return (0);
1803 }
1804
1805 /*
1806 * Mark lro enabled or disabled in all qsets for this port
1807 */
1808 static int
cxgb_set_lro(struct port_info * p,int enabled)1809 cxgb_set_lro(struct port_info *p, int enabled)
1810 {
1811 int i;
1812 struct adapter *adp = p->adapter;
1813 struct sge_qset *q;
1814
1815 for (i = 0; i < p->nqsets; i++) {
1816 q = &adp->sge.qs[p->first_qset + i];
1817 q->lro.enabled = (enabled != 0);
1818 }
1819 return (0);
1820 }
1821
1822 static int
cxgb_ioctl(if_t ifp,unsigned long command,caddr_t data)1823 cxgb_ioctl(if_t ifp, unsigned long command, caddr_t data)
1824 {
1825 struct port_info *p = if_getsoftc(ifp);
1826 struct adapter *sc = p->adapter;
1827 struct ifreq *ifr = (struct ifreq *)data;
1828 int flags, error = 0, mtu;
1829 uint32_t mask;
1830
1831 switch (command) {
1832 case SIOCSIFMTU:
1833 ADAPTER_LOCK(sc);
1834 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1835 if (error) {
1836 fail:
1837 ADAPTER_UNLOCK(sc);
1838 return (error);
1839 }
1840
1841 mtu = ifr->ifr_mtu;
1842 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1843 error = EINVAL;
1844 } else {
1845 if_setmtu(ifp, mtu);
1846 PORT_LOCK(p);
1847 cxgb_update_mac_settings(p);
1848 PORT_UNLOCK(p);
1849 }
1850 ADAPTER_UNLOCK(sc);
1851 break;
1852 case SIOCSIFFLAGS:
1853 ADAPTER_LOCK(sc);
1854 if (IS_DOOMED(p)) {
1855 error = ENXIO;
1856 goto fail;
1857 }
1858 if (if_getflags(ifp) & IFF_UP) {
1859 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1860 flags = p->if_flags;
1861 if (((if_getflags(ifp) ^ flags) & IFF_PROMISC) ||
1862 ((if_getflags(ifp) ^ flags) & IFF_ALLMULTI)) {
1863 if (IS_BUSY(sc)) {
1864 error = EBUSY;
1865 goto fail;
1866 }
1867 PORT_LOCK(p);
1868 cxgb_update_mac_settings(p);
1869 PORT_UNLOCK(p);
1870 }
1871 ADAPTER_UNLOCK(sc);
1872 } else
1873 error = cxgb_init_locked(p);
1874 p->if_flags = if_getflags(ifp);
1875 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1876 error = cxgb_uninit_locked(p);
1877 else
1878 ADAPTER_UNLOCK(sc);
1879
1880 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1881 break;
1882 case SIOCADDMULTI:
1883 case SIOCDELMULTI:
1884 ADAPTER_LOCK(sc);
1885 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1886 if (error)
1887 goto fail;
1888
1889 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1890 PORT_LOCK(p);
1891 cxgb_update_mac_settings(p);
1892 PORT_UNLOCK(p);
1893 }
1894 ADAPTER_UNLOCK(sc);
1895
1896 break;
1897 case SIOCSIFCAP:
1898 ADAPTER_LOCK(sc);
1899 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1900 if (error)
1901 goto fail;
1902
1903 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1904 if (mask & IFCAP_TXCSUM) {
1905 if_togglecapenable(ifp, IFCAP_TXCSUM);
1906 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP);
1907
1908 if (IFCAP_TSO4 & if_getcapenable(ifp) &&
1909 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
1910 mask &= ~IFCAP_TSO4;
1911 if_setcapenablebit(ifp, 0, IFCAP_TSO4);
1912 if_printf(ifp,
1913 "tso4 disabled due to -txcsum.\n");
1914 }
1915 }
1916 if (mask & IFCAP_TXCSUM_IPV6) {
1917 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
1918 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1919
1920 if (IFCAP_TSO6 & if_getcapenable(ifp) &&
1921 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
1922 mask &= ~IFCAP_TSO6;
1923 if_setcapenablebit(ifp, 0, IFCAP_TSO6);
1924 if_printf(ifp,
1925 "tso6 disabled due to -txcsum6.\n");
1926 }
1927 }
1928 if (mask & IFCAP_RXCSUM)
1929 if_togglecapenable(ifp, IFCAP_RXCSUM);
1930 if (mask & IFCAP_RXCSUM_IPV6)
1931 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
1932
1933 /*
1934 * Note that we leave CSUM_TSO alone (it is always set). The
1935 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1936 * sending a TSO request our way, so it's sufficient to toggle
1937 * IFCAP_TSOx only.
1938 */
1939 if (mask & IFCAP_TSO4) {
1940 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
1941 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
1942 if_printf(ifp, "enable txcsum first.\n");
1943 error = EAGAIN;
1944 goto fail;
1945 }
1946 if_togglecapenable(ifp, IFCAP_TSO4);
1947 }
1948 if (mask & IFCAP_TSO6) {
1949 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
1950 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
1951 if_printf(ifp, "enable txcsum6 first.\n");
1952 error = EAGAIN;
1953 goto fail;
1954 }
1955 if_togglecapenable(ifp, IFCAP_TSO6);
1956 }
1957 if (mask & IFCAP_LRO) {
1958 if_togglecapenable(ifp, IFCAP_LRO);
1959
1960 /* Safe to do this even if cxgb_up not called yet */
1961 cxgb_set_lro(p, if_getcapenable(ifp) & IFCAP_LRO);
1962 }
1963 #ifdef TCP_OFFLOAD
1964 if (mask & IFCAP_TOE4) {
1965 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE4;
1966
1967 error = toe_capability(p, enable);
1968 if (error == 0)
1969 if_togglecapenable(ifp, mask);
1970 }
1971 #endif
1972 if (mask & IFCAP_VLAN_HWTAGGING) {
1973 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1974 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1975 PORT_LOCK(p);
1976 cxgb_update_mac_settings(p);
1977 PORT_UNLOCK(p);
1978 }
1979 }
1980 if (mask & IFCAP_VLAN_MTU) {
1981 if_togglecapenable(ifp, IFCAP_VLAN_MTU);
1982 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1983 PORT_LOCK(p);
1984 cxgb_update_mac_settings(p);
1985 PORT_UNLOCK(p);
1986 }
1987 }
1988 if (mask & IFCAP_VLAN_HWTSO)
1989 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
1990 if (mask & IFCAP_VLAN_HWCSUM)
1991 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM);
1992
1993 #ifdef VLAN_CAPABILITIES
1994 VLAN_CAPABILITIES(ifp);
1995 #endif
1996 ADAPTER_UNLOCK(sc);
1997 break;
1998 case SIOCSIFMEDIA:
1999 case SIOCGIFMEDIA:
2000 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2001 break;
2002 default:
2003 error = ether_ioctl(ifp, command, data);
2004 }
2005
2006 return (error);
2007 }
2008
2009 static int
cxgb_media_change(if_t ifp)2010 cxgb_media_change(if_t ifp)
2011 {
2012 return (EOPNOTSUPP);
2013 }
2014
2015 /*
2016 * Translates phy->modtype to the correct Ethernet media subtype.
2017 */
2018 static int
cxgb_ifm_type(int mod)2019 cxgb_ifm_type(int mod)
2020 {
2021 switch (mod) {
2022 case phy_modtype_sr:
2023 return (IFM_10G_SR);
2024 case phy_modtype_lr:
2025 return (IFM_10G_LR);
2026 case phy_modtype_lrm:
2027 return (IFM_10G_LRM);
2028 case phy_modtype_twinax:
2029 return (IFM_10G_TWINAX);
2030 case phy_modtype_twinax_long:
2031 return (IFM_10G_TWINAX_LONG);
2032 case phy_modtype_none:
2033 return (IFM_NONE);
2034 case phy_modtype_unknown:
2035 return (IFM_UNKNOWN);
2036 }
2037
2038 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2039 return (IFM_UNKNOWN);
2040 }
2041
2042 /*
2043 * Rebuilds the ifmedia list for this port, and sets the current media.
2044 */
2045 static void
cxgb_build_medialist(struct port_info * p)2046 cxgb_build_medialist(struct port_info *p)
2047 {
2048 struct cphy *phy = &p->phy;
2049 struct ifmedia *media = &p->media;
2050 int mod = phy->modtype;
2051 int m = IFM_ETHER | IFM_FDX;
2052
2053 PORT_LOCK(p);
2054
2055 ifmedia_removeall(media);
2056 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2057 /* Copper (RJ45) */
2058
2059 if (phy->caps & SUPPORTED_10000baseT_Full)
2060 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2061
2062 if (phy->caps & SUPPORTED_1000baseT_Full)
2063 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2064
2065 if (phy->caps & SUPPORTED_100baseT_Full)
2066 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2067
2068 if (phy->caps & SUPPORTED_10baseT_Full)
2069 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2070
2071 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2072 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2073
2074 } else if (phy->caps & SUPPORTED_TP) {
2075 /* Copper (CX4) */
2076
2077 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2078 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2079
2080 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2081 ifmedia_set(media, m | IFM_10G_CX4);
2082
2083 } else if (phy->caps & SUPPORTED_FIBRE &&
2084 phy->caps & SUPPORTED_10000baseT_Full) {
2085 /* 10G optical (but includes SFP+ twinax) */
2086
2087 m |= cxgb_ifm_type(mod);
2088 if (IFM_SUBTYPE(m) == IFM_NONE)
2089 m &= ~IFM_FDX;
2090
2091 ifmedia_add(media, m, mod, NULL);
2092 ifmedia_set(media, m);
2093
2094 } else if (phy->caps & SUPPORTED_FIBRE &&
2095 phy->caps & SUPPORTED_1000baseT_Full) {
2096 /* 1G optical */
2097
2098 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2099 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2100 ifmedia_set(media, m | IFM_1000_SX);
2101
2102 } else {
2103 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2104 phy->caps));
2105 }
2106
2107 PORT_UNLOCK(p);
2108 }
2109
2110 static void
cxgb_media_status(if_t ifp,struct ifmediareq * ifmr)2111 cxgb_media_status(if_t ifp, struct ifmediareq *ifmr)
2112 {
2113 struct port_info *p = if_getsoftc(ifp);
2114 struct ifmedia_entry *cur = p->media.ifm_cur;
2115 int speed = p->link_config.speed;
2116
2117 if (cur->ifm_data != p->phy.modtype) {
2118 cxgb_build_medialist(p);
2119 cur = p->media.ifm_cur;
2120 }
2121
2122 ifmr->ifm_status = IFM_AVALID;
2123 if (!p->link_config.link_ok)
2124 return;
2125
2126 ifmr->ifm_status |= IFM_ACTIVE;
2127
2128 /*
2129 * active and current will differ iff current media is autoselect. That
2130 * can happen only for copper RJ45.
2131 */
2132 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2133 return;
2134 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2135 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2136
2137 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2138 if (speed == SPEED_10000)
2139 ifmr->ifm_active |= IFM_10G_T;
2140 else if (speed == SPEED_1000)
2141 ifmr->ifm_active |= IFM_1000_T;
2142 else if (speed == SPEED_100)
2143 ifmr->ifm_active |= IFM_100_TX;
2144 else if (speed == SPEED_10)
2145 ifmr->ifm_active |= IFM_10_T;
2146 else
2147 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2148 speed));
2149 }
2150
2151 static uint64_t
cxgb_get_counter(if_t ifp,ift_counter c)2152 cxgb_get_counter(if_t ifp, ift_counter c)
2153 {
2154 struct port_info *pi = if_getsoftc(ifp);
2155 struct adapter *sc = pi->adapter;
2156 struct cmac *mac = &pi->mac;
2157 struct mac_stats *mstats = &mac->stats;
2158
2159 cxgb_refresh_stats(pi);
2160
2161 switch (c) {
2162 case IFCOUNTER_IPACKETS:
2163 return (mstats->rx_frames);
2164
2165 case IFCOUNTER_IERRORS:
2166 return (mstats->rx_jabber + mstats->rx_data_errs +
2167 mstats->rx_sequence_errs + mstats->rx_runt +
2168 mstats->rx_too_long + mstats->rx_mac_internal_errs +
2169 mstats->rx_short + mstats->rx_fcs_errs);
2170
2171 case IFCOUNTER_OPACKETS:
2172 return (mstats->tx_frames);
2173
2174 case IFCOUNTER_OERRORS:
2175 return (mstats->tx_excess_collisions + mstats->tx_underrun +
2176 mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2177 mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2178
2179 case IFCOUNTER_COLLISIONS:
2180 return (mstats->tx_total_collisions);
2181
2182 case IFCOUNTER_IBYTES:
2183 return (mstats->rx_octets);
2184
2185 case IFCOUNTER_OBYTES:
2186 return (mstats->tx_octets);
2187
2188 case IFCOUNTER_IMCASTS:
2189 return (mstats->rx_mcast_frames);
2190
2191 case IFCOUNTER_OMCASTS:
2192 return (mstats->tx_mcast_frames);
2193
2194 case IFCOUNTER_IQDROPS:
2195 return (mstats->rx_cong_drops);
2196
2197 case IFCOUNTER_OQDROPS: {
2198 int i;
2199 uint64_t drops;
2200
2201 drops = 0;
2202 if (sc->flags & FULL_INIT_DONE) {
2203 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2204 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2205 }
2206
2207 return (drops);
2208
2209 }
2210
2211 default:
2212 return (if_get_counter_default(ifp, c));
2213 }
2214 }
2215
2216 static void
cxgb_async_intr(void * data)2217 cxgb_async_intr(void *data)
2218 {
2219 adapter_t *sc = data;
2220
2221 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2222 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2223 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2224 }
2225
2226 static void
link_check_callout(void * arg)2227 link_check_callout(void *arg)
2228 {
2229 struct port_info *pi = arg;
2230 struct adapter *sc = pi->adapter;
2231
2232 if (!isset(&sc->open_device_map, pi->port_id))
2233 return;
2234
2235 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2236 }
2237
2238 static void
check_link_status(void * arg,int pending)2239 check_link_status(void *arg, int pending)
2240 {
2241 struct port_info *pi = arg;
2242 struct adapter *sc = pi->adapter;
2243
2244 if (!isset(&sc->open_device_map, pi->port_id))
2245 return;
2246
2247 t3_link_changed(sc, pi->port_id);
2248
2249 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2250 pi->link_config.link_ok == 0)
2251 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2252 }
2253
2254 void
t3_os_link_intr(struct port_info * pi)2255 t3_os_link_intr(struct port_info *pi)
2256 {
2257 /*
2258 * Schedule a link check in the near future. If the link is flapping
2259 * rapidly we'll keep resetting the callout and delaying the check until
2260 * things stabilize a bit.
2261 */
2262 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2263 }
2264
2265 static void
check_t3b2_mac(struct adapter * sc)2266 check_t3b2_mac(struct adapter *sc)
2267 {
2268 int i;
2269
2270 if (sc->flags & CXGB_SHUTDOWN)
2271 return;
2272
2273 for_each_port(sc, i) {
2274 struct port_info *p = &sc->port[i];
2275 int status;
2276 #ifdef INVARIANTS
2277 if_t ifp = p->ifp;
2278 #endif
2279
2280 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2281 !p->link_config.link_ok)
2282 continue;
2283
2284 KASSERT(if_getdrvflags(ifp) & IFF_DRV_RUNNING,
2285 ("%s: state mismatch (drv_flags %x, device_map %x)",
2286 __func__, if_getdrvflags(ifp), sc->open_device_map));
2287
2288 PORT_LOCK(p);
2289 status = t3b2_mac_watchdog_task(&p->mac);
2290 if (status == 1)
2291 p->mac.stats.num_toggled++;
2292 else if (status == 2) {
2293 struct cmac *mac = &p->mac;
2294
2295 cxgb_update_mac_settings(p);
2296 t3_link_start(&p->phy, mac, &p->link_config);
2297 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2298 t3_port_intr_enable(sc, p->port_id);
2299 p->mac.stats.num_resets++;
2300 }
2301 PORT_UNLOCK(p);
2302 }
2303 }
2304
2305 static void
cxgb_tick(void * arg)2306 cxgb_tick(void *arg)
2307 {
2308 adapter_t *sc = (adapter_t *)arg;
2309
2310 if (sc->flags & CXGB_SHUTDOWN)
2311 return;
2312
2313 taskqueue_enqueue(sc->tq, &sc->tick_task);
2314 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2315 }
2316
2317 void
cxgb_refresh_stats(struct port_info * pi)2318 cxgb_refresh_stats(struct port_info *pi)
2319 {
2320 struct timeval tv;
2321 const struct timeval interval = {0, 250000}; /* 250ms */
2322
2323 getmicrotime(&tv);
2324 timevalsub(&tv, &interval);
2325 if (timevalcmp(&tv, &pi->last_refreshed, <))
2326 return;
2327
2328 PORT_LOCK(pi);
2329 t3_mac_update_stats(&pi->mac);
2330 PORT_UNLOCK(pi);
2331 getmicrotime(&pi->last_refreshed);
2332 }
2333
2334 static void
cxgb_tick_handler(void * arg,int count)2335 cxgb_tick_handler(void *arg, int count)
2336 {
2337 adapter_t *sc = (adapter_t *)arg;
2338 const struct adapter_params *p = &sc->params;
2339 int i;
2340 uint32_t cause, reset;
2341
2342 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2343 return;
2344
2345 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2346 check_t3b2_mac(sc);
2347
2348 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2349 if (cause) {
2350 struct sge_qset *qs = &sc->sge.qs[0];
2351 uint32_t mask, v;
2352
2353 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2354
2355 mask = 1;
2356 for (i = 0; i < SGE_QSETS; i++) {
2357 if (v & mask)
2358 qs[i].rspq.starved++;
2359 mask <<= 1;
2360 }
2361
2362 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2363
2364 for (i = 0; i < SGE_QSETS * 2; i++) {
2365 if (v & mask) {
2366 qs[i / 2].fl[i % 2].empty++;
2367 }
2368 mask <<= 1;
2369 }
2370
2371 /* clear */
2372 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2373 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2374 }
2375
2376 for (i = 0; i < sc->params.nports; i++) {
2377 struct port_info *pi = &sc->port[i];
2378 struct cmac *mac = &pi->mac;
2379
2380 if (!isset(&sc->open_device_map, pi->port_id))
2381 continue;
2382
2383 cxgb_refresh_stats(pi);
2384
2385 if (mac->multiport)
2386 continue;
2387
2388 /* Count rx fifo overflows, once per second */
2389 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2390 reset = 0;
2391 if (cause & F_RXFIFO_OVERFLOW) {
2392 mac->stats.rx_fifo_ovfl++;
2393 reset |= F_RXFIFO_OVERFLOW;
2394 }
2395 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2396 }
2397 }
2398
2399 static void
touch_bars(device_t dev)2400 touch_bars(device_t dev)
2401 {
2402 /*
2403 * Don't enable yet
2404 */
2405 #if !defined(__LP64__) && 0
2406 u32 v;
2407
2408 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2409 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2410 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2411 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2412 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2413 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2414 #endif
2415 }
2416
2417 static int
set_eeprom(struct port_info * pi,const uint8_t * data,int len,int offset)2418 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2419 {
2420 uint8_t *buf;
2421 int err = 0;
2422 u32 aligned_offset, aligned_len, *p;
2423 struct adapter *adapter = pi->adapter;
2424
2425
2426 aligned_offset = offset & ~3;
2427 aligned_len = (len + (offset & 3) + 3) & ~3;
2428
2429 if (aligned_offset != offset || aligned_len != len) {
2430 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK | M_ZERO);
2431 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2432 if (!err && aligned_len > 4)
2433 err = t3_seeprom_read(adapter,
2434 aligned_offset + aligned_len - 4,
2435 (u32 *)&buf[aligned_len - 4]);
2436 if (err)
2437 goto out;
2438 memcpy(buf + (offset & 3), data, len);
2439 } else
2440 buf = (uint8_t *)(uintptr_t)data;
2441
2442 err = t3_seeprom_wp(adapter, 0);
2443 if (err)
2444 goto out;
2445
2446 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2447 err = t3_seeprom_write(adapter, aligned_offset, *p);
2448 aligned_offset += 4;
2449 }
2450
2451 if (!err)
2452 err = t3_seeprom_wp(adapter, 1);
2453 out:
2454 if (buf != data)
2455 free(buf, M_DEVBUF);
2456 return err;
2457 }
2458
2459
2460 static int
in_range(int val,int lo,int hi)2461 in_range(int val, int lo, int hi)
2462 {
2463 return val < 0 || (val <= hi && val >= lo);
2464 }
2465
2466 static int
cxgb_extension_open(struct cdev * dev,int flags,int fmp,struct thread * td)2467 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2468 {
2469 return (0);
2470 }
2471
2472 static int
cxgb_extension_close(struct cdev * dev,int flags,int fmt,struct thread * td)2473 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2474 {
2475 return (0);
2476 }
2477
2478 static int
cxgb_extension_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)2479 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2480 int fflag, struct thread *td)
2481 {
2482 int mmd, error = 0;
2483 struct port_info *pi = dev->si_drv1;
2484 adapter_t *sc = pi->adapter;
2485
2486 #ifdef PRIV_SUPPORTED
2487 if (priv_check(td, PRIV_DRIVER)) {
2488 if (cxgb_debug)
2489 printf("user does not have access to privileged ioctls\n");
2490 return (EPERM);
2491 }
2492 #else
2493 if (suser(td)) {
2494 if (cxgb_debug)
2495 printf("user does not have access to privileged ioctls\n");
2496 return (EPERM);
2497 }
2498 #endif
2499
2500 switch (cmd) {
2501 case CHELSIO_GET_MIIREG: {
2502 uint32_t val;
2503 struct cphy *phy = &pi->phy;
2504 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2505
2506 if (!phy->mdio_read)
2507 return (EOPNOTSUPP);
2508 if (is_10G(sc)) {
2509 mmd = mid->phy_id >> 8;
2510 if (!mmd)
2511 mmd = MDIO_DEV_PCS;
2512 else if (mmd > MDIO_DEV_VEND2)
2513 return (EINVAL);
2514
2515 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2516 mid->reg_num, &val);
2517 } else
2518 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2519 mid->reg_num & 0x1f, &val);
2520 if (error == 0)
2521 mid->val_out = val;
2522 break;
2523 }
2524 case CHELSIO_SET_MIIREG: {
2525 struct cphy *phy = &pi->phy;
2526 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2527
2528 if (!phy->mdio_write)
2529 return (EOPNOTSUPP);
2530 if (is_10G(sc)) {
2531 mmd = mid->phy_id >> 8;
2532 if (!mmd)
2533 mmd = MDIO_DEV_PCS;
2534 else if (mmd > MDIO_DEV_VEND2)
2535 return (EINVAL);
2536
2537 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2538 mmd, mid->reg_num, mid->val_in);
2539 } else
2540 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2541 mid->reg_num & 0x1f,
2542 mid->val_in);
2543 break;
2544 }
2545 case CHELSIO_SETREG: {
2546 struct ch_reg *edata = (struct ch_reg *)data;
2547 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2548 return (EFAULT);
2549 t3_write_reg(sc, edata->addr, edata->val);
2550 break;
2551 }
2552 case CHELSIO_GETREG: {
2553 struct ch_reg *edata = (struct ch_reg *)data;
2554 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2555 return (EFAULT);
2556 edata->val = t3_read_reg(sc, edata->addr);
2557 break;
2558 }
2559 case CHELSIO_GET_SGE_CONTEXT: {
2560 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2561 mtx_lock_spin(&sc->sge.reg_lock);
2562 switch (ecntxt->cntxt_type) {
2563 case CNTXT_TYPE_EGRESS:
2564 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2565 ecntxt->data);
2566 break;
2567 case CNTXT_TYPE_FL:
2568 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2569 ecntxt->data);
2570 break;
2571 case CNTXT_TYPE_RSP:
2572 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2573 ecntxt->data);
2574 break;
2575 case CNTXT_TYPE_CQ:
2576 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2577 ecntxt->data);
2578 break;
2579 default:
2580 error = EINVAL;
2581 break;
2582 }
2583 mtx_unlock_spin(&sc->sge.reg_lock);
2584 break;
2585 }
2586 case CHELSIO_GET_SGE_DESC: {
2587 struct ch_desc *edesc = (struct ch_desc *)data;
2588 int ret;
2589 if (edesc->queue_num >= SGE_QSETS * 6)
2590 return (EINVAL);
2591 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2592 edesc->queue_num % 6, edesc->idx, edesc->data);
2593 if (ret < 0)
2594 return (EINVAL);
2595 edesc->size = ret;
2596 break;
2597 }
2598 case CHELSIO_GET_QSET_PARAMS: {
2599 struct qset_params *q;
2600 struct ch_qset_params *t = (struct ch_qset_params *)data;
2601 int q1 = pi->first_qset;
2602 int nqsets = pi->nqsets;
2603 int i;
2604
2605 if (t->qset_idx >= nqsets)
2606 return EINVAL;
2607
2608 i = q1 + t->qset_idx;
2609 q = &sc->params.sge.qset[i];
2610 t->rspq_size = q->rspq_size;
2611 t->txq_size[0] = q->txq_size[0];
2612 t->txq_size[1] = q->txq_size[1];
2613 t->txq_size[2] = q->txq_size[2];
2614 t->fl_size[0] = q->fl_size;
2615 t->fl_size[1] = q->jumbo_size;
2616 t->polling = q->polling;
2617 t->lro = q->lro;
2618 t->intr_lat = q->coalesce_usecs;
2619 t->cong_thres = q->cong_thres;
2620 t->qnum = i;
2621
2622 if ((sc->flags & FULL_INIT_DONE) == 0)
2623 t->vector = 0;
2624 else if (sc->flags & USING_MSIX)
2625 t->vector = rman_get_start(sc->msix_irq_res[i]);
2626 else
2627 t->vector = rman_get_start(sc->irq_res);
2628
2629 break;
2630 }
2631 case CHELSIO_GET_QSET_NUM: {
2632 struct ch_reg *edata = (struct ch_reg *)data;
2633 edata->val = pi->nqsets;
2634 break;
2635 }
2636 case CHELSIO_LOAD_FW: {
2637 uint8_t *fw_data;
2638 uint32_t vers;
2639 struct ch_mem_range *t = (struct ch_mem_range *)data;
2640
2641 /*
2642 * You're allowed to load a firmware only before FULL_INIT_DONE
2643 *
2644 * FW_UPTODATE is also set so the rest of the initialization
2645 * will not overwrite what was loaded here. This gives you the
2646 * flexibility to load any firmware (and maybe shoot yourself in
2647 * the foot).
2648 */
2649
2650 ADAPTER_LOCK(sc);
2651 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2652 ADAPTER_UNLOCK(sc);
2653 return (EBUSY);
2654 }
2655
2656 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2657 if (!fw_data)
2658 error = ENOMEM;
2659 else
2660 error = copyin(t->buf, fw_data, t->len);
2661
2662 if (!error)
2663 error = -t3_load_fw(sc, fw_data, t->len);
2664
2665 if (t3_get_fw_version(sc, &vers) == 0) {
2666 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2667 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2668 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2669 }
2670
2671 if (!error)
2672 sc->flags |= FW_UPTODATE;
2673
2674 free(fw_data, M_DEVBUF);
2675 ADAPTER_UNLOCK(sc);
2676 break;
2677 }
2678 case CHELSIO_LOAD_BOOT: {
2679 uint8_t *boot_data;
2680 struct ch_mem_range *t = (struct ch_mem_range *)data;
2681
2682 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2683 if (!boot_data)
2684 return ENOMEM;
2685
2686 error = copyin(t->buf, boot_data, t->len);
2687 if (!error)
2688 error = -t3_load_boot(sc, boot_data, t->len);
2689
2690 free(boot_data, M_DEVBUF);
2691 break;
2692 }
2693 case CHELSIO_GET_PM: {
2694 struct ch_pm *m = (struct ch_pm *)data;
2695 struct tp_params *p = &sc->params.tp;
2696
2697 if (!is_offload(sc))
2698 return (EOPNOTSUPP);
2699
2700 m->tx_pg_sz = p->tx_pg_size;
2701 m->tx_num_pg = p->tx_num_pgs;
2702 m->rx_pg_sz = p->rx_pg_size;
2703 m->rx_num_pg = p->rx_num_pgs;
2704 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2705
2706 break;
2707 }
2708 case CHELSIO_SET_PM: {
2709 struct ch_pm *m = (struct ch_pm *)data;
2710 struct tp_params *p = &sc->params.tp;
2711
2712 if (!is_offload(sc))
2713 return (EOPNOTSUPP);
2714 if (sc->flags & FULL_INIT_DONE)
2715 return (EBUSY);
2716
2717 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2718 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2719 return (EINVAL); /* not power of 2 */
2720 if (!(m->rx_pg_sz & 0x14000))
2721 return (EINVAL); /* not 16KB or 64KB */
2722 if (!(m->tx_pg_sz & 0x1554000))
2723 return (EINVAL);
2724 if (m->tx_num_pg == -1)
2725 m->tx_num_pg = p->tx_num_pgs;
2726 if (m->rx_num_pg == -1)
2727 m->rx_num_pg = p->rx_num_pgs;
2728 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2729 return (EINVAL);
2730 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2731 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2732 return (EINVAL);
2733
2734 p->rx_pg_size = m->rx_pg_sz;
2735 p->tx_pg_size = m->tx_pg_sz;
2736 p->rx_num_pgs = m->rx_num_pg;
2737 p->tx_num_pgs = m->tx_num_pg;
2738 break;
2739 }
2740 case CHELSIO_SETMTUTAB: {
2741 struct ch_mtus *m = (struct ch_mtus *)data;
2742 int i;
2743
2744 if (!is_offload(sc))
2745 return (EOPNOTSUPP);
2746 if (offload_running(sc))
2747 return (EBUSY);
2748 if (m->nmtus != NMTUS)
2749 return (EINVAL);
2750 if (m->mtus[0] < 81) /* accommodate SACK */
2751 return (EINVAL);
2752
2753 /*
2754 * MTUs must be in ascending order
2755 */
2756 for (i = 1; i < NMTUS; ++i)
2757 if (m->mtus[i] < m->mtus[i - 1])
2758 return (EINVAL);
2759
2760 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2761 break;
2762 }
2763 case CHELSIO_GETMTUTAB: {
2764 struct ch_mtus *m = (struct ch_mtus *)data;
2765
2766 if (!is_offload(sc))
2767 return (EOPNOTSUPP);
2768
2769 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2770 m->nmtus = NMTUS;
2771 break;
2772 }
2773 case CHELSIO_GET_MEM: {
2774 struct ch_mem_range *t = (struct ch_mem_range *)data;
2775 struct mc7 *mem;
2776 uint8_t *useraddr;
2777 u64 buf[32];
2778
2779 /*
2780 * Use these to avoid modifying len/addr in the return
2781 * struct
2782 */
2783 uint32_t len = t->len, addr = t->addr;
2784
2785 if (!is_offload(sc))
2786 return (EOPNOTSUPP);
2787 if (!(sc->flags & FULL_INIT_DONE))
2788 return (EIO); /* need the memory controllers */
2789 if ((addr & 0x7) || (len & 0x7))
2790 return (EINVAL);
2791 if (t->mem_id == MEM_CM)
2792 mem = &sc->cm;
2793 else if (t->mem_id == MEM_PMRX)
2794 mem = &sc->pmrx;
2795 else if (t->mem_id == MEM_PMTX)
2796 mem = &sc->pmtx;
2797 else
2798 return (EINVAL);
2799
2800 /*
2801 * Version scheme:
2802 * bits 0..9: chip version
2803 * bits 10..15: chip revision
2804 */
2805 t->version = 3 | (sc->params.rev << 10);
2806
2807 /*
2808 * Read 256 bytes at a time as len can be large and we don't
2809 * want to use huge intermediate buffers.
2810 */
2811 useraddr = (uint8_t *)t->buf;
2812 while (len) {
2813 unsigned int chunk = min(len, sizeof(buf));
2814
2815 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2816 if (error)
2817 return (-error);
2818 if (copyout(buf, useraddr, chunk))
2819 return (EFAULT);
2820 useraddr += chunk;
2821 addr += chunk;
2822 len -= chunk;
2823 }
2824 break;
2825 }
2826 case CHELSIO_READ_TCAM_WORD: {
2827 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2828
2829 if (!is_offload(sc))
2830 return (EOPNOTSUPP);
2831 if (!(sc->flags & FULL_INIT_DONE))
2832 return (EIO); /* need MC5 */
2833 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2834 break;
2835 }
2836 case CHELSIO_SET_TRACE_FILTER: {
2837 struct ch_trace *t = (struct ch_trace *)data;
2838 const struct trace_params *tp;
2839
2840 tp = (const struct trace_params *)&t->sip;
2841 if (t->config_tx)
2842 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2843 t->trace_tx);
2844 if (t->config_rx)
2845 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2846 t->trace_rx);
2847 break;
2848 }
2849 case CHELSIO_SET_PKTSCHED: {
2850 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2851 if (sc->open_device_map == 0)
2852 return (EAGAIN);
2853 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2854 p->binding);
2855 break;
2856 }
2857 case CHELSIO_IFCONF_GETREGS: {
2858 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2859 int reglen = cxgb_get_regs_len();
2860 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2861 if (buf == NULL) {
2862 return (ENOMEM);
2863 }
2864 if (regs->len > reglen)
2865 regs->len = reglen;
2866 else if (regs->len < reglen)
2867 error = ENOBUFS;
2868
2869 if (!error) {
2870 cxgb_get_regs(sc, regs, buf);
2871 error = copyout(buf, regs->data, reglen);
2872 }
2873 free(buf, M_DEVBUF);
2874
2875 break;
2876 }
2877 case CHELSIO_SET_HW_SCHED: {
2878 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2879 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2880
2881 if ((sc->flags & FULL_INIT_DONE) == 0)
2882 return (EAGAIN); /* need TP to be initialized */
2883 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2884 !in_range(t->channel, 0, 1) ||
2885 !in_range(t->kbps, 0, 10000000) ||
2886 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2887 !in_range(t->flow_ipg, 0,
2888 dack_ticks_to_usec(sc, 0x7ff)))
2889 return (EINVAL);
2890
2891 if (t->kbps >= 0) {
2892 error = t3_config_sched(sc, t->kbps, t->sched);
2893 if (error < 0)
2894 return (-error);
2895 }
2896 if (t->class_ipg >= 0)
2897 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2898 if (t->flow_ipg >= 0) {
2899 t->flow_ipg *= 1000; /* us -> ns */
2900 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2901 }
2902 if (t->mode >= 0) {
2903 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2904
2905 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2906 bit, t->mode ? bit : 0);
2907 }
2908 if (t->channel >= 0)
2909 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2910 1 << t->sched, t->channel << t->sched);
2911 break;
2912 }
2913 case CHELSIO_GET_EEPROM: {
2914 int i;
2915 struct ch_eeprom *e = (struct ch_eeprom *)data;
2916 uint8_t *buf;
2917
2918 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
2919 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
2920 return (EINVAL);
2921 }
2922
2923 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2924 if (buf == NULL) {
2925 return (ENOMEM);
2926 }
2927 e->magic = EEPROM_MAGIC;
2928 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2929 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2930
2931 if (!error)
2932 error = copyout(buf + e->offset, e->data, e->len);
2933
2934 free(buf, M_DEVBUF);
2935 break;
2936 }
2937 case CHELSIO_CLEAR_STATS: {
2938 if (!(sc->flags & FULL_INIT_DONE))
2939 return EAGAIN;
2940
2941 PORT_LOCK(pi);
2942 t3_mac_update_stats(&pi->mac);
2943 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2944 PORT_UNLOCK(pi);
2945 break;
2946 }
2947 case CHELSIO_GET_UP_LA: {
2948 struct ch_up_la *la = (struct ch_up_la *)data;
2949 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
2950 if (buf == NULL) {
2951 return (ENOMEM);
2952 }
2953 if (la->bufsize < LA_BUFSIZE)
2954 error = ENOBUFS;
2955
2956 if (!error)
2957 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
2958 &la->bufsize, buf);
2959 if (!error)
2960 error = copyout(buf, la->data, la->bufsize);
2961
2962 free(buf, M_DEVBUF);
2963 break;
2964 }
2965 case CHELSIO_GET_UP_IOQS: {
2966 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
2967 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
2968 uint32_t *v;
2969
2970 if (buf == NULL) {
2971 return (ENOMEM);
2972 }
2973 if (ioqs->bufsize < IOQS_BUFSIZE)
2974 error = ENOBUFS;
2975
2976 if (!error)
2977 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
2978
2979 if (!error) {
2980 v = (uint32_t *)buf;
2981
2982 ioqs->ioq_rx_enable = *v++;
2983 ioqs->ioq_tx_enable = *v++;
2984 ioqs->ioq_rx_status = *v++;
2985 ioqs->ioq_tx_status = *v++;
2986
2987 error = copyout(v, ioqs->data, ioqs->bufsize);
2988 }
2989
2990 free(buf, M_DEVBUF);
2991 break;
2992 }
2993 case CHELSIO_SET_FILTER: {
2994 struct ch_filter *f = (struct ch_filter *)data;
2995 struct filter_info *p;
2996 unsigned int nfilters = sc->params.mc5.nfilters;
2997
2998 if (!is_offload(sc))
2999 return (EOPNOTSUPP); /* No TCAM */
3000 if (!(sc->flags & FULL_INIT_DONE))
3001 return (EAGAIN); /* mc5 not setup yet */
3002 if (nfilters == 0)
3003 return (EBUSY); /* TOE will use TCAM */
3004
3005 /* sanity checks */
3006 if (f->filter_id >= nfilters ||
3007 (f->val.dip && f->mask.dip != 0xffffffff) ||
3008 (f->val.sport && f->mask.sport != 0xffff) ||
3009 (f->val.dport && f->mask.dport != 0xffff) ||
3010 (f->val.vlan && f->mask.vlan != 0xfff) ||
3011 (f->val.vlan_prio &&
3012 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3013 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3014 f->qset >= SGE_QSETS ||
3015 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3016 return (EINVAL);
3017
3018 /* Was allocated with M_WAITOK */
3019 KASSERT(sc->filters, ("filter table NULL\n"));
3020
3021 p = &sc->filters[f->filter_id];
3022 if (p->locked)
3023 return (EPERM);
3024
3025 bzero(p, sizeof(*p));
3026 p->sip = f->val.sip;
3027 p->sip_mask = f->mask.sip;
3028 p->dip = f->val.dip;
3029 p->sport = f->val.sport;
3030 p->dport = f->val.dport;
3031 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3032 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3033 FILTER_NO_VLAN_PRI;
3034 p->mac_hit = f->mac_hit;
3035 p->mac_vld = f->mac_addr_idx != 0xffff;
3036 p->mac_idx = f->mac_addr_idx;
3037 p->pkt_type = f->proto;
3038 p->report_filter_id = f->want_filter_id;
3039 p->pass = f->pass;
3040 p->rss = f->rss;
3041 p->qset = f->qset;
3042
3043 error = set_filter(sc, f->filter_id, p);
3044 if (error == 0)
3045 p->valid = 1;
3046 break;
3047 }
3048 case CHELSIO_DEL_FILTER: {
3049 struct ch_filter *f = (struct ch_filter *)data;
3050 struct filter_info *p;
3051 unsigned int nfilters = sc->params.mc5.nfilters;
3052
3053 if (!is_offload(sc))
3054 return (EOPNOTSUPP);
3055 if (!(sc->flags & FULL_INIT_DONE))
3056 return (EAGAIN);
3057 if (nfilters == 0 || sc->filters == NULL)
3058 return (EINVAL);
3059 if (f->filter_id >= nfilters)
3060 return (EINVAL);
3061
3062 p = &sc->filters[f->filter_id];
3063 if (p->locked)
3064 return (EPERM);
3065 if (!p->valid)
3066 return (EFAULT); /* Read "Bad address" as "Bad index" */
3067
3068 bzero(p, sizeof(*p));
3069 p->sip = p->sip_mask = 0xffffffff;
3070 p->vlan = 0xfff;
3071 p->vlan_prio = FILTER_NO_VLAN_PRI;
3072 p->pkt_type = 1;
3073 error = set_filter(sc, f->filter_id, p);
3074 break;
3075 }
3076 case CHELSIO_GET_FILTER: {
3077 struct ch_filter *f = (struct ch_filter *)data;
3078 struct filter_info *p;
3079 unsigned int i, nfilters = sc->params.mc5.nfilters;
3080
3081 if (!is_offload(sc))
3082 return (EOPNOTSUPP);
3083 if (!(sc->flags & FULL_INIT_DONE))
3084 return (EAGAIN);
3085 if (nfilters == 0 || sc->filters == NULL)
3086 return (EINVAL);
3087
3088 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3089 for (; i < nfilters; i++) {
3090 p = &sc->filters[i];
3091 if (!p->valid)
3092 continue;
3093
3094 bzero(f, sizeof(*f));
3095
3096 f->filter_id = i;
3097 f->val.sip = p->sip;
3098 f->mask.sip = p->sip_mask;
3099 f->val.dip = p->dip;
3100 f->mask.dip = p->dip ? 0xffffffff : 0;
3101 f->val.sport = p->sport;
3102 f->mask.sport = p->sport ? 0xffff : 0;
3103 f->val.dport = p->dport;
3104 f->mask.dport = p->dport ? 0xffff : 0;
3105 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3106 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3107 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3108 0 : p->vlan_prio;
3109 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3110 0 : FILTER_NO_VLAN_PRI;
3111 f->mac_hit = p->mac_hit;
3112 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3113 f->proto = p->pkt_type;
3114 f->want_filter_id = p->report_filter_id;
3115 f->pass = p->pass;
3116 f->rss = p->rss;
3117 f->qset = p->qset;
3118
3119 break;
3120 }
3121
3122 if (i == nfilters)
3123 f->filter_id = 0xffffffff;
3124 break;
3125 }
3126 default:
3127 return (EOPNOTSUPP);
3128 break;
3129 }
3130
3131 return (error);
3132 }
3133
3134 static __inline void
reg_block_dump(struct adapter * ap,uint8_t * buf,unsigned int start,unsigned int end)3135 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3136 unsigned int end)
3137 {
3138 uint32_t *p = (uint32_t *)(buf + start);
3139
3140 for ( ; start <= end; start += sizeof(uint32_t))
3141 *p++ = t3_read_reg(ap, start);
3142 }
3143
3144 #define T3_REGMAP_SIZE (3 * 1024)
3145 static int
cxgb_get_regs_len(void)3146 cxgb_get_regs_len(void)
3147 {
3148 return T3_REGMAP_SIZE;
3149 }
3150
3151 static void
cxgb_get_regs(adapter_t * sc,struct ch_ifconf_regs * regs,uint8_t * buf)3152 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3153 {
3154
3155 /*
3156 * Version scheme:
3157 * bits 0..9: chip version
3158 * bits 10..15: chip revision
3159 * bit 31: set for PCIe cards
3160 */
3161 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3162
3163 /*
3164 * We skip the MAC statistics registers because they are clear-on-read.
3165 * Also reading multi-register stats would need to synchronize with the
3166 * periodic mac stats accumulation. Hard to justify the complexity.
3167 */
3168 memset(buf, 0, cxgb_get_regs_len());
3169 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3170 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3171 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3172 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3173 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3174 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3175 XGM_REG(A_XGM_SERDES_STAT3, 1));
3176 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3177 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3178 }
3179
3180 static int
alloc_filters(struct adapter * sc)3181 alloc_filters(struct adapter *sc)
3182 {
3183 struct filter_info *p;
3184 unsigned int nfilters = sc->params.mc5.nfilters;
3185
3186 if (nfilters == 0)
3187 return (0);
3188
3189 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3190 sc->filters = p;
3191
3192 p = &sc->filters[nfilters - 1];
3193 p->vlan = 0xfff;
3194 p->vlan_prio = FILTER_NO_VLAN_PRI;
3195 p->pass = p->rss = p->valid = p->locked = 1;
3196
3197 return (0);
3198 }
3199
3200 static int
setup_hw_filters(struct adapter * sc)3201 setup_hw_filters(struct adapter *sc)
3202 {
3203 int i, rc;
3204 unsigned int nfilters = sc->params.mc5.nfilters;
3205
3206 if (!sc->filters)
3207 return (0);
3208
3209 t3_enable_filters(sc);
3210
3211 for (i = rc = 0; i < nfilters && !rc; i++) {
3212 if (sc->filters[i].locked)
3213 rc = set_filter(sc, i, &sc->filters[i]);
3214 }
3215
3216 return (rc);
3217 }
3218
3219 static int
set_filter(struct adapter * sc,int id,const struct filter_info * f)3220 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3221 {
3222 int len;
3223 struct mbuf *m;
3224 struct ulp_txpkt *txpkt;
3225 struct work_request_hdr *wr;
3226 struct cpl_pass_open_req *oreq;
3227 struct cpl_set_tcb_field *sreq;
3228
3229 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3230 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3231
3232 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3233 sc->params.mc5.nfilters;
3234
3235 m = m_gethdr(M_WAITOK, MT_DATA);
3236 m->m_len = m->m_pkthdr.len = len;
3237 bzero(mtod(m, char *), len);
3238
3239 wr = mtod(m, struct work_request_hdr *);
3240 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3241
3242 oreq = (struct cpl_pass_open_req *)(wr + 1);
3243 txpkt = (struct ulp_txpkt *)oreq;
3244 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3245 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3246 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3247 oreq->local_port = htons(f->dport);
3248 oreq->peer_port = htons(f->sport);
3249 oreq->local_ip = htonl(f->dip);
3250 oreq->peer_ip = htonl(f->sip);
3251 oreq->peer_netmask = htonl(f->sip_mask);
3252 oreq->opt0h = 0;
3253 oreq->opt0l = htonl(F_NO_OFFLOAD);
3254 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3255 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3256 V_VLAN_PRI(f->vlan_prio >> 1) |
3257 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3258 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3259 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3260
3261 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3262 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3263 (f->report_filter_id << 15) | (1 << 23) |
3264 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3265 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3266 t3_mgmt_tx(sc, m);
3267
3268 if (f->pass && !f->rss) {
3269 len = sizeof(*sreq);
3270 m = m_gethdr(M_WAITOK, MT_DATA);
3271 m->m_len = m->m_pkthdr.len = len;
3272 bzero(mtod(m, char *), len);
3273 sreq = mtod(m, struct cpl_set_tcb_field *);
3274 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3275 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3276 (u64)sc->rrss_map[f->qset] << 19);
3277 t3_mgmt_tx(sc, m);
3278 }
3279 return 0;
3280 }
3281
3282 static inline void
mk_set_tcb_field(struct cpl_set_tcb_field * req,unsigned int tid,unsigned int word,u64 mask,u64 val)3283 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3284 unsigned int word, u64 mask, u64 val)
3285 {
3286 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3287 req->reply = V_NO_REPLY(1);
3288 req->cpu_idx = 0;
3289 req->word = htons(word);
3290 req->mask = htobe64(mask);
3291 req->val = htobe64(val);
3292 }
3293
3294 static inline void
set_tcb_field_ulp(struct cpl_set_tcb_field * req,unsigned int tid,unsigned int word,u64 mask,u64 val)3295 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3296 unsigned int word, u64 mask, u64 val)
3297 {
3298 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3299
3300 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3301 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3302 mk_set_tcb_field(req, tid, word, mask, val);
3303 }
3304
3305 void
t3_iterate(void (* func)(struct adapter *,void *),void * arg)3306 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3307 {
3308 struct adapter *sc;
3309
3310 mtx_lock(&t3_list_lock);
3311 SLIST_FOREACH(sc, &t3_list, link) {
3312 /*
3313 * func should not make any assumptions about what state sc is
3314 * in - the only guarantee is that sc->sc_lock is a valid lock.
3315 */
3316 func(sc, arg);
3317 }
3318 mtx_unlock(&t3_list_lock);
3319 }
3320
3321 #ifdef TCP_OFFLOAD
3322 static int
toe_capability(struct port_info * pi,int enable)3323 toe_capability(struct port_info *pi, int enable)
3324 {
3325 int rc;
3326 struct adapter *sc = pi->adapter;
3327
3328 ADAPTER_LOCK_ASSERT_OWNED(sc);
3329
3330 if (!is_offload(sc))
3331 return (ENODEV);
3332
3333 if (enable) {
3334 if (!(sc->flags & FULL_INIT_DONE)) {
3335 log(LOG_WARNING,
3336 "You must enable a cxgb interface first\n");
3337 return (EAGAIN);
3338 }
3339
3340 if (isset(&sc->offload_map, pi->port_id))
3341 return (0);
3342
3343 if (!(sc->flags & TOM_INIT_DONE)) {
3344 rc = t3_activate_uld(sc, ULD_TOM);
3345 if (rc == EAGAIN) {
3346 log(LOG_WARNING,
3347 "You must kldload t3_tom.ko before trying "
3348 "to enable TOE on a cxgb interface.\n");
3349 }
3350 if (rc != 0)
3351 return (rc);
3352 KASSERT(sc->tom_softc != NULL,
3353 ("%s: TOM activated but softc NULL", __func__));
3354 KASSERT(sc->flags & TOM_INIT_DONE,
3355 ("%s: TOM activated but flag not set", __func__));
3356 }
3357
3358 setbit(&sc->offload_map, pi->port_id);
3359
3360 /*
3361 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3362 * enabled on any port. Need to figure out how to enable,
3363 * disable, load, and unload iWARP cleanly.
3364 */
3365 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3366 t3_activate_uld(sc, ULD_IWARP) == 0)
3367 setbit(&sc->offload_map, MAX_NPORTS);
3368 } else {
3369 if (!isset(&sc->offload_map, pi->port_id))
3370 return (0);
3371
3372 KASSERT(sc->flags & TOM_INIT_DONE,
3373 ("%s: TOM never initialized?", __func__));
3374 clrbit(&sc->offload_map, pi->port_id);
3375 }
3376
3377 return (0);
3378 }
3379
3380 /*
3381 * Add an upper layer driver to the global list.
3382 */
3383 int
t3_register_uld(struct uld_info * ui)3384 t3_register_uld(struct uld_info *ui)
3385 {
3386 int rc = 0;
3387 struct uld_info *u;
3388
3389 mtx_lock(&t3_uld_list_lock);
3390 SLIST_FOREACH(u, &t3_uld_list, link) {
3391 if (u->uld_id == ui->uld_id) {
3392 rc = EEXIST;
3393 goto done;
3394 }
3395 }
3396
3397 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3398 ui->refcount = 0;
3399 done:
3400 mtx_unlock(&t3_uld_list_lock);
3401 return (rc);
3402 }
3403
3404 int
t3_unregister_uld(struct uld_info * ui)3405 t3_unregister_uld(struct uld_info *ui)
3406 {
3407 int rc = EINVAL;
3408 struct uld_info *u;
3409
3410 mtx_lock(&t3_uld_list_lock);
3411
3412 SLIST_FOREACH(u, &t3_uld_list, link) {
3413 if (u == ui) {
3414 if (ui->refcount > 0) {
3415 rc = EBUSY;
3416 goto done;
3417 }
3418
3419 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3420 rc = 0;
3421 goto done;
3422 }
3423 }
3424 done:
3425 mtx_unlock(&t3_uld_list_lock);
3426 return (rc);
3427 }
3428
3429 int
t3_activate_uld(struct adapter * sc,int id)3430 t3_activate_uld(struct adapter *sc, int id)
3431 {
3432 int rc = EAGAIN;
3433 struct uld_info *ui;
3434
3435 mtx_lock(&t3_uld_list_lock);
3436
3437 SLIST_FOREACH(ui, &t3_uld_list, link) {
3438 if (ui->uld_id == id) {
3439 rc = ui->activate(sc);
3440 if (rc == 0)
3441 ui->refcount++;
3442 goto done;
3443 }
3444 }
3445 done:
3446 mtx_unlock(&t3_uld_list_lock);
3447
3448 return (rc);
3449 }
3450
3451 int
t3_deactivate_uld(struct adapter * sc,int id)3452 t3_deactivate_uld(struct adapter *sc, int id)
3453 {
3454 int rc = EINVAL;
3455 struct uld_info *ui;
3456
3457 mtx_lock(&t3_uld_list_lock);
3458
3459 SLIST_FOREACH(ui, &t3_uld_list, link) {
3460 if (ui->uld_id == id) {
3461 rc = ui->deactivate(sc);
3462 if (rc == 0)
3463 ui->refcount--;
3464 goto done;
3465 }
3466 }
3467 done:
3468 mtx_unlock(&t3_uld_list_lock);
3469
3470 return (rc);
3471 }
3472
3473 static int
cpl_not_handled(struct sge_qset * qs __unused,struct rsp_desc * r __unused,struct mbuf * m)3474 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3475 struct mbuf *m)
3476 {
3477 m_freem(m);
3478 return (EDOOFUS);
3479 }
3480
3481 int
t3_register_cpl_handler(struct adapter * sc,int opcode,cpl_handler_t h)3482 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3483 {
3484 uintptr_t *loc, new;
3485
3486 if (opcode >= NUM_CPL_HANDLERS)
3487 return (EINVAL);
3488
3489 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3490 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3491 atomic_store_rel_ptr(loc, new);
3492
3493 return (0);
3494 }
3495 #endif
3496
3497 static int
cxgbc_mod_event(module_t mod,int cmd,void * arg)3498 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3499 {
3500 int rc = 0;
3501
3502 switch (cmd) {
3503 case MOD_LOAD:
3504 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3505 SLIST_INIT(&t3_list);
3506 #ifdef TCP_OFFLOAD
3507 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3508 SLIST_INIT(&t3_uld_list);
3509 #endif
3510 break;
3511
3512 case MOD_UNLOAD:
3513 #ifdef TCP_OFFLOAD
3514 mtx_lock(&t3_uld_list_lock);
3515 if (!SLIST_EMPTY(&t3_uld_list)) {
3516 rc = EBUSY;
3517 mtx_unlock(&t3_uld_list_lock);
3518 break;
3519 }
3520 mtx_unlock(&t3_uld_list_lock);
3521 mtx_destroy(&t3_uld_list_lock);
3522 #endif
3523 mtx_lock(&t3_list_lock);
3524 if (!SLIST_EMPTY(&t3_list)) {
3525 rc = EBUSY;
3526 mtx_unlock(&t3_list_lock);
3527 break;
3528 }
3529 mtx_unlock(&t3_list_lock);
3530 mtx_destroy(&t3_list_lock);
3531 break;
3532 }
3533
3534 return (rc);
3535 }
3536
3537 #ifdef DEBUGNET
3538 static void
cxgb_debugnet_init(if_t ifp,int * nrxr,int * ncl,int * clsize)3539 cxgb_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize)
3540 {
3541 struct port_info *pi;
3542 adapter_t *adap;
3543
3544 pi = if_getsoftc(ifp);
3545 adap = pi->adapter;
3546 ADAPTER_LOCK(adap);
3547 *nrxr = adap->nqsets;
3548 *ncl = adap->sge.qs[0].fl[1].size;
3549 *clsize = adap->sge.qs[0].fl[1].buf_size;
3550 ADAPTER_UNLOCK(adap);
3551 }
3552
3553 static void
cxgb_debugnet_event(if_t ifp,enum debugnet_ev event)3554 cxgb_debugnet_event(if_t ifp, enum debugnet_ev event)
3555 {
3556 struct port_info *pi;
3557 struct sge_qset *qs;
3558 int i;
3559
3560 pi = if_getsoftc(ifp);
3561 if (event == DEBUGNET_START)
3562 for (i = 0; i < pi->adapter->nqsets; i++) {
3563 qs = &pi->adapter->sge.qs[i];
3564
3565 /* Need to reinit after debugnet_mbuf_start(). */
3566 qs->fl[0].zone = zone_pack;
3567 qs->fl[1].zone = zone_clust;
3568 qs->lro.enabled = 0;
3569 }
3570 }
3571
3572 static int
cxgb_debugnet_transmit(if_t ifp,struct mbuf * m)3573 cxgb_debugnet_transmit(if_t ifp, struct mbuf *m)
3574 {
3575 struct port_info *pi;
3576 struct sge_qset *qs;
3577
3578 pi = if_getsoftc(ifp);
3579 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
3580 IFF_DRV_RUNNING)
3581 return (ENOENT);
3582
3583 qs = &pi->adapter->sge.qs[pi->first_qset];
3584 return (cxgb_debugnet_encap(qs, &m));
3585 }
3586
3587 static int
cxgb_debugnet_poll(if_t ifp,int count)3588 cxgb_debugnet_poll(if_t ifp, int count)
3589 {
3590 struct port_info *pi;
3591 adapter_t *adap;
3592 int i;
3593
3594 pi = if_getsoftc(ifp);
3595 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
3596 return (ENOENT);
3597
3598 adap = pi->adapter;
3599 for (i = 0; i < adap->nqsets; i++)
3600 (void)cxgb_debugnet_poll_rx(adap, &adap->sge.qs[i]);
3601 (void)cxgb_debugnet_poll_tx(&adap->sge.qs[pi->first_qset]);
3602 return (0);
3603 }
3604 #endif /* DEBUGNET */
3605