1 /**************************************************************************
2 SPDX-License-Identifier: BSD-2-Clause
3
4 Copyright (c) 2007-2009, Chelsio Inc.
5 All rights reserved.
6
7 Redistribution and use in source and binary forms, with or without
8 modification, are permitted provided that the following conditions are met:
9
10 1. Redistributions of source code must retain the above copyright notice,
11 this list of conditions and the following disclaimer.
12
13 2. Neither the name of the Chelsio Corporation nor the names of its
14 contributors may be used to endorse or promote products derived from
15 this software without specific prior written permission.
16
17 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 POSSIBILITY OF SUCH DAMAGE.
28
29 ***************************************************************************/
30
31 #include <sys/cdefs.h>
32 #include "opt_inet.h"
33
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/bus.h>
38 #include <sys/module.h>
39 #include <sys/conf.h>
40 #include <machine/bus.h>
41 #include <machine/resource.h>
42 #include <sys/ktr.h>
43 #include <sys/rman.h>
44 #include <sys/ioccom.h>
45 #include <sys/mbuf.h>
46 #include <sys/linker.h>
47 #include <sys/firmware.h>
48 #include <sys/socket.h>
49 #include <sys/sockio.h>
50 #include <sys/smp.h>
51 #include <sys/sysctl.h>
52 #include <sys/syslog.h>
53 #include <sys/queue.h>
54 #include <sys/taskqueue.h>
55 #include <sys/proc.h>
56
57 #include <net/bpf.h>
58 #include <net/debugnet.h>
59 #include <net/ethernet.h>
60 #include <net/if.h>
61 #include <net/if_var.h>
62 #include <net/if_arp.h>
63 #include <net/if_dl.h>
64 #include <net/if_media.h>
65 #include <net/if_types.h>
66 #include <net/if_vlan_var.h>
67
68 #include <netinet/in_systm.h>
69 #include <netinet/in.h>
70 #include <netinet/if_ether.h>
71 #include <netinet/ip.h>
72 #include <netinet/ip.h>
73 #include <netinet/tcp.h>
74 #include <netinet/udp.h>
75
76 #include <dev/pci/pcireg.h>
77 #include <dev/pci/pcivar.h>
78
79 #include <cxgb_include.h>
80
81 #ifdef PRIV_SUPPORTED
82 #include <sys/priv.h>
83 #endif
84
85 static int cxgb_setup_interrupts(adapter_t *);
86 static void cxgb_teardown_interrupts(adapter_t *);
87 static void cxgb_init(void *);
88 static int cxgb_init_locked(struct port_info *);
89 static int cxgb_uninit_locked(struct port_info *);
90 static int cxgb_uninit_synchronized(struct port_info *);
91 static int cxgb_ioctl(if_t, unsigned long, caddr_t);
92 static int cxgb_media_change(if_t);
93 static int cxgb_ifm_type(int);
94 static void cxgb_build_medialist(struct port_info *);
95 static void cxgb_media_status(if_t, struct ifmediareq *);
96 static uint64_t cxgb_get_counter(if_t, ift_counter);
97 static int setup_sge_qsets(adapter_t *);
98 static void cxgb_async_intr(void *);
99 static void cxgb_tick_handler(void *, int);
100 static void cxgb_tick(void *);
101 static void link_check_callout(void *);
102 static void check_link_status(void *, int);
103 static void setup_rss(adapter_t *sc);
104 static int alloc_filters(struct adapter *);
105 static int setup_hw_filters(struct adapter *);
106 static int set_filter(struct adapter *, int, const struct filter_info *);
107 static inline void mk_set_tcb_field(struct cpl_set_tcb_field *, unsigned int,
108 unsigned int, u64, u64);
109 static inline void set_tcb_field_ulp(struct cpl_set_tcb_field *, unsigned int,
110 unsigned int, u64, u64);
111 #ifdef TCP_OFFLOAD
112 static int cpl_not_handled(struct sge_qset *, struct rsp_desc *, struct mbuf *);
113 #endif
114
115 /* Attachment glue for the PCI controller end of the device. Each port of
116 * the device is attached separately, as defined later.
117 */
118 static int cxgb_controller_probe(device_t);
119 static int cxgb_controller_attach(device_t);
120 static int cxgb_controller_detach(device_t);
121 static void cxgb_free(struct adapter *);
122 static __inline void reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
123 unsigned int end);
124 static void cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf);
125 static int cxgb_get_regs_len(void);
126 static void touch_bars(device_t dev);
127 static void cxgb_update_mac_settings(struct port_info *p);
128 #ifdef TCP_OFFLOAD
129 static int toe_capability(struct port_info *, int);
130 #endif
131
132 /* Table for probing the cards. The desc field isn't actually used */
133 struct cxgb_ident {
134 uint16_t vendor;
135 uint16_t device;
136 int index;
137 char *desc;
138 } cxgb_identifiers[] = {
139 {PCI_VENDOR_ID_CHELSIO, 0x0020, 0, "PE9000"},
140 {PCI_VENDOR_ID_CHELSIO, 0x0021, 1, "T302E"},
141 {PCI_VENDOR_ID_CHELSIO, 0x0022, 2, "T310E"},
142 {PCI_VENDOR_ID_CHELSIO, 0x0023, 3, "T320X"},
143 {PCI_VENDOR_ID_CHELSIO, 0x0024, 1, "T302X"},
144 {PCI_VENDOR_ID_CHELSIO, 0x0025, 3, "T320E"},
145 {PCI_VENDOR_ID_CHELSIO, 0x0026, 2, "T310X"},
146 {PCI_VENDOR_ID_CHELSIO, 0x0030, 2, "T3B10"},
147 {PCI_VENDOR_ID_CHELSIO, 0x0031, 3, "T3B20"},
148 {PCI_VENDOR_ID_CHELSIO, 0x0032, 1, "T3B02"},
149 {PCI_VENDOR_ID_CHELSIO, 0x0033, 4, "T3B04"},
150 {PCI_VENDOR_ID_CHELSIO, 0x0035, 6, "T3C10"},
151 {PCI_VENDOR_ID_CHELSIO, 0x0036, 3, "S320E-CR"},
152 {PCI_VENDOR_ID_CHELSIO, 0x0037, 7, "N320E-G2"},
153 {0, 0, 0, NULL}
154 };
155
156 static device_method_t cxgb_controller_methods[] = {
157 DEVMETHOD(device_probe, cxgb_controller_probe),
158 DEVMETHOD(device_attach, cxgb_controller_attach),
159 DEVMETHOD(device_detach, cxgb_controller_detach),
160
161 DEVMETHOD_END
162 };
163
164 static driver_t cxgb_controller_driver = {
165 "cxgbc",
166 cxgb_controller_methods,
167 sizeof(struct adapter)
168 };
169
170 static int cxgbc_mod_event(module_t, int, void *);
171
172 DRIVER_MODULE(cxgbc, pci, cxgb_controller_driver, cxgbc_mod_event, NULL);
173 MODULE_PNP_INFO("U16:vendor;U16:device", pci, cxgbc, cxgb_identifiers,
174 nitems(cxgb_identifiers) - 1);
175 MODULE_VERSION(cxgbc, 1);
176 MODULE_DEPEND(cxgbc, firmware, 1, 1, 1);
177
178 /*
179 * Attachment glue for the ports. Attachment is done directly to the
180 * controller device.
181 */
182 static int cxgb_port_probe(device_t);
183 static int cxgb_port_attach(device_t);
184 static int cxgb_port_detach(device_t);
185
186 static device_method_t cxgb_port_methods[] = {
187 DEVMETHOD(device_probe, cxgb_port_probe),
188 DEVMETHOD(device_attach, cxgb_port_attach),
189 DEVMETHOD(device_detach, cxgb_port_detach),
190 { 0, 0 }
191 };
192
193 static driver_t cxgb_port_driver = {
194 "cxgb",
195 cxgb_port_methods,
196 0
197 };
198
199 static d_ioctl_t cxgb_extension_ioctl;
200 static d_open_t cxgb_extension_open;
201 static d_close_t cxgb_extension_close;
202
203 static struct cdevsw cxgb_cdevsw = {
204 .d_version = D_VERSION,
205 .d_flags = 0,
206 .d_open = cxgb_extension_open,
207 .d_close = cxgb_extension_close,
208 .d_ioctl = cxgb_extension_ioctl,
209 .d_name = "cxgb",
210 };
211
212 DRIVER_MODULE(cxgb, cxgbc, cxgb_port_driver, 0, 0);
213 MODULE_VERSION(cxgb, 1);
214
215 DEBUGNET_DEFINE(cxgb);
216
217 static struct mtx t3_list_lock;
218 static SLIST_HEAD(, adapter) t3_list;
219 #ifdef TCP_OFFLOAD
220 static struct mtx t3_uld_list_lock;
221 static SLIST_HEAD(, uld_info) t3_uld_list;
222 #endif
223
224 /*
225 * The driver uses the best interrupt scheme available on a platform in the
226 * order MSI-X, MSI, legacy pin interrupts. This parameter determines which
227 * of these schemes the driver may consider as follows:
228 *
229 * msi = 2: choose from among all three options
230 * msi = 1 : only consider MSI and pin interrupts
231 * msi = 0: force pin interrupts
232 */
233 static int msi_allowed = 2;
234
235 SYSCTL_NODE(_hw, OID_AUTO, cxgb, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
236 "CXGB driver parameters");
237 SYSCTL_INT(_hw_cxgb, OID_AUTO, msi_allowed, CTLFLAG_RDTUN, &msi_allowed, 0,
238 "MSI-X, MSI, INTx selector");
239
240 /*
241 * The driver uses an auto-queue algorithm by default.
242 * To disable it and force a single queue-set per port, use multiq = 0
243 */
244 static int multiq = 1;
245 SYSCTL_INT(_hw_cxgb, OID_AUTO, multiq, CTLFLAG_RDTUN, &multiq, 0,
246 "use min(ncpus/ports, 8) queue-sets per port");
247
248 /*
249 * By default the driver will not update the firmware unless
250 * it was compiled against a newer version
251 *
252 */
253 static int force_fw_update = 0;
254 SYSCTL_INT(_hw_cxgb, OID_AUTO, force_fw_update, CTLFLAG_RDTUN, &force_fw_update, 0,
255 "update firmware even if up to date");
256
257 int cxgb_use_16k_clusters = -1;
258 SYSCTL_INT(_hw_cxgb, OID_AUTO, use_16k_clusters, CTLFLAG_RDTUN,
259 &cxgb_use_16k_clusters, 0, "use 16kB clusters for the jumbo queue ");
260
261 static int nfilters = -1;
262 SYSCTL_INT(_hw_cxgb, OID_AUTO, nfilters, CTLFLAG_RDTUN,
263 &nfilters, 0, "max number of entries in the filter table");
264
265 enum {
266 MAX_TXQ_ENTRIES = 16384,
267 MAX_CTRL_TXQ_ENTRIES = 1024,
268 MAX_RSPQ_ENTRIES = 16384,
269 MAX_RX_BUFFERS = 16384,
270 MAX_RX_JUMBO_BUFFERS = 16384,
271 MIN_TXQ_ENTRIES = 4,
272 MIN_CTRL_TXQ_ENTRIES = 4,
273 MIN_RSPQ_ENTRIES = 32,
274 MIN_FL_ENTRIES = 32,
275 MIN_FL_JUMBO_ENTRIES = 32
276 };
277
278 struct filter_info {
279 u32 sip;
280 u32 sip_mask;
281 u32 dip;
282 u16 sport;
283 u16 dport;
284 u32 vlan:12;
285 u32 vlan_prio:3;
286 u32 mac_hit:1;
287 u32 mac_idx:4;
288 u32 mac_vld:1;
289 u32 pkt_type:2;
290 u32 report_filter_id:1;
291 u32 pass:1;
292 u32 rss:1;
293 u32 qset:3;
294 u32 locked:1;
295 u32 valid:1;
296 };
297
298 enum { FILTER_NO_VLAN_PRI = 7 };
299
300 #define EEPROM_MAGIC 0x38E2F10C
301
302 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
303
304
305 static int set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset);
306
307
308 static __inline char
t3rev2char(struct adapter * adapter)309 t3rev2char(struct adapter *adapter)
310 {
311 char rev = 'z';
312
313 switch(adapter->params.rev) {
314 case T3_REV_A:
315 rev = 'a';
316 break;
317 case T3_REV_B:
318 case T3_REV_B2:
319 rev = 'b';
320 break;
321 case T3_REV_C:
322 rev = 'c';
323 break;
324 }
325 return rev;
326 }
327
328 static struct cxgb_ident *
cxgb_get_ident(device_t dev)329 cxgb_get_ident(device_t dev)
330 {
331 struct cxgb_ident *id;
332
333 for (id = cxgb_identifiers; id->desc != NULL; id++) {
334 if ((id->vendor == pci_get_vendor(dev)) &&
335 (id->device == pci_get_device(dev))) {
336 return (id);
337 }
338 }
339 return (NULL);
340 }
341
342 static const struct adapter_info *
cxgb_get_adapter_info(device_t dev)343 cxgb_get_adapter_info(device_t dev)
344 {
345 struct cxgb_ident *id;
346 const struct adapter_info *ai;
347
348 id = cxgb_get_ident(dev);
349 if (id == NULL)
350 return (NULL);
351
352 ai = t3_get_adapter_info(id->index);
353
354 return (ai);
355 }
356
357 static int
cxgb_controller_probe(device_t dev)358 cxgb_controller_probe(device_t dev)
359 {
360 const struct adapter_info *ai;
361 const char *ports;
362 int nports;
363
364 ai = cxgb_get_adapter_info(dev);
365 if (ai == NULL)
366 return (ENXIO);
367
368 nports = ai->nports0 + ai->nports1;
369 if (nports == 1)
370 ports = "port";
371 else
372 ports = "ports";
373
374 device_set_descf(dev, "%s, %d %s", ai->desc, nports, ports);
375 return (BUS_PROBE_DEFAULT);
376 }
377
378 #define FW_FNAME "cxgb_t3fw"
379 #define TPEEPROM_NAME "cxgb_t3%c_tp_eeprom"
380 #define TPSRAM_NAME "cxgb_t3%c_protocol_sram"
381
382 static int
upgrade_fw(adapter_t * sc)383 upgrade_fw(adapter_t *sc)
384 {
385 const struct firmware *fw;
386 int status;
387 u32 vers;
388
389 if ((fw = firmware_get(FW_FNAME)) == NULL) {
390 device_printf(sc->dev, "Could not find firmware image %s\n", FW_FNAME);
391 return (ENOENT);
392 } else
393 device_printf(sc->dev, "installing firmware on card\n");
394 status = t3_load_fw(sc, (const uint8_t *)fw->data, fw->datasize);
395
396 if (status != 0) {
397 device_printf(sc->dev, "failed to install firmware: %d\n",
398 status);
399 } else {
400 t3_get_fw_version(sc, &vers);
401 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
402 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
403 G_FW_VERSION_MICRO(vers));
404 }
405
406 firmware_put(fw, FIRMWARE_UNLOAD);
407
408 return (status);
409 }
410
411 /*
412 * The cxgb_controller_attach function is responsible for the initial
413 * bringup of the device. Its responsibilities include:
414 *
415 * 1. Determine if the device supports MSI or MSI-X.
416 * 2. Allocate bus resources so that we can access the Base Address Register
417 * 3. Create and initialize mutexes for the controller and its control
418 * logic such as SGE and MDIO.
419 * 4. Call hardware specific setup routine for the adapter as a whole.
420 * 5. Allocate the BAR for doing MSI-X.
421 * 6. Setup the line interrupt iff MSI-X is not supported.
422 * 7. Create the driver's taskq.
423 * 8. Start one task queue service thread.
424 * 9. Check if the firmware and SRAM are up-to-date. They will be
425 * auto-updated later (before FULL_INIT_DONE), if required.
426 * 10. Create a child device for each MAC (port)
427 * 11. Initialize T3 private state.
428 * 12. Trigger the LED
429 * 13. Setup offload iff supported.
430 * 14. Reset/restart the tick callout.
431 * 15. Attach sysctls
432 *
433 * NOTE: Any modification or deviation from this list MUST be reflected in
434 * the above comment. Failure to do so will result in problems on various
435 * error conditions including link flapping.
436 */
437 static int
cxgb_controller_attach(device_t dev)438 cxgb_controller_attach(device_t dev)
439 {
440 device_t child;
441 const struct adapter_info *ai;
442 struct adapter *sc;
443 int i, error = 0;
444 uint32_t vers;
445 int port_qsets = 1;
446 int msi_needed, reg;
447
448 sc = device_get_softc(dev);
449 sc->dev = dev;
450 sc->msi_count = 0;
451 ai = cxgb_get_adapter_info(dev);
452
453 snprintf(sc->lockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb controller lock %d",
454 device_get_unit(dev));
455 ADAPTER_LOCK_INIT(sc, sc->lockbuf);
456
457 snprintf(sc->reglockbuf, ADAPTER_LOCK_NAME_LEN, "SGE reg lock %d",
458 device_get_unit(dev));
459 snprintf(sc->mdiolockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb mdio lock %d",
460 device_get_unit(dev));
461 snprintf(sc->elmerlockbuf, ADAPTER_LOCK_NAME_LEN, "cxgb elmer lock %d",
462 device_get_unit(dev));
463
464 MTX_INIT(&sc->sge.reg_lock, sc->reglockbuf, NULL, MTX_SPIN);
465 MTX_INIT(&sc->mdio_lock, sc->mdiolockbuf, NULL, MTX_DEF);
466 MTX_INIT(&sc->elmer_lock, sc->elmerlockbuf, NULL, MTX_DEF);
467
468 mtx_lock(&t3_list_lock);
469 SLIST_INSERT_HEAD(&t3_list, sc, link);
470 mtx_unlock(&t3_list_lock);
471
472 /* find the PCIe link width and set max read request to 4KB*/
473 if (pci_find_cap(dev, PCIY_EXPRESS, ®) == 0) {
474 uint16_t lnk;
475
476 lnk = pci_read_config(dev, reg + PCIER_LINK_STA, 2);
477 sc->link_width = (lnk & PCIEM_LINK_STA_WIDTH) >> 4;
478 if (sc->link_width < 8 &&
479 (ai->caps & SUPPORTED_10000baseT_Full)) {
480 device_printf(sc->dev,
481 "PCIe x%d Link, expect reduced performance\n",
482 sc->link_width);
483 }
484
485 pci_set_max_read_req(dev, 4096);
486 }
487
488 touch_bars(dev);
489 pci_enable_busmaster(dev);
490 /*
491 * Allocate the registers and make them available to the driver.
492 * The registers that we care about for NIC mode are in BAR 0
493 */
494 sc->regs_rid = PCIR_BAR(0);
495 if ((sc->regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
496 &sc->regs_rid, RF_ACTIVE)) == NULL) {
497 device_printf(dev, "Cannot allocate BAR region 0\n");
498 error = ENXIO;
499 goto out;
500 }
501
502 sc->bt = rman_get_bustag(sc->regs_res);
503 sc->bh = rman_get_bushandle(sc->regs_res);
504 sc->mmio_len = rman_get_size(sc->regs_res);
505
506 for (i = 0; i < MAX_NPORTS; i++)
507 sc->port[i].adapter = sc;
508
509 if (t3_prep_adapter(sc, ai, 1) < 0) {
510 printf("prep adapter failed\n");
511 error = ENODEV;
512 goto out;
513 }
514
515 sc->udbs_rid = PCIR_BAR(2);
516 sc->udbs_res = NULL;
517 if (is_offload(sc) &&
518 ((sc->udbs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
519 &sc->udbs_rid, RF_ACTIVE)) == NULL)) {
520 device_printf(dev, "Cannot allocate BAR region 1\n");
521 error = ENXIO;
522 goto out;
523 }
524
525 /* Allocate the BAR for doing MSI-X. If it succeeds, try to allocate
526 * enough messages for the queue sets. If that fails, try falling
527 * back to MSI. If that fails, then try falling back to the legacy
528 * interrupt pin model.
529 */
530 sc->msix_regs_rid = 0x20;
531 if ((msi_allowed >= 2) &&
532 (sc->msix_regs_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
533 &sc->msix_regs_rid, RF_ACTIVE)) != NULL) {
534
535 if (multiq)
536 port_qsets = min(SGE_QSETS/sc->params.nports, mp_ncpus);
537 msi_needed = sc->msi_count = sc->params.nports * port_qsets + 1;
538
539 if (pci_msix_count(dev) == 0 ||
540 (error = pci_alloc_msix(dev, &sc->msi_count)) != 0 ||
541 sc->msi_count != msi_needed) {
542 device_printf(dev, "alloc msix failed - "
543 "msi_count=%d, msi_needed=%d, err=%d; "
544 "will try MSI\n", sc->msi_count,
545 msi_needed, error);
546 sc->msi_count = 0;
547 port_qsets = 1;
548 pci_release_msi(dev);
549 bus_release_resource(dev, SYS_RES_MEMORY,
550 sc->msix_regs_rid, sc->msix_regs_res);
551 sc->msix_regs_res = NULL;
552 } else {
553 sc->flags |= USING_MSIX;
554 sc->cxgb_intr = cxgb_async_intr;
555 device_printf(dev,
556 "using MSI-X interrupts (%u vectors)\n",
557 sc->msi_count);
558 }
559 }
560
561 if ((msi_allowed >= 1) && (sc->msi_count == 0)) {
562 sc->msi_count = 1;
563 if ((error = pci_alloc_msi(dev, &sc->msi_count)) != 0) {
564 device_printf(dev, "alloc msi failed - "
565 "err=%d; will try INTx\n", error);
566 sc->msi_count = 0;
567 port_qsets = 1;
568 pci_release_msi(dev);
569 } else {
570 sc->flags |= USING_MSI;
571 sc->cxgb_intr = t3_intr_msi;
572 device_printf(dev, "using MSI interrupts\n");
573 }
574 }
575 if (sc->msi_count == 0) {
576 device_printf(dev, "using line interrupts\n");
577 sc->cxgb_intr = t3b_intr;
578 }
579
580 /* Create a private taskqueue thread for handling driver events */
581 sc->tq = taskqueue_create("cxgb_taskq", M_NOWAIT,
582 taskqueue_thread_enqueue, &sc->tq);
583 if (sc->tq == NULL) {
584 device_printf(dev, "failed to allocate controller task queue\n");
585 goto out;
586 }
587
588 taskqueue_start_threads(&sc->tq, 1, PI_NET, "%s taskq",
589 device_get_nameunit(dev));
590 TASK_INIT(&sc->tick_task, 0, cxgb_tick_handler, sc);
591
592
593 /* Create a periodic callout for checking adapter status */
594 callout_init(&sc->cxgb_tick_ch, 1);
595
596 if (t3_check_fw_version(sc) < 0 || force_fw_update) {
597 /*
598 * Warn user that a firmware update will be attempted in init.
599 */
600 device_printf(dev, "firmware needs to be updated to version %d.%d.%d\n",
601 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
602 sc->flags &= ~FW_UPTODATE;
603 } else {
604 sc->flags |= FW_UPTODATE;
605 }
606
607 if (t3_check_tpsram_version(sc) < 0) {
608 /*
609 * Warn user that a firmware update will be attempted in init.
610 */
611 device_printf(dev, "SRAM needs to be updated to version %c-%d.%d.%d\n",
612 t3rev2char(sc), TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
613 sc->flags &= ~TPS_UPTODATE;
614 } else {
615 sc->flags |= TPS_UPTODATE;
616 }
617
618 /*
619 * Create a child device for each MAC. The ethernet attachment
620 * will be done in these children.
621 */
622 for (i = 0; i < (sc)->params.nports; i++) {
623 struct port_info *pi;
624
625 if ((child = device_add_child(dev, "cxgb", -1)) == NULL) {
626 device_printf(dev, "failed to add child port\n");
627 error = EINVAL;
628 goto out;
629 }
630 pi = &sc->port[i];
631 pi->adapter = sc;
632 pi->nqsets = port_qsets;
633 pi->first_qset = i*port_qsets;
634 pi->port_id = i;
635 pi->tx_chan = i >= ai->nports0;
636 pi->txpkt_intf = pi->tx_chan ? 2 * (i - ai->nports0) + 1 : 2 * i;
637 sc->rxpkt_map[pi->txpkt_intf] = i;
638 sc->port[i].tx_chan = i >= ai->nports0;
639 sc->portdev[i] = child;
640 device_set_softc(child, pi);
641 }
642 bus_attach_children(dev);
643
644 /* initialize sge private state */
645 t3_sge_init_adapter(sc);
646
647 t3_led_ready(sc);
648
649 error = t3_get_fw_version(sc, &vers);
650 if (error)
651 goto out;
652
653 snprintf(&sc->fw_version[0], sizeof(sc->fw_version), "%d.%d.%d",
654 G_FW_VERSION_MAJOR(vers), G_FW_VERSION_MINOR(vers),
655 G_FW_VERSION_MICRO(vers));
656
657 device_set_descf(dev, "%s %sNIC\t E/C: %s S/N: %s",
658 ai->desc, is_offload(sc) ? "R" : "",
659 sc->params.vpd.ec, sc->params.vpd.sn);
660
661 snprintf(&sc->port_types[0], sizeof(sc->port_types), "%x%x%x%x",
662 sc->params.vpd.port_type[0], sc->params.vpd.port_type[1],
663 sc->params.vpd.port_type[2], sc->params.vpd.port_type[3]);
664
665 device_printf(sc->dev, "Firmware Version %s\n", &sc->fw_version[0]);
666 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
667 t3_add_attach_sysctls(sc);
668
669 #ifdef TCP_OFFLOAD
670 for (i = 0; i < NUM_CPL_HANDLERS; i++)
671 sc->cpl_handler[i] = cpl_not_handled;
672 #endif
673
674 t3_intr_clear(sc);
675 error = cxgb_setup_interrupts(sc);
676 out:
677 if (error)
678 cxgb_free(sc);
679
680 return (error);
681 }
682
683 /*
684 * The cxgb_controller_detach routine is called with the device is
685 * unloaded from the system.
686 */
687
688 static int
cxgb_controller_detach(device_t dev)689 cxgb_controller_detach(device_t dev)
690 {
691 struct adapter *sc;
692
693 sc = device_get_softc(dev);
694
695 cxgb_free(sc);
696
697 return (0);
698 }
699
700 /*
701 * The cxgb_free() is called by the cxgb_controller_detach() routine
702 * to tear down the structures that were built up in
703 * cxgb_controller_attach(), and should be the final piece of work
704 * done when fully unloading the driver.
705 *
706 *
707 * 1. Shutting down the threads started by the cxgb_controller_attach()
708 * routine.
709 * 2. Stopping the lower level device and all callouts (cxgb_down_locked()).
710 * 3. Detaching all of the port devices created during the
711 * cxgb_controller_attach() routine.
712 * 4. Removing the device children created via cxgb_controller_attach().
713 * 5. Releasing PCI resources associated with the device.
714 * 6. Turning off the offload support, iff it was turned on.
715 * 7. Destroying the mutexes created in cxgb_controller_attach().
716 *
717 */
718 static void
cxgb_free(struct adapter * sc)719 cxgb_free(struct adapter *sc)
720 {
721 int i, nqsets = 0;
722
723 ADAPTER_LOCK(sc);
724 sc->flags |= CXGB_SHUTDOWN;
725 ADAPTER_UNLOCK(sc);
726
727 /*
728 * Make sure all child devices are gone.
729 */
730 bus_detach_children(sc->dev);
731 for (i = 0; i < (sc)->params.nports; i++) {
732 if (sc->portdev[i] &&
733 device_delete_child(sc->dev, sc->portdev[i]) != 0)
734 device_printf(sc->dev, "failed to delete child port\n");
735 nqsets += sc->port[i].nqsets;
736 }
737
738 /*
739 * At this point, it is as if cxgb_port_detach has run on all ports, and
740 * cxgb_down has run on the adapter. All interrupts have been silenced,
741 * all open devices have been closed.
742 */
743 KASSERT(sc->open_device_map == 0, ("%s: device(s) still open (%x)",
744 __func__, sc->open_device_map));
745 for (i = 0; i < sc->params.nports; i++) {
746 KASSERT(sc->port[i].ifp == NULL, ("%s: port %i undead!",
747 __func__, i));
748 }
749
750 /*
751 * Finish off the adapter's callouts.
752 */
753 callout_drain(&sc->cxgb_tick_ch);
754 callout_drain(&sc->sge_timer_ch);
755
756 /*
757 * Release resources grabbed under FULL_INIT_DONE by cxgb_up. The
758 * sysctls are cleaned up by the kernel linker.
759 */
760 if (sc->flags & FULL_INIT_DONE) {
761 t3_free_sge_resources(sc, nqsets);
762 sc->flags &= ~FULL_INIT_DONE;
763 }
764
765 /*
766 * Release all interrupt resources.
767 */
768 cxgb_teardown_interrupts(sc);
769 if (sc->flags & (USING_MSI | USING_MSIX)) {
770 device_printf(sc->dev, "releasing msi message(s)\n");
771 pci_release_msi(sc->dev);
772 } else {
773 device_printf(sc->dev, "no msi message to release\n");
774 }
775
776 if (sc->msix_regs_res != NULL) {
777 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->msix_regs_rid,
778 sc->msix_regs_res);
779 }
780
781 /*
782 * Free the adapter's taskqueue.
783 */
784 if (sc->tq != NULL) {
785 taskqueue_free(sc->tq);
786 sc->tq = NULL;
787 }
788
789 free(sc->filters, M_DEVBUF);
790 t3_sge_free(sc);
791
792 if (sc->udbs_res != NULL)
793 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->udbs_rid,
794 sc->udbs_res);
795
796 if (sc->regs_res != NULL)
797 bus_release_resource(sc->dev, SYS_RES_MEMORY, sc->regs_rid,
798 sc->regs_res);
799
800 MTX_DESTROY(&sc->mdio_lock);
801 MTX_DESTROY(&sc->sge.reg_lock);
802 MTX_DESTROY(&sc->elmer_lock);
803 mtx_lock(&t3_list_lock);
804 SLIST_REMOVE(&t3_list, sc, adapter, link);
805 mtx_unlock(&t3_list_lock);
806 ADAPTER_LOCK_DEINIT(sc);
807 }
808
809 /**
810 * setup_sge_qsets - configure SGE Tx/Rx/response queues
811 * @sc: the controller softc
812 *
813 * Determines how many sets of SGE queues to use and initializes them.
814 * We support multiple queue sets per port if we have MSI-X, otherwise
815 * just one queue set per port.
816 */
817 static int
setup_sge_qsets(adapter_t * sc)818 setup_sge_qsets(adapter_t *sc)
819 {
820 int i, j, err, irq_idx = 0, qset_idx = 0;
821 u_int ntxq = SGE_TXQ_PER_SET;
822
823 if ((err = t3_sge_alloc(sc)) != 0) {
824 device_printf(sc->dev, "t3_sge_alloc returned %d\n", err);
825 return (err);
826 }
827
828 if (sc->params.rev > 0 && !(sc->flags & USING_MSI))
829 irq_idx = -1;
830
831 for (i = 0; i < (sc)->params.nports; i++) {
832 struct port_info *pi = &sc->port[i];
833
834 for (j = 0; j < pi->nqsets; j++, qset_idx++) {
835 err = t3_sge_alloc_qset(sc, qset_idx, (sc)->params.nports,
836 (sc->flags & USING_MSIX) ? qset_idx + 1 : irq_idx,
837 &sc->params.sge.qset[qset_idx], ntxq, pi);
838 if (err) {
839 t3_free_sge_resources(sc, qset_idx);
840 device_printf(sc->dev,
841 "t3_sge_alloc_qset failed with %d\n", err);
842 return (err);
843 }
844 }
845 }
846
847 sc->nqsets = qset_idx;
848
849 return (0);
850 }
851
852 static void
cxgb_teardown_interrupts(adapter_t * sc)853 cxgb_teardown_interrupts(adapter_t *sc)
854 {
855 int i;
856
857 for (i = 0; i < SGE_QSETS; i++) {
858 if (sc->msix_intr_tag[i] == NULL) {
859
860 /* Should have been setup fully or not at all */
861 KASSERT(sc->msix_irq_res[i] == NULL &&
862 sc->msix_irq_rid[i] == 0,
863 ("%s: half-done interrupt (%d).", __func__, i));
864
865 continue;
866 }
867
868 bus_teardown_intr(sc->dev, sc->msix_irq_res[i],
869 sc->msix_intr_tag[i]);
870 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->msix_irq_rid[i],
871 sc->msix_irq_res[i]);
872
873 sc->msix_irq_res[i] = sc->msix_intr_tag[i] = NULL;
874 sc->msix_irq_rid[i] = 0;
875 }
876
877 if (sc->intr_tag) {
878 KASSERT(sc->irq_res != NULL,
879 ("%s: half-done interrupt.", __func__));
880
881 bus_teardown_intr(sc->dev, sc->irq_res, sc->intr_tag);
882 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
883 sc->irq_res);
884
885 sc->irq_res = sc->intr_tag = NULL;
886 sc->irq_rid = 0;
887 }
888 }
889
890 static int
cxgb_setup_interrupts(adapter_t * sc)891 cxgb_setup_interrupts(adapter_t *sc)
892 {
893 struct resource *res;
894 void *tag;
895 int i, rid, err, intr_flag = sc->flags & (USING_MSI | USING_MSIX);
896
897 sc->irq_rid = intr_flag ? 1 : 0;
898 sc->irq_res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &sc->irq_rid,
899 RF_SHAREABLE | RF_ACTIVE);
900 if (sc->irq_res == NULL) {
901 device_printf(sc->dev, "Cannot allocate interrupt (%x, %u)\n",
902 intr_flag, sc->irq_rid);
903 err = EINVAL;
904 sc->irq_rid = 0;
905 } else {
906 err = bus_setup_intr(sc->dev, sc->irq_res,
907 INTR_MPSAFE | INTR_TYPE_NET, NULL,
908 sc->cxgb_intr, sc, &sc->intr_tag);
909
910 if (err) {
911 device_printf(sc->dev,
912 "Cannot set up interrupt (%x, %u, %d)\n",
913 intr_flag, sc->irq_rid, err);
914 bus_release_resource(sc->dev, SYS_RES_IRQ, sc->irq_rid,
915 sc->irq_res);
916 sc->irq_res = sc->intr_tag = NULL;
917 sc->irq_rid = 0;
918 }
919 }
920
921 /* That's all for INTx or MSI */
922 if (!(intr_flag & USING_MSIX) || err)
923 return (err);
924
925 bus_describe_intr(sc->dev, sc->irq_res, sc->intr_tag, "err");
926 for (i = 0; i < sc->msi_count - 1; i++) {
927 rid = i + 2;
928 res = bus_alloc_resource_any(sc->dev, SYS_RES_IRQ, &rid,
929 RF_SHAREABLE | RF_ACTIVE);
930 if (res == NULL) {
931 device_printf(sc->dev, "Cannot allocate interrupt "
932 "for message %d\n", rid);
933 err = EINVAL;
934 break;
935 }
936
937 err = bus_setup_intr(sc->dev, res, INTR_MPSAFE | INTR_TYPE_NET,
938 NULL, t3_intr_msix, &sc->sge.qs[i], &tag);
939 if (err) {
940 device_printf(sc->dev, "Cannot set up interrupt "
941 "for message %d (%d)\n", rid, err);
942 bus_release_resource(sc->dev, SYS_RES_IRQ, rid, res);
943 break;
944 }
945
946 sc->msix_irq_rid[i] = rid;
947 sc->msix_irq_res[i] = res;
948 sc->msix_intr_tag[i] = tag;
949 bus_describe_intr(sc->dev, res, tag, "qs%d", i);
950 }
951
952 if (err)
953 cxgb_teardown_interrupts(sc);
954
955 return (err);
956 }
957
958
959 static int
cxgb_port_probe(device_t dev)960 cxgb_port_probe(device_t dev)
961 {
962 struct port_info *p;
963 const char *desc;
964
965 p = device_get_softc(dev);
966 desc = p->phy.desc;
967 device_set_descf(dev, "Port %d %s", p->port_id, desc);
968 return (0);
969 }
970
971
972 static int
cxgb_makedev(struct port_info * pi)973 cxgb_makedev(struct port_info *pi)
974 {
975
976 pi->port_cdev = make_dev(&cxgb_cdevsw, if_getdunit(pi->ifp),
977 UID_ROOT, GID_WHEEL, 0600, "%s", if_name(pi->ifp));
978
979 if (pi->port_cdev == NULL)
980 return (ENOMEM);
981
982 pi->port_cdev->si_drv1 = (void *)pi;
983
984 return (0);
985 }
986
987 #define CXGB_CAP (IFCAP_VLAN_HWTAGGING | IFCAP_VLAN_MTU | IFCAP_HWCSUM | \
988 IFCAP_VLAN_HWCSUM | IFCAP_TSO | IFCAP_JUMBO_MTU | IFCAP_LRO | \
989 IFCAP_VLAN_HWTSO | IFCAP_LINKSTATE | IFCAP_HWCSUM_IPV6)
990 #define CXGB_CAP_ENABLE CXGB_CAP
991
992 static int
cxgb_port_attach(device_t dev)993 cxgb_port_attach(device_t dev)
994 {
995 struct port_info *p;
996 if_t ifp;
997 int err;
998 struct adapter *sc;
999
1000 p = device_get_softc(dev);
1001 sc = p->adapter;
1002 snprintf(p->lockbuf, PORT_NAME_LEN, "cxgb port lock %d:%d",
1003 device_get_unit(device_get_parent(dev)), p->port_id);
1004 PORT_LOCK_INIT(p, p->lockbuf);
1005
1006 callout_init(&p->link_check_ch, 1);
1007 TASK_INIT(&p->link_check_task, 0, check_link_status, p);
1008
1009 /* Allocate an ifnet object and set it up */
1010 ifp = p->ifp = if_alloc(IFT_ETHER);
1011 if_initname(ifp, device_get_name(dev), device_get_unit(dev));
1012 if_setinitfn(ifp, cxgb_init);
1013 if_setsoftc(ifp, p);
1014 if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
1015 if_setioctlfn(ifp, cxgb_ioctl);
1016 if_settransmitfn(ifp, cxgb_transmit);
1017 if_setqflushfn(ifp, cxgb_qflush);
1018 if_setgetcounterfn(ifp, cxgb_get_counter);
1019
1020 if_setcapabilities(ifp, CXGB_CAP);
1021 #ifdef TCP_OFFLOAD
1022 if (is_offload(sc))
1023 if_setcapabilitiesbit(ifp, IFCAP_TOE4, 0);
1024 #endif
1025 if_setcapenable(ifp, CXGB_CAP_ENABLE);
1026 if_sethwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP | CSUM_TSO |
1027 CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1028 if_sethwtsomax(ifp, IP_MAXPACKET);
1029 if_sethwtsomaxsegcount(ifp, 36);
1030 if_sethwtsomaxsegsize(ifp, 65536);
1031
1032 /*
1033 * Disable TSO on 4-port - it isn't supported by the firmware.
1034 */
1035 if (sc->params.nports > 2) {
1036 if_setcapabilitiesbit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO);
1037 if_setcapenablebit(ifp, 0, IFCAP_TSO | IFCAP_VLAN_HWTSO);
1038 if_sethwassistbits(ifp, 0, CSUM_TSO);
1039 }
1040
1041 /* Create a list of media supported by this port */
1042 ifmedia_init(&p->media, IFM_IMASK, cxgb_media_change,
1043 cxgb_media_status);
1044 cxgb_build_medialist(p);
1045
1046 ether_ifattach(ifp, p->hw_addr);
1047
1048 /* Attach driver debugnet methods. */
1049 DEBUGNET_SET(ifp, cxgb);
1050
1051 #ifdef DEFAULT_JUMBO
1052 if (sc->params.nports <= 2)
1053 if_setmtu(ifp, ETHERMTU_JUMBO);
1054 #endif
1055 if ((err = cxgb_makedev(p)) != 0) {
1056 printf("makedev failed %d\n", err);
1057 return (err);
1058 }
1059
1060 t3_sge_init_port(p);
1061
1062 return (err);
1063 }
1064
1065 /*
1066 * cxgb_port_detach() is called via the device_detach methods when
1067 * cxgb_free() calls the bus_detach_children. It is responsible for
1068 * removing the device from the view of the kernel, i.e. from all
1069 * interfaces lists etc. This routine is only called when the driver is
1070 * being unloaded, not when the link goes down.
1071 */
1072 static int
cxgb_port_detach(device_t dev)1073 cxgb_port_detach(device_t dev)
1074 {
1075 struct port_info *p;
1076 struct adapter *sc;
1077 int i;
1078
1079 p = device_get_softc(dev);
1080 sc = p->adapter;
1081
1082 /* Tell cxgb_ioctl and if_init that the port is going away */
1083 ADAPTER_LOCK(sc);
1084 SET_DOOMED(p);
1085 wakeup(&sc->flags);
1086 while (IS_BUSY(sc))
1087 mtx_sleep(&sc->flags, &sc->lock, 0, "cxgbdtch", 0);
1088 SET_BUSY(sc);
1089 ADAPTER_UNLOCK(sc);
1090
1091 if (p->port_cdev != NULL)
1092 destroy_dev(p->port_cdev);
1093
1094 cxgb_uninit_synchronized(p);
1095 ether_ifdetach(p->ifp);
1096
1097 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1098 struct sge_qset *qs = &sc->sge.qs[i];
1099 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1100
1101 callout_drain(&txq->txq_watchdog);
1102 callout_drain(&txq->txq_timer);
1103 }
1104
1105 PORT_LOCK_DEINIT(p);
1106 if_free(p->ifp);
1107 p->ifp = NULL;
1108
1109 ADAPTER_LOCK(sc);
1110 CLR_BUSY(sc);
1111 wakeup_one(&sc->flags);
1112 ADAPTER_UNLOCK(sc);
1113 return (0);
1114 }
1115
1116 void
t3_fatal_err(struct adapter * sc)1117 t3_fatal_err(struct adapter *sc)
1118 {
1119 u_int fw_status[4];
1120
1121 if (sc->flags & FULL_INIT_DONE) {
1122 t3_sge_stop(sc);
1123 t3_write_reg(sc, A_XGM_TX_CTRL, 0);
1124 t3_write_reg(sc, A_XGM_RX_CTRL, 0);
1125 t3_write_reg(sc, XGM_REG(A_XGM_TX_CTRL, 1), 0);
1126 t3_write_reg(sc, XGM_REG(A_XGM_RX_CTRL, 1), 0);
1127 t3_intr_disable(sc);
1128 }
1129 device_printf(sc->dev,"encountered fatal error, operation suspended\n");
1130 if (!t3_cim_ctl_blk_read(sc, 0xa0, 4, fw_status))
1131 device_printf(sc->dev, "FW_ status: 0x%x, 0x%x, 0x%x, 0x%x\n",
1132 fw_status[0], fw_status[1], fw_status[2], fw_status[3]);
1133 }
1134
1135 int
t3_os_find_pci_capability(adapter_t * sc,int cap)1136 t3_os_find_pci_capability(adapter_t *sc, int cap)
1137 {
1138 int rc, reg = 0;
1139
1140 rc = pci_find_cap(sc->dev, cap, ®);
1141 return (rc == 0 ? reg : 0);
1142 }
1143
1144 int
t3_os_pci_save_state(struct adapter * sc)1145 t3_os_pci_save_state(struct adapter *sc)
1146 {
1147 pci_save_state(sc->dev);
1148 return (0);
1149 }
1150
1151 int
t3_os_pci_restore_state(struct adapter * sc)1152 t3_os_pci_restore_state(struct adapter *sc)
1153 {
1154 pci_restore_state(sc->dev);
1155 return (0);
1156 }
1157
1158 /**
1159 * t3_os_link_changed - handle link status changes
1160 * @sc: the adapter associated with the link change
1161 * @port_id: the port index whose link status has changed
1162 * @link_status: the new status of the link
1163 * @speed: the new speed setting
1164 * @duplex: the new duplex setting
1165 * @fc: the new flow-control setting
1166 *
1167 * This is the OS-dependent handler for link status changes. The OS
1168 * neutral handler takes care of most of the processing for these events,
1169 * then calls this handler for any OS-specific processing.
1170 */
1171 void
t3_os_link_changed(adapter_t * adapter,int port_id,int link_status,int speed,int duplex,int fc,int mac_was_reset)1172 t3_os_link_changed(adapter_t *adapter, int port_id, int link_status, int speed,
1173 int duplex, int fc, int mac_was_reset)
1174 {
1175 struct port_info *pi = &adapter->port[port_id];
1176 if_t ifp = pi->ifp;
1177
1178 /* no race with detach, so ifp should always be good */
1179 KASSERT(ifp, ("%s: if detached.", __func__));
1180
1181 /* Reapply mac settings if they were lost due to a reset */
1182 if (mac_was_reset) {
1183 PORT_LOCK(pi);
1184 cxgb_update_mac_settings(pi);
1185 PORT_UNLOCK(pi);
1186 }
1187
1188 if (link_status) {
1189 if_setbaudrate(ifp, IF_Mbps(speed));
1190 if_link_state_change(ifp, LINK_STATE_UP);
1191 } else
1192 if_link_state_change(ifp, LINK_STATE_DOWN);
1193 }
1194
1195 /**
1196 * t3_os_phymod_changed - handle PHY module changes
1197 * @phy: the PHY reporting the module change
1198 * @mod_type: new module type
1199 *
1200 * This is the OS-dependent handler for PHY module changes. It is
1201 * invoked when a PHY module is removed or inserted for any OS-specific
1202 * processing.
1203 */
t3_os_phymod_changed(struct adapter * adap,int port_id)1204 void t3_os_phymod_changed(struct adapter *adap, int port_id)
1205 {
1206 static const char *mod_str[] = {
1207 NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX-L", "unknown"
1208 };
1209 struct port_info *pi = &adap->port[port_id];
1210 int mod = pi->phy.modtype;
1211
1212 if (mod != pi->media.ifm_cur->ifm_data)
1213 cxgb_build_medialist(pi);
1214
1215 if (mod == phy_modtype_none)
1216 if_printf(pi->ifp, "PHY module unplugged\n");
1217 else {
1218 KASSERT(mod < ARRAY_SIZE(mod_str),
1219 ("invalid PHY module type %d", mod));
1220 if_printf(pi->ifp, "%s PHY module inserted\n", mod_str[mod]);
1221 }
1222 }
1223
1224 void
t3_os_set_hw_addr(adapter_t * adapter,int port_idx,u8 hw_addr[])1225 t3_os_set_hw_addr(adapter_t *adapter, int port_idx, u8 hw_addr[])
1226 {
1227
1228 /*
1229 * The ifnet might not be allocated before this gets called,
1230 * as this is called early on in attach by t3_prep_adapter
1231 * save the address off in the port structure
1232 */
1233 if (cxgb_debug)
1234 printf("set_hw_addr on idx %d addr %6D\n", port_idx, hw_addr, ":");
1235 bcopy(hw_addr, adapter->port[port_idx].hw_addr, ETHER_ADDR_LEN);
1236 }
1237
1238 /*
1239 * Programs the XGMAC based on the settings in the ifnet. These settings
1240 * include MTU, MAC address, mcast addresses, etc.
1241 */
1242 static void
cxgb_update_mac_settings(struct port_info * p)1243 cxgb_update_mac_settings(struct port_info *p)
1244 {
1245 if_t ifp = p->ifp;
1246 struct t3_rx_mode rm;
1247 struct cmac *mac = &p->mac;
1248 int mtu, hwtagging;
1249
1250 PORT_LOCK_ASSERT_OWNED(p);
1251
1252 bcopy(if_getlladdr(ifp), p->hw_addr, ETHER_ADDR_LEN);
1253
1254 mtu = if_getmtu(ifp);
1255 if (if_getcapenable(ifp) & IFCAP_VLAN_MTU)
1256 mtu += ETHER_VLAN_ENCAP_LEN;
1257
1258 hwtagging = (if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0;
1259
1260 t3_mac_set_mtu(mac, mtu);
1261 t3_set_vlan_accel(p->adapter, 1 << p->tx_chan, hwtagging);
1262 t3_mac_set_address(mac, 0, p->hw_addr);
1263 t3_init_rx_mode(&rm, p);
1264 t3_mac_set_rx_mode(mac, &rm);
1265 }
1266
1267
1268 static int
await_mgmt_replies(struct adapter * adap,unsigned long init_cnt,unsigned long n)1269 await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
1270 unsigned long n)
1271 {
1272 int attempts = 5;
1273
1274 while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
1275 if (!--attempts)
1276 return (ETIMEDOUT);
1277 t3_os_sleep(10);
1278 }
1279 return 0;
1280 }
1281
1282 static int
init_tp_parity(struct adapter * adap)1283 init_tp_parity(struct adapter *adap)
1284 {
1285 int i;
1286 struct mbuf *m;
1287 struct cpl_set_tcb_field *greq;
1288 unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
1289
1290 t3_tp_set_offload_mode(adap, 1);
1291
1292 for (i = 0; i < 16; i++) {
1293 struct cpl_smt_write_req *req;
1294
1295 m = m_gethdr(M_WAITOK, MT_DATA);
1296 req = mtod(m, struct cpl_smt_write_req *);
1297 m->m_len = m->m_pkthdr.len = sizeof(*req);
1298 memset(req, 0, sizeof(*req));
1299 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1300 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
1301 req->iff = i;
1302 t3_mgmt_tx(adap, m);
1303 }
1304
1305 for (i = 0; i < 2048; i++) {
1306 struct cpl_l2t_write_req *req;
1307
1308 m = m_gethdr(M_WAITOK, MT_DATA);
1309 req = mtod(m, struct cpl_l2t_write_req *);
1310 m->m_len = m->m_pkthdr.len = sizeof(*req);
1311 memset(req, 0, sizeof(*req));
1312 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1313 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
1314 req->params = htonl(V_L2T_W_IDX(i));
1315 t3_mgmt_tx(adap, m);
1316 }
1317
1318 for (i = 0; i < 2048; i++) {
1319 struct cpl_rte_write_req *req;
1320
1321 m = m_gethdr(M_WAITOK, MT_DATA);
1322 req = mtod(m, struct cpl_rte_write_req *);
1323 m->m_len = m->m_pkthdr.len = sizeof(*req);
1324 memset(req, 0, sizeof(*req));
1325 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1326 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
1327 req->l2t_idx = htonl(V_L2T_W_IDX(i));
1328 t3_mgmt_tx(adap, m);
1329 }
1330
1331 m = m_gethdr(M_WAITOK, MT_DATA);
1332 greq = mtod(m, struct cpl_set_tcb_field *);
1333 m->m_len = m->m_pkthdr.len = sizeof(*greq);
1334 memset(greq, 0, sizeof(*greq));
1335 greq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
1336 OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
1337 greq->mask = htobe64(1);
1338 t3_mgmt_tx(adap, m);
1339
1340 i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
1341 t3_tp_set_offload_mode(adap, 0);
1342 return (i);
1343 }
1344
1345 /**
1346 * setup_rss - configure Receive Side Steering (per-queue connection demux)
1347 * @adap: the adapter
1348 *
1349 * Sets up RSS to distribute packets to multiple receive queues. We
1350 * configure the RSS CPU lookup table to distribute to the number of HW
1351 * receive queues, and the response queue lookup table to narrow that
1352 * down to the response queues actually configured for each port.
1353 * We always configure the RSS mapping for two ports since the mapping
1354 * table has plenty of entries.
1355 */
1356 static void
setup_rss(adapter_t * adap)1357 setup_rss(adapter_t *adap)
1358 {
1359 int i;
1360 u_int nq[2];
1361 uint8_t cpus[SGE_QSETS + 1];
1362 uint16_t rspq_map[RSS_TABLE_SIZE];
1363
1364 for (i = 0; i < SGE_QSETS; ++i)
1365 cpus[i] = i;
1366 cpus[SGE_QSETS] = 0xff;
1367
1368 nq[0] = nq[1] = 0;
1369 for_each_port(adap, i) {
1370 const struct port_info *pi = adap2pinfo(adap, i);
1371
1372 nq[pi->tx_chan] += pi->nqsets;
1373 }
1374 for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
1375 rspq_map[i] = nq[0] ? i % nq[0] : 0;
1376 rspq_map[i + RSS_TABLE_SIZE / 2] = nq[1] ? i % nq[1] + nq[0] : 0;
1377 }
1378
1379 /* Calculate the reverse RSS map table */
1380 for (i = 0; i < SGE_QSETS; ++i)
1381 adap->rrss_map[i] = 0xff;
1382 for (i = 0; i < RSS_TABLE_SIZE; ++i)
1383 if (adap->rrss_map[rspq_map[i]] == 0xff)
1384 adap->rrss_map[rspq_map[i]] = i;
1385
1386 t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
1387 F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN | F_OFDMAPEN |
1388 F_RRCPLMAPEN | V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ,
1389 cpus, rspq_map);
1390
1391 }
1392 static void
send_pktsched_cmd(struct adapter * adap,int sched,int qidx,int lo,int hi,int port)1393 send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
1394 int hi, int port)
1395 {
1396 struct mbuf *m;
1397 struct mngt_pktsched_wr *req;
1398
1399 m = m_gethdr(M_NOWAIT, MT_DATA);
1400 if (m) {
1401 req = mtod(m, struct mngt_pktsched_wr *);
1402 req->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
1403 req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
1404 req->sched = sched;
1405 req->idx = qidx;
1406 req->min = lo;
1407 req->max = hi;
1408 req->binding = port;
1409 m->m_len = m->m_pkthdr.len = sizeof(*req);
1410 t3_mgmt_tx(adap, m);
1411 }
1412 }
1413
1414 static void
bind_qsets(adapter_t * sc)1415 bind_qsets(adapter_t *sc)
1416 {
1417 int i, j;
1418
1419 for (i = 0; i < (sc)->params.nports; ++i) {
1420 const struct port_info *pi = adap2pinfo(sc, i);
1421
1422 for (j = 0; j < pi->nqsets; ++j) {
1423 send_pktsched_cmd(sc, 1, pi->first_qset + j, -1,
1424 -1, pi->tx_chan);
1425
1426 }
1427 }
1428 }
1429
1430 static void
update_tpeeprom(struct adapter * adap)1431 update_tpeeprom(struct adapter *adap)
1432 {
1433 const struct firmware *tpeeprom;
1434
1435 uint32_t version;
1436 unsigned int major, minor;
1437 int ret, len;
1438 char rev, name[32];
1439
1440 t3_seeprom_read(adap, TP_SRAM_OFFSET, &version);
1441
1442 major = G_TP_VERSION_MAJOR(version);
1443 minor = G_TP_VERSION_MINOR(version);
1444 if (major == TP_VERSION_MAJOR && minor == TP_VERSION_MINOR)
1445 return;
1446
1447 rev = t3rev2char(adap);
1448 snprintf(name, sizeof(name), TPEEPROM_NAME, rev);
1449
1450 tpeeprom = firmware_get(name);
1451 if (tpeeprom == NULL) {
1452 device_printf(adap->dev,
1453 "could not load TP EEPROM: unable to load %s\n",
1454 name);
1455 return;
1456 }
1457
1458 len = tpeeprom->datasize - 4;
1459
1460 ret = t3_check_tpsram(adap, tpeeprom->data, tpeeprom->datasize);
1461 if (ret)
1462 goto release_tpeeprom;
1463
1464 if (len != TP_SRAM_LEN) {
1465 device_printf(adap->dev,
1466 "%s length is wrong len=%d expected=%d\n", name,
1467 len, TP_SRAM_LEN);
1468 return;
1469 }
1470
1471 ret = set_eeprom(&adap->port[0], tpeeprom->data, tpeeprom->datasize,
1472 TP_SRAM_OFFSET);
1473
1474 if (!ret) {
1475 device_printf(adap->dev,
1476 "Protocol SRAM image updated in EEPROM to %d.%d.%d\n",
1477 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1478 } else
1479 device_printf(adap->dev,
1480 "Protocol SRAM image update in EEPROM failed\n");
1481
1482 release_tpeeprom:
1483 firmware_put(tpeeprom, FIRMWARE_UNLOAD);
1484
1485 return;
1486 }
1487
1488 static int
update_tpsram(struct adapter * adap)1489 update_tpsram(struct adapter *adap)
1490 {
1491 const struct firmware *tpsram;
1492 int ret;
1493 char rev, name[32];
1494
1495 rev = t3rev2char(adap);
1496 snprintf(name, sizeof(name), TPSRAM_NAME, rev);
1497
1498 update_tpeeprom(adap);
1499
1500 tpsram = firmware_get(name);
1501 if (tpsram == NULL){
1502 device_printf(adap->dev, "could not load TP SRAM\n");
1503 return (EINVAL);
1504 } else
1505 device_printf(adap->dev, "updating TP SRAM\n");
1506
1507 ret = t3_check_tpsram(adap, tpsram->data, tpsram->datasize);
1508 if (ret)
1509 goto release_tpsram;
1510
1511 ret = t3_set_proto_sram(adap, tpsram->data);
1512 if (ret)
1513 device_printf(adap->dev, "loading protocol SRAM failed\n");
1514
1515 release_tpsram:
1516 firmware_put(tpsram, FIRMWARE_UNLOAD);
1517
1518 return ret;
1519 }
1520
1521 /**
1522 * cxgb_up - enable the adapter
1523 * @adap: adapter being enabled
1524 *
1525 * Called when the first port is enabled, this function performs the
1526 * actions necessary to make an adapter operational, such as completing
1527 * the initialization of HW modules, and enabling interrupts.
1528 */
1529 static int
cxgb_up(struct adapter * sc)1530 cxgb_up(struct adapter *sc)
1531 {
1532 int err = 0;
1533 unsigned int mxf = t3_mc5_size(&sc->mc5) - MC5_MIN_TIDS;
1534
1535 KASSERT(sc->open_device_map == 0, ("%s: device(s) already open (%x)",
1536 __func__, sc->open_device_map));
1537
1538 if ((sc->flags & FULL_INIT_DONE) == 0) {
1539
1540 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1541
1542 if ((sc->flags & FW_UPTODATE) == 0)
1543 if ((err = upgrade_fw(sc)))
1544 goto out;
1545
1546 if ((sc->flags & TPS_UPTODATE) == 0)
1547 if ((err = update_tpsram(sc)))
1548 goto out;
1549
1550 if (is_offload(sc) && nfilters != 0) {
1551 sc->params.mc5.nservers = 0;
1552
1553 if (nfilters < 0)
1554 sc->params.mc5.nfilters = mxf;
1555 else
1556 sc->params.mc5.nfilters = min(nfilters, mxf);
1557 }
1558
1559 err = t3_init_hw(sc, 0);
1560 if (err)
1561 goto out;
1562
1563 t3_set_reg_field(sc, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1564 t3_write_reg(sc, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1565
1566 err = setup_sge_qsets(sc);
1567 if (err)
1568 goto out;
1569
1570 alloc_filters(sc);
1571 setup_rss(sc);
1572
1573 t3_add_configured_sysctls(sc);
1574 sc->flags |= FULL_INIT_DONE;
1575 }
1576
1577 t3_intr_clear(sc);
1578 t3_sge_start(sc);
1579 t3_intr_enable(sc);
1580
1581 if (sc->params.rev >= T3_REV_C && !(sc->flags & TP_PARITY_INIT) &&
1582 is_offload(sc) && init_tp_parity(sc) == 0)
1583 sc->flags |= TP_PARITY_INIT;
1584
1585 if (sc->flags & TP_PARITY_INIT) {
1586 t3_write_reg(sc, A_TP_INT_CAUSE, F_CMCACHEPERR | F_ARPLUTPERR);
1587 t3_write_reg(sc, A_TP_INT_ENABLE, 0x7fbfffff);
1588 }
1589
1590 if (!(sc->flags & QUEUES_BOUND)) {
1591 bind_qsets(sc);
1592 setup_hw_filters(sc);
1593 sc->flags |= QUEUES_BOUND;
1594 }
1595
1596 t3_sge_reset_adapter(sc);
1597 out:
1598 return (err);
1599 }
1600
1601 /*
1602 * Called when the last open device is closed. Does NOT undo all of cxgb_up's
1603 * work. Specifically, the resources grabbed under FULL_INIT_DONE are released
1604 * during controller_detach, not here.
1605 */
1606 static void
cxgb_down(struct adapter * sc)1607 cxgb_down(struct adapter *sc)
1608 {
1609 t3_sge_stop(sc);
1610 t3_intr_disable(sc);
1611 }
1612
1613 /*
1614 * if_init for cxgb ports.
1615 */
1616 static void
cxgb_init(void * arg)1617 cxgb_init(void *arg)
1618 {
1619 struct port_info *p = arg;
1620 struct adapter *sc = p->adapter;
1621
1622 ADAPTER_LOCK(sc);
1623 cxgb_init_locked(p); /* releases adapter lock */
1624 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1625 }
1626
1627 static int
cxgb_init_locked(struct port_info * p)1628 cxgb_init_locked(struct port_info *p)
1629 {
1630 struct adapter *sc = p->adapter;
1631 if_t ifp = p->ifp;
1632 struct cmac *mac = &p->mac;
1633 int i, rc = 0, may_sleep = 0, gave_up_lock = 0;
1634
1635 ADAPTER_LOCK_ASSERT_OWNED(sc);
1636
1637 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1638 gave_up_lock = 1;
1639 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbinit", 0)) {
1640 rc = EINTR;
1641 goto done;
1642 }
1643 }
1644 if (IS_DOOMED(p)) {
1645 rc = ENXIO;
1646 goto done;
1647 }
1648 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1649
1650 /*
1651 * The code that runs during one-time adapter initialization can sleep
1652 * so it's important not to hold any locks across it.
1653 */
1654 may_sleep = sc->flags & FULL_INIT_DONE ? 0 : 1;
1655
1656 if (may_sleep) {
1657 SET_BUSY(sc);
1658 gave_up_lock = 1;
1659 ADAPTER_UNLOCK(sc);
1660 }
1661
1662 if (sc->open_device_map == 0 && ((rc = cxgb_up(sc)) != 0))
1663 goto done;
1664
1665 PORT_LOCK(p);
1666 if (isset(&sc->open_device_map, p->port_id) &&
1667 (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
1668 PORT_UNLOCK(p);
1669 goto done;
1670 }
1671 t3_port_intr_enable(sc, p->port_id);
1672 if (!mac->multiport)
1673 t3_mac_init(mac);
1674 cxgb_update_mac_settings(p);
1675 t3_link_start(&p->phy, mac, &p->link_config);
1676 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
1677 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, IFF_DRV_OACTIVE);
1678 PORT_UNLOCK(p);
1679
1680 for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1681 struct sge_qset *qs = &sc->sge.qs[i];
1682 struct sge_txq *txq = &qs->txq[TXQ_ETH];
1683
1684 callout_reset_on(&txq->txq_watchdog, hz, cxgb_tx_watchdog, qs,
1685 txq->txq_watchdog.c_cpu);
1686 }
1687
1688 /* all ok */
1689 setbit(&sc->open_device_map, p->port_id);
1690 callout_reset(&p->link_check_ch,
1691 p->phy.caps & SUPPORTED_LINK_IRQ ? hz * 3 : hz / 4,
1692 link_check_callout, p);
1693
1694 done:
1695 if (may_sleep) {
1696 ADAPTER_LOCK(sc);
1697 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1698 CLR_BUSY(sc);
1699 }
1700 if (gave_up_lock)
1701 wakeup_one(&sc->flags);
1702 ADAPTER_UNLOCK(sc);
1703 return (rc);
1704 }
1705
1706 static int
cxgb_uninit_locked(struct port_info * p)1707 cxgb_uninit_locked(struct port_info *p)
1708 {
1709 struct adapter *sc = p->adapter;
1710 int rc;
1711
1712 ADAPTER_LOCK_ASSERT_OWNED(sc);
1713
1714 while (!IS_DOOMED(p) && IS_BUSY(sc)) {
1715 if (mtx_sleep(&sc->flags, &sc->lock, PCATCH, "cxgbunin", 0)) {
1716 rc = EINTR;
1717 goto done;
1718 }
1719 }
1720 if (IS_DOOMED(p)) {
1721 rc = ENXIO;
1722 goto done;
1723 }
1724 KASSERT(!IS_BUSY(sc), ("%s: controller busy.", __func__));
1725 SET_BUSY(sc);
1726 ADAPTER_UNLOCK(sc);
1727
1728 rc = cxgb_uninit_synchronized(p);
1729
1730 ADAPTER_LOCK(sc);
1731 KASSERT(IS_BUSY(sc), ("%s: controller not busy.", __func__));
1732 CLR_BUSY(sc);
1733 wakeup_one(&sc->flags);
1734 done:
1735 ADAPTER_UNLOCK(sc);
1736 return (rc);
1737 }
1738
1739 /*
1740 * Called on "ifconfig down", and from port_detach
1741 */
1742 static int
cxgb_uninit_synchronized(struct port_info * pi)1743 cxgb_uninit_synchronized(struct port_info *pi)
1744 {
1745 struct adapter *sc = pi->adapter;
1746 if_t ifp = pi->ifp;
1747
1748 /*
1749 * taskqueue_drain may cause a deadlock if the adapter lock is held.
1750 */
1751 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1752
1753 /*
1754 * Clear this port's bit from the open device map, and then drain all
1755 * the tasks that can access/manipulate this port's port_info or ifp.
1756 * We disable this port's interrupts here and so the slow/ext
1757 * interrupt tasks won't be enqueued. The tick task will continue to
1758 * be enqueued every second but the runs after this drain will not see
1759 * this port in the open device map.
1760 *
1761 * A well behaved task must take open_device_map into account and ignore
1762 * ports that are not open.
1763 */
1764 clrbit(&sc->open_device_map, pi->port_id);
1765 t3_port_intr_disable(sc, pi->port_id);
1766 taskqueue_drain(sc->tq, &sc->slow_intr_task);
1767 taskqueue_drain(sc->tq, &sc->tick_task);
1768
1769 callout_drain(&pi->link_check_ch);
1770 taskqueue_drain(sc->tq, &pi->link_check_task);
1771
1772 PORT_LOCK(pi);
1773 if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
1774
1775 /* disable pause frames */
1776 t3_set_reg_field(sc, A_XGM_TX_CFG + pi->mac.offset, F_TXPAUSEEN, 0);
1777
1778 /* Reset RX FIFO HWM */
1779 t3_set_reg_field(sc, A_XGM_RXFIFO_CFG + pi->mac.offset,
1780 V_RXFIFOPAUSEHWM(M_RXFIFOPAUSEHWM), 0);
1781
1782 DELAY(100 * 1000);
1783
1784 /* Wait for TXFIFO empty */
1785 t3_wait_op_done(sc, A_XGM_TXFIFO_CFG + pi->mac.offset,
1786 F_TXFIFO_EMPTY, 1, 20, 5);
1787
1788 DELAY(100 * 1000);
1789 t3_mac_disable(&pi->mac, MAC_DIRECTION_RX);
1790
1791 pi->phy.ops->power_down(&pi->phy, 1);
1792
1793 PORT_UNLOCK(pi);
1794
1795 pi->link_config.link_ok = 0;
1796 t3_os_link_changed(sc, pi->port_id, 0, 0, 0, 0, 0);
1797
1798 if (sc->open_device_map == 0)
1799 cxgb_down(pi->adapter);
1800
1801 return (0);
1802 }
1803
1804 /*
1805 * Mark lro enabled or disabled in all qsets for this port
1806 */
1807 static int
cxgb_set_lro(struct port_info * p,int enabled)1808 cxgb_set_lro(struct port_info *p, int enabled)
1809 {
1810 int i;
1811 struct adapter *adp = p->adapter;
1812 struct sge_qset *q;
1813
1814 for (i = 0; i < p->nqsets; i++) {
1815 q = &adp->sge.qs[p->first_qset + i];
1816 q->lro.enabled = (enabled != 0);
1817 }
1818 return (0);
1819 }
1820
1821 static int
cxgb_ioctl(if_t ifp,unsigned long command,caddr_t data)1822 cxgb_ioctl(if_t ifp, unsigned long command, caddr_t data)
1823 {
1824 struct port_info *p = if_getsoftc(ifp);
1825 struct adapter *sc = p->adapter;
1826 struct ifreq *ifr = (struct ifreq *)data;
1827 int flags, error = 0, mtu;
1828 uint32_t mask;
1829
1830 switch (command) {
1831 case SIOCSIFMTU:
1832 ADAPTER_LOCK(sc);
1833 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1834 if (error) {
1835 fail:
1836 ADAPTER_UNLOCK(sc);
1837 return (error);
1838 }
1839
1840 mtu = ifr->ifr_mtu;
1841 if ((mtu < ETHERMIN) || (mtu > ETHERMTU_JUMBO)) {
1842 error = EINVAL;
1843 } else {
1844 if_setmtu(ifp, mtu);
1845 PORT_LOCK(p);
1846 cxgb_update_mac_settings(p);
1847 PORT_UNLOCK(p);
1848 }
1849 ADAPTER_UNLOCK(sc);
1850 break;
1851 case SIOCSIFFLAGS:
1852 ADAPTER_LOCK(sc);
1853 if (IS_DOOMED(p)) {
1854 error = ENXIO;
1855 goto fail;
1856 }
1857 if (if_getflags(ifp) & IFF_UP) {
1858 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1859 flags = p->if_flags;
1860 if (((if_getflags(ifp) ^ flags) & IFF_PROMISC) ||
1861 ((if_getflags(ifp) ^ flags) & IFF_ALLMULTI)) {
1862 if (IS_BUSY(sc)) {
1863 error = EBUSY;
1864 goto fail;
1865 }
1866 PORT_LOCK(p);
1867 cxgb_update_mac_settings(p);
1868 PORT_UNLOCK(p);
1869 }
1870 ADAPTER_UNLOCK(sc);
1871 } else
1872 error = cxgb_init_locked(p);
1873 p->if_flags = if_getflags(ifp);
1874 } else if (if_getdrvflags(ifp) & IFF_DRV_RUNNING)
1875 error = cxgb_uninit_locked(p);
1876 else
1877 ADAPTER_UNLOCK(sc);
1878
1879 ADAPTER_LOCK_ASSERT_NOTOWNED(sc);
1880 break;
1881 case SIOCADDMULTI:
1882 case SIOCDELMULTI:
1883 ADAPTER_LOCK(sc);
1884 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1885 if (error)
1886 goto fail;
1887
1888 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1889 PORT_LOCK(p);
1890 cxgb_update_mac_settings(p);
1891 PORT_UNLOCK(p);
1892 }
1893 ADAPTER_UNLOCK(sc);
1894
1895 break;
1896 case SIOCSIFCAP:
1897 ADAPTER_LOCK(sc);
1898 error = IS_DOOMED(p) ? ENXIO : (IS_BUSY(sc) ? EBUSY : 0);
1899 if (error)
1900 goto fail;
1901
1902 mask = ifr->ifr_reqcap ^ if_getcapenable(ifp);
1903 if (mask & IFCAP_TXCSUM) {
1904 if_togglecapenable(ifp, IFCAP_TXCSUM);
1905 if_togglehwassist(ifp, CSUM_TCP | CSUM_UDP | CSUM_IP);
1906
1907 if (IFCAP_TSO4 & if_getcapenable(ifp) &&
1908 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
1909 mask &= ~IFCAP_TSO4;
1910 if_setcapenablebit(ifp, 0, IFCAP_TSO4);
1911 if_printf(ifp,
1912 "tso4 disabled due to -txcsum.\n");
1913 }
1914 }
1915 if (mask & IFCAP_TXCSUM_IPV6) {
1916 if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
1917 if_togglehwassist(ifp, CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
1918
1919 if (IFCAP_TSO6 & if_getcapenable(ifp) &&
1920 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
1921 mask &= ~IFCAP_TSO6;
1922 if_setcapenablebit(ifp, 0, IFCAP_TSO6);
1923 if_printf(ifp,
1924 "tso6 disabled due to -txcsum6.\n");
1925 }
1926 }
1927 if (mask & IFCAP_RXCSUM)
1928 if_togglecapenable(ifp, IFCAP_RXCSUM);
1929 if (mask & IFCAP_RXCSUM_IPV6)
1930 if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
1931
1932 /*
1933 * Note that we leave CSUM_TSO alone (it is always set). The
1934 * kernel takes both IFCAP_TSOx and CSUM_TSO into account before
1935 * sending a TSO request our way, so it's sufficient to toggle
1936 * IFCAP_TSOx only.
1937 */
1938 if (mask & IFCAP_TSO4) {
1939 if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
1940 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
1941 if_printf(ifp, "enable txcsum first.\n");
1942 error = EAGAIN;
1943 goto fail;
1944 }
1945 if_togglecapenable(ifp, IFCAP_TSO4);
1946 }
1947 if (mask & IFCAP_TSO6) {
1948 if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
1949 !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
1950 if_printf(ifp, "enable txcsum6 first.\n");
1951 error = EAGAIN;
1952 goto fail;
1953 }
1954 if_togglecapenable(ifp, IFCAP_TSO6);
1955 }
1956 if (mask & IFCAP_LRO) {
1957 if_togglecapenable(ifp, IFCAP_LRO);
1958
1959 /* Safe to do this even if cxgb_up not called yet */
1960 cxgb_set_lro(p, if_getcapenable(ifp) & IFCAP_LRO);
1961 }
1962 #ifdef TCP_OFFLOAD
1963 if (mask & IFCAP_TOE4) {
1964 int enable = (if_getcapenable(ifp) ^ mask) & IFCAP_TOE4;
1965
1966 error = toe_capability(p, enable);
1967 if (error == 0)
1968 if_togglecapenable(ifp, mask);
1969 }
1970 #endif
1971 if (mask & IFCAP_VLAN_HWTAGGING) {
1972 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
1973 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1974 PORT_LOCK(p);
1975 cxgb_update_mac_settings(p);
1976 PORT_UNLOCK(p);
1977 }
1978 }
1979 if (mask & IFCAP_VLAN_MTU) {
1980 if_togglecapenable(ifp, IFCAP_VLAN_MTU);
1981 if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
1982 PORT_LOCK(p);
1983 cxgb_update_mac_settings(p);
1984 PORT_UNLOCK(p);
1985 }
1986 }
1987 if (mask & IFCAP_VLAN_HWTSO)
1988 if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
1989 if (mask & IFCAP_VLAN_HWCSUM)
1990 if_togglecapenable(ifp, IFCAP_VLAN_HWCSUM);
1991
1992 #ifdef VLAN_CAPABILITIES
1993 VLAN_CAPABILITIES(ifp);
1994 #endif
1995 ADAPTER_UNLOCK(sc);
1996 break;
1997 case SIOCSIFMEDIA:
1998 case SIOCGIFMEDIA:
1999 error = ifmedia_ioctl(ifp, ifr, &p->media, command);
2000 break;
2001 default:
2002 error = ether_ioctl(ifp, command, data);
2003 }
2004
2005 return (error);
2006 }
2007
2008 static int
cxgb_media_change(if_t ifp)2009 cxgb_media_change(if_t ifp)
2010 {
2011 return (EOPNOTSUPP);
2012 }
2013
2014 /*
2015 * Translates phy->modtype to the correct Ethernet media subtype.
2016 */
2017 static int
cxgb_ifm_type(int mod)2018 cxgb_ifm_type(int mod)
2019 {
2020 switch (mod) {
2021 case phy_modtype_sr:
2022 return (IFM_10G_SR);
2023 case phy_modtype_lr:
2024 return (IFM_10G_LR);
2025 case phy_modtype_lrm:
2026 return (IFM_10G_LRM);
2027 case phy_modtype_twinax:
2028 return (IFM_10G_TWINAX);
2029 case phy_modtype_twinax_long:
2030 return (IFM_10G_TWINAX_LONG);
2031 case phy_modtype_none:
2032 return (IFM_NONE);
2033 case phy_modtype_unknown:
2034 return (IFM_UNKNOWN);
2035 }
2036
2037 KASSERT(0, ("%s: modtype %d unknown", __func__, mod));
2038 return (IFM_UNKNOWN);
2039 }
2040
2041 /*
2042 * Rebuilds the ifmedia list for this port, and sets the current media.
2043 */
2044 static void
cxgb_build_medialist(struct port_info * p)2045 cxgb_build_medialist(struct port_info *p)
2046 {
2047 struct cphy *phy = &p->phy;
2048 struct ifmedia *media = &p->media;
2049 int mod = phy->modtype;
2050 int m = IFM_ETHER | IFM_FDX;
2051
2052 PORT_LOCK(p);
2053
2054 ifmedia_removeall(media);
2055 if (phy->caps & SUPPORTED_TP && phy->caps & SUPPORTED_Autoneg) {
2056 /* Copper (RJ45) */
2057
2058 if (phy->caps & SUPPORTED_10000baseT_Full)
2059 ifmedia_add(media, m | IFM_10G_T, mod, NULL);
2060
2061 if (phy->caps & SUPPORTED_1000baseT_Full)
2062 ifmedia_add(media, m | IFM_1000_T, mod, NULL);
2063
2064 if (phy->caps & SUPPORTED_100baseT_Full)
2065 ifmedia_add(media, m | IFM_100_TX, mod, NULL);
2066
2067 if (phy->caps & SUPPORTED_10baseT_Full)
2068 ifmedia_add(media, m | IFM_10_T, mod, NULL);
2069
2070 ifmedia_add(media, IFM_ETHER | IFM_AUTO, mod, NULL);
2071 ifmedia_set(media, IFM_ETHER | IFM_AUTO);
2072
2073 } else if (phy->caps & SUPPORTED_TP) {
2074 /* Copper (CX4) */
2075
2076 KASSERT(phy->caps & SUPPORTED_10000baseT_Full,
2077 ("%s: unexpected cap 0x%x", __func__, phy->caps));
2078
2079 ifmedia_add(media, m | IFM_10G_CX4, mod, NULL);
2080 ifmedia_set(media, m | IFM_10G_CX4);
2081
2082 } else if (phy->caps & SUPPORTED_FIBRE &&
2083 phy->caps & SUPPORTED_10000baseT_Full) {
2084 /* 10G optical (but includes SFP+ twinax) */
2085
2086 m |= cxgb_ifm_type(mod);
2087 if (IFM_SUBTYPE(m) == IFM_NONE)
2088 m &= ~IFM_FDX;
2089
2090 ifmedia_add(media, m, mod, NULL);
2091 ifmedia_set(media, m);
2092
2093 } else if (phy->caps & SUPPORTED_FIBRE &&
2094 phy->caps & SUPPORTED_1000baseT_Full) {
2095 /* 1G optical */
2096
2097 /* XXX: Lie and claim to be SX, could actually be any 1G-X */
2098 ifmedia_add(media, m | IFM_1000_SX, mod, NULL);
2099 ifmedia_set(media, m | IFM_1000_SX);
2100
2101 } else {
2102 KASSERT(0, ("%s: don't know how to handle 0x%x.", __func__,
2103 phy->caps));
2104 }
2105
2106 PORT_UNLOCK(p);
2107 }
2108
2109 static void
cxgb_media_status(if_t ifp,struct ifmediareq * ifmr)2110 cxgb_media_status(if_t ifp, struct ifmediareq *ifmr)
2111 {
2112 struct port_info *p = if_getsoftc(ifp);
2113 struct ifmedia_entry *cur = p->media.ifm_cur;
2114 int speed = p->link_config.speed;
2115
2116 if (cur->ifm_data != p->phy.modtype) {
2117 cxgb_build_medialist(p);
2118 cur = p->media.ifm_cur;
2119 }
2120
2121 ifmr->ifm_status = IFM_AVALID;
2122 if (!p->link_config.link_ok)
2123 return;
2124
2125 ifmr->ifm_status |= IFM_ACTIVE;
2126
2127 /*
2128 * active and current will differ iff current media is autoselect. That
2129 * can happen only for copper RJ45.
2130 */
2131 if (IFM_SUBTYPE(cur->ifm_media) != IFM_AUTO)
2132 return;
2133 KASSERT(p->phy.caps & SUPPORTED_TP && p->phy.caps & SUPPORTED_Autoneg,
2134 ("%s: unexpected PHY caps 0x%x", __func__, p->phy.caps));
2135
2136 ifmr->ifm_active = IFM_ETHER | IFM_FDX;
2137 if (speed == SPEED_10000)
2138 ifmr->ifm_active |= IFM_10G_T;
2139 else if (speed == SPEED_1000)
2140 ifmr->ifm_active |= IFM_1000_T;
2141 else if (speed == SPEED_100)
2142 ifmr->ifm_active |= IFM_100_TX;
2143 else if (speed == SPEED_10)
2144 ifmr->ifm_active |= IFM_10_T;
2145 else
2146 KASSERT(0, ("%s: link up but speed unknown (%u)", __func__,
2147 speed));
2148 }
2149
2150 static uint64_t
cxgb_get_counter(if_t ifp,ift_counter c)2151 cxgb_get_counter(if_t ifp, ift_counter c)
2152 {
2153 struct port_info *pi = if_getsoftc(ifp);
2154 struct adapter *sc = pi->adapter;
2155 struct cmac *mac = &pi->mac;
2156 struct mac_stats *mstats = &mac->stats;
2157
2158 cxgb_refresh_stats(pi);
2159
2160 switch (c) {
2161 case IFCOUNTER_IPACKETS:
2162 return (mstats->rx_frames);
2163
2164 case IFCOUNTER_IERRORS:
2165 return (mstats->rx_jabber + mstats->rx_data_errs +
2166 mstats->rx_sequence_errs + mstats->rx_runt +
2167 mstats->rx_too_long + mstats->rx_mac_internal_errs +
2168 mstats->rx_short + mstats->rx_fcs_errs);
2169
2170 case IFCOUNTER_OPACKETS:
2171 return (mstats->tx_frames);
2172
2173 case IFCOUNTER_OERRORS:
2174 return (mstats->tx_excess_collisions + mstats->tx_underrun +
2175 mstats->tx_len_errs + mstats->tx_mac_internal_errs +
2176 mstats->tx_excess_deferral + mstats->tx_fcs_errs);
2177
2178 case IFCOUNTER_COLLISIONS:
2179 return (mstats->tx_total_collisions);
2180
2181 case IFCOUNTER_IBYTES:
2182 return (mstats->rx_octets);
2183
2184 case IFCOUNTER_OBYTES:
2185 return (mstats->tx_octets);
2186
2187 case IFCOUNTER_IMCASTS:
2188 return (mstats->rx_mcast_frames);
2189
2190 case IFCOUNTER_OMCASTS:
2191 return (mstats->tx_mcast_frames);
2192
2193 case IFCOUNTER_IQDROPS:
2194 return (mstats->rx_cong_drops);
2195
2196 case IFCOUNTER_OQDROPS: {
2197 int i;
2198 uint64_t drops;
2199
2200 drops = 0;
2201 if (sc->flags & FULL_INIT_DONE) {
2202 for (i = pi->first_qset; i < pi->first_qset + pi->nqsets; i++)
2203 drops += sc->sge.qs[i].txq[TXQ_ETH].txq_mr->br_drops;
2204 }
2205
2206 return (drops);
2207
2208 }
2209
2210 default:
2211 return (if_get_counter_default(ifp, c));
2212 }
2213 }
2214
2215 static void
cxgb_async_intr(void * data)2216 cxgb_async_intr(void *data)
2217 {
2218 adapter_t *sc = data;
2219
2220 t3_write_reg(sc, A_PL_INT_ENABLE0, 0);
2221 (void) t3_read_reg(sc, A_PL_INT_ENABLE0);
2222 taskqueue_enqueue(sc->tq, &sc->slow_intr_task);
2223 }
2224
2225 static void
link_check_callout(void * arg)2226 link_check_callout(void *arg)
2227 {
2228 struct port_info *pi = arg;
2229 struct adapter *sc = pi->adapter;
2230
2231 if (!isset(&sc->open_device_map, pi->port_id))
2232 return;
2233
2234 taskqueue_enqueue(sc->tq, &pi->link_check_task);
2235 }
2236
2237 static void
check_link_status(void * arg,int pending)2238 check_link_status(void *arg, int pending)
2239 {
2240 struct port_info *pi = arg;
2241 struct adapter *sc = pi->adapter;
2242
2243 if (!isset(&sc->open_device_map, pi->port_id))
2244 return;
2245
2246 t3_link_changed(sc, pi->port_id);
2247
2248 if (pi->link_fault || !(pi->phy.caps & SUPPORTED_LINK_IRQ) ||
2249 pi->link_config.link_ok == 0)
2250 callout_reset(&pi->link_check_ch, hz, link_check_callout, pi);
2251 }
2252
2253 void
t3_os_link_intr(struct port_info * pi)2254 t3_os_link_intr(struct port_info *pi)
2255 {
2256 /*
2257 * Schedule a link check in the near future. If the link is flapping
2258 * rapidly we'll keep resetting the callout and delaying the check until
2259 * things stabilize a bit.
2260 */
2261 callout_reset(&pi->link_check_ch, hz / 4, link_check_callout, pi);
2262 }
2263
2264 static void
check_t3b2_mac(struct adapter * sc)2265 check_t3b2_mac(struct adapter *sc)
2266 {
2267 int i;
2268
2269 if (sc->flags & CXGB_SHUTDOWN)
2270 return;
2271
2272 for_each_port(sc, i) {
2273 struct port_info *p = &sc->port[i];
2274 int status;
2275 #ifdef INVARIANTS
2276 if_t ifp = p->ifp;
2277 #endif
2278
2279 if (!isset(&sc->open_device_map, p->port_id) || p->link_fault ||
2280 !p->link_config.link_ok)
2281 continue;
2282
2283 KASSERT(if_getdrvflags(ifp) & IFF_DRV_RUNNING,
2284 ("%s: state mismatch (drv_flags %x, device_map %x)",
2285 __func__, if_getdrvflags(ifp), sc->open_device_map));
2286
2287 PORT_LOCK(p);
2288 status = t3b2_mac_watchdog_task(&p->mac);
2289 if (status == 1)
2290 p->mac.stats.num_toggled++;
2291 else if (status == 2) {
2292 struct cmac *mac = &p->mac;
2293
2294 cxgb_update_mac_settings(p);
2295 t3_link_start(&p->phy, mac, &p->link_config);
2296 t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2297 t3_port_intr_enable(sc, p->port_id);
2298 p->mac.stats.num_resets++;
2299 }
2300 PORT_UNLOCK(p);
2301 }
2302 }
2303
2304 static void
cxgb_tick(void * arg)2305 cxgb_tick(void *arg)
2306 {
2307 adapter_t *sc = (adapter_t *)arg;
2308
2309 if (sc->flags & CXGB_SHUTDOWN)
2310 return;
2311
2312 taskqueue_enqueue(sc->tq, &sc->tick_task);
2313 callout_reset(&sc->cxgb_tick_ch, hz, cxgb_tick, sc);
2314 }
2315
2316 void
cxgb_refresh_stats(struct port_info * pi)2317 cxgb_refresh_stats(struct port_info *pi)
2318 {
2319 struct timeval tv;
2320 const struct timeval interval = {0, 250000}; /* 250ms */
2321
2322 getmicrotime(&tv);
2323 timevalsub(&tv, &interval);
2324 if (timevalcmp(&tv, &pi->last_refreshed, <))
2325 return;
2326
2327 PORT_LOCK(pi);
2328 t3_mac_update_stats(&pi->mac);
2329 PORT_UNLOCK(pi);
2330 getmicrotime(&pi->last_refreshed);
2331 }
2332
2333 static void
cxgb_tick_handler(void * arg,int count)2334 cxgb_tick_handler(void *arg, int count)
2335 {
2336 adapter_t *sc = (adapter_t *)arg;
2337 const struct adapter_params *p = &sc->params;
2338 int i;
2339 uint32_t cause, reset;
2340
2341 if (sc->flags & CXGB_SHUTDOWN || !(sc->flags & FULL_INIT_DONE))
2342 return;
2343
2344 if (p->rev == T3_REV_B2 && p->nports < 4 && sc->open_device_map)
2345 check_t3b2_mac(sc);
2346
2347 cause = t3_read_reg(sc, A_SG_INT_CAUSE) & (F_RSPQSTARVE | F_FLEMPTY);
2348 if (cause) {
2349 struct sge_qset *qs = &sc->sge.qs[0];
2350 uint32_t mask, v;
2351
2352 v = t3_read_reg(sc, A_SG_RSPQ_FL_STATUS) & ~0xff00;
2353
2354 mask = 1;
2355 for (i = 0; i < SGE_QSETS; i++) {
2356 if (v & mask)
2357 qs[i].rspq.starved++;
2358 mask <<= 1;
2359 }
2360
2361 mask <<= SGE_QSETS; /* skip RSPQXDISABLED */
2362
2363 for (i = 0; i < SGE_QSETS * 2; i++) {
2364 if (v & mask) {
2365 qs[i / 2].fl[i % 2].empty++;
2366 }
2367 mask <<= 1;
2368 }
2369
2370 /* clear */
2371 t3_write_reg(sc, A_SG_RSPQ_FL_STATUS, v);
2372 t3_write_reg(sc, A_SG_INT_CAUSE, cause);
2373 }
2374
2375 for (i = 0; i < sc->params.nports; i++) {
2376 struct port_info *pi = &sc->port[i];
2377 struct cmac *mac = &pi->mac;
2378
2379 if (!isset(&sc->open_device_map, pi->port_id))
2380 continue;
2381
2382 cxgb_refresh_stats(pi);
2383
2384 if (mac->multiport)
2385 continue;
2386
2387 /* Count rx fifo overflows, once per second */
2388 cause = t3_read_reg(sc, A_XGM_INT_CAUSE + mac->offset);
2389 reset = 0;
2390 if (cause & F_RXFIFO_OVERFLOW) {
2391 mac->stats.rx_fifo_ovfl++;
2392 reset |= F_RXFIFO_OVERFLOW;
2393 }
2394 t3_write_reg(sc, A_XGM_INT_CAUSE + mac->offset, reset);
2395 }
2396 }
2397
2398 static void
touch_bars(device_t dev)2399 touch_bars(device_t dev)
2400 {
2401 /*
2402 * Don't enable yet
2403 */
2404 #if !defined(__LP64__) && 0
2405 u32 v;
2406
2407 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_1, &v);
2408 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_1, v);
2409 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_3, &v);
2410 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_3, v);
2411 pci_read_config_dword(pdev, PCI_BASE_ADDRESS_5, &v);
2412 pci_write_config_dword(pdev, PCI_BASE_ADDRESS_5, v);
2413 #endif
2414 }
2415
2416 static int
set_eeprom(struct port_info * pi,const uint8_t * data,int len,int offset)2417 set_eeprom(struct port_info *pi, const uint8_t *data, int len, int offset)
2418 {
2419 uint8_t *buf;
2420 int err = 0;
2421 u32 aligned_offset, aligned_len, *p;
2422 struct adapter *adapter = pi->adapter;
2423
2424
2425 aligned_offset = offset & ~3;
2426 aligned_len = (len + (offset & 3) + 3) & ~3;
2427
2428 if (aligned_offset != offset || aligned_len != len) {
2429 buf = malloc(aligned_len, M_DEVBUF, M_WAITOK | M_ZERO);
2430 err = t3_seeprom_read(adapter, aligned_offset, (u32 *)buf);
2431 if (!err && aligned_len > 4)
2432 err = t3_seeprom_read(adapter,
2433 aligned_offset + aligned_len - 4,
2434 (u32 *)&buf[aligned_len - 4]);
2435 if (err)
2436 goto out;
2437 memcpy(buf + (offset & 3), data, len);
2438 } else
2439 buf = (uint8_t *)(uintptr_t)data;
2440
2441 err = t3_seeprom_wp(adapter, 0);
2442 if (err)
2443 goto out;
2444
2445 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2446 err = t3_seeprom_write(adapter, aligned_offset, *p);
2447 aligned_offset += 4;
2448 }
2449
2450 if (!err)
2451 err = t3_seeprom_wp(adapter, 1);
2452 out:
2453 if (buf != data)
2454 free(buf, M_DEVBUF);
2455 return err;
2456 }
2457
2458
2459 static int
in_range(int val,int lo,int hi)2460 in_range(int val, int lo, int hi)
2461 {
2462 return val < 0 || (val <= hi && val >= lo);
2463 }
2464
2465 static int
cxgb_extension_open(struct cdev * dev,int flags,int fmp,struct thread * td)2466 cxgb_extension_open(struct cdev *dev, int flags, int fmp, struct thread *td)
2467 {
2468 return (0);
2469 }
2470
2471 static int
cxgb_extension_close(struct cdev * dev,int flags,int fmt,struct thread * td)2472 cxgb_extension_close(struct cdev *dev, int flags, int fmt, struct thread *td)
2473 {
2474 return (0);
2475 }
2476
2477 static int
cxgb_extension_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)2478 cxgb_extension_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data,
2479 int fflag, struct thread *td)
2480 {
2481 int mmd, error = 0;
2482 struct port_info *pi = dev->si_drv1;
2483 adapter_t *sc = pi->adapter;
2484
2485 #ifdef PRIV_SUPPORTED
2486 if (priv_check(td, PRIV_DRIVER)) {
2487 if (cxgb_debug)
2488 printf("user does not have access to privileged ioctls\n");
2489 return (EPERM);
2490 }
2491 #else
2492 if (suser(td)) {
2493 if (cxgb_debug)
2494 printf("user does not have access to privileged ioctls\n");
2495 return (EPERM);
2496 }
2497 #endif
2498
2499 switch (cmd) {
2500 case CHELSIO_GET_MIIREG: {
2501 uint32_t val;
2502 struct cphy *phy = &pi->phy;
2503 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2504
2505 if (!phy->mdio_read)
2506 return (EOPNOTSUPP);
2507 if (is_10G(sc)) {
2508 mmd = mid->phy_id >> 8;
2509 if (!mmd)
2510 mmd = MDIO_DEV_PCS;
2511 else if (mmd > MDIO_DEV_VEND2)
2512 return (EINVAL);
2513
2514 error = phy->mdio_read(sc, mid->phy_id & 0x1f, mmd,
2515 mid->reg_num, &val);
2516 } else
2517 error = phy->mdio_read(sc, mid->phy_id & 0x1f, 0,
2518 mid->reg_num & 0x1f, &val);
2519 if (error == 0)
2520 mid->val_out = val;
2521 break;
2522 }
2523 case CHELSIO_SET_MIIREG: {
2524 struct cphy *phy = &pi->phy;
2525 struct ch_mii_data *mid = (struct ch_mii_data *)data;
2526
2527 if (!phy->mdio_write)
2528 return (EOPNOTSUPP);
2529 if (is_10G(sc)) {
2530 mmd = mid->phy_id >> 8;
2531 if (!mmd)
2532 mmd = MDIO_DEV_PCS;
2533 else if (mmd > MDIO_DEV_VEND2)
2534 return (EINVAL);
2535
2536 error = phy->mdio_write(sc, mid->phy_id & 0x1f,
2537 mmd, mid->reg_num, mid->val_in);
2538 } else
2539 error = phy->mdio_write(sc, mid->phy_id & 0x1f, 0,
2540 mid->reg_num & 0x1f,
2541 mid->val_in);
2542 break;
2543 }
2544 case CHELSIO_SETREG: {
2545 struct ch_reg *edata = (struct ch_reg *)data;
2546 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2547 return (EFAULT);
2548 t3_write_reg(sc, edata->addr, edata->val);
2549 break;
2550 }
2551 case CHELSIO_GETREG: {
2552 struct ch_reg *edata = (struct ch_reg *)data;
2553 if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
2554 return (EFAULT);
2555 edata->val = t3_read_reg(sc, edata->addr);
2556 break;
2557 }
2558 case CHELSIO_GET_SGE_CONTEXT: {
2559 struct ch_cntxt *ecntxt = (struct ch_cntxt *)data;
2560 mtx_lock_spin(&sc->sge.reg_lock);
2561 switch (ecntxt->cntxt_type) {
2562 case CNTXT_TYPE_EGRESS:
2563 error = -t3_sge_read_ecntxt(sc, ecntxt->cntxt_id,
2564 ecntxt->data);
2565 break;
2566 case CNTXT_TYPE_FL:
2567 error = -t3_sge_read_fl(sc, ecntxt->cntxt_id,
2568 ecntxt->data);
2569 break;
2570 case CNTXT_TYPE_RSP:
2571 error = -t3_sge_read_rspq(sc, ecntxt->cntxt_id,
2572 ecntxt->data);
2573 break;
2574 case CNTXT_TYPE_CQ:
2575 error = -t3_sge_read_cq(sc, ecntxt->cntxt_id,
2576 ecntxt->data);
2577 break;
2578 default:
2579 error = EINVAL;
2580 break;
2581 }
2582 mtx_unlock_spin(&sc->sge.reg_lock);
2583 break;
2584 }
2585 case CHELSIO_GET_SGE_DESC: {
2586 struct ch_desc *edesc = (struct ch_desc *)data;
2587 int ret;
2588 if (edesc->queue_num >= SGE_QSETS * 6)
2589 return (EINVAL);
2590 ret = t3_get_desc(&sc->sge.qs[edesc->queue_num / 6],
2591 edesc->queue_num % 6, edesc->idx, edesc->data);
2592 if (ret < 0)
2593 return (EINVAL);
2594 edesc->size = ret;
2595 break;
2596 }
2597 case CHELSIO_GET_QSET_PARAMS: {
2598 struct qset_params *q;
2599 struct ch_qset_params *t = (struct ch_qset_params *)data;
2600 int q1 = pi->first_qset;
2601 int nqsets = pi->nqsets;
2602 int i;
2603
2604 if (t->qset_idx >= nqsets)
2605 return EINVAL;
2606
2607 i = q1 + t->qset_idx;
2608 q = &sc->params.sge.qset[i];
2609 t->rspq_size = q->rspq_size;
2610 t->txq_size[0] = q->txq_size[0];
2611 t->txq_size[1] = q->txq_size[1];
2612 t->txq_size[2] = q->txq_size[2];
2613 t->fl_size[0] = q->fl_size;
2614 t->fl_size[1] = q->jumbo_size;
2615 t->polling = q->polling;
2616 t->lro = q->lro;
2617 t->intr_lat = q->coalesce_usecs;
2618 t->cong_thres = q->cong_thres;
2619 t->qnum = i;
2620
2621 if ((sc->flags & FULL_INIT_DONE) == 0)
2622 t->vector = 0;
2623 else if (sc->flags & USING_MSIX)
2624 t->vector = rman_get_start(sc->msix_irq_res[i]);
2625 else
2626 t->vector = rman_get_start(sc->irq_res);
2627
2628 break;
2629 }
2630 case CHELSIO_GET_QSET_NUM: {
2631 struct ch_reg *edata = (struct ch_reg *)data;
2632 edata->val = pi->nqsets;
2633 break;
2634 }
2635 case CHELSIO_LOAD_FW: {
2636 uint8_t *fw_data;
2637 uint32_t vers;
2638 struct ch_mem_range *t = (struct ch_mem_range *)data;
2639
2640 /*
2641 * You're allowed to load a firmware only before FULL_INIT_DONE
2642 *
2643 * FW_UPTODATE is also set so the rest of the initialization
2644 * will not overwrite what was loaded here. This gives you the
2645 * flexibility to load any firmware (and maybe shoot yourself in
2646 * the foot).
2647 */
2648
2649 ADAPTER_LOCK(sc);
2650 if (sc->open_device_map || sc->flags & FULL_INIT_DONE) {
2651 ADAPTER_UNLOCK(sc);
2652 return (EBUSY);
2653 }
2654
2655 fw_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2656 if (!fw_data)
2657 error = ENOMEM;
2658 else
2659 error = copyin(t->buf, fw_data, t->len);
2660
2661 if (!error)
2662 error = -t3_load_fw(sc, fw_data, t->len);
2663
2664 if (t3_get_fw_version(sc, &vers) == 0) {
2665 snprintf(&sc->fw_version[0], sizeof(sc->fw_version),
2666 "%d.%d.%d", G_FW_VERSION_MAJOR(vers),
2667 G_FW_VERSION_MINOR(vers), G_FW_VERSION_MICRO(vers));
2668 }
2669
2670 if (!error)
2671 sc->flags |= FW_UPTODATE;
2672
2673 free(fw_data, M_DEVBUF);
2674 ADAPTER_UNLOCK(sc);
2675 break;
2676 }
2677 case CHELSIO_LOAD_BOOT: {
2678 uint8_t *boot_data;
2679 struct ch_mem_range *t = (struct ch_mem_range *)data;
2680
2681 boot_data = malloc(t->len, M_DEVBUF, M_NOWAIT);
2682 if (!boot_data)
2683 return ENOMEM;
2684
2685 error = copyin(t->buf, boot_data, t->len);
2686 if (!error)
2687 error = -t3_load_boot(sc, boot_data, t->len);
2688
2689 free(boot_data, M_DEVBUF);
2690 break;
2691 }
2692 case CHELSIO_GET_PM: {
2693 struct ch_pm *m = (struct ch_pm *)data;
2694 struct tp_params *p = &sc->params.tp;
2695
2696 if (!is_offload(sc))
2697 return (EOPNOTSUPP);
2698
2699 m->tx_pg_sz = p->tx_pg_size;
2700 m->tx_num_pg = p->tx_num_pgs;
2701 m->rx_pg_sz = p->rx_pg_size;
2702 m->rx_num_pg = p->rx_num_pgs;
2703 m->pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2704
2705 break;
2706 }
2707 case CHELSIO_SET_PM: {
2708 struct ch_pm *m = (struct ch_pm *)data;
2709 struct tp_params *p = &sc->params.tp;
2710
2711 if (!is_offload(sc))
2712 return (EOPNOTSUPP);
2713 if (sc->flags & FULL_INIT_DONE)
2714 return (EBUSY);
2715
2716 if (!m->rx_pg_sz || (m->rx_pg_sz & (m->rx_pg_sz - 1)) ||
2717 !m->tx_pg_sz || (m->tx_pg_sz & (m->tx_pg_sz - 1)))
2718 return (EINVAL); /* not power of 2 */
2719 if (!(m->rx_pg_sz & 0x14000))
2720 return (EINVAL); /* not 16KB or 64KB */
2721 if (!(m->tx_pg_sz & 0x1554000))
2722 return (EINVAL);
2723 if (m->tx_num_pg == -1)
2724 m->tx_num_pg = p->tx_num_pgs;
2725 if (m->rx_num_pg == -1)
2726 m->rx_num_pg = p->rx_num_pgs;
2727 if (m->tx_num_pg % 24 || m->rx_num_pg % 24)
2728 return (EINVAL);
2729 if (m->rx_num_pg * m->rx_pg_sz > p->chan_rx_size ||
2730 m->tx_num_pg * m->tx_pg_sz > p->chan_tx_size)
2731 return (EINVAL);
2732
2733 p->rx_pg_size = m->rx_pg_sz;
2734 p->tx_pg_size = m->tx_pg_sz;
2735 p->rx_num_pgs = m->rx_num_pg;
2736 p->tx_num_pgs = m->tx_num_pg;
2737 break;
2738 }
2739 case CHELSIO_SETMTUTAB: {
2740 struct ch_mtus *m = (struct ch_mtus *)data;
2741 int i;
2742
2743 if (!is_offload(sc))
2744 return (EOPNOTSUPP);
2745 if (offload_running(sc))
2746 return (EBUSY);
2747 if (m->nmtus != NMTUS)
2748 return (EINVAL);
2749 if (m->mtus[0] < 81) /* accommodate SACK */
2750 return (EINVAL);
2751
2752 /*
2753 * MTUs must be in ascending order
2754 */
2755 for (i = 1; i < NMTUS; ++i)
2756 if (m->mtus[i] < m->mtus[i - 1])
2757 return (EINVAL);
2758
2759 memcpy(sc->params.mtus, m->mtus, sizeof(sc->params.mtus));
2760 break;
2761 }
2762 case CHELSIO_GETMTUTAB: {
2763 struct ch_mtus *m = (struct ch_mtus *)data;
2764
2765 if (!is_offload(sc))
2766 return (EOPNOTSUPP);
2767
2768 memcpy(m->mtus, sc->params.mtus, sizeof(m->mtus));
2769 m->nmtus = NMTUS;
2770 break;
2771 }
2772 case CHELSIO_GET_MEM: {
2773 struct ch_mem_range *t = (struct ch_mem_range *)data;
2774 struct mc7 *mem;
2775 uint8_t *useraddr;
2776 u64 buf[32];
2777
2778 /*
2779 * Use these to avoid modifying len/addr in the return
2780 * struct
2781 */
2782 uint32_t len = t->len, addr = t->addr;
2783
2784 if (!is_offload(sc))
2785 return (EOPNOTSUPP);
2786 if (!(sc->flags & FULL_INIT_DONE))
2787 return (EIO); /* need the memory controllers */
2788 if ((addr & 0x7) || (len & 0x7))
2789 return (EINVAL);
2790 if (t->mem_id == MEM_CM)
2791 mem = &sc->cm;
2792 else if (t->mem_id == MEM_PMRX)
2793 mem = &sc->pmrx;
2794 else if (t->mem_id == MEM_PMTX)
2795 mem = &sc->pmtx;
2796 else
2797 return (EINVAL);
2798
2799 /*
2800 * Version scheme:
2801 * bits 0..9: chip version
2802 * bits 10..15: chip revision
2803 */
2804 t->version = 3 | (sc->params.rev << 10);
2805
2806 /*
2807 * Read 256 bytes at a time as len can be large and we don't
2808 * want to use huge intermediate buffers.
2809 */
2810 useraddr = (uint8_t *)t->buf;
2811 while (len) {
2812 unsigned int chunk = min(len, sizeof(buf));
2813
2814 error = t3_mc7_bd_read(mem, addr / 8, chunk / 8, buf);
2815 if (error)
2816 return (-error);
2817 if (copyout(buf, useraddr, chunk))
2818 return (EFAULT);
2819 useraddr += chunk;
2820 addr += chunk;
2821 len -= chunk;
2822 }
2823 break;
2824 }
2825 case CHELSIO_READ_TCAM_WORD: {
2826 struct ch_tcam_word *t = (struct ch_tcam_word *)data;
2827
2828 if (!is_offload(sc))
2829 return (EOPNOTSUPP);
2830 if (!(sc->flags & FULL_INIT_DONE))
2831 return (EIO); /* need MC5 */
2832 return -t3_read_mc5_range(&sc->mc5, t->addr, 1, t->buf);
2833 break;
2834 }
2835 case CHELSIO_SET_TRACE_FILTER: {
2836 struct ch_trace *t = (struct ch_trace *)data;
2837 const struct trace_params *tp;
2838
2839 tp = (const struct trace_params *)&t->sip;
2840 if (t->config_tx)
2841 t3_config_trace_filter(sc, tp, 0, t->invert_match,
2842 t->trace_tx);
2843 if (t->config_rx)
2844 t3_config_trace_filter(sc, tp, 1, t->invert_match,
2845 t->trace_rx);
2846 break;
2847 }
2848 case CHELSIO_SET_PKTSCHED: {
2849 struct ch_pktsched_params *p = (struct ch_pktsched_params *)data;
2850 if (sc->open_device_map == 0)
2851 return (EAGAIN);
2852 send_pktsched_cmd(sc, p->sched, p->idx, p->min, p->max,
2853 p->binding);
2854 break;
2855 }
2856 case CHELSIO_IFCONF_GETREGS: {
2857 struct ch_ifconf_regs *regs = (struct ch_ifconf_regs *)data;
2858 int reglen = cxgb_get_regs_len();
2859 uint8_t *buf = malloc(reglen, M_DEVBUF, M_NOWAIT);
2860 if (buf == NULL) {
2861 return (ENOMEM);
2862 }
2863 if (regs->len > reglen)
2864 regs->len = reglen;
2865 else if (regs->len < reglen)
2866 error = ENOBUFS;
2867
2868 if (!error) {
2869 cxgb_get_regs(sc, regs, buf);
2870 error = copyout(buf, regs->data, reglen);
2871 }
2872 free(buf, M_DEVBUF);
2873
2874 break;
2875 }
2876 case CHELSIO_SET_HW_SCHED: {
2877 struct ch_hw_sched *t = (struct ch_hw_sched *)data;
2878 unsigned int ticks_per_usec = core_ticks_per_usec(sc);
2879
2880 if ((sc->flags & FULL_INIT_DONE) == 0)
2881 return (EAGAIN); /* need TP to be initialized */
2882 if (t->sched >= NTX_SCHED || !in_range(t->mode, 0, 1) ||
2883 !in_range(t->channel, 0, 1) ||
2884 !in_range(t->kbps, 0, 10000000) ||
2885 !in_range(t->class_ipg, 0, 10000 * 65535 / ticks_per_usec) ||
2886 !in_range(t->flow_ipg, 0,
2887 dack_ticks_to_usec(sc, 0x7ff)))
2888 return (EINVAL);
2889
2890 if (t->kbps >= 0) {
2891 error = t3_config_sched(sc, t->kbps, t->sched);
2892 if (error < 0)
2893 return (-error);
2894 }
2895 if (t->class_ipg >= 0)
2896 t3_set_sched_ipg(sc, t->sched, t->class_ipg);
2897 if (t->flow_ipg >= 0) {
2898 t->flow_ipg *= 1000; /* us -> ns */
2899 t3_set_pace_tbl(sc, &t->flow_ipg, t->sched, 1);
2900 }
2901 if (t->mode >= 0) {
2902 int bit = 1 << (S_TX_MOD_TIMER_MODE + t->sched);
2903
2904 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2905 bit, t->mode ? bit : 0);
2906 }
2907 if (t->channel >= 0)
2908 t3_set_reg_field(sc, A_TP_TX_MOD_QUEUE_REQ_MAP,
2909 1 << t->sched, t->channel << t->sched);
2910 break;
2911 }
2912 case CHELSIO_GET_EEPROM: {
2913 int i;
2914 struct ch_eeprom *e = (struct ch_eeprom *)data;
2915 uint8_t *buf;
2916
2917 if (e->offset & 3 || e->offset >= EEPROMSIZE ||
2918 e->len > EEPROMSIZE || e->offset + e->len > EEPROMSIZE) {
2919 return (EINVAL);
2920 }
2921
2922 buf = malloc(EEPROMSIZE, M_DEVBUF, M_NOWAIT);
2923 if (buf == NULL) {
2924 return (ENOMEM);
2925 }
2926 e->magic = EEPROM_MAGIC;
2927 for (i = e->offset & ~3; !error && i < e->offset + e->len; i += 4)
2928 error = -t3_seeprom_read(sc, i, (uint32_t *)&buf[i]);
2929
2930 if (!error)
2931 error = copyout(buf + e->offset, e->data, e->len);
2932
2933 free(buf, M_DEVBUF);
2934 break;
2935 }
2936 case CHELSIO_CLEAR_STATS: {
2937 if (!(sc->flags & FULL_INIT_DONE))
2938 return EAGAIN;
2939
2940 PORT_LOCK(pi);
2941 t3_mac_update_stats(&pi->mac);
2942 memset(&pi->mac.stats, 0, sizeof(pi->mac.stats));
2943 PORT_UNLOCK(pi);
2944 break;
2945 }
2946 case CHELSIO_GET_UP_LA: {
2947 struct ch_up_la *la = (struct ch_up_la *)data;
2948 uint8_t *buf = malloc(LA_BUFSIZE, M_DEVBUF, M_NOWAIT);
2949 if (buf == NULL) {
2950 return (ENOMEM);
2951 }
2952 if (la->bufsize < LA_BUFSIZE)
2953 error = ENOBUFS;
2954
2955 if (!error)
2956 error = -t3_get_up_la(sc, &la->stopped, &la->idx,
2957 &la->bufsize, buf);
2958 if (!error)
2959 error = copyout(buf, la->data, la->bufsize);
2960
2961 free(buf, M_DEVBUF);
2962 break;
2963 }
2964 case CHELSIO_GET_UP_IOQS: {
2965 struct ch_up_ioqs *ioqs = (struct ch_up_ioqs *)data;
2966 uint8_t *buf = malloc(IOQS_BUFSIZE, M_DEVBUF, M_NOWAIT);
2967 uint32_t *v;
2968
2969 if (buf == NULL) {
2970 return (ENOMEM);
2971 }
2972 if (ioqs->bufsize < IOQS_BUFSIZE)
2973 error = ENOBUFS;
2974
2975 if (!error)
2976 error = -t3_get_up_ioqs(sc, &ioqs->bufsize, buf);
2977
2978 if (!error) {
2979 v = (uint32_t *)buf;
2980
2981 ioqs->ioq_rx_enable = *v++;
2982 ioqs->ioq_tx_enable = *v++;
2983 ioqs->ioq_rx_status = *v++;
2984 ioqs->ioq_tx_status = *v++;
2985
2986 error = copyout(v, ioqs->data, ioqs->bufsize);
2987 }
2988
2989 free(buf, M_DEVBUF);
2990 break;
2991 }
2992 case CHELSIO_SET_FILTER: {
2993 struct ch_filter *f = (struct ch_filter *)data;
2994 struct filter_info *p;
2995 unsigned int nfilters = sc->params.mc5.nfilters;
2996
2997 if (!is_offload(sc))
2998 return (EOPNOTSUPP); /* No TCAM */
2999 if (!(sc->flags & FULL_INIT_DONE))
3000 return (EAGAIN); /* mc5 not setup yet */
3001 if (nfilters == 0)
3002 return (EBUSY); /* TOE will use TCAM */
3003
3004 /* sanity checks */
3005 if (f->filter_id >= nfilters ||
3006 (f->val.dip && f->mask.dip != 0xffffffff) ||
3007 (f->val.sport && f->mask.sport != 0xffff) ||
3008 (f->val.dport && f->mask.dport != 0xffff) ||
3009 (f->val.vlan && f->mask.vlan != 0xfff) ||
3010 (f->val.vlan_prio &&
3011 f->mask.vlan_prio != FILTER_NO_VLAN_PRI) ||
3012 (f->mac_addr_idx != 0xffff && f->mac_addr_idx > 15) ||
3013 f->qset >= SGE_QSETS ||
3014 sc->rrss_map[f->qset] >= RSS_TABLE_SIZE)
3015 return (EINVAL);
3016
3017 /* Was allocated with M_WAITOK */
3018 KASSERT(sc->filters, ("filter table NULL\n"));
3019
3020 p = &sc->filters[f->filter_id];
3021 if (p->locked)
3022 return (EPERM);
3023
3024 bzero(p, sizeof(*p));
3025 p->sip = f->val.sip;
3026 p->sip_mask = f->mask.sip;
3027 p->dip = f->val.dip;
3028 p->sport = f->val.sport;
3029 p->dport = f->val.dport;
3030 p->vlan = f->mask.vlan ? f->val.vlan : 0xfff;
3031 p->vlan_prio = f->mask.vlan_prio ? (f->val.vlan_prio & 6) :
3032 FILTER_NO_VLAN_PRI;
3033 p->mac_hit = f->mac_hit;
3034 p->mac_vld = f->mac_addr_idx != 0xffff;
3035 p->mac_idx = f->mac_addr_idx;
3036 p->pkt_type = f->proto;
3037 p->report_filter_id = f->want_filter_id;
3038 p->pass = f->pass;
3039 p->rss = f->rss;
3040 p->qset = f->qset;
3041
3042 error = set_filter(sc, f->filter_id, p);
3043 if (error == 0)
3044 p->valid = 1;
3045 break;
3046 }
3047 case CHELSIO_DEL_FILTER: {
3048 struct ch_filter *f = (struct ch_filter *)data;
3049 struct filter_info *p;
3050 unsigned int nfilters = sc->params.mc5.nfilters;
3051
3052 if (!is_offload(sc))
3053 return (EOPNOTSUPP);
3054 if (!(sc->flags & FULL_INIT_DONE))
3055 return (EAGAIN);
3056 if (nfilters == 0 || sc->filters == NULL)
3057 return (EINVAL);
3058 if (f->filter_id >= nfilters)
3059 return (EINVAL);
3060
3061 p = &sc->filters[f->filter_id];
3062 if (p->locked)
3063 return (EPERM);
3064 if (!p->valid)
3065 return (EFAULT); /* Read "Bad address" as "Bad index" */
3066
3067 bzero(p, sizeof(*p));
3068 p->sip = p->sip_mask = 0xffffffff;
3069 p->vlan = 0xfff;
3070 p->vlan_prio = FILTER_NO_VLAN_PRI;
3071 p->pkt_type = 1;
3072 error = set_filter(sc, f->filter_id, p);
3073 break;
3074 }
3075 case CHELSIO_GET_FILTER: {
3076 struct ch_filter *f = (struct ch_filter *)data;
3077 struct filter_info *p;
3078 unsigned int i, nfilters = sc->params.mc5.nfilters;
3079
3080 if (!is_offload(sc))
3081 return (EOPNOTSUPP);
3082 if (!(sc->flags & FULL_INIT_DONE))
3083 return (EAGAIN);
3084 if (nfilters == 0 || sc->filters == NULL)
3085 return (EINVAL);
3086
3087 i = f->filter_id == 0xffffffff ? 0 : f->filter_id + 1;
3088 for (; i < nfilters; i++) {
3089 p = &sc->filters[i];
3090 if (!p->valid)
3091 continue;
3092
3093 bzero(f, sizeof(*f));
3094
3095 f->filter_id = i;
3096 f->val.sip = p->sip;
3097 f->mask.sip = p->sip_mask;
3098 f->val.dip = p->dip;
3099 f->mask.dip = p->dip ? 0xffffffff : 0;
3100 f->val.sport = p->sport;
3101 f->mask.sport = p->sport ? 0xffff : 0;
3102 f->val.dport = p->dport;
3103 f->mask.dport = p->dport ? 0xffff : 0;
3104 f->val.vlan = p->vlan == 0xfff ? 0 : p->vlan;
3105 f->mask.vlan = p->vlan == 0xfff ? 0 : 0xfff;
3106 f->val.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3107 0 : p->vlan_prio;
3108 f->mask.vlan_prio = p->vlan_prio == FILTER_NO_VLAN_PRI ?
3109 0 : FILTER_NO_VLAN_PRI;
3110 f->mac_hit = p->mac_hit;
3111 f->mac_addr_idx = p->mac_vld ? p->mac_idx : 0xffff;
3112 f->proto = p->pkt_type;
3113 f->want_filter_id = p->report_filter_id;
3114 f->pass = p->pass;
3115 f->rss = p->rss;
3116 f->qset = p->qset;
3117
3118 break;
3119 }
3120
3121 if (i == nfilters)
3122 f->filter_id = 0xffffffff;
3123 break;
3124 }
3125 default:
3126 return (EOPNOTSUPP);
3127 break;
3128 }
3129
3130 return (error);
3131 }
3132
3133 static __inline void
reg_block_dump(struct adapter * ap,uint8_t * buf,unsigned int start,unsigned int end)3134 reg_block_dump(struct adapter *ap, uint8_t *buf, unsigned int start,
3135 unsigned int end)
3136 {
3137 uint32_t *p = (uint32_t *)(buf + start);
3138
3139 for ( ; start <= end; start += sizeof(uint32_t))
3140 *p++ = t3_read_reg(ap, start);
3141 }
3142
3143 #define T3_REGMAP_SIZE (3 * 1024)
3144 static int
cxgb_get_regs_len(void)3145 cxgb_get_regs_len(void)
3146 {
3147 return T3_REGMAP_SIZE;
3148 }
3149
3150 static void
cxgb_get_regs(adapter_t * sc,struct ch_ifconf_regs * regs,uint8_t * buf)3151 cxgb_get_regs(adapter_t *sc, struct ch_ifconf_regs *regs, uint8_t *buf)
3152 {
3153
3154 /*
3155 * Version scheme:
3156 * bits 0..9: chip version
3157 * bits 10..15: chip revision
3158 * bit 31: set for PCIe cards
3159 */
3160 regs->version = 3 | (sc->params.rev << 10) | (is_pcie(sc) << 31);
3161
3162 /*
3163 * We skip the MAC statistics registers because they are clear-on-read.
3164 * Also reading multi-register stats would need to synchronize with the
3165 * periodic mac stats accumulation. Hard to justify the complexity.
3166 */
3167 memset(buf, 0, cxgb_get_regs_len());
3168 reg_block_dump(sc, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
3169 reg_block_dump(sc, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
3170 reg_block_dump(sc, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
3171 reg_block_dump(sc, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
3172 reg_block_dump(sc, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
3173 reg_block_dump(sc, buf, A_XGM_SERDES_STATUS0,
3174 XGM_REG(A_XGM_SERDES_STAT3, 1));
3175 reg_block_dump(sc, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
3176 XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
3177 }
3178
3179 static int
alloc_filters(struct adapter * sc)3180 alloc_filters(struct adapter *sc)
3181 {
3182 struct filter_info *p;
3183 unsigned int nfilters = sc->params.mc5.nfilters;
3184
3185 if (nfilters == 0)
3186 return (0);
3187
3188 p = malloc(sizeof(*p) * nfilters, M_DEVBUF, M_WAITOK | M_ZERO);
3189 sc->filters = p;
3190
3191 p = &sc->filters[nfilters - 1];
3192 p->vlan = 0xfff;
3193 p->vlan_prio = FILTER_NO_VLAN_PRI;
3194 p->pass = p->rss = p->valid = p->locked = 1;
3195
3196 return (0);
3197 }
3198
3199 static int
setup_hw_filters(struct adapter * sc)3200 setup_hw_filters(struct adapter *sc)
3201 {
3202 int i, rc;
3203 unsigned int nfilters = sc->params.mc5.nfilters;
3204
3205 if (!sc->filters)
3206 return (0);
3207
3208 t3_enable_filters(sc);
3209
3210 for (i = rc = 0; i < nfilters && !rc; i++) {
3211 if (sc->filters[i].locked)
3212 rc = set_filter(sc, i, &sc->filters[i]);
3213 }
3214
3215 return (rc);
3216 }
3217
3218 static int
set_filter(struct adapter * sc,int id,const struct filter_info * f)3219 set_filter(struct adapter *sc, int id, const struct filter_info *f)
3220 {
3221 int len;
3222 struct mbuf *m;
3223 struct ulp_txpkt *txpkt;
3224 struct work_request_hdr *wr;
3225 struct cpl_pass_open_req *oreq;
3226 struct cpl_set_tcb_field *sreq;
3227
3228 len = sizeof(*wr) + sizeof(*oreq) + 2 * sizeof(*sreq);
3229 KASSERT(len <= MHLEN, ("filter request too big for an mbuf"));
3230
3231 id += t3_mc5_size(&sc->mc5) - sc->params.mc5.nroutes -
3232 sc->params.mc5.nfilters;
3233
3234 m = m_gethdr(M_WAITOK, MT_DATA);
3235 m->m_len = m->m_pkthdr.len = len;
3236 bzero(mtod(m, char *), len);
3237
3238 wr = mtod(m, struct work_request_hdr *);
3239 wr->wrh_hi = htonl(V_WR_OP(FW_WROPCODE_BYPASS) | F_WR_ATOMIC);
3240
3241 oreq = (struct cpl_pass_open_req *)(wr + 1);
3242 txpkt = (struct ulp_txpkt *)oreq;
3243 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3244 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*oreq) / 8));
3245 OPCODE_TID(oreq) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, id));
3246 oreq->local_port = htons(f->dport);
3247 oreq->peer_port = htons(f->sport);
3248 oreq->local_ip = htonl(f->dip);
3249 oreq->peer_ip = htonl(f->sip);
3250 oreq->peer_netmask = htonl(f->sip_mask);
3251 oreq->opt0h = 0;
3252 oreq->opt0l = htonl(F_NO_OFFLOAD);
3253 oreq->opt1 = htonl(V_MAC_MATCH_VALID(f->mac_vld) |
3254 V_CONN_POLICY(CPL_CONN_POLICY_FILTER) |
3255 V_VLAN_PRI(f->vlan_prio >> 1) |
3256 V_VLAN_PRI_VALID(f->vlan_prio != FILTER_NO_VLAN_PRI) |
3257 V_PKT_TYPE(f->pkt_type) | V_OPT1_VLAN(f->vlan) |
3258 V_MAC_MATCH(f->mac_idx | (f->mac_hit << 4)));
3259
3260 sreq = (struct cpl_set_tcb_field *)(oreq + 1);
3261 set_tcb_field_ulp(sreq, id, 1, 0x1800808000ULL,
3262 (f->report_filter_id << 15) | (1 << 23) |
3263 ((u64)f->pass << 35) | ((u64)!f->rss << 36));
3264 set_tcb_field_ulp(sreq + 1, id, 0, 0xffffffff, (2 << 19) | 1);
3265 t3_mgmt_tx(sc, m);
3266
3267 if (f->pass && !f->rss) {
3268 len = sizeof(*sreq);
3269 m = m_gethdr(M_WAITOK, MT_DATA);
3270 m->m_len = m->m_pkthdr.len = len;
3271 bzero(mtod(m, char *), len);
3272 sreq = mtod(m, struct cpl_set_tcb_field *);
3273 sreq->wr.wrh_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
3274 mk_set_tcb_field(sreq, id, 25, 0x3f80000,
3275 (u64)sc->rrss_map[f->qset] << 19);
3276 t3_mgmt_tx(sc, m);
3277 }
3278 return 0;
3279 }
3280
3281 static inline void
mk_set_tcb_field(struct cpl_set_tcb_field * req,unsigned int tid,unsigned int word,u64 mask,u64 val)3282 mk_set_tcb_field(struct cpl_set_tcb_field *req, unsigned int tid,
3283 unsigned int word, u64 mask, u64 val)
3284 {
3285 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
3286 req->reply = V_NO_REPLY(1);
3287 req->cpu_idx = 0;
3288 req->word = htons(word);
3289 req->mask = htobe64(mask);
3290 req->val = htobe64(val);
3291 }
3292
3293 static inline void
set_tcb_field_ulp(struct cpl_set_tcb_field * req,unsigned int tid,unsigned int word,u64 mask,u64 val)3294 set_tcb_field_ulp(struct cpl_set_tcb_field *req, unsigned int tid,
3295 unsigned int word, u64 mask, u64 val)
3296 {
3297 struct ulp_txpkt *txpkt = (struct ulp_txpkt *)req;
3298
3299 txpkt->cmd_dest = htonl(V_ULPTX_CMD(ULP_TXPKT));
3300 txpkt->len = htonl(V_ULPTX_NFLITS(sizeof(*req) / 8));
3301 mk_set_tcb_field(req, tid, word, mask, val);
3302 }
3303
3304 void
t3_iterate(void (* func)(struct adapter *,void *),void * arg)3305 t3_iterate(void (*func)(struct adapter *, void *), void *arg)
3306 {
3307 struct adapter *sc;
3308
3309 mtx_lock(&t3_list_lock);
3310 SLIST_FOREACH(sc, &t3_list, link) {
3311 /*
3312 * func should not make any assumptions about what state sc is
3313 * in - the only guarantee is that sc->sc_lock is a valid lock.
3314 */
3315 func(sc, arg);
3316 }
3317 mtx_unlock(&t3_list_lock);
3318 }
3319
3320 #ifdef TCP_OFFLOAD
3321 static int
toe_capability(struct port_info * pi,int enable)3322 toe_capability(struct port_info *pi, int enable)
3323 {
3324 int rc;
3325 struct adapter *sc = pi->adapter;
3326
3327 ADAPTER_LOCK_ASSERT_OWNED(sc);
3328
3329 if (!is_offload(sc))
3330 return (ENODEV);
3331
3332 if (enable) {
3333 if (!(sc->flags & FULL_INIT_DONE)) {
3334 log(LOG_WARNING,
3335 "You must enable a cxgb interface first\n");
3336 return (EAGAIN);
3337 }
3338
3339 if (isset(&sc->offload_map, pi->port_id))
3340 return (0);
3341
3342 if (!(sc->flags & TOM_INIT_DONE)) {
3343 rc = t3_activate_uld(sc, ULD_TOM);
3344 if (rc == EAGAIN) {
3345 log(LOG_WARNING,
3346 "You must kldload t3_tom.ko before trying "
3347 "to enable TOE on a cxgb interface.\n");
3348 }
3349 if (rc != 0)
3350 return (rc);
3351 KASSERT(sc->tom_softc != NULL,
3352 ("%s: TOM activated but softc NULL", __func__));
3353 KASSERT(sc->flags & TOM_INIT_DONE,
3354 ("%s: TOM activated but flag not set", __func__));
3355 }
3356
3357 setbit(&sc->offload_map, pi->port_id);
3358
3359 /*
3360 * XXX: Temporary code to allow iWARP to be enabled when TOE is
3361 * enabled on any port. Need to figure out how to enable,
3362 * disable, load, and unload iWARP cleanly.
3363 */
3364 if (!isset(&sc->offload_map, MAX_NPORTS) &&
3365 t3_activate_uld(sc, ULD_IWARP) == 0)
3366 setbit(&sc->offload_map, MAX_NPORTS);
3367 } else {
3368 if (!isset(&sc->offload_map, pi->port_id))
3369 return (0);
3370
3371 KASSERT(sc->flags & TOM_INIT_DONE,
3372 ("%s: TOM never initialized?", __func__));
3373 clrbit(&sc->offload_map, pi->port_id);
3374 }
3375
3376 return (0);
3377 }
3378
3379 /*
3380 * Add an upper layer driver to the global list.
3381 */
3382 int
t3_register_uld(struct uld_info * ui)3383 t3_register_uld(struct uld_info *ui)
3384 {
3385 int rc = 0;
3386 struct uld_info *u;
3387
3388 mtx_lock(&t3_uld_list_lock);
3389 SLIST_FOREACH(u, &t3_uld_list, link) {
3390 if (u->uld_id == ui->uld_id) {
3391 rc = EEXIST;
3392 goto done;
3393 }
3394 }
3395
3396 SLIST_INSERT_HEAD(&t3_uld_list, ui, link);
3397 ui->refcount = 0;
3398 done:
3399 mtx_unlock(&t3_uld_list_lock);
3400 return (rc);
3401 }
3402
3403 int
t3_unregister_uld(struct uld_info * ui)3404 t3_unregister_uld(struct uld_info *ui)
3405 {
3406 int rc = EINVAL;
3407 struct uld_info *u;
3408
3409 mtx_lock(&t3_uld_list_lock);
3410
3411 SLIST_FOREACH(u, &t3_uld_list, link) {
3412 if (u == ui) {
3413 if (ui->refcount > 0) {
3414 rc = EBUSY;
3415 goto done;
3416 }
3417
3418 SLIST_REMOVE(&t3_uld_list, ui, uld_info, link);
3419 rc = 0;
3420 goto done;
3421 }
3422 }
3423 done:
3424 mtx_unlock(&t3_uld_list_lock);
3425 return (rc);
3426 }
3427
3428 int
t3_activate_uld(struct adapter * sc,int id)3429 t3_activate_uld(struct adapter *sc, int id)
3430 {
3431 int rc = EAGAIN;
3432 struct uld_info *ui;
3433
3434 mtx_lock(&t3_uld_list_lock);
3435
3436 SLIST_FOREACH(ui, &t3_uld_list, link) {
3437 if (ui->uld_id == id) {
3438 rc = ui->activate(sc);
3439 if (rc == 0)
3440 ui->refcount++;
3441 goto done;
3442 }
3443 }
3444 done:
3445 mtx_unlock(&t3_uld_list_lock);
3446
3447 return (rc);
3448 }
3449
3450 int
t3_deactivate_uld(struct adapter * sc,int id)3451 t3_deactivate_uld(struct adapter *sc, int id)
3452 {
3453 int rc = EINVAL;
3454 struct uld_info *ui;
3455
3456 mtx_lock(&t3_uld_list_lock);
3457
3458 SLIST_FOREACH(ui, &t3_uld_list, link) {
3459 if (ui->uld_id == id) {
3460 rc = ui->deactivate(sc);
3461 if (rc == 0)
3462 ui->refcount--;
3463 goto done;
3464 }
3465 }
3466 done:
3467 mtx_unlock(&t3_uld_list_lock);
3468
3469 return (rc);
3470 }
3471
3472 static int
cpl_not_handled(struct sge_qset * qs __unused,struct rsp_desc * r __unused,struct mbuf * m)3473 cpl_not_handled(struct sge_qset *qs __unused, struct rsp_desc *r __unused,
3474 struct mbuf *m)
3475 {
3476 m_freem(m);
3477 return (EDOOFUS);
3478 }
3479
3480 int
t3_register_cpl_handler(struct adapter * sc,int opcode,cpl_handler_t h)3481 t3_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h)
3482 {
3483 uintptr_t *loc, new;
3484
3485 if (opcode >= NUM_CPL_HANDLERS)
3486 return (EINVAL);
3487
3488 new = h ? (uintptr_t)h : (uintptr_t)cpl_not_handled;
3489 loc = (uintptr_t *) &sc->cpl_handler[opcode];
3490 atomic_store_rel_ptr(loc, new);
3491
3492 return (0);
3493 }
3494 #endif
3495
3496 static int
cxgbc_mod_event(module_t mod,int cmd,void * arg)3497 cxgbc_mod_event(module_t mod, int cmd, void *arg)
3498 {
3499 int rc = 0;
3500
3501 switch (cmd) {
3502 case MOD_LOAD:
3503 mtx_init(&t3_list_lock, "T3 adapters", 0, MTX_DEF);
3504 SLIST_INIT(&t3_list);
3505 #ifdef TCP_OFFLOAD
3506 mtx_init(&t3_uld_list_lock, "T3 ULDs", 0, MTX_DEF);
3507 SLIST_INIT(&t3_uld_list);
3508 #endif
3509 break;
3510
3511 case MOD_UNLOAD:
3512 #ifdef TCP_OFFLOAD
3513 mtx_lock(&t3_uld_list_lock);
3514 if (!SLIST_EMPTY(&t3_uld_list)) {
3515 rc = EBUSY;
3516 mtx_unlock(&t3_uld_list_lock);
3517 break;
3518 }
3519 mtx_unlock(&t3_uld_list_lock);
3520 mtx_destroy(&t3_uld_list_lock);
3521 #endif
3522 mtx_lock(&t3_list_lock);
3523 if (!SLIST_EMPTY(&t3_list)) {
3524 rc = EBUSY;
3525 mtx_unlock(&t3_list_lock);
3526 break;
3527 }
3528 mtx_unlock(&t3_list_lock);
3529 mtx_destroy(&t3_list_lock);
3530 break;
3531 }
3532
3533 return (rc);
3534 }
3535
3536 #ifdef DEBUGNET
3537 static void
cxgb_debugnet_init(if_t ifp,int * nrxr,int * ncl,int * clsize)3538 cxgb_debugnet_init(if_t ifp, int *nrxr, int *ncl, int *clsize)
3539 {
3540 struct port_info *pi;
3541 adapter_t *adap;
3542
3543 pi = if_getsoftc(ifp);
3544 adap = pi->adapter;
3545 ADAPTER_LOCK(adap);
3546 *nrxr = adap->nqsets;
3547 *ncl = adap->sge.qs[0].fl[1].size;
3548 *clsize = adap->sge.qs[0].fl[1].buf_size;
3549 ADAPTER_UNLOCK(adap);
3550 }
3551
3552 static void
cxgb_debugnet_event(if_t ifp,enum debugnet_ev event)3553 cxgb_debugnet_event(if_t ifp, enum debugnet_ev event)
3554 {
3555 struct port_info *pi;
3556 struct sge_qset *qs;
3557 int i;
3558
3559 pi = if_getsoftc(ifp);
3560 if (event == DEBUGNET_START)
3561 for (i = 0; i < pi->adapter->nqsets; i++) {
3562 qs = &pi->adapter->sge.qs[i];
3563
3564 /* Need to reinit after debugnet_mbuf_start(). */
3565 qs->fl[0].zone = zone_pack;
3566 qs->fl[1].zone = zone_clust;
3567 qs->lro.enabled = 0;
3568 }
3569 }
3570
3571 static int
cxgb_debugnet_transmit(if_t ifp,struct mbuf * m)3572 cxgb_debugnet_transmit(if_t ifp, struct mbuf *m)
3573 {
3574 struct port_info *pi;
3575 struct sge_qset *qs;
3576
3577 pi = if_getsoftc(ifp);
3578 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
3579 IFF_DRV_RUNNING)
3580 return (ENOENT);
3581
3582 qs = &pi->adapter->sge.qs[pi->first_qset];
3583 return (cxgb_debugnet_encap(qs, &m));
3584 }
3585
3586 static int
cxgb_debugnet_poll(if_t ifp,int count)3587 cxgb_debugnet_poll(if_t ifp, int count)
3588 {
3589 struct port_info *pi;
3590 adapter_t *adap;
3591 int i;
3592
3593 pi = if_getsoftc(ifp);
3594 if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0)
3595 return (ENOENT);
3596
3597 adap = pi->adapter;
3598 for (i = 0; i < adap->nqsets; i++)
3599 (void)cxgb_debugnet_poll_rx(adap, &adap->sge.qs[i]);
3600 (void)cxgb_debugnet_poll_tx(&adap->sge.qs[pi->first_qset]);
3601 return (0);
3602 }
3603 #endif /* DEBUGNET */
3604