1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/modctl.h> 27 #include <sys/conf.h> 28 #include <sys/devops.h> 29 #include <sys/pci.h> 30 #include <sys/atomic.h> 31 #include <sys/types.h> 32 #include <sys/file.h> 33 #include <sys/errno.h> 34 #include <sys/open.h> 35 #include <sys/cred.h> 36 #include <sys/stat.h> 37 #include <sys/mkdev.h> 38 #include <sys/queue.h> 39 #include <sys/containerof.h> 40 41 #include "version.h" 42 #include "common/common.h" 43 #include "common/t4_msg.h" 44 #include "common/t4_regs.h" 45 #include "firmware/t4_fw.h" 46 #include "firmware/t4_cfg.h" 47 #include "firmware/t5_fw.h" 48 #include "firmware/t5_cfg.h" 49 #include "firmware/t6_fw.h" 50 #include "firmware/t6_cfg.h" 51 #include "t4_l2t.h" 52 53 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp); 54 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp); 55 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, 56 int *rp); 57 struct cb_ops t4_cb_ops = { 58 .cb_open = t4_cb_open, 59 .cb_close = t4_cb_close, 60 .cb_strategy = nodev, 61 .cb_print = nodev, 62 .cb_dump = nodev, 63 .cb_read = nodev, 64 .cb_write = nodev, 65 .cb_ioctl = t4_cb_ioctl, 66 .cb_devmap = nodev, 67 .cb_mmap = nodev, 68 .cb_segmap = nodev, 69 .cb_chpoll = nochpoll, 70 .cb_prop_op = ddi_prop_op, 71 .cb_flag = D_MP, 72 .cb_rev = CB_REV, 73 .cb_aread = nodev, 74 .cb_awrite = nodev 75 }; 76 77 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, 78 void *arg, void *result); 79 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 80 void *arg, dev_info_t **cdipp); 81 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags, 82 ddi_bus_config_op_t op, void *arg); 83 struct bus_ops t4_bus_ops = { 84 .busops_rev = BUSO_REV, 85 .bus_ctl = t4_bus_ctl, 86 .bus_prop_op = ddi_bus_prop_op, 87 .bus_config = t4_bus_config, 88 .bus_unconfig = t4_bus_unconfig, 89 }; 90 91 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 92 void **rp); 93 static int t4_devo_probe(dev_info_t *dip); 94 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 95 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 96 static int t4_devo_quiesce(dev_info_t *dip); 97 struct dev_ops t4_dev_ops = { 98 .devo_rev = DEVO_REV, 99 .devo_getinfo = t4_devo_getinfo, 100 .devo_identify = nulldev, 101 .devo_probe = t4_devo_probe, 102 .devo_attach = t4_devo_attach, 103 .devo_detach = t4_devo_detach, 104 .devo_reset = nodev, 105 .devo_cb_ops = &t4_cb_ops, 106 .devo_bus_ops = &t4_bus_ops, 107 .devo_quiesce = &t4_devo_quiesce, 108 }; 109 110 static struct modldrv modldrv = { 111 .drv_modops = &mod_driverops, 112 .drv_linkinfo = "Chelsio T4 nexus " DRV_VERSION, 113 .drv_dev_ops = &t4_dev_ops 114 }; 115 116 static struct modlinkage modlinkage = { 117 .ml_rev = MODREV_1, 118 .ml_linkage = {&modldrv, NULL}, 119 }; 120 121 void *t4_list; 122 123 struct intrs_and_queues { 124 int intr_type; /* DDI_INTR_TYPE_* */ 125 int nirq; /* Number of vectors */ 126 int intr_fwd; /* Interrupts forwarded */ 127 int ntxq10g; /* # of NIC txq's for each 10G port */ 128 int nrxq10g; /* # of NIC rxq's for each 10G port */ 129 int ntxq1g; /* # of NIC txq's for each 1G port */ 130 int nrxq1g; /* # of NIC rxq's for each 1G port */ 131 #ifdef TCP_OFFLOAD_ENABLE 132 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 133 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 134 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 135 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 136 #endif 137 }; 138 139 struct fw_info fi[3]; 140 141 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, 142 mblk_t *m); 143 static int fw_msg_not_handled(struct adapter *, const __be64 *); 144 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h); 145 static unsigned int getpf(struct adapter *sc); 146 static int prep_firmware(struct adapter *sc); 147 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma); 148 static int partition_resources(struct adapter *sc); 149 static int adap__pre_init_tweaks(struct adapter *sc); 150 static int get_params__pre_init(struct adapter *sc); 151 static int get_params__post_init(struct adapter *sc); 152 static int set_params__post_init(struct adapter *); 153 static void setup_memwin(struct adapter *sc); 154 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 155 uint32_t *); 156 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 157 uint32_t position_memwin(struct adapter *, int, uint32_t); 158 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 159 uint_t count); 160 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 161 uint_t count); 162 static int init_driver_props(struct adapter *sc, struct driver_properties *p); 163 static int remove_extra_props(struct adapter *sc, int n10g, int n1g); 164 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 165 struct intrs_and_queues *iaq); 166 static int add_child_node(struct adapter *sc, int idx); 167 static int remove_child_node(struct adapter *sc, int idx); 168 static kstat_t *setup_kstats(struct adapter *sc); 169 static kstat_t *setup_wc_kstats(struct adapter *); 170 static int update_wc_kstats(kstat_t *, int); 171 #ifdef TCP_OFFLOAD_ENABLE 172 static int toe_capability(struct port_info *pi, int enable); 173 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc); 174 static int deactivate_uld(struct uld_softc *usc); 175 #endif 176 static kmutex_t t4_adapter_list_lock; 177 static SLIST_HEAD(, adapter) t4_adapter_list; 178 #ifdef TCP_OFFLOAD_ENABLE 179 static kmutex_t t4_uld_list_lock; 180 static SLIST_HEAD(, uld_info) t4_uld_list; 181 #endif 182 183 int 184 _init(void) 185 { 186 int rc; 187 188 rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0); 189 if (rc != 0) 190 return (rc); 191 192 rc = mod_install(&modlinkage); 193 if (rc != 0) 194 ddi_soft_state_fini(&t4_list); 195 196 mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL); 197 SLIST_INIT(&t4_adapter_list); 198 199 #ifdef TCP_OFFLOAD_ENABLE 200 mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL); 201 SLIST_INIT(&t4_uld_list); 202 #endif 203 204 return (rc); 205 } 206 207 int 208 _fini(void) 209 { 210 int rc; 211 212 rc = mod_remove(&modlinkage); 213 if (rc != 0) 214 return (rc); 215 216 ddi_soft_state_fini(&t4_list); 217 return (0); 218 } 219 220 int 221 _info(struct modinfo *mi) 222 { 223 return (mod_info(&modlinkage, mi)); 224 } 225 226 /* ARGSUSED */ 227 static int 228 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp) 229 { 230 struct adapter *sc; 231 minor_t minor; 232 233 minor = getminor((dev_t)arg); /* same as instance# in our case */ 234 235 if (cmd == DDI_INFO_DEVT2DEVINFO) { 236 sc = ddi_get_soft_state(t4_list, minor); 237 if (sc == NULL) 238 return (DDI_FAILURE); 239 240 ASSERT(sc->dev == (dev_t)arg); 241 *rp = (void *)sc->dip; 242 } else if (cmd == DDI_INFO_DEVT2INSTANCE) 243 *rp = (void *) (unsigned long) minor; 244 else 245 ASSERT(0); 246 247 return (DDI_SUCCESS); 248 } 249 250 static int 251 t4_devo_probe(dev_info_t *dip) 252 { 253 int rc, id, *reg; 254 uint_t n, pf; 255 256 id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 257 "device-id", 0xffff); 258 if (id == 0xffff) 259 return (DDI_PROBE_DONTCARE); 260 261 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 262 "reg", ®, &n); 263 if (rc != DDI_SUCCESS) 264 return (DDI_PROBE_DONTCARE); 265 266 pf = PCI_REG_FUNC_G(reg[0]); 267 ddi_prop_free(reg); 268 269 /* Prevent driver attachment on any PF except 0 on the FPGA */ 270 if (id == 0xa000 && pf != 0) 271 return (DDI_PROBE_FAILURE); 272 273 return (DDI_PROBE_DONTCARE); 274 } 275 276 static int 277 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 278 { 279 struct adapter *sc = NULL; 280 struct sge *s; 281 int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q; 282 int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g; 283 #ifdef TCP_OFFLOAD_ENABLE 284 int ofld_rqidx, ofld_tqidx; 285 #endif 286 char name[16]; 287 struct driver_properties *prp; 288 struct intrs_and_queues iaq; 289 ddi_device_acc_attr_t da = { 290 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 291 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 292 .devacc_attr_dataorder = DDI_UNORDERED_OK_ACC 293 }; 294 ddi_device_acc_attr_t da1 = { 295 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 296 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 297 .devacc_attr_dataorder = DDI_MERGING_OK_ACC 298 }; 299 300 if (cmd != DDI_ATTACH) 301 return (DDI_FAILURE); 302 303 /* 304 * Allocate space for soft state. 305 */ 306 instance = ddi_get_instance(dip); 307 rc = ddi_soft_state_zalloc(t4_list, instance); 308 if (rc != DDI_SUCCESS) { 309 cxgb_printf(dip, CE_WARN, 310 "failed to allocate soft state: %d", rc); 311 return (DDI_FAILURE); 312 } 313 314 sc = ddi_get_soft_state(t4_list, instance); 315 sc->dip = dip; 316 sc->dev = makedevice(ddi_driver_major(dip), instance); 317 mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL); 318 cv_init(&sc->cv, NULL, CV_DRIVER, NULL); 319 mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL); 320 TAILQ_INIT(&sc->sfl); 321 322 mutex_enter(&t4_adapter_list_lock); 323 SLIST_INSERT_HEAD(&t4_adapter_list, sc, link); 324 mutex_exit(&t4_adapter_list_lock); 325 326 sc->pf = getpf(sc); 327 if (sc->pf > 8) { 328 rc = EINVAL; 329 cxgb_printf(dip, CE_WARN, 330 "failed to determine PCI PF# of device"); 331 goto done; 332 } 333 sc->mbox = sc->pf; 334 335 /* Initialize the driver properties */ 336 prp = &sc->props; 337 (void)init_driver_props(sc, prp); 338 339 /* 340 * Enable access to the PCI config space. 341 */ 342 rc = pci_config_setup(dip, &sc->pci_regh); 343 if (rc != DDI_SUCCESS) { 344 cxgb_printf(dip, CE_WARN, 345 "failed to enable PCI config space access: %d", rc); 346 goto done; 347 } 348 349 /* TODO: Set max read request to 4K */ 350 351 /* 352 * Enable MMIO access. 353 */ 354 rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh); 355 if (rc != DDI_SUCCESS) { 356 cxgb_printf(dip, CE_WARN, 357 "failed to map device registers: %d", rc); 358 goto done; 359 } 360 361 (void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map)); 362 363 /* 364 * Initialize cpl handler. 365 */ 366 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) { 367 sc->cpl_handler[i] = cpl_not_handled; 368 } 369 370 for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) { 371 sc->fw_msg_handler[i] = fw_msg_not_handled; 372 } 373 374 for (i = 0; i < NCHAN; i++) { 375 (void) snprintf(name, sizeof (name), "%s-%d", 376 "reclaim", i); 377 sc->tq[i] = ddi_taskq_create(sc->dip, 378 name, 1, TASKQ_DEFAULTPRI, 0); 379 380 if (sc->tq[i] == NULL) { 381 cxgb_printf(dip, CE_WARN, 382 "failed to create task queues"); 383 rc = DDI_FAILURE; 384 goto done; 385 } 386 } 387 388 /* 389 * Prepare the adapter for operation. 390 */ 391 rc = -t4_prep_adapter(sc, false); 392 if (rc != 0) { 393 cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc); 394 goto done; 395 } 396 397 /* 398 * Enable BAR1 access. 399 */ 400 sc->doorbells |= DOORBELL_KDB; 401 rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h); 402 if (rc != DDI_SUCCESS) { 403 cxgb_printf(dip, CE_WARN, 404 "failed to map BAR1 device registers: %d", rc); 405 goto done; 406 } else { 407 if (is_t5(sc->params.chip)) { 408 sc->doorbells |= DOORBELL_UDB; 409 if (prp->wc) { 410 /* 411 * Enable write combining on BAR2. This is the 412 * userspace doorbell BAR and is split into 128B 413 * (UDBS_SEG_SIZE) doorbell regions, each associated 414 * with an egress queue. The first 64B has the doorbell 415 * and the second 64B can be used to submit a tx work 416 * request with an implicit doorbell. 417 */ 418 sc->doorbells &= ~DOORBELL_UDB; 419 sc->doorbells |= (DOORBELL_WCWR | 420 DOORBELL_UDBWC); 421 t4_write_reg(sc, A_SGE_STAT_CFG, 422 V_STATSOURCE_T5(7) | V_STATMODE(0)); 423 } 424 } 425 } 426 427 /* 428 * Do this really early. Note that minor number = instance. 429 */ 430 (void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance); 431 rc = ddi_create_minor_node(dip, name, S_IFCHR, instance, 432 DDI_NT_NEXUS, 0); 433 if (rc != DDI_SUCCESS) { 434 cxgb_printf(dip, CE_WARN, 435 "failed to create device node: %d", rc); 436 rc = DDI_SUCCESS; /* carry on */ 437 } 438 439 /* Do this early. Memory window is required for loading config file. */ 440 setup_memwin(sc); 441 442 /* Prepare the firmware for operation */ 443 rc = prep_firmware(sc); 444 if (rc != 0) 445 goto done; /* error message displayed already */ 446 447 rc = adap__pre_init_tweaks(sc); 448 if (rc != 0) 449 goto done; 450 451 rc = get_params__pre_init(sc); 452 if (rc != 0) 453 goto done; /* error message displayed already */ 454 455 t4_sge_init(sc); 456 457 if (sc->flags & MASTER_PF) { 458 /* get basic stuff going */ 459 rc = -t4_fw_initialize(sc, sc->mbox); 460 if (rc != 0) { 461 cxgb_printf(sc->dip, CE_WARN, 462 "early init failed: %d.\n", rc); 463 goto done; 464 } 465 } 466 467 rc = get_params__post_init(sc); 468 if (rc != 0) 469 goto done; /* error message displayed already */ 470 471 rc = set_params__post_init(sc); 472 if (rc != 0) 473 goto done; /* error message displayed already */ 474 475 /* 476 * TODO: This is the place to call t4_set_filter_mode() 477 */ 478 479 /* tweak some settings */ 480 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) | 481 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | 482 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9)); 483 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 484 485 /* 486 * Work-around for bug 2619 487 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the 488 * VLAN tag extraction is disabled. 489 */ 490 t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN); 491 492 /* Store filter mode */ 493 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 494 A_TP_VLAN_PRI_MAP); 495 496 /* 497 * First pass over all the ports - allocate VIs and initialize some 498 * basic parameters like mac address, port type, etc. We also figure 499 * out whether a port is 10G or 1G and use that information when 500 * calculating how many interrupts to attempt to allocate. 501 */ 502 n100g = n40g = n25g = n10g = n1g = 0; 503 for_each_port(sc, i) { 504 struct port_info *pi; 505 506 pi = kmem_zalloc(sizeof (*pi), KM_SLEEP); 507 sc->port[i] = pi; 508 509 /* These must be set before t4_port_init */ 510 pi->adapter = sc; 511 /* LINTED: E_ASSIGN_NARROW_CONV */ 512 pi->port_id = i; 513 } 514 515 /* Allocate the vi and initialize parameters like mac addr */ 516 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0); 517 if (rc) { 518 cxgb_printf(dip, CE_WARN, 519 "unable to initialize port: %d", rc); 520 goto done; 521 } 522 523 for_each_port(sc, i) { 524 struct port_info *pi = sc->port[i]; 525 526 mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL); 527 pi->mtu = ETHERMTU; 528 529 if (is_100G_port(pi)) { 530 n100g++; 531 pi->tmr_idx = prp->tmr_idx_10g; 532 pi->pktc_idx = prp->pktc_idx_10g; 533 } else if (is_40G_port(pi)) { 534 n40g++; 535 pi->tmr_idx = prp->tmr_idx_10g; 536 pi->pktc_idx = prp->pktc_idx_10g; 537 } else if (is_25G_port(pi)) { 538 n25g++; 539 pi->tmr_idx = prp->tmr_idx_10g; 540 pi->pktc_idx = prp->pktc_idx_10g; 541 } else if (is_10G_port(pi)) { 542 n10g++; 543 pi->tmr_idx = prp->tmr_idx_10g; 544 pi->pktc_idx = prp->pktc_idx_10g; 545 } else { 546 n1g++; 547 pi->tmr_idx = prp->tmr_idx_1g; 548 pi->pktc_idx = prp->pktc_idx_1g; 549 } 550 551 pi->xact_addr_filt = -1; 552 t4_mc_init(pi); 553 554 setbit(&sc->registered_device_map, i); 555 } 556 557 nxg = n10g + n25g + n40g + n100g; 558 (void) remove_extra_props(sc, nxg, n1g); 559 560 if (sc->registered_device_map == 0) { 561 cxgb_printf(dip, CE_WARN, "no usable ports"); 562 rc = DDI_FAILURE; 563 goto done; 564 } 565 566 rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq); 567 if (rc != 0) 568 goto done; /* error message displayed already */ 569 570 sc->intr_type = iaq.intr_type; 571 sc->intr_count = iaq.nirq; 572 573 if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) { 574 sc->props.multi_rings = 0; 575 cxgb_printf(dip, CE_WARN, 576 "Multiple rings disabled as interrupt type is not MSI-X"); 577 } 578 579 if (sc->props.multi_rings && iaq.intr_fwd) { 580 sc->props.multi_rings = 0; 581 cxgb_printf(dip, CE_WARN, 582 "Multiple rings disabled as interrupts are forwarded"); 583 } 584 585 if (!sc->props.multi_rings) { 586 iaq.ntxq10g = 1; 587 iaq.ntxq1g = 1; 588 } 589 s = &sc->sge; 590 s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g; 591 s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g; 592 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */ 593 #ifdef TCP_OFFLOAD_ENABLE 594 /* control queues, 1 per port + 1 mgmtq */ 595 s->neq += sc->params.nports + 1; 596 #endif 597 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 598 if (iaq.intr_fwd != 0) 599 sc->flags |= INTR_FWD; 600 #ifdef TCP_OFFLOAD_ENABLE 601 if (is_offload(sc) != 0) { 602 603 s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 604 s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 605 s->neq += s->nofldtxq + s->nofldrxq; 606 s->niq += s->nofldrxq; 607 608 s->ofld_rxq = kmem_zalloc(s->nofldrxq * 609 sizeof (struct sge_ofld_rxq), KM_SLEEP); 610 s->ofld_txq = kmem_zalloc(s->nofldtxq * 611 sizeof (struct sge_wrq), KM_SLEEP); 612 s->ctrlq = kmem_zalloc(sc->params.nports * 613 sizeof (struct sge_wrq), KM_SLEEP); 614 615 } 616 #endif 617 s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP); 618 s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP); 619 s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP); 620 s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP); 621 622 sc->intr_handle = kmem_zalloc(sc->intr_count * 623 sizeof (ddi_intr_handle_t), KM_SLEEP); 624 625 /* 626 * Second pass over the ports. This time we know the number of rx and 627 * tx queues that each port should get. 628 */ 629 rqidx = tqidx = 0; 630 #ifdef TCP_OFFLOAD_ENABLE 631 ofld_rqidx = ofld_tqidx = 0; 632 #endif 633 for_each_port(sc, i) { 634 struct port_info *pi = sc->port[i]; 635 636 if (pi == NULL) 637 continue; 638 639 t4_mc_cb_init(pi); 640 /* LINTED: E_ASSIGN_NARROW_CONV */ 641 pi->first_rxq = rqidx; 642 /* LINTED: E_ASSIGN_NARROW_CONV */ 643 pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g 644 : iaq.nrxq1g; 645 /* LINTED: E_ASSIGN_NARROW_CONV */ 646 pi->first_txq = tqidx; 647 /* LINTED: E_ASSIGN_NARROW_CONV */ 648 pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g 649 : iaq.ntxq1g; 650 651 rqidx += pi->nrxq; 652 tqidx += pi->ntxq; 653 654 #ifdef TCP_OFFLOAD_ENABLE 655 if (is_offload(sc) != 0) { 656 /* LINTED: E_ASSIGN_NARROW_CONV */ 657 pi->first_ofld_rxq = ofld_rqidx; 658 pi->nofldrxq = max(1, pi->nrxq / 4); 659 660 /* LINTED: E_ASSIGN_NARROW_CONV */ 661 pi->first_ofld_txq = ofld_tqidx; 662 pi->nofldtxq = max(1, pi->ntxq / 2); 663 664 ofld_rqidx += pi->nofldrxq; 665 ofld_tqidx += pi->nofldtxq; 666 } 667 #endif 668 669 /* 670 * Enable hw checksumming and LSO for all ports by default. 671 * They can be disabled using ndd (hw_csum and hw_lso). 672 */ 673 pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO); 674 } 675 676 #ifdef TCP_OFFLOAD_ENABLE 677 sc->l2t = t4_init_l2t(sc); 678 #endif 679 680 /* 681 * Setup Interrupts. 682 */ 683 684 i = 0; 685 rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0, 686 sc->intr_count, &i, DDI_INTR_ALLOC_STRICT); 687 if (rc != DDI_SUCCESS) { 688 cxgb_printf(dip, CE_WARN, 689 "failed to allocate %d interrupt(s) of type %d: %d, %d", 690 sc->intr_count, sc->intr_type, rc, i); 691 goto done; 692 } 693 ASSERT(sc->intr_count == i); /* allocation was STRICT */ 694 (void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap); 695 (void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri); 696 if (sc->intr_count == 1) { 697 ASSERT(sc->flags & INTR_FWD); 698 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc, 699 &s->fwq); 700 } else { 701 /* Multiple interrupts. The first one is always error intr */ 702 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc, 703 NULL); 704 irq++; 705 706 /* The second one is always the firmware event queue */ 707 (void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc, 708 &s->fwq); 709 irq++; 710 /* 711 * Note that if INTR_FWD is set then either the NIC rx 712 * queues or (exclusive or) the TOE rx queueus will be taking 713 * direct interrupts. 714 * 715 * There is no need to check for is_offload(sc) as nofldrxq 716 * will be 0 if offload is disabled. 717 */ 718 for_each_port(sc, i) { 719 struct port_info *pi = sc->port[i]; 720 struct sge_rxq *rxq; 721 #ifdef TCP_OFFLOAD_ENABLE 722 struct sge_ofld_rxq *ofld_rxq; 723 724 /* 725 * Skip over the NIC queues if they aren't taking direct 726 * interrupts. 727 */ 728 if ((sc->flags & INTR_FWD) && 729 pi->nofldrxq > pi->nrxq) 730 goto ofld_queues; 731 #endif 732 rxq = &s->rxq[pi->first_rxq]; 733 for (q = 0; q < pi->nrxq; q++, rxq++) { 734 (void) ddi_intr_add_handler( 735 sc->intr_handle[irq], t4_intr, sc, 736 &rxq->iq); 737 irq++; 738 } 739 740 #ifdef TCP_OFFLOAD_ENABLE 741 /* 742 * Skip over the offload queues if they aren't taking 743 * direct interrupts. 744 */ 745 if ((sc->flags & INTR_FWD)) 746 continue; 747 ofld_queues: 748 ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq]; 749 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 750 (void) ddi_intr_add_handler( 751 sc->intr_handle[irq], t4_intr, sc, 752 &ofld_rxq->iq); 753 irq++; 754 } 755 #endif 756 } 757 758 } 759 sc->flags |= INTR_ALLOCATED; 760 761 ASSERT(rc == DDI_SUCCESS); 762 ddi_report_dev(dip); 763 764 /* 765 * Hardware/Firmware/etc. Version/Revision IDs. 766 */ 767 t4_dump_version_info(sc); 768 769 if (n100g) { 770 cxgb_printf(dip, CE_NOTE, 771 "%dx100G (%d rxq, %d txq total) %d %s.", 772 n100g, rqidx, tqidx, sc->intr_count, 773 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 774 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 775 "fixed interrupt"); 776 } else if (n40g) { 777 cxgb_printf(dip, CE_NOTE, 778 "%dx40G (%d rxq, %d txq total) %d %s.", 779 n40g, rqidx, tqidx, sc->intr_count, 780 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 781 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 782 "fixed interrupt"); 783 } else if (n25g) { 784 cxgb_printf(dip, CE_NOTE, 785 "%dx25G (%d rxq, %d txq total) %d %s.", 786 n25g, rqidx, tqidx, sc->intr_count, 787 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 788 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 789 "fixed interrupt"); 790 } else if (n10g && n1g) { 791 cxgb_printf(dip, CE_NOTE, 792 "%dx10G %dx1G (%d rxq, %d txq total) %d %s.", 793 n10g, n1g, rqidx, tqidx, sc->intr_count, 794 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 795 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 796 "fixed interrupt"); 797 } else { 798 cxgb_printf(dip, CE_NOTE, 799 "%dx%sG (%d rxq, %d txq per port) %d %s.", 800 n10g ? n10g : n1g, 801 n10g ? "10" : "1", 802 n10g ? iaq.nrxq10g : iaq.nrxq1g, 803 n10g ? iaq.ntxq10g : iaq.ntxq1g, 804 sc->intr_count, 805 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 806 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 807 "fixed interrupt"); 808 } 809 810 sc->ksp = setup_kstats(sc); 811 sc->ksp_stat = setup_wc_kstats(sc); 812 sc->params.drv_memwin = MEMWIN_NIC; 813 814 done: 815 if (rc != DDI_SUCCESS) { 816 (void) t4_devo_detach(dip, DDI_DETACH); 817 818 /* rc may have errno style errors or DDI errors */ 819 rc = DDI_FAILURE; 820 } 821 822 return (rc); 823 } 824 825 static int 826 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 827 { 828 int instance, i; 829 struct adapter *sc; 830 struct port_info *pi; 831 struct sge *s; 832 833 if (cmd != DDI_DETACH) 834 return (DDI_FAILURE); 835 836 instance = ddi_get_instance(dip); 837 sc = ddi_get_soft_state(t4_list, instance); 838 if (sc == NULL) 839 return (DDI_SUCCESS); 840 841 if (sc->flags & FULL_INIT_DONE) { 842 t4_intr_disable(sc); 843 for_each_port(sc, i) { 844 pi = sc->port[i]; 845 if (pi && pi->flags & PORT_INIT_DONE) 846 (void) port_full_uninit(pi); 847 } 848 (void) adapter_full_uninit(sc); 849 } 850 851 /* Safe to call no matter what */ 852 ddi_prop_remove_all(dip); 853 ddi_remove_minor_node(dip, NULL); 854 855 for (i = 0; i < NCHAN; i++) { 856 if (sc->tq[i]) { 857 ddi_taskq_wait(sc->tq[i]); 858 ddi_taskq_destroy(sc->tq[i]); 859 } 860 } 861 862 if (sc->ksp != NULL) 863 kstat_delete(sc->ksp); 864 if (sc->ksp_stat != NULL) 865 kstat_delete(sc->ksp_stat); 866 867 s = &sc->sge; 868 if (s->rxq != NULL) 869 kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq)); 870 #ifdef TCP_OFFLOAD_ENABLE 871 if (s->ofld_txq != NULL) 872 kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq)); 873 if (s->ofld_rxq != NULL) 874 kmem_free(s->ofld_rxq, 875 s->nofldrxq * sizeof (struct sge_ofld_rxq)); 876 if (s->ctrlq != NULL) 877 kmem_free(s->ctrlq, 878 sc->params.nports * sizeof (struct sge_wrq)); 879 #endif 880 if (s->txq != NULL) 881 kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq)); 882 if (s->iqmap != NULL) 883 kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *)); 884 if (s->eqmap != NULL) 885 kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *)); 886 887 if (s->rxbuf_cache != NULL) 888 rxbuf_cache_destroy(s->rxbuf_cache); 889 890 if (sc->flags & INTR_ALLOCATED) { 891 for (i = 0; i < sc->intr_count; i++) { 892 (void) ddi_intr_remove_handler(sc->intr_handle[i]); 893 (void) ddi_intr_free(sc->intr_handle[i]); 894 } 895 sc->flags &= ~INTR_ALLOCATED; 896 } 897 898 if (sc->intr_handle != NULL) { 899 kmem_free(sc->intr_handle, 900 sc->intr_count * sizeof (*sc->intr_handle)); 901 } 902 903 for_each_port(sc, i) { 904 pi = sc->port[i]; 905 if (pi != NULL) { 906 mutex_destroy(&pi->lock); 907 kmem_free(pi, sizeof (*pi)); 908 clrbit(&sc->registered_device_map, i); 909 } 910 } 911 912 if (sc->flags & FW_OK) 913 (void) t4_fw_bye(sc, sc->mbox); 914 915 if (sc->reg1h != NULL) 916 ddi_regs_map_free(&sc->reg1h); 917 918 if (sc->regh != NULL) 919 ddi_regs_map_free(&sc->regh); 920 921 if (sc->pci_regh != NULL) 922 pci_config_teardown(&sc->pci_regh); 923 924 mutex_enter(&t4_adapter_list_lock); 925 SLIST_REMOVE_HEAD(&t4_adapter_list, link); 926 mutex_exit(&t4_adapter_list_lock); 927 928 mutex_destroy(&sc->lock); 929 cv_destroy(&sc->cv); 930 mutex_destroy(&sc->sfl_lock); 931 932 #ifdef DEBUG 933 bzero(sc, sizeof (*sc)); 934 #endif 935 ddi_soft_state_free(t4_list, instance); 936 937 return (DDI_SUCCESS); 938 } 939 940 static int 941 t4_devo_quiesce(dev_info_t *dip) 942 { 943 int instance; 944 struct adapter *sc; 945 946 instance = ddi_get_instance(dip); 947 sc = ddi_get_soft_state(t4_list, instance); 948 if (sc == NULL) 949 return (DDI_SUCCESS); 950 951 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 952 t4_intr_disable(sc); 953 t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST); 954 955 return (DDI_SUCCESS); 956 } 957 958 static int 959 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg, 960 void *result) 961 { 962 char s[4]; 963 struct port_info *pi; 964 dev_info_t *child = (dev_info_t *)arg; 965 966 switch (op) { 967 case DDI_CTLOPS_REPORTDEV: 968 pi = ddi_get_parent_data(rdip); 969 pi->instance = ddi_get_instance(dip); 970 pi->child_inst = ddi_get_instance(rdip); 971 cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n", 972 ddi_node_name(rdip), ddi_get_instance(rdip), 973 ddi_get_name_addr(rdip), ddi_driver_name(dip), 974 ddi_get_instance(dip)); 975 return (DDI_SUCCESS); 976 977 case DDI_CTLOPS_INITCHILD: 978 pi = ddi_get_parent_data(child); 979 if (pi == NULL) 980 return (DDI_NOT_WELL_FORMED); 981 (void) snprintf(s, sizeof (s), "%d", pi->port_id); 982 ddi_set_name_addr(child, s); 983 return (DDI_SUCCESS); 984 985 case DDI_CTLOPS_UNINITCHILD: 986 ddi_set_name_addr(child, NULL); 987 return (DDI_SUCCESS); 988 989 case DDI_CTLOPS_ATTACH: 990 case DDI_CTLOPS_DETACH: 991 return (DDI_SUCCESS); 992 993 default: 994 return (ddi_ctlops(dip, rdip, op, arg, result)); 995 } 996 } 997 998 static int 999 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg, 1000 dev_info_t **cdipp) 1001 { 1002 int instance, i; 1003 struct adapter *sc; 1004 1005 instance = ddi_get_instance(dip); 1006 sc = ddi_get_soft_state(t4_list, instance); 1007 1008 if (op == BUS_CONFIG_ONE) { 1009 char *c; 1010 1011 /* 1012 * arg is something like "cxgb@0" where 0 is the port_id hanging 1013 * off this nexus. 1014 */ 1015 1016 c = arg; 1017 while (*(c + 1)) 1018 c++; 1019 1020 /* There should be exactly 1 digit after '@' */ 1021 if (*(c - 1) != '@') 1022 return (NDI_FAILURE); 1023 1024 i = *c - '0'; 1025 1026 if (add_child_node(sc, i) != 0) 1027 return (NDI_FAILURE); 1028 1029 flags |= NDI_ONLINE_ATTACH; 1030 1031 } else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) { 1032 /* Allocate and bind all child device nodes */ 1033 for_each_port(sc, i) 1034 (void) add_child_node(sc, i); 1035 flags |= NDI_ONLINE_ATTACH; 1036 } 1037 1038 return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0)); 1039 } 1040 1041 static int 1042 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 1043 void *arg) 1044 { 1045 int instance, i, rc; 1046 struct adapter *sc; 1047 1048 instance = ddi_get_instance(dip); 1049 sc = ddi_get_soft_state(t4_list, instance); 1050 1051 if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL || 1052 op == BUS_UNCONFIG_DRIVER) 1053 flags |= NDI_UNCONFIG; 1054 1055 rc = ndi_busop_bus_unconfig(dip, flags, op, arg); 1056 if (rc != 0) 1057 return (rc); 1058 1059 if (op == BUS_UNCONFIG_ONE) { 1060 char *c; 1061 1062 c = arg; 1063 while (*(c + 1)) 1064 c++; 1065 1066 if (*(c - 1) != '@') 1067 return (NDI_SUCCESS); 1068 1069 i = *c - '0'; 1070 1071 rc = remove_child_node(sc, i); 1072 1073 } else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) { 1074 1075 for_each_port(sc, i) 1076 (void) remove_child_node(sc, i); 1077 } 1078 1079 return (rc); 1080 } 1081 1082 /* ARGSUSED */ 1083 static int 1084 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp) 1085 { 1086 struct adapter *sc; 1087 1088 if (otyp != OTYP_CHR) 1089 return (EINVAL); 1090 1091 sc = ddi_get_soft_state(t4_list, getminor(*devp)); 1092 if (sc == NULL) 1093 return (ENXIO); 1094 1095 return (atomic_cas_uint(&sc->open, 0, EBUSY)); 1096 } 1097 1098 /* ARGSUSED */ 1099 static int 1100 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp) 1101 { 1102 struct adapter *sc; 1103 1104 sc = ddi_get_soft_state(t4_list, getminor(dev)); 1105 if (sc == NULL) 1106 return (EINVAL); 1107 1108 (void) atomic_swap_uint(&sc->open, 0); 1109 return (0); 1110 } 1111 1112 /* ARGSUSED */ 1113 static int 1114 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp) 1115 { 1116 int instance; 1117 struct adapter *sc; 1118 void *data = (void *)d; 1119 1120 if (crgetuid(credp) != 0) 1121 return (EPERM); 1122 1123 instance = getminor(dev); 1124 sc = ddi_get_soft_state(t4_list, instance); 1125 if (sc == NULL) 1126 return (EINVAL); 1127 1128 return (t4_ioctl(sc, cmd, data, mode)); 1129 } 1130 1131 static unsigned int 1132 getpf(struct adapter *sc) 1133 { 1134 int rc, *data; 1135 uint_t n, pf; 1136 1137 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1138 DDI_PROP_DONTPASS, "reg", &data, &n); 1139 if (rc != DDI_SUCCESS) { 1140 cxgb_printf(sc->dip, CE_WARN, 1141 "failed to lookup \"reg\" property: %d", rc); 1142 return (0xff); 1143 } 1144 1145 pf = PCI_REG_FUNC_G(data[0]); 1146 ddi_prop_free(data); 1147 1148 return (pf); 1149 } 1150 1151 1152 static struct fw_info * 1153 find_fw_info(int chip) 1154 { 1155 u32 i; 1156 1157 fi[0].chip = CHELSIO_T4; 1158 fi[0].fw_hdr.chip = FW_HDR_CHIP_T4; 1159 fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4)); 1160 fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC); 1161 fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC); 1162 fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD); 1163 fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI); 1164 fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU); 1165 fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI); 1166 fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU); 1167 fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE); 1168 1169 fi[1].chip = CHELSIO_T5; 1170 fi[1].fw_hdr.chip = FW_HDR_CHIP_T5; 1171 fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5)); 1172 fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC); 1173 fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC); 1174 fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD); 1175 fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI); 1176 fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU); 1177 fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI); 1178 fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU); 1179 fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE); 1180 1181 fi[2].chip = CHELSIO_T6; 1182 fi[2].fw_hdr.chip = FW_HDR_CHIP_T6; 1183 fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6)); 1184 fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC); 1185 fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC); 1186 fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD); 1187 fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI); 1188 fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU); 1189 fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI); 1190 fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU); 1191 fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE); 1192 1193 for (i = 0; i < ARRAY_SIZE(fi); i++) { 1194 if (fi[i].chip == chip) 1195 return &fi[i]; 1196 } 1197 1198 return NULL; 1199 } 1200 1201 /* 1202 * Install a compatible firmware (if required), establish contact with it, 1203 * become the master, and reset the device. 1204 */ 1205 static int 1206 prep_firmware(struct adapter *sc) 1207 { 1208 int rc; 1209 int fw_size; 1210 int reset = 1; 1211 enum dev_state state; 1212 unsigned char *fw_data; 1213 struct fw_info *fw_info; 1214 struct fw_hdr *card_fw; 1215 1216 struct driver_properties *p = &sc->props; 1217 1218 /* Contact firmware, request master */ 1219 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state); 1220 if (rc < 0) { 1221 rc = -rc; 1222 cxgb_printf(sc->dip, CE_WARN, 1223 "failed to connect to the firmware: %d.", rc); 1224 return (rc); 1225 } 1226 1227 if (rc == sc->mbox) 1228 sc->flags |= MASTER_PF; 1229 1230 /* We may need FW version info for later reporting */ 1231 t4_get_version_info(sc); 1232 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip)); 1233 /* allocate memory to read the header of the firmware on the 1234 * card 1235 */ 1236 if (!fw_info) { 1237 cxgb_printf(sc->dip, CE_WARN, 1238 "unable to look up firmware information for chip %d.\n", 1239 CHELSIO_CHIP_VERSION(sc->params.chip)); 1240 return EINVAL; 1241 } 1242 card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP); 1243 if(!card_fw) { 1244 cxgb_printf(sc->dip, CE_WARN, 1245 "Memory allocation for card FW header failed\n"); 1246 return ENOMEM; 1247 } 1248 switch(CHELSIO_CHIP_VERSION(sc->params.chip)) { 1249 case CHELSIO_T4: 1250 fw_data = t4fw_data; 1251 fw_size = t4fw_size; 1252 break; 1253 case CHELSIO_T5: 1254 fw_data = t5fw_data; 1255 fw_size = t5fw_size; 1256 break; 1257 case CHELSIO_T6: 1258 fw_data = t6fw_data; 1259 fw_size = t6fw_size; 1260 break; 1261 default: 1262 cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n"); 1263 kmem_free(card_fw, sizeof(*card_fw)); 1264 return EINVAL; 1265 } 1266 1267 rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw, 1268 p->t4_fw_install, state, &reset); 1269 1270 kmem_free(card_fw, sizeof(*card_fw)); 1271 1272 if (rc != 0) { 1273 cxgb_printf(sc->dip, CE_WARN, 1274 "failed to install firmware: %d", rc); 1275 return (rc); 1276 } else { 1277 /* refresh */ 1278 (void) t4_check_fw_version(sc); 1279 } 1280 1281 /* Reset device */ 1282 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1283 if (rc != 0) { 1284 cxgb_printf(sc->dip, CE_WARN, 1285 "firmware reset failed: %d.", rc); 1286 if (rc != ETIMEDOUT && rc != EIO) 1287 (void) t4_fw_bye(sc, sc->mbox); 1288 return (rc); 1289 } 1290 1291 /* Partition adapter resources as specified in the config file. */ 1292 if (sc->flags & MASTER_PF) { 1293 /* Handle default vs special T4 config file */ 1294 1295 rc = partition_resources(sc); 1296 if (rc != 0) 1297 goto err; /* error message displayed already */ 1298 } 1299 1300 sc->flags |= FW_OK; 1301 return (0); 1302 err: 1303 return (rc); 1304 1305 } 1306 1307 static const struct memwin t4_memwin[] = { 1308 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1309 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1310 { MEMWIN2_BASE, MEMWIN2_APERTURE } 1311 }; 1312 1313 static const struct memwin t5_memwin[] = { 1314 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1315 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1316 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1317 }; 1318 1319 #define FW_PARAM_DEV(param) \ 1320 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1321 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1322 #define FW_PARAM_PFVF(param) \ 1323 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1324 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1325 1326 /* 1327 * Verify that the memory range specified by the memtype/offset/len pair is 1328 * valid and lies entirely within the memtype specified. The global address of 1329 * the start of the range is returned in addr. 1330 */ 1331 int 1332 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1333 uint32_t *addr) 1334 { 1335 uint32_t em, addr_len, maddr, mlen; 1336 1337 /* Memory can only be accessed in naturally aligned 4 byte units */ 1338 if (off & 3 || len & 3 || len == 0) 1339 return (EINVAL); 1340 1341 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1342 switch (mtype) { 1343 case MEM_EDC0: 1344 if (!(em & F_EDRAM0_ENABLE)) 1345 return (EINVAL); 1346 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1347 maddr = G_EDRAM0_BASE(addr_len) << 20; 1348 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1349 break; 1350 case MEM_EDC1: 1351 if (!(em & F_EDRAM1_ENABLE)) 1352 return (EINVAL); 1353 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1354 maddr = G_EDRAM1_BASE(addr_len) << 20; 1355 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1356 break; 1357 case MEM_MC: 1358 if (!(em & F_EXT_MEM_ENABLE)) 1359 return (EINVAL); 1360 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1361 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1362 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1363 break; 1364 case MEM_MC1: 1365 if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE)) 1366 return (EINVAL); 1367 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1368 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1369 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1370 break; 1371 default: 1372 return (EINVAL); 1373 } 1374 1375 if (mlen > 0 && off < mlen && off + len <= mlen) { 1376 *addr = maddr + off; /* global address */ 1377 return (0); 1378 } 1379 1380 return (EFAULT); 1381 } 1382 1383 void 1384 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1385 { 1386 const struct memwin *mw; 1387 1388 if (is_t4(sc->params.chip)) { 1389 mw = &t4_memwin[win]; 1390 } else { 1391 mw = &t5_memwin[win]; 1392 } 1393 1394 if (base != NULL) 1395 *base = mw->base; 1396 if (aperture != NULL) 1397 *aperture = mw->aperture; 1398 } 1399 1400 /* 1401 * Upload configuration file to card's memory. 1402 */ 1403 static int 1404 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma) 1405 { 1406 int rc = 0, cflen; 1407 u_int i, n; 1408 uint32_t param, val, addr, mtype, maddr; 1409 uint32_t off, mw_base, mw_aperture; 1410 const uint32_t *cfdata; 1411 1412 /* Figure out where the firmware wants us to upload it. */ 1413 param = FW_PARAM_DEV(CF); 1414 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1415 if (rc != 0) { 1416 /* Firmwares without config file support will fail this way */ 1417 cxgb_printf(sc->dip, CE_WARN, 1418 "failed to query config file location: %d.\n", rc); 1419 return (rc); 1420 } 1421 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1422 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1423 1424 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 1425 case CHELSIO_T4: 1426 cflen = t4cfg_size & ~3; 1427 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1428 cfdata = (const uint32_t *)t4cfg_data; 1429 break; 1430 case CHELSIO_T5: 1431 cflen = t5cfg_size & ~3; 1432 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1433 cfdata = (const uint32_t *)t5cfg_data; 1434 break; 1435 case CHELSIO_T6: 1436 cflen = t6cfg_size & ~3; 1437 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1438 cfdata = (const uint32_t *)t6cfg_data; 1439 break; 1440 default: 1441 cxgb_printf(sc->dip, CE_WARN, 1442 "Invalid Adapter detected\n"); 1443 return EINVAL; 1444 } 1445 1446 if (cflen > FLASH_CFG_MAX_SIZE) { 1447 cxgb_printf(sc->dip, CE_WARN, 1448 "config file too long (%d, max allowed is %d). ", 1449 cflen, FLASH_CFG_MAX_SIZE); 1450 return (EFBIG); 1451 } 1452 1453 rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr); 1454 if (rc != 0) { 1455 1456 cxgb_printf(sc->dip, CE_WARN, 1457 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 1458 "Will try to use the config on the card, if any.\n", 1459 __func__, mtype, maddr, cflen, rc); 1460 return (EFAULT); 1461 } 1462 1463 memwin_info(sc, 2, &mw_base, &mw_aperture); 1464 while (cflen) { 1465 off = position_memwin(sc, 2, addr); 1466 n = min(cflen, mw_aperture - off); 1467 for (i = 0; i < n; i += 4) 1468 t4_write_reg(sc, mw_base + off + i, *cfdata++); 1469 cflen -= n; 1470 addr += n; 1471 } 1472 1473 return (rc); 1474 } 1475 1476 /* 1477 * Partition chip resources for use between various PFs, VFs, etc. This is done 1478 * by uploading the firmware configuration file to the adapter and instructing 1479 * the firmware to process it. 1480 */ 1481 static int 1482 partition_resources(struct adapter *sc) 1483 { 1484 int rc; 1485 struct fw_caps_config_cmd caps; 1486 uint32_t mtype, maddr, finicsum, cfcsum; 1487 1488 rc = upload_config_file(sc, &mtype, &maddr); 1489 if (rc != 0) { 1490 mtype = FW_MEMTYPE_CF_FLASH; 1491 maddr = t4_flash_cfg_addr(sc); 1492 } 1493 1494 bzero(&caps, sizeof (caps)); 1495 caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1496 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1497 caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1498 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1499 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1500 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1501 if (rc != 0) { 1502 cxgb_printf(sc->dip, CE_WARN, 1503 "failed to pre-process config file: %d.\n", rc); 1504 return (rc); 1505 } 1506 1507 finicsum = ntohl(caps.finicsum); 1508 cfcsum = ntohl(caps.cfcsum); 1509 if (finicsum != cfcsum) { 1510 cxgb_printf(sc->dip, CE_WARN, 1511 "WARNING: config file checksum mismatch: %08x %08x\n", 1512 finicsum, cfcsum); 1513 } 1514 sc->cfcsum = cfcsum; 1515 1516 /* TODO: Need to configure this correctly */ 1517 caps.toecaps = htons(FW_CAPS_CONFIG_TOE); 1518 caps.iscsicaps = 0; 1519 caps.rdmacaps = 0; 1520 caps.fcoecaps = 0; 1521 /* TODO: Disable VNIC cap for now */ 1522 caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 1523 1524 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1525 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1526 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1527 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL); 1528 if (rc != 0) { 1529 cxgb_printf(sc->dip, CE_WARN, 1530 "failed to process config file: %d.\n", rc); 1531 return (rc); 1532 } 1533 1534 return (0); 1535 } 1536 1537 /* 1538 * Tweak configuration based on module parameters, etc. Most of these have 1539 * defaults assigned to them by Firmware Configuration Files (if we're using 1540 * them) but need to be explicitly set if we're using hard-coded 1541 * initialization. But even in the case of using Firmware Configuration 1542 * Files, we'd like to expose the ability to change these via module 1543 * parameters so these are essentially common tweaks/settings for 1544 * Configuration Files and hard-coded initialization ... 1545 */ 1546 static int 1547 adap__pre_init_tweaks(struct adapter *sc) 1548 { 1549 int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */ 1550 1551 /* 1552 * Fix up various Host-Dependent Parameters like Page Size, Cache 1553 * Line Size, etc. The firmware default is for a 4KB Page Size and 1554 * 64B Cache Line Size ... 1555 */ 1556 (void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV); 1557 1558 t4_set_reg_field(sc, A_SGE_CONTROL, 1559 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset)); 1560 1561 return 0; 1562 } 1563 /* 1564 * Retrieve parameters that are needed (or nice to have) prior to calling 1565 * t4_sge_init and t4_fw_initialize. 1566 */ 1567 static int 1568 get_params__pre_init(struct adapter *sc) 1569 { 1570 int rc; 1571 uint32_t param[2], val[2]; 1572 struct fw_devlog_cmd cmd; 1573 struct devlog_params *dlog = &sc->params.devlog; 1574 1575 /* 1576 * Grab the raw VPD parameters. 1577 */ 1578 rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd); 1579 if (rc != 0) { 1580 cxgb_printf(sc->dip, CE_WARN, 1581 "failed to query VPD parameters (pre_init): %d.\n", rc); 1582 return (rc); 1583 } 1584 1585 param[0] = FW_PARAM_DEV(PORTVEC); 1586 param[1] = FW_PARAM_DEV(CCLK); 1587 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1588 if (rc != 0) { 1589 cxgb_printf(sc->dip, CE_WARN, 1590 "failed to query parameters (pre_init): %d.\n", rc); 1591 return (rc); 1592 } 1593 1594 sc->params.portvec = val[0]; 1595 sc->params.nports = 0; 1596 while (val[0]) { 1597 sc->params.nports++; 1598 val[0] &= val[0] - 1; 1599 } 1600 1601 sc->params.vpd.cclk = val[1]; 1602 1603 /* Read device log parameters. */ 1604 bzero(&cmd, sizeof (cmd)); 1605 cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1606 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1607 cmd.retval_len16 = htonl(FW_LEN16(cmd)); 1608 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd); 1609 if (rc != 0) { 1610 cxgb_printf(sc->dip, CE_WARN, 1611 "failed to get devlog parameters: %d.\n", rc); 1612 bzero(dlog, sizeof (*dlog)); 1613 rc = 0; /* devlog isn't critical for device operation */ 1614 } else { 1615 val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog); 1616 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1617 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1618 dlog->size = ntohl(cmd.memsize_devlog); 1619 } 1620 1621 return (rc); 1622 } 1623 1624 /* 1625 * Retrieve various parameters that are of interest to the driver. The device 1626 * has been initialized by the firmware at this point. 1627 */ 1628 static int 1629 get_params__post_init(struct adapter *sc) 1630 { 1631 int rc; 1632 uint32_t param[7], val[7]; 1633 struct fw_caps_config_cmd caps; 1634 1635 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1636 param[1] = FW_PARAM_PFVF(EQ_START); 1637 param[2] = FW_PARAM_PFVF(FILTER_START); 1638 param[3] = FW_PARAM_PFVF(FILTER_END); 1639 param[4] = FW_PARAM_PFVF(L2T_START); 1640 param[5] = FW_PARAM_PFVF(L2T_END); 1641 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1642 if (rc != 0) { 1643 cxgb_printf(sc->dip, CE_WARN, 1644 "failed to query parameters (post_init): %d.\n", rc); 1645 return (rc); 1646 } 1647 1648 /* LINTED: E_ASSIGN_NARROW_CONV */ 1649 sc->sge.iq_start = val[0]; 1650 sc->sge.eq_start = val[1]; 1651 sc->tids.ftid_base = val[2]; 1652 sc->tids.nftids = val[3] - val[2] + 1; 1653 sc->vres.l2t.start = val[4]; 1654 sc->vres.l2t.size = val[5] - val[4] + 1; 1655 1656 /* get capabilites */ 1657 bzero(&caps, sizeof (caps)); 1658 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1659 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1660 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1661 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1662 if (rc != 0) { 1663 cxgb_printf(sc->dip, CE_WARN, 1664 "failed to get card capabilities: %d.\n", rc); 1665 return (rc); 1666 } 1667 1668 if (caps.toecaps != 0) { 1669 /* query offload-related parameters */ 1670 param[0] = FW_PARAM_DEV(NTID); 1671 param[1] = FW_PARAM_PFVF(SERVER_START); 1672 param[2] = FW_PARAM_PFVF(SERVER_END); 1673 param[3] = FW_PARAM_PFVF(TDDP_START); 1674 param[4] = FW_PARAM_PFVF(TDDP_END); 1675 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1676 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1677 if (rc != 0) { 1678 cxgb_printf(sc->dip, CE_WARN, 1679 "failed to query TOE parameters: %d.\n", rc); 1680 return (rc); 1681 } 1682 sc->tids.ntids = val[0]; 1683 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1684 sc->tids.stid_base = val[1]; 1685 sc->tids.nstids = val[2] - val[1] + 1; 1686 sc->vres.ddp.start = val[3]; 1687 sc->vres.ddp.size = val[4] - val[3] + 1; 1688 sc->params.ofldq_wr_cred = val[5]; 1689 sc->params.offload = 1; 1690 } 1691 1692 /* These are finalized by FW initialization, load their values now */ 1693 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1694 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1695 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1696 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1697 1698 return (rc); 1699 } 1700 1701 static int 1702 set_params__post_init(struct adapter *sc) 1703 { 1704 uint32_t param, val; 1705 1706 /* ask for encapsulated CPLs */ 1707 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 1708 val = 1; 1709 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1710 1711 return (0); 1712 } 1713 1714 /* TODO: verify */ 1715 static void 1716 setup_memwin(struct adapter *sc) 1717 { 1718 pci_regspec_t *data; 1719 int rc; 1720 uint_t n; 1721 uintptr_t bar0; 1722 uintptr_t mem_win0_base, mem_win1_base, mem_win2_base; 1723 uintptr_t mem_win2_aperture; 1724 1725 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1726 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n); 1727 if (rc != DDI_SUCCESS) { 1728 cxgb_printf(sc->dip, CE_WARN, 1729 "failed to lookup \"assigned-addresses\" property: %d", rc); 1730 return; 1731 } 1732 n /= sizeof (*data); 1733 1734 bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low; 1735 ddi_prop_free(data); 1736 1737 if (is_t4(sc->params.chip)) { 1738 mem_win0_base = bar0 + MEMWIN0_BASE; 1739 mem_win1_base = bar0 + MEMWIN1_BASE; 1740 mem_win2_base = bar0 + MEMWIN2_BASE; 1741 mem_win2_aperture = MEMWIN2_APERTURE; 1742 } else { 1743 /* For T5, only relative offset inside the PCIe BAR is passed */ 1744 mem_win0_base = MEMWIN0_BASE; 1745 mem_win1_base = MEMWIN1_BASE; 1746 mem_win2_base = MEMWIN2_BASE_T5; 1747 mem_win2_aperture = MEMWIN2_APERTURE_T5; 1748 } 1749 1750 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1751 mem_win0_base | V_BIR(0) | 1752 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1753 1754 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1755 mem_win1_base | V_BIR(0) | 1756 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1757 1758 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1759 mem_win2_base | V_BIR(0) | 1760 V_WINDOW(ilog2(mem_win2_aperture) - 10)); 1761 1762 /* flush */ 1763 (void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1764 } 1765 1766 /* 1767 * Positions the memory window such that it can be used to access the specified 1768 * address in the chip's address space. The return value is the offset of addr 1769 * from the start of the window. 1770 */ 1771 uint32_t 1772 position_memwin(struct adapter *sc, int n, uint32_t addr) 1773 { 1774 uint32_t start, pf; 1775 uint32_t reg; 1776 1777 if (addr & 3) { 1778 cxgb_printf(sc->dip, CE_WARN, 1779 "addr (0x%x) is not at a 4B boundary.\n", addr); 1780 return (EFAULT); 1781 } 1782 1783 if (is_t4(sc->params.chip)) { 1784 pf = 0; 1785 start = addr & ~0xf; /* start must be 16B aligned */ 1786 } else { 1787 pf = V_PFNUM(sc->pf); 1788 start = addr & ~0x7f; /* start must be 128B aligned */ 1789 } 1790 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1791 1792 t4_write_reg(sc, reg, start | pf); 1793 (void) t4_read_reg(sc, reg); 1794 1795 return (addr - start); 1796 } 1797 1798 1799 /* 1800 * Reads the named property and fills up the "data" array (which has at least 1801 * "count" elements). We first try and lookup the property for our dev_t and 1802 * then retry with DDI_DEV_T_ANY if it's not found. 1803 * 1804 * Returns non-zero if the property was found and "data" has been updated. 1805 */ 1806 static int 1807 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count) 1808 { 1809 dev_info_t *dip = sc->dip; 1810 dev_t dev = sc->dev; 1811 int rc, *d; 1812 uint_t i, n; 1813 1814 rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS, 1815 name, &d, &n); 1816 if (rc == DDI_PROP_SUCCESS) 1817 goto found; 1818 1819 if (rc != DDI_PROP_NOT_FOUND) { 1820 cxgb_printf(dip, CE_WARN, 1821 "failed to lookup property %s for minor %d: %d.", 1822 name, getminor(dev), rc); 1823 return (0); 1824 } 1825 1826 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1827 name, &d, &n); 1828 if (rc == DDI_PROP_SUCCESS) 1829 goto found; 1830 1831 if (rc != DDI_PROP_NOT_FOUND) { 1832 cxgb_printf(dip, CE_WARN, 1833 "failed to lookup property %s: %d.", name, rc); 1834 return (0); 1835 } 1836 1837 return (0); 1838 1839 found: 1840 if (n > count) { 1841 cxgb_printf(dip, CE_NOTE, 1842 "property %s has too many elements (%d), ignoring extras", 1843 name, n); 1844 } 1845 1846 for (i = 0; i < n && i < count; i++) 1847 data[i] = d[i]; 1848 ddi_prop_free(d); 1849 1850 return (1); 1851 } 1852 1853 static int 1854 prop_lookup_int(struct adapter *sc, char *name, int defval) 1855 { 1856 int rc; 1857 1858 rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1); 1859 if (rc != -1) 1860 return (rc); 1861 1862 return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS, 1863 name, defval)); 1864 } 1865 1866 static int 1867 init_driver_props(struct adapter *sc, struct driver_properties *p) 1868 { 1869 dev_t dev = sc->dev; 1870 dev_info_t *dip = sc->dip; 1871 int i, *data; 1872 uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200}; 1873 uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 1874 1875 /* 1876 * Holdoff timer 1877 */ 1878 data = &p->timer_val[0]; 1879 for (i = 0; i < SGE_NTIMERS; i++) 1880 data[i] = tmr[i]; 1881 (void) prop_lookup_int_array(sc, "holdoff-timer-values", data, 1882 SGE_NTIMERS); 1883 for (i = 0; i < SGE_NTIMERS; i++) { 1884 int limit = 200U; 1885 if (data[i] > limit) { 1886 cxgb_printf(dip, CE_WARN, 1887 "holdoff timer %d is too high (%d), lowered to %d.", 1888 i, data[i], limit); 1889 data[i] = limit; 1890 } 1891 } 1892 (void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values", 1893 data, SGE_NTIMERS); 1894 1895 /* 1896 * Holdoff packet counter 1897 */ 1898 data = &p->counter_val[0]; 1899 for (i = 0; i < SGE_NCOUNTERS; i++) 1900 data[i] = cnt[i]; 1901 (void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data, 1902 SGE_NCOUNTERS); 1903 for (i = 0; i < SGE_NCOUNTERS; i++) { 1904 int limit = M_THRESHOLD_0; 1905 if (data[i] > limit) { 1906 cxgb_printf(dip, CE_WARN, 1907 "holdoff pkt-counter %d is too high (%d), " 1908 "lowered to %d.", i, data[i], limit); 1909 data[i] = limit; 1910 } 1911 } 1912 (void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values", 1913 data, SGE_NCOUNTERS); 1914 1915 /* 1916 * Maximum # of tx and rx queues to use for each 1917 * 100G, 40G, 25G, 10G and 1G port. 1918 */ 1919 p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8); 1920 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1921 p->max_ntxq_10g); 1922 1923 p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8); 1924 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1925 p->max_nrxq_10g); 1926 1927 p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2); 1928 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1929 p->max_ntxq_1g); 1930 1931 p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2); 1932 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1933 p->max_nrxq_1g); 1934 1935 #ifdef TCP_OFFLOAD_ENABLE 1936 p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8); 1937 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1938 p->max_nofldtxq_10g); 1939 1940 p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2); 1941 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1942 p->max_nofldrxq_10g); 1943 1944 p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2); 1945 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1946 p->max_nofldtxq_1g); 1947 1948 p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1); 1949 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1950 p->max_nofldrxq_1g); 1951 #endif 1952 1953 /* 1954 * Holdoff parameters for 10G and 1G ports. 1955 */ 1956 p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0); 1957 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G", 1958 p->tmr_idx_10g); 1959 1960 p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2); 1961 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G", 1962 p->pktc_idx_10g); 1963 1964 p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0); 1965 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G", 1966 p->tmr_idx_1g); 1967 1968 p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2); 1969 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G", 1970 p->pktc_idx_1g); 1971 1972 /* 1973 * Size (number of entries) of each tx and rx queue. 1974 */ 1975 i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE); 1976 p->qsize_txq = max(i, 128); 1977 if (p->qsize_txq != i) { 1978 cxgb_printf(dip, CE_WARN, 1979 "using %d instead of %d as the tx queue size", 1980 p->qsize_txq, i); 1981 } 1982 (void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq); 1983 1984 i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE); 1985 p->qsize_rxq = max(i, 128); 1986 while (p->qsize_rxq & 7) 1987 p->qsize_rxq--; 1988 if (p->qsize_rxq != i) { 1989 cxgb_printf(dip, CE_WARN, 1990 "using %d instead of %d as the rx queue size", 1991 p->qsize_rxq, i); 1992 } 1993 (void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq); 1994 1995 /* 1996 * Interrupt types allowed. 1997 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively. See sys/ddi_intr.h 1998 */ 1999 p->intr_types = prop_lookup_int(sc, "interrupt-types", 2000 DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED); 2001 (void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types); 2002 2003 /* 2004 * Forwarded interrupt queues. Create this property to force the driver 2005 * to use forwarded interrupt queues. 2006 */ 2007 if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS, 2008 "interrupt-forwarding") != 0 || 2009 ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2010 "interrupt-forwarding") != 0) { 2011 UNIMPLEMENTED(); 2012 (void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP, 2013 "interrupt-forwarding", NULL, 0); 2014 } 2015 2016 /* 2017 * Write combining 2018 * 0 to disable, 1 to enable 2019 */ 2020 p->wc = prop_lookup_int(sc, "write-combine", 1); 2021 cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc); 2022 if (p->wc != 0 && p->wc != 1) { 2023 cxgb_printf(dip, CE_WARN, 2024 "write-combine: using 1 instead of %d", p->wc); 2025 p->wc = 1; 2026 } 2027 (void) ddi_prop_update_int(dev, dip, "write-combine", p->wc); 2028 2029 p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1); 2030 if (p->t4_fw_install != 0 && p->t4_fw_install != 2) 2031 p->t4_fw_install = 1; 2032 (void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install); 2033 2034 /* Multiple Rings */ 2035 p->multi_rings = prop_lookup_int(sc, "multi-rings", 1); 2036 if (p->multi_rings != 0 && p->multi_rings != 1) { 2037 cxgb_printf(dip, CE_NOTE, 2038 "multi-rings: using value 1 instead of %d", p->multi_rings); 2039 p->multi_rings = 1; 2040 } 2041 2042 (void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings); 2043 2044 return (0); 2045 } 2046 2047 static int 2048 remove_extra_props(struct adapter *sc, int n10g, int n1g) 2049 { 2050 if (n10g == 0) { 2051 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port"); 2052 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port"); 2053 (void) ddi_prop_remove(sc->dev, sc->dip, 2054 "holdoff-timer-idx-10G"); 2055 (void) ddi_prop_remove(sc->dev, sc->dip, 2056 "holdoff-pktc-idx-10G"); 2057 } 2058 2059 if (n1g == 0) { 2060 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port"); 2061 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port"); 2062 (void) ddi_prop_remove(sc->dev, sc->dip, 2063 "holdoff-timer-idx-1G"); 2064 (void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G"); 2065 } 2066 2067 return (0); 2068 } 2069 2070 static int 2071 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 2072 struct intrs_and_queues *iaq) 2073 { 2074 struct driver_properties *p = &sc->props; 2075 int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n; 2076 int nofldrxq10g = 0, nofldrxq1g = 0; 2077 2078 bzero(iaq, sizeof (*iaq)); 2079 nc = ncpus; /* our snapshot of the number of CPUs */ 2080 iaq->ntxq10g = min(nc, p->max_ntxq_10g); 2081 iaq->ntxq1g = min(nc, p->max_ntxq_1g); 2082 iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g); 2083 iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g); 2084 #ifdef TCP_OFFLOAD_ENABLE 2085 iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g); 2086 iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g); 2087 iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g); 2088 iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g); 2089 #endif 2090 2091 rc = ddi_intr_get_supported_types(sc->dip, &itypes); 2092 if (rc != DDI_SUCCESS) { 2093 cxgb_printf(sc->dip, CE_WARN, 2094 "failed to determine supported interrupt types: %d", rc); 2095 return (rc); 2096 } 2097 2098 for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) { 2099 ASSERT(itype == DDI_INTR_TYPE_MSIX || 2100 itype == DDI_INTR_TYPE_MSI || 2101 itype == DDI_INTR_TYPE_FIXED); 2102 2103 if ((itype & itypes & p->intr_types) == 0) 2104 continue; /* not supported or not allowed */ 2105 2106 navail = 0; 2107 rc = ddi_intr_get_navail(sc->dip, itype, &navail); 2108 if (rc != DDI_SUCCESS || navail == 0) { 2109 cxgb_printf(sc->dip, CE_WARN, 2110 "failed to get # of interrupts for type %d: %d", 2111 itype, rc); 2112 continue; /* carry on */ 2113 } 2114 2115 iaq->intr_type = itype; 2116 if (navail == 0) 2117 continue; 2118 2119 /* 2120 * Best option: an interrupt vector for errors, one for the 2121 * firmware event queue, and one each for each rxq (NIC as well 2122 * as offload). 2123 */ 2124 iaq->nirq = T4_EXTRA_INTR; 2125 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 2126 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 2127 2128 if (iaq->nirq <= navail && 2129 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2130 iaq->intr_fwd = 0; 2131 goto allocate; 2132 } 2133 2134 /* 2135 * Second best option: an interrupt vector for errors, one for 2136 * the firmware event queue, and one each for either NIC or 2137 * offload rxq's. 2138 */ 2139 iaq->nirq = T4_EXTRA_INTR; 2140 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 2141 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 2142 if (iaq->nirq <= navail && 2143 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2144 iaq->intr_fwd = 1; 2145 goto allocate; 2146 } 2147 2148 /* 2149 * Next best option: an interrupt vector for errors, one for the 2150 * firmware event queue, and at least one per port. At this 2151 * point we know we'll have to downsize nrxq or nofldrxq to fit 2152 * what's available to us. 2153 */ 2154 iaq->nirq = T4_EXTRA_INTR; 2155 iaq->nirq += n10g + n1g; 2156 if (iaq->nirq <= navail) { 2157 int leftover = navail - iaq->nirq; 2158 2159 if (n10g > 0) { 2160 int target = max(nrxq10g, nofldrxq10g); 2161 2162 n = 1; 2163 while (n < target && leftover >= n10g) { 2164 leftover -= n10g; 2165 iaq->nirq += n10g; 2166 n++; 2167 } 2168 iaq->nrxq10g = min(n, nrxq10g); 2169 #ifdef TCP_OFFLOAD_ENABLE 2170 iaq->nofldrxq10g = min(n, nofldrxq10g); 2171 #endif 2172 } 2173 2174 if (n1g > 0) { 2175 int target = max(nrxq1g, nofldrxq1g); 2176 2177 n = 1; 2178 while (n < target && leftover >= n1g) { 2179 leftover -= n1g; 2180 iaq->nirq += n1g; 2181 n++; 2182 } 2183 iaq->nrxq1g = min(n, nrxq1g); 2184 #ifdef TCP_OFFLOAD_ENABLE 2185 iaq->nofldrxq1g = min(n, nofldrxq1g); 2186 #endif 2187 } 2188 2189 /* We have arrived at a minimum value required to enable 2190 * per queue irq(either NIC or offload). Thus for non- 2191 * offload case, we will get a vector per queue, while 2192 * offload case, we will get a vector per offload/NIC q. 2193 * Hence enable Interrupt forwarding only for offload 2194 * case. 2195 */ 2196 #ifdef TCP_OFFLOAD_ENABLE 2197 if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) { 2198 iaq->intr_fwd = 1; 2199 #else 2200 if (itype != DDI_INTR_TYPE_MSI) { 2201 #endif 2202 goto allocate; 2203 } 2204 } 2205 2206 /* 2207 * Least desirable option: one interrupt vector for everything. 2208 */ 2209 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2210 #ifdef TCP_OFFLOAD_ENABLE 2211 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2212 #endif 2213 iaq->intr_fwd = 1; 2214 2215 allocate: 2216 return (0); 2217 } 2218 2219 cxgb_printf(sc->dip, CE_WARN, 2220 "failed to find a usable interrupt type. supported=%d, allowed=%d", 2221 itypes, p->intr_types); 2222 return (DDI_FAILURE); 2223 } 2224 2225 static int 2226 add_child_node(struct adapter *sc, int idx) 2227 { 2228 int rc; 2229 struct port_info *pi; 2230 2231 if (idx < 0 || idx >= sc->params.nports) 2232 return (EINVAL); 2233 2234 pi = sc->port[idx]; 2235 if (pi == NULL) 2236 return (ENODEV); /* t4_port_init failed earlier */ 2237 2238 PORT_LOCK(pi); 2239 if (pi->dip != NULL) { 2240 rc = 0; /* EEXIST really, but then bus_config fails */ 2241 goto done; 2242 } 2243 2244 rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip); 2245 if (rc != DDI_SUCCESS || pi->dip == NULL) { 2246 rc = ENOMEM; 2247 goto done; 2248 } 2249 2250 (void) ddi_set_parent_data(pi->dip, pi); 2251 (void) ndi_devi_bind_driver(pi->dip, 0); 2252 rc = 0; 2253 done: 2254 PORT_UNLOCK(pi); 2255 return (rc); 2256 } 2257 2258 static int 2259 remove_child_node(struct adapter *sc, int idx) 2260 { 2261 int rc; 2262 struct port_info *pi; 2263 2264 if (idx < 0 || idx >= sc->params.nports) 2265 return (EINVAL); 2266 2267 pi = sc->port[idx]; 2268 if (pi == NULL) 2269 return (ENODEV); 2270 2271 PORT_LOCK(pi); 2272 if (pi->dip == NULL) { 2273 rc = ENODEV; 2274 goto done; 2275 } 2276 2277 rc = ndi_devi_free(pi->dip); 2278 if (rc == 0) 2279 pi->dip = NULL; 2280 done: 2281 PORT_UNLOCK(pi); 2282 return (rc); 2283 } 2284 2285 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 2286 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 2287 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 2288 #define KS_C_SET(x, ...) \ 2289 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 2290 2291 /* 2292 * t4nex:X:config 2293 */ 2294 struct t4_kstats { 2295 kstat_named_t chip_ver; 2296 kstat_named_t fw_vers; 2297 kstat_named_t tp_vers; 2298 kstat_named_t driver_version; 2299 kstat_named_t serial_number; 2300 kstat_named_t ec_level; 2301 kstat_named_t id; 2302 kstat_named_t bus_type; 2303 kstat_named_t bus_width; 2304 kstat_named_t bus_speed; 2305 kstat_named_t core_clock; 2306 kstat_named_t port_cnt; 2307 kstat_named_t port_type; 2308 kstat_named_t pci_vendor_id; 2309 kstat_named_t pci_device_id; 2310 }; 2311 static kstat_t * 2312 setup_kstats(struct adapter *sc) 2313 { 2314 kstat_t *ksp; 2315 struct t4_kstats *kstatp; 2316 int ndata; 2317 struct pci_params *p = &sc->params.pci; 2318 struct vpd_params *v = &sc->params.vpd; 2319 uint16_t pci_vendor, pci_device; 2320 2321 ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t); 2322 2323 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config", 2324 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2325 if (ksp == NULL) { 2326 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2327 return (NULL); 2328 } 2329 2330 kstatp = (struct t4_kstats *)ksp->ks_data; 2331 2332 KS_UINIT(chip_ver); 2333 KS_CINIT(fw_vers); 2334 KS_CINIT(tp_vers); 2335 KS_CINIT(driver_version); 2336 KS_CINIT(serial_number); 2337 KS_CINIT(ec_level); 2338 KS_CINIT(id); 2339 KS_CINIT(bus_type); 2340 KS_CINIT(bus_width); 2341 KS_CINIT(bus_speed); 2342 KS_UINIT(core_clock); 2343 KS_UINIT(port_cnt); 2344 KS_CINIT(port_type); 2345 KS_CINIT(pci_vendor_id); 2346 KS_CINIT(pci_device_id); 2347 2348 KS_U_SET(chip_ver, sc->params.chip); 2349 KS_C_SET(fw_vers, "%d.%d.%d.%d", 2350 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2351 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2352 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2353 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2354 KS_C_SET(tp_vers, "%d.%d.%d.%d", 2355 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 2356 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 2357 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 2358 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 2359 KS_C_SET(driver_version, DRV_VERSION); 2360 KS_C_SET(serial_number, "%s", v->sn); 2361 KS_C_SET(ec_level, "%s", v->ec); 2362 KS_C_SET(id, "%s", v->id); 2363 KS_C_SET(bus_type, "pci-express"); 2364 KS_C_SET(bus_width, "x%d lanes", p->width); 2365 KS_C_SET(bus_speed, "%d", p->speed); 2366 KS_U_SET(core_clock, v->cclk); 2367 KS_U_SET(port_cnt, sc->params.nports); 2368 2369 t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor); 2370 KS_C_SET(pci_vendor_id, "0x%x", pci_vendor); 2371 2372 t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device); 2373 KS_C_SET(pci_device_id, "0x%x", pci_device); 2374 2375 KS_C_SET(port_type, "%s/%s/%s/%s", 2376 print_port_speed(sc->port[0]), 2377 print_port_speed(sc->port[1]), 2378 print_port_speed(sc->port[2]), 2379 print_port_speed(sc->port[3])); 2380 2381 /* Do NOT set ksp->ks_update. These kstats do not change. */ 2382 2383 /* Install the kstat */ 2384 ksp->ks_private = (void *)sc; 2385 kstat_install(ksp); 2386 2387 return (ksp); 2388 } 2389 2390 /* 2391 * t4nex:X:stat 2392 */ 2393 struct t4_wc_kstats { 2394 kstat_named_t write_coal_success; 2395 kstat_named_t write_coal_failure; 2396 }; 2397 static kstat_t * 2398 setup_wc_kstats(struct adapter *sc) 2399 { 2400 kstat_t *ksp; 2401 struct t4_wc_kstats *kstatp; 2402 int ndata; 2403 2404 ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t); 2405 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats", 2406 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2407 if (ksp == NULL) { 2408 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2409 return (NULL); 2410 } 2411 2412 kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2413 2414 KS_UINIT(write_coal_success); 2415 KS_UINIT(write_coal_failure); 2416 2417 ksp->ks_update = update_wc_kstats; 2418 /* Install the kstat */ 2419 ksp->ks_private = (void *)sc; 2420 kstat_install(ksp); 2421 2422 return (ksp); 2423 } 2424 2425 static int 2426 update_wc_kstats(kstat_t *ksp, int rw) 2427 { 2428 struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2429 struct adapter *sc = ksp->ks_private; 2430 uint32_t wc_total, wc_success, wc_failure; 2431 2432 if (rw == KSTAT_WRITE) 2433 return (0); 2434 2435 if (is_t5(sc->params.chip)) { 2436 wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL); 2437 wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH); 2438 wc_success = wc_total - wc_failure; 2439 } else { 2440 wc_success = 0; 2441 wc_failure = 0; 2442 } 2443 2444 KS_U_SET(write_coal_success, wc_success); 2445 KS_U_SET(write_coal_failure, wc_failure); 2446 2447 return (0); 2448 } 2449 2450 int 2451 adapter_full_init(struct adapter *sc) 2452 { 2453 int i, rc = 0; 2454 2455 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2456 2457 rc = t4_setup_adapter_queues(sc); 2458 if (rc != 0) 2459 goto done; 2460 2461 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2462 (void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count); 2463 else { 2464 for (i = 0; i < sc->intr_count; i++) 2465 (void) ddi_intr_enable(sc->intr_handle[i]); 2466 } 2467 t4_intr_enable(sc); 2468 sc->flags |= FULL_INIT_DONE; 2469 2470 #ifdef TCP_OFFLOAD_ENABLE 2471 /* TODO: wrong place to enable TOE capability */ 2472 if (is_offload(sc) != 0) { 2473 for_each_port(sc, i) { 2474 struct port_info *pi = sc->port[i]; 2475 rc = toe_capability(pi, 1); 2476 if (rc != 0) { 2477 cxgb_printf(pi->dip, CE_WARN, 2478 "Failed to activate toe capability: %d", 2479 rc); 2480 rc = 0; /* not a fatal error */ 2481 } 2482 } 2483 } 2484 #endif 2485 2486 done: 2487 if (rc != 0) 2488 (void) adapter_full_uninit(sc); 2489 2490 return (rc); 2491 } 2492 2493 int 2494 adapter_full_uninit(struct adapter *sc) 2495 { 2496 int i, rc = 0; 2497 2498 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2499 2500 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2501 (void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count); 2502 else { 2503 for (i = 0; i < sc->intr_count; i++) 2504 (void) ddi_intr_disable(sc->intr_handle[i]); 2505 } 2506 2507 rc = t4_teardown_adapter_queues(sc); 2508 if (rc != 0) 2509 return (rc); 2510 2511 sc->flags &= ~FULL_INIT_DONE; 2512 2513 return (0); 2514 } 2515 2516 int 2517 port_full_init(struct port_info *pi) 2518 { 2519 struct adapter *sc = pi->adapter; 2520 uint16_t *rss; 2521 struct sge_rxq *rxq; 2522 int rc, i; 2523 2524 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2525 ASSERT((pi->flags & PORT_INIT_DONE) == 0); 2526 2527 /* 2528 * Allocate tx/rx/fl queues for this port. 2529 */ 2530 rc = t4_setup_port_queues(pi); 2531 if (rc != 0) 2532 goto done; /* error message displayed already */ 2533 2534 /* 2535 * Setup RSS for this port. 2536 */ 2537 rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP); 2538 for_each_rxq(pi, i, rxq) { 2539 rss[i] = rxq->iq.abs_id; 2540 } 2541 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2542 pi->rss_size, rss, pi->nrxq); 2543 kmem_free(rss, pi->nrxq * sizeof (*rss)); 2544 if (rc != 0) { 2545 cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc); 2546 goto done; 2547 } 2548 2549 pi->flags |= PORT_INIT_DONE; 2550 done: 2551 if (rc != 0) 2552 (void) port_full_uninit(pi); 2553 2554 return (rc); 2555 } 2556 2557 /* 2558 * Idempotent. 2559 */ 2560 int 2561 port_full_uninit(struct port_info *pi) 2562 { 2563 2564 ASSERT(pi->flags & PORT_INIT_DONE); 2565 2566 (void) t4_teardown_port_queues(pi); 2567 pi->flags &= ~PORT_INIT_DONE; 2568 2569 return (0); 2570 } 2571 2572 void 2573 enable_port_queues(struct port_info *pi) 2574 { 2575 struct adapter *sc = pi->adapter; 2576 int i; 2577 struct sge_iq *iq; 2578 struct sge_rxq *rxq; 2579 #ifdef TCP_OFFLOAD_ENABLE 2580 struct sge_ofld_rxq *ofld_rxq; 2581 #endif 2582 2583 ASSERT(pi->flags & PORT_INIT_DONE); 2584 2585 /* 2586 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED 2587 * back in disable_port_queues will be processed now, after an unbounded 2588 * delay. This can't be good. 2589 */ 2590 2591 #ifdef TCP_OFFLOAD_ENABLE 2592 for_each_ofld_rxq(pi, i, ofld_rxq) { 2593 iq = &ofld_rxq->iq; 2594 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2595 IQS_DISABLED) 2596 panic("%s: iq %p wasn't disabled", __func__, 2597 (void *)iq); 2598 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2599 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2600 } 2601 #endif 2602 2603 for_each_rxq(pi, i, rxq) { 2604 iq = &rxq->iq; 2605 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2606 IQS_DISABLED) 2607 panic("%s: iq %p wasn't disabled", __func__, 2608 (void *) iq); 2609 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2610 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2611 } 2612 } 2613 2614 void 2615 disable_port_queues(struct port_info *pi) 2616 { 2617 int i; 2618 struct adapter *sc = pi->adapter; 2619 struct sge_rxq *rxq; 2620 #ifdef TCP_OFFLOAD_ENABLE 2621 struct sge_ofld_rxq *ofld_rxq; 2622 #endif 2623 2624 ASSERT(pi->flags & PORT_INIT_DONE); 2625 2626 /* 2627 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld). 2628 */ 2629 2630 #ifdef TCP_OFFLOAD_ENABLE 2631 for_each_ofld_rxq(pi, i, ofld_rxq) { 2632 while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE, 2633 IQS_DISABLED) != IQS_IDLE) 2634 msleep(1); 2635 } 2636 #endif 2637 2638 for_each_rxq(pi, i, rxq) { 2639 while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE, 2640 IQS_DISABLED) != IQS_IDLE) 2641 msleep(1); 2642 } 2643 2644 mutex_enter(&sc->sfl_lock); 2645 #ifdef TCP_OFFLOAD_ENABLE 2646 for_each_ofld_rxq(pi, i, ofld_rxq) 2647 ofld_rxq->fl.flags |= FL_DOOMED; 2648 #endif 2649 for_each_rxq(pi, i, rxq) 2650 rxq->fl.flags |= FL_DOOMED; 2651 mutex_exit(&sc->sfl_lock); 2652 /* TODO: need to wait for all fl's to be removed from sc->sfl */ 2653 } 2654 2655 void 2656 t4_fatal_err(struct adapter *sc) 2657 { 2658 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 2659 t4_intr_disable(sc); 2660 cxgb_printf(sc->dip, CE_WARN, 2661 "encountered fatal error, adapter stopped."); 2662 } 2663 2664 int 2665 t4_os_find_pci_capability(struct adapter *sc, int cap) 2666 { 2667 uint16_t stat; 2668 uint8_t cap_ptr, cap_id; 2669 2670 t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat); 2671 if ((stat & PCI_STAT_CAP) == 0) 2672 return (0); /* does not implement capabilities */ 2673 2674 t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr); 2675 while (cap_ptr) { 2676 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id); 2677 if (cap_id == cap) 2678 return (cap_ptr); /* found */ 2679 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr); 2680 } 2681 2682 return (0); /* not found */ 2683 } 2684 2685 void 2686 t4_os_portmod_changed(const struct adapter *sc, int idx) 2687 { 2688 static const char *mod_str[] = { 2689 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 2690 }; 2691 const struct port_info *pi = sc->port[idx]; 2692 2693 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 2694 cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged."); 2695 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 2696 cxgb_printf(pi->dip, CE_NOTE, 2697 "unknown transceiver inserted.\n"); 2698 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 2699 cxgb_printf(pi->dip, CE_NOTE, 2700 "unsupported transceiver inserted.\n"); 2701 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 2702 cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n", 2703 mod_str[pi->mod_type]); 2704 else 2705 cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.", 2706 pi->mod_type); 2707 } 2708 2709 /* ARGSUSED */ 2710 static int 2711 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 2712 { 2713 if (m != NULL) 2714 freemsg(m); 2715 return (0); 2716 } 2717 2718 int 2719 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2720 { 2721 uint_t *loc, new; 2722 2723 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2724 return (EINVAL); 2725 2726 new = (uint_t)(unsigned long) (h ? h : cpl_not_handled); 2727 loc = (uint_t *)&sc->cpl_handler[opcode]; 2728 (void) atomic_swap_uint(loc, new); 2729 2730 return (0); 2731 } 2732 2733 static int 2734 fw_msg_not_handled(struct adapter *sc, const __be64 *data) 2735 { 2736 struct cpl_fw6_msg *cpl; 2737 2738 cpl = __containerof((void *)data, struct cpl_fw6_msg, data); 2739 2740 cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type); 2741 return (0); 2742 } 2743 2744 int 2745 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 2746 { 2747 fw_msg_handler_t *loc, new; 2748 2749 if (type >= ARRAY_SIZE(sc->fw_msg_handler)) 2750 return (EINVAL); 2751 2752 /* 2753 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 2754 * handler dispatch table. Reject any attempt to install a handler for 2755 * this subtype. 2756 */ 2757 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 2758 return (EINVAL); 2759 2760 new = h ? h : fw_msg_not_handled; 2761 loc = &sc->fw_msg_handler[type]; 2762 (void)atomic_swap_ptr(loc, (void *)new); 2763 2764 return (0); 2765 } 2766 2767 #ifdef TCP_OFFLOAD_ENABLE 2768 static int 2769 toe_capability(struct port_info *pi, int enable) 2770 { 2771 int rc; 2772 struct adapter *sc = pi->adapter; 2773 2774 if (!is_offload(sc)) 2775 return (ENODEV); 2776 2777 if (enable != 0) { 2778 if (isset(&sc->offload_map, pi->port_id) != 0) 2779 return (0); 2780 2781 if (sc->offload_map == 0) { 2782 rc = activate_uld(sc, ULD_TOM, &sc->tom); 2783 if (rc != 0) 2784 return (rc); 2785 } 2786 2787 setbit(&sc->offload_map, pi->port_id); 2788 } else { 2789 if (!isset(&sc->offload_map, pi->port_id)) 2790 return (0); 2791 2792 clrbit(&sc->offload_map, pi->port_id); 2793 2794 if (sc->offload_map == 0) { 2795 rc = deactivate_uld(&sc->tom); 2796 if (rc != 0) { 2797 setbit(&sc->offload_map, pi->port_id); 2798 return (rc); 2799 } 2800 } 2801 } 2802 2803 return (0); 2804 } 2805 2806 /* 2807 * Add an upper layer driver to the global list. 2808 */ 2809 int 2810 t4_register_uld(struct uld_info *ui) 2811 { 2812 int rc = 0; 2813 struct uld_info *u; 2814 2815 mutex_enter(&t4_uld_list_lock); 2816 SLIST_FOREACH(u, &t4_uld_list, link) { 2817 if (u->uld_id == ui->uld_id) { 2818 rc = EEXIST; 2819 goto done; 2820 } 2821 } 2822 2823 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 2824 ui->refcount = 0; 2825 done: 2826 mutex_exit(&t4_uld_list_lock); 2827 return (rc); 2828 } 2829 2830 int 2831 t4_unregister_uld(struct uld_info *ui) 2832 { 2833 int rc = EINVAL; 2834 struct uld_info *u; 2835 2836 mutex_enter(&t4_uld_list_lock); 2837 2838 SLIST_FOREACH(u, &t4_uld_list, link) { 2839 if (u == ui) { 2840 if (ui->refcount > 0) { 2841 rc = EBUSY; 2842 goto done; 2843 } 2844 2845 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 2846 rc = 0; 2847 goto done; 2848 } 2849 } 2850 done: 2851 mutex_exit(&t4_uld_list_lock); 2852 return (rc); 2853 } 2854 2855 static int 2856 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 2857 { 2858 int rc = EAGAIN; 2859 struct uld_info *ui; 2860 2861 mutex_enter(&t4_uld_list_lock); 2862 2863 SLIST_FOREACH(ui, &t4_uld_list, link) { 2864 if (ui->uld_id == id) { 2865 rc = ui->attach(sc, &usc->softc); 2866 if (rc == 0) { 2867 ASSERT(usc->softc != NULL); 2868 ui->refcount++; 2869 usc->uld = ui; 2870 } 2871 goto done; 2872 } 2873 } 2874 done: 2875 mutex_exit(&t4_uld_list_lock); 2876 2877 return (rc); 2878 } 2879 2880 static int 2881 deactivate_uld(struct uld_softc *usc) 2882 { 2883 int rc; 2884 2885 mutex_enter(&t4_uld_list_lock); 2886 2887 if (usc->uld == NULL || usc->softc == NULL) { 2888 rc = EINVAL; 2889 goto done; 2890 } 2891 2892 rc = usc->uld->detach(usc->softc); 2893 if (rc == 0) { 2894 ASSERT(usc->uld->refcount > 0); 2895 usc->uld->refcount--; 2896 usc->uld = NULL; 2897 usc->softc = NULL; 2898 } 2899 done: 2900 mutex_exit(&t4_uld_list_lock); 2901 2902 return (rc); 2903 } 2904 2905 void 2906 t4_iterate(void (*func)(int, void *), void *arg) 2907 { 2908 struct adapter *sc; 2909 2910 mutex_enter(&t4_adapter_list_lock); 2911 SLIST_FOREACH(sc, &t4_adapter_list, link) { 2912 /* 2913 * func should not make any assumptions about what state sc is 2914 * in - the only guarantee is that sc->sc_lock is a valid lock. 2915 */ 2916 func(ddi_get_instance(sc->dip), arg); 2917 } 2918 mutex_exit(&t4_adapter_list_lock); 2919 } 2920 2921 #endif 2922