1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * This file is part of the Chelsio T4 support code. 14 * 15 * Copyright (C) 2010-2013 Chelsio Communications. All rights reserved. 16 * 17 * This program is distributed in the hope that it will be useful, but WITHOUT 18 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 19 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this 20 * release for licensing terms and conditions. 21 */ 22 23 #include <sys/ddi.h> 24 #include <sys/sunddi.h> 25 #include <sys/sunndi.h> 26 #include <sys/modctl.h> 27 #include <sys/conf.h> 28 #include <sys/devops.h> 29 #include <sys/pci.h> 30 #include <sys/atomic.h> 31 #include <sys/types.h> 32 #include <sys/file.h> 33 #include <sys/errno.h> 34 #include <sys/open.h> 35 #include <sys/cred.h> 36 #include <sys/stat.h> 37 #include <sys/mkdev.h> 38 #include <sys/queue.h> 39 #include <sys/containerof.h> 40 #include <sys/sensors.h> 41 42 #include "version.h" 43 #include "common/common.h" 44 #include "common/t4_msg.h" 45 #include "common/t4_regs.h" 46 #include "firmware/t4_fw.h" 47 #include "firmware/t4_cfg.h" 48 #include "firmware/t5_fw.h" 49 #include "firmware/t5_cfg.h" 50 #include "firmware/t6_fw.h" 51 #include "firmware/t6_cfg.h" 52 #include "t4_l2t.h" 53 54 static int t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp); 55 static int t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp); 56 static int t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, 57 int *rp); 58 struct cb_ops t4_cb_ops = { 59 .cb_open = t4_cb_open, 60 .cb_close = t4_cb_close, 61 .cb_strategy = nodev, 62 .cb_print = nodev, 63 .cb_dump = nodev, 64 .cb_read = nodev, 65 .cb_write = nodev, 66 .cb_ioctl = t4_cb_ioctl, 67 .cb_devmap = nodev, 68 .cb_mmap = nodev, 69 .cb_segmap = nodev, 70 .cb_chpoll = nochpoll, 71 .cb_prop_op = ddi_prop_op, 72 .cb_flag = D_MP, 73 .cb_rev = CB_REV, 74 .cb_aread = nodev, 75 .cb_awrite = nodev 76 }; 77 78 static int t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, 79 void *arg, void *result); 80 static int t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 81 void *arg, dev_info_t **cdipp); 82 static int t4_bus_unconfig(dev_info_t *dip, uint_t flags, 83 ddi_bus_config_op_t op, void *arg); 84 struct bus_ops t4_bus_ops = { 85 .busops_rev = BUSO_REV, 86 .bus_ctl = t4_bus_ctl, 87 .bus_prop_op = ddi_bus_prop_op, 88 .bus_config = t4_bus_config, 89 .bus_unconfig = t4_bus_unconfig, 90 }; 91 92 static int t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, 93 void **rp); 94 static int t4_devo_probe(dev_info_t *dip); 95 static int t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 96 static int t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 97 static int t4_devo_quiesce(dev_info_t *dip); 98 struct dev_ops t4_dev_ops = { 99 .devo_rev = DEVO_REV, 100 .devo_getinfo = t4_devo_getinfo, 101 .devo_identify = nulldev, 102 .devo_probe = t4_devo_probe, 103 .devo_attach = t4_devo_attach, 104 .devo_detach = t4_devo_detach, 105 .devo_reset = nodev, 106 .devo_cb_ops = &t4_cb_ops, 107 .devo_bus_ops = &t4_bus_ops, 108 .devo_quiesce = &t4_devo_quiesce, 109 }; 110 111 static struct modldrv modldrv = { 112 .drv_modops = &mod_driverops, 113 .drv_linkinfo = "Chelsio T4 nexus " DRV_VERSION, 114 .drv_dev_ops = &t4_dev_ops 115 }; 116 117 static struct modlinkage modlinkage = { 118 .ml_rev = MODREV_1, 119 .ml_linkage = {&modldrv, NULL}, 120 }; 121 122 void *t4_list; 123 124 struct intrs_and_queues { 125 int intr_type; /* DDI_INTR_TYPE_* */ 126 int nirq; /* Number of vectors */ 127 int intr_fwd; /* Interrupts forwarded */ 128 int ntxq10g; /* # of NIC txq's for each 10G port */ 129 int nrxq10g; /* # of NIC rxq's for each 10G port */ 130 int ntxq1g; /* # of NIC txq's for each 1G port */ 131 int nrxq1g; /* # of NIC rxq's for each 1G port */ 132 #ifdef TCP_OFFLOAD_ENABLE 133 int nofldtxq10g; /* # of TOE txq's for each 10G port */ 134 int nofldrxq10g; /* # of TOE rxq's for each 10G port */ 135 int nofldtxq1g; /* # of TOE txq's for each 1G port */ 136 int nofldrxq1g; /* # of TOE rxq's for each 1G port */ 137 #endif 138 }; 139 140 struct fw_info fi[3]; 141 142 static int cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, 143 mblk_t *m); 144 static int fw_msg_not_handled(struct adapter *, const __be64 *); 145 int t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h); 146 static unsigned int getpf(struct adapter *sc); 147 static int prep_firmware(struct adapter *sc); 148 static int upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma); 149 static int partition_resources(struct adapter *sc); 150 static int adap__pre_init_tweaks(struct adapter *sc); 151 static int get_params__pre_init(struct adapter *sc); 152 static int get_params__post_init(struct adapter *sc); 153 static int set_params__post_init(struct adapter *); 154 static void setup_memwin(struct adapter *sc); 155 static int validate_mt_off_len(struct adapter *, int, uint32_t, int, 156 uint32_t *); 157 void memwin_info(struct adapter *, int, uint32_t *, uint32_t *); 158 uint32_t position_memwin(struct adapter *, int, uint32_t); 159 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 160 uint_t count); 161 static int prop_lookup_int_array(struct adapter *sc, char *name, int *data, 162 uint_t count); 163 static int init_driver_props(struct adapter *sc, struct driver_properties *p); 164 static int remove_extra_props(struct adapter *sc, int n10g, int n1g); 165 static int cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 166 struct intrs_and_queues *iaq); 167 static int add_child_node(struct adapter *sc, int idx); 168 static int remove_child_node(struct adapter *sc, int idx); 169 static kstat_t *setup_kstats(struct adapter *sc); 170 static kstat_t *setup_wc_kstats(struct adapter *); 171 static int update_wc_kstats(kstat_t *, int); 172 #ifdef TCP_OFFLOAD_ENABLE 173 static int toe_capability(struct port_info *pi, int enable); 174 static int activate_uld(struct adapter *sc, int id, struct uld_softc *usc); 175 static int deactivate_uld(struct uld_softc *usc); 176 #endif 177 static kmutex_t t4_adapter_list_lock; 178 static SLIST_HEAD(, adapter) t4_adapter_list; 179 #ifdef TCP_OFFLOAD_ENABLE 180 static kmutex_t t4_uld_list_lock; 181 static SLIST_HEAD(, uld_info) t4_uld_list; 182 #endif 183 184 static int t4_temperature_read(void *, sensor_ioctl_scalar_t *); 185 static int t4_voltage_read(void *, sensor_ioctl_scalar_t *); 186 static const ksensor_ops_t t4_temp_ops = { 187 .kso_kind = ksensor_kind_temperature, 188 .kso_scalar = t4_temperature_read 189 }; 190 191 static const ksensor_ops_t t4_volt_ops = { 192 .kso_kind = ksensor_kind_voltage, 193 .kso_scalar = t4_voltage_read 194 }; 195 196 int 197 _init(void) 198 { 199 int rc; 200 201 rc = ddi_soft_state_init(&t4_list, sizeof (struct adapter), 0); 202 if (rc != 0) 203 return (rc); 204 205 rc = mod_install(&modlinkage); 206 if (rc != 0) 207 ddi_soft_state_fini(&t4_list); 208 209 mutex_init(&t4_adapter_list_lock, NULL, MUTEX_DRIVER, NULL); 210 SLIST_INIT(&t4_adapter_list); 211 212 #ifdef TCP_OFFLOAD_ENABLE 213 mutex_init(&t4_uld_list_lock, NULL, MUTEX_DRIVER, NULL); 214 SLIST_INIT(&t4_uld_list); 215 #endif 216 217 return (rc); 218 } 219 220 int 221 _fini(void) 222 { 223 int rc; 224 225 rc = mod_remove(&modlinkage); 226 if (rc != 0) 227 return (rc); 228 229 ddi_soft_state_fini(&t4_list); 230 return (0); 231 } 232 233 int 234 _info(struct modinfo *mi) 235 { 236 return (mod_info(&modlinkage, mi)); 237 } 238 239 /* ARGSUSED */ 240 static int 241 t4_devo_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **rp) 242 { 243 struct adapter *sc; 244 minor_t minor; 245 246 minor = getminor((dev_t)arg); /* same as instance# in our case */ 247 248 if (cmd == DDI_INFO_DEVT2DEVINFO) { 249 sc = ddi_get_soft_state(t4_list, minor); 250 if (sc == NULL) 251 return (DDI_FAILURE); 252 253 ASSERT(sc->dev == (dev_t)arg); 254 *rp = (void *)sc->dip; 255 } else if (cmd == DDI_INFO_DEVT2INSTANCE) 256 *rp = (void *) (unsigned long) minor; 257 else 258 ASSERT(0); 259 260 return (DDI_SUCCESS); 261 } 262 263 static int 264 t4_devo_probe(dev_info_t *dip) 265 { 266 int rc, id, *reg; 267 uint_t n, pf; 268 269 id = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 270 "device-id", 0xffff); 271 if (id == 0xffff) 272 return (DDI_PROBE_DONTCARE); 273 274 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 275 "reg", ®, &n); 276 if (rc != DDI_SUCCESS) 277 return (DDI_PROBE_DONTCARE); 278 279 pf = PCI_REG_FUNC_G(reg[0]); 280 ddi_prop_free(reg); 281 282 /* Prevent driver attachment on any PF except 0 on the FPGA */ 283 if (id == 0xa000 && pf != 0) 284 return (DDI_PROBE_FAILURE); 285 286 return (DDI_PROBE_DONTCARE); 287 } 288 289 static int 290 t4_devo_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 291 { 292 struct adapter *sc = NULL; 293 struct sge *s; 294 int i, instance, rc = DDI_SUCCESS, rqidx, tqidx, q; 295 int irq = 0, nxg, n100g, n40g, n25g, n10g, n1g; 296 #ifdef TCP_OFFLOAD_ENABLE 297 int ofld_rqidx, ofld_tqidx; 298 #endif 299 char name[16]; 300 struct driver_properties *prp; 301 struct intrs_and_queues iaq; 302 ddi_device_acc_attr_t da = { 303 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 304 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 305 .devacc_attr_dataorder = DDI_UNORDERED_OK_ACC 306 }; 307 ddi_device_acc_attr_t da1 = { 308 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 309 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 310 .devacc_attr_dataorder = DDI_MERGING_OK_ACC 311 }; 312 313 if (cmd != DDI_ATTACH) 314 return (DDI_FAILURE); 315 316 /* 317 * Allocate space for soft state. 318 */ 319 instance = ddi_get_instance(dip); 320 rc = ddi_soft_state_zalloc(t4_list, instance); 321 if (rc != DDI_SUCCESS) { 322 cxgb_printf(dip, CE_WARN, 323 "failed to allocate soft state: %d", rc); 324 return (DDI_FAILURE); 325 } 326 327 sc = ddi_get_soft_state(t4_list, instance); 328 sc->dip = dip; 329 sc->dev = makedevice(ddi_driver_major(dip), instance); 330 mutex_init(&sc->lock, NULL, MUTEX_DRIVER, NULL); 331 cv_init(&sc->cv, NULL, CV_DRIVER, NULL); 332 mutex_init(&sc->sfl_lock, NULL, MUTEX_DRIVER, NULL); 333 TAILQ_INIT(&sc->sfl); 334 335 mutex_enter(&t4_adapter_list_lock); 336 SLIST_INSERT_HEAD(&t4_adapter_list, sc, link); 337 mutex_exit(&t4_adapter_list_lock); 338 339 sc->pf = getpf(sc); 340 if (sc->pf > 8) { 341 rc = EINVAL; 342 cxgb_printf(dip, CE_WARN, 343 "failed to determine PCI PF# of device"); 344 goto done; 345 } 346 sc->mbox = sc->pf; 347 348 /* Initialize the driver properties */ 349 prp = &sc->props; 350 (void)init_driver_props(sc, prp); 351 352 /* 353 * Enable access to the PCI config space. 354 */ 355 rc = pci_config_setup(dip, &sc->pci_regh); 356 if (rc != DDI_SUCCESS) { 357 cxgb_printf(dip, CE_WARN, 358 "failed to enable PCI config space access: %d", rc); 359 goto done; 360 } 361 362 /* TODO: Set max read request to 4K */ 363 364 /* 365 * Enable MMIO access. 366 */ 367 rc = ddi_regs_map_setup(dip, 1, &sc->regp, 0, 0, &da, &sc->regh); 368 if (rc != DDI_SUCCESS) { 369 cxgb_printf(dip, CE_WARN, 370 "failed to map device registers: %d", rc); 371 goto done; 372 } 373 374 (void) memset(sc->chan_map, 0xff, sizeof (sc->chan_map)); 375 376 /* 377 * Initialize cpl handler. 378 */ 379 for (i = 0; i < ARRAY_SIZE(sc->cpl_handler); i++) { 380 sc->cpl_handler[i] = cpl_not_handled; 381 } 382 383 for (i = 0; i < ARRAY_SIZE(sc->fw_msg_handler); i++) { 384 sc->fw_msg_handler[i] = fw_msg_not_handled; 385 } 386 387 for (i = 0; i < NCHAN; i++) { 388 (void) snprintf(name, sizeof (name), "%s-%d", 389 "reclaim", i); 390 sc->tq[i] = ddi_taskq_create(sc->dip, 391 name, 1, TASKQ_DEFAULTPRI, 0); 392 393 if (sc->tq[i] == NULL) { 394 cxgb_printf(dip, CE_WARN, 395 "failed to create task queues"); 396 rc = DDI_FAILURE; 397 goto done; 398 } 399 } 400 401 /* 402 * Prepare the adapter for operation. 403 */ 404 rc = -t4_prep_adapter(sc, false); 405 if (rc != 0) { 406 cxgb_printf(dip, CE_WARN, "failed to prepare adapter: %d", rc); 407 goto done; 408 } 409 410 /* 411 * Enable BAR1 access. 412 */ 413 sc->doorbells |= DOORBELL_KDB; 414 rc = ddi_regs_map_setup(dip, 2, &sc->reg1p, 0, 0, &da1, &sc->reg1h); 415 if (rc != DDI_SUCCESS) { 416 cxgb_printf(dip, CE_WARN, 417 "failed to map BAR1 device registers: %d", rc); 418 goto done; 419 } else { 420 if (is_t5(sc->params.chip)) { 421 sc->doorbells |= DOORBELL_UDB; 422 if (prp->wc) { 423 /* 424 * Enable write combining on BAR2. This is the 425 * userspace doorbell BAR and is split into 128B 426 * (UDBS_SEG_SIZE) doorbell regions, each associated 427 * with an egress queue. The first 64B has the doorbell 428 * and the second 64B can be used to submit a tx work 429 * request with an implicit doorbell. 430 */ 431 sc->doorbells &= ~DOORBELL_UDB; 432 sc->doorbells |= (DOORBELL_WCWR | 433 DOORBELL_UDBWC); 434 t4_write_reg(sc, A_SGE_STAT_CFG, 435 V_STATSOURCE_T5(7) | V_STATMODE(0)); 436 } 437 } 438 } 439 440 /* 441 * Do this really early. Note that minor number = instance. 442 */ 443 (void) snprintf(name, sizeof (name), "%s,%d", T4_NEXUS_NAME, instance); 444 rc = ddi_create_minor_node(dip, name, S_IFCHR, instance, 445 DDI_NT_NEXUS, 0); 446 if (rc != DDI_SUCCESS) { 447 cxgb_printf(dip, CE_WARN, 448 "failed to create device node: %d", rc); 449 rc = DDI_SUCCESS; /* carry on */ 450 } 451 452 /* Do this early. Memory window is required for loading config file. */ 453 setup_memwin(sc); 454 455 /* Prepare the firmware for operation */ 456 rc = prep_firmware(sc); 457 if (rc != 0) 458 goto done; /* error message displayed already */ 459 460 rc = adap__pre_init_tweaks(sc); 461 if (rc != 0) 462 goto done; 463 464 rc = get_params__pre_init(sc); 465 if (rc != 0) 466 goto done; /* error message displayed already */ 467 468 t4_sge_init(sc); 469 470 if (sc->flags & MASTER_PF) { 471 /* get basic stuff going */ 472 rc = -t4_fw_initialize(sc, sc->mbox); 473 if (rc != 0) { 474 cxgb_printf(sc->dip, CE_WARN, 475 "early init failed: %d.\n", rc); 476 goto done; 477 } 478 } 479 480 rc = get_params__post_init(sc); 481 if (rc != 0) 482 goto done; /* error message displayed already */ 483 484 rc = set_params__post_init(sc); 485 if (rc != 0) 486 goto done; /* error message displayed already */ 487 488 /* 489 * TODO: This is the place to call t4_set_filter_mode() 490 */ 491 492 /* tweak some settings */ 493 t4_write_reg(sc, A_TP_SHIFT_CNT, V_SYNSHIFTMAX(6) | V_RXTSHIFTMAXR1(4) | 494 V_RXTSHIFTMAXR2(15) | V_PERSHIFTBACKOFFMAX(8) | V_PERSHIFTMAX(8) | 495 V_KEEPALIVEMAXR1(4) | V_KEEPALIVEMAXR2(9)); 496 t4_write_reg(sc, A_ULP_RX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12)); 497 498 /* 499 * Work-around for bug 2619 500 * Set DisableVlan field in TP_RSS_CONFIG_VRT register so that the 501 * VLAN tag extraction is disabled. 502 */ 503 t4_set_reg_field(sc, A_TP_RSS_CONFIG_VRT, F_DISABLEVLAN, F_DISABLEVLAN); 504 505 /* Store filter mode */ 506 t4_read_indirect(sc, A_TP_PIO_ADDR, A_TP_PIO_DATA, &sc->filter_mode, 1, 507 A_TP_VLAN_PRI_MAP); 508 509 /* 510 * First pass over all the ports - allocate VIs and initialize some 511 * basic parameters like mac address, port type, etc. We also figure 512 * out whether a port is 10G or 1G and use that information when 513 * calculating how many interrupts to attempt to allocate. 514 */ 515 n100g = n40g = n25g = n10g = n1g = 0; 516 for_each_port(sc, i) { 517 struct port_info *pi; 518 519 pi = kmem_zalloc(sizeof (*pi), KM_SLEEP); 520 sc->port[i] = pi; 521 522 /* These must be set before t4_port_init */ 523 pi->adapter = sc; 524 /* LINTED: E_ASSIGN_NARROW_CONV */ 525 pi->port_id = i; 526 } 527 528 /* Allocate the vi and initialize parameters like mac addr */ 529 rc = -t4_port_init(sc, sc->mbox, sc->pf, 0); 530 if (rc) { 531 cxgb_printf(dip, CE_WARN, 532 "unable to initialize port: %d", rc); 533 goto done; 534 } 535 536 for_each_port(sc, i) { 537 struct port_info *pi = sc->port[i]; 538 539 mutex_init(&pi->lock, NULL, MUTEX_DRIVER, NULL); 540 pi->mtu = ETHERMTU; 541 542 if (is_100G_port(pi)) { 543 n100g++; 544 pi->tmr_idx = prp->tmr_idx_10g; 545 pi->pktc_idx = prp->pktc_idx_10g; 546 } else if (is_40G_port(pi)) { 547 n40g++; 548 pi->tmr_idx = prp->tmr_idx_10g; 549 pi->pktc_idx = prp->pktc_idx_10g; 550 } else if (is_25G_port(pi)) { 551 n25g++; 552 pi->tmr_idx = prp->tmr_idx_10g; 553 pi->pktc_idx = prp->pktc_idx_10g; 554 } else if (is_10G_port(pi)) { 555 n10g++; 556 pi->tmr_idx = prp->tmr_idx_10g; 557 pi->pktc_idx = prp->pktc_idx_10g; 558 } else { 559 n1g++; 560 pi->tmr_idx = prp->tmr_idx_1g; 561 pi->pktc_idx = prp->pktc_idx_1g; 562 } 563 564 pi->xact_addr_filt = -1; 565 t4_mc_init(pi); 566 567 setbit(&sc->registered_device_map, i); 568 } 569 570 nxg = n10g + n25g + n40g + n100g; 571 (void) remove_extra_props(sc, nxg, n1g); 572 573 if (sc->registered_device_map == 0) { 574 cxgb_printf(dip, CE_WARN, "no usable ports"); 575 rc = DDI_FAILURE; 576 goto done; 577 } 578 579 rc = cfg_itype_and_nqueues(sc, nxg, n1g, &iaq); 580 if (rc != 0) 581 goto done; /* error message displayed already */ 582 583 sc->intr_type = iaq.intr_type; 584 sc->intr_count = iaq.nirq; 585 586 if (sc->props.multi_rings && (sc->intr_type != DDI_INTR_TYPE_MSIX)) { 587 sc->props.multi_rings = 0; 588 cxgb_printf(dip, CE_WARN, 589 "Multiple rings disabled as interrupt type is not MSI-X"); 590 } 591 592 if (sc->props.multi_rings && iaq.intr_fwd) { 593 sc->props.multi_rings = 0; 594 cxgb_printf(dip, CE_WARN, 595 "Multiple rings disabled as interrupts are forwarded"); 596 } 597 598 if (!sc->props.multi_rings) { 599 iaq.ntxq10g = 1; 600 iaq.ntxq1g = 1; 601 } 602 s = &sc->sge; 603 s->nrxq = nxg * iaq.nrxq10g + n1g * iaq.nrxq1g; 604 s->ntxq = nxg * iaq.ntxq10g + n1g * iaq.ntxq1g; 605 s->neq = s->ntxq + s->nrxq; /* the fl in an rxq is an eq */ 606 #ifdef TCP_OFFLOAD_ENABLE 607 /* control queues, 1 per port + 1 mgmtq */ 608 s->neq += sc->params.nports + 1; 609 #endif 610 s->niq = s->nrxq + 1; /* 1 extra for firmware event queue */ 611 if (iaq.intr_fwd != 0) 612 sc->flags |= INTR_FWD; 613 #ifdef TCP_OFFLOAD_ENABLE 614 if (is_offload(sc) != 0) { 615 616 s->nofldrxq = nxg * iaq.nofldrxq10g + n1g * iaq.nofldrxq1g; 617 s->nofldtxq = nxg * iaq.nofldtxq10g + n1g * iaq.nofldtxq1g; 618 s->neq += s->nofldtxq + s->nofldrxq; 619 s->niq += s->nofldrxq; 620 621 s->ofld_rxq = kmem_zalloc(s->nofldrxq * 622 sizeof (struct sge_ofld_rxq), KM_SLEEP); 623 s->ofld_txq = kmem_zalloc(s->nofldtxq * 624 sizeof (struct sge_wrq), KM_SLEEP); 625 s->ctrlq = kmem_zalloc(sc->params.nports * 626 sizeof (struct sge_wrq), KM_SLEEP); 627 628 } 629 #endif 630 s->rxq = kmem_zalloc(s->nrxq * sizeof (struct sge_rxq), KM_SLEEP); 631 s->txq = kmem_zalloc(s->ntxq * sizeof (struct sge_txq), KM_SLEEP); 632 s->iqmap = kmem_zalloc(s->niq * sizeof (struct sge_iq *), KM_SLEEP); 633 s->eqmap = kmem_zalloc(s->neq * sizeof (struct sge_eq *), KM_SLEEP); 634 635 sc->intr_handle = kmem_zalloc(sc->intr_count * 636 sizeof (ddi_intr_handle_t), KM_SLEEP); 637 638 /* 639 * Second pass over the ports. This time we know the number of rx and 640 * tx queues that each port should get. 641 */ 642 rqidx = tqidx = 0; 643 #ifdef TCP_OFFLOAD_ENABLE 644 ofld_rqidx = ofld_tqidx = 0; 645 #endif 646 for_each_port(sc, i) { 647 struct port_info *pi = sc->port[i]; 648 649 if (pi == NULL) 650 continue; 651 652 t4_mc_cb_init(pi); 653 /* LINTED: E_ASSIGN_NARROW_CONV */ 654 pi->first_rxq = rqidx; 655 /* LINTED: E_ASSIGN_NARROW_CONV */ 656 pi->nrxq = (is_10XG_port(pi)) ? iaq.nrxq10g 657 : iaq.nrxq1g; 658 /* LINTED: E_ASSIGN_NARROW_CONV */ 659 pi->first_txq = tqidx; 660 /* LINTED: E_ASSIGN_NARROW_CONV */ 661 pi->ntxq = (is_10XG_port(pi)) ? iaq.ntxq10g 662 : iaq.ntxq1g; 663 664 rqidx += pi->nrxq; 665 tqidx += pi->ntxq; 666 667 #ifdef TCP_OFFLOAD_ENABLE 668 if (is_offload(sc) != 0) { 669 /* LINTED: E_ASSIGN_NARROW_CONV */ 670 pi->first_ofld_rxq = ofld_rqidx; 671 pi->nofldrxq = max(1, pi->nrxq / 4); 672 673 /* LINTED: E_ASSIGN_NARROW_CONV */ 674 pi->first_ofld_txq = ofld_tqidx; 675 pi->nofldtxq = max(1, pi->ntxq / 2); 676 677 ofld_rqidx += pi->nofldrxq; 678 ofld_tqidx += pi->nofldtxq; 679 } 680 #endif 681 682 /* 683 * Enable hw checksumming and LSO for all ports by default. 684 * They can be disabled using ndd (hw_csum and hw_lso). 685 */ 686 pi->features |= (CXGBE_HW_CSUM | CXGBE_HW_LSO); 687 } 688 689 #ifdef TCP_OFFLOAD_ENABLE 690 sc->l2t = t4_init_l2t(sc); 691 #endif 692 693 /* 694 * Setup Interrupts. 695 */ 696 697 i = 0; 698 rc = ddi_intr_alloc(dip, sc->intr_handle, sc->intr_type, 0, 699 sc->intr_count, &i, DDI_INTR_ALLOC_STRICT); 700 if (rc != DDI_SUCCESS) { 701 cxgb_printf(dip, CE_WARN, 702 "failed to allocate %d interrupt(s) of type %d: %d, %d", 703 sc->intr_count, sc->intr_type, rc, i); 704 goto done; 705 } 706 ASSERT(sc->intr_count == i); /* allocation was STRICT */ 707 (void) ddi_intr_get_cap(sc->intr_handle[0], &sc->intr_cap); 708 (void) ddi_intr_get_pri(sc->intr_handle[0], &sc->intr_pri); 709 if (sc->intr_count == 1) { 710 ASSERT(sc->flags & INTR_FWD); 711 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_all, sc, 712 &s->fwq); 713 } else { 714 /* Multiple interrupts. The first one is always error intr */ 715 (void) ddi_intr_add_handler(sc->intr_handle[0], t4_intr_err, sc, 716 NULL); 717 irq++; 718 719 /* The second one is always the firmware event queue */ 720 (void) ddi_intr_add_handler(sc->intr_handle[1], t4_intr, sc, 721 &s->fwq); 722 irq++; 723 /* 724 * Note that if INTR_FWD is set then either the NIC rx 725 * queues or (exclusive or) the TOE rx queueus will be taking 726 * direct interrupts. 727 * 728 * There is no need to check for is_offload(sc) as nofldrxq 729 * will be 0 if offload is disabled. 730 */ 731 for_each_port(sc, i) { 732 struct port_info *pi = sc->port[i]; 733 struct sge_rxq *rxq; 734 #ifdef TCP_OFFLOAD_ENABLE 735 struct sge_ofld_rxq *ofld_rxq; 736 737 /* 738 * Skip over the NIC queues if they aren't taking direct 739 * interrupts. 740 */ 741 if ((sc->flags & INTR_FWD) && 742 pi->nofldrxq > pi->nrxq) 743 goto ofld_queues; 744 #endif 745 rxq = &s->rxq[pi->first_rxq]; 746 for (q = 0; q < pi->nrxq; q++, rxq++) { 747 (void) ddi_intr_add_handler( 748 sc->intr_handle[irq], t4_intr, sc, 749 &rxq->iq); 750 irq++; 751 } 752 753 #ifdef TCP_OFFLOAD_ENABLE 754 /* 755 * Skip over the offload queues if they aren't taking 756 * direct interrupts. 757 */ 758 if ((sc->flags & INTR_FWD)) 759 continue; 760 ofld_queues: 761 ofld_rxq = &s->ofld_rxq[pi->first_ofld_rxq]; 762 for (q = 0; q < pi->nofldrxq; q++, ofld_rxq++) { 763 (void) ddi_intr_add_handler( 764 sc->intr_handle[irq], t4_intr, sc, 765 &ofld_rxq->iq); 766 irq++; 767 } 768 #endif 769 } 770 771 } 772 sc->flags |= INTR_ALLOCATED; 773 774 if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_TEMPERATURE, 775 &t4_temp_ops, sc, "temp", &sc->temp_sensor)) != 0) { 776 cxgb_printf(dip, CE_WARN, "failed to create temperature " 777 "sensor: %d", rc); 778 rc = DDI_FAILURE; 779 goto done; 780 } 781 782 if ((rc = ksensor_create_scalar_pcidev(dip, SENSOR_KIND_VOLTAGE, 783 &t4_volt_ops, sc, "vdd", &sc->volt_sensor)) != 0) { 784 cxgb_printf(dip, CE_WARN, "failed to create voltage " 785 "sensor: %d", rc); 786 rc = DDI_FAILURE; 787 goto done; 788 } 789 790 791 ddi_report_dev(dip); 792 793 /* 794 * Hardware/Firmware/etc. Version/Revision IDs. 795 */ 796 t4_dump_version_info(sc); 797 798 if (n100g) { 799 cxgb_printf(dip, CE_NOTE, 800 "%dx100G (%d rxq, %d txq total) %d %s.", 801 n100g, rqidx, tqidx, sc->intr_count, 802 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 803 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 804 "fixed interrupt"); 805 } else if (n40g) { 806 cxgb_printf(dip, CE_NOTE, 807 "%dx40G (%d rxq, %d txq total) %d %s.", 808 n40g, rqidx, tqidx, sc->intr_count, 809 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 810 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 811 "fixed interrupt"); 812 } else if (n25g) { 813 cxgb_printf(dip, CE_NOTE, 814 "%dx25G (%d rxq, %d txq total) %d %s.", 815 n25g, rqidx, tqidx, sc->intr_count, 816 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 817 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 818 "fixed interrupt"); 819 } else if (n10g && n1g) { 820 cxgb_printf(dip, CE_NOTE, 821 "%dx10G %dx1G (%d rxq, %d txq total) %d %s.", 822 n10g, n1g, rqidx, tqidx, sc->intr_count, 823 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 824 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 825 "fixed interrupt"); 826 } else { 827 cxgb_printf(dip, CE_NOTE, 828 "%dx%sG (%d rxq, %d txq per port) %d %s.", 829 n10g ? n10g : n1g, 830 n10g ? "10" : "1", 831 n10g ? iaq.nrxq10g : iaq.nrxq1g, 832 n10g ? iaq.ntxq10g : iaq.ntxq1g, 833 sc->intr_count, 834 sc->intr_type == DDI_INTR_TYPE_MSIX ? "MSI-X interrupts" : 835 sc->intr_type == DDI_INTR_TYPE_MSI ? "MSI interrupts" : 836 "fixed interrupt"); 837 } 838 839 sc->ksp = setup_kstats(sc); 840 sc->ksp_stat = setup_wc_kstats(sc); 841 sc->params.drv_memwin = MEMWIN_NIC; 842 843 done: 844 if (rc != DDI_SUCCESS) { 845 (void) t4_devo_detach(dip, DDI_DETACH); 846 847 /* rc may have errno style errors or DDI errors */ 848 rc = DDI_FAILURE; 849 } 850 851 return (rc); 852 } 853 854 static int 855 t4_devo_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 856 { 857 int instance, i; 858 struct adapter *sc; 859 struct port_info *pi; 860 struct sge *s; 861 862 if (cmd != DDI_DETACH) 863 return (DDI_FAILURE); 864 865 instance = ddi_get_instance(dip); 866 sc = ddi_get_soft_state(t4_list, instance); 867 if (sc == NULL) 868 return (DDI_SUCCESS); 869 870 if (sc->flags & FULL_INIT_DONE) { 871 t4_intr_disable(sc); 872 for_each_port(sc, i) { 873 pi = sc->port[i]; 874 if (pi && pi->flags & PORT_INIT_DONE) 875 (void) port_full_uninit(pi); 876 } 877 (void) adapter_full_uninit(sc); 878 } 879 880 /* Safe to call no matter what */ 881 (void) ksensor_remove(dip, KSENSOR_ALL_IDS); 882 ddi_prop_remove_all(dip); 883 ddi_remove_minor_node(dip, NULL); 884 885 for (i = 0; i < NCHAN; i++) { 886 if (sc->tq[i]) { 887 ddi_taskq_wait(sc->tq[i]); 888 ddi_taskq_destroy(sc->tq[i]); 889 } 890 } 891 892 if (sc->ksp != NULL) 893 kstat_delete(sc->ksp); 894 if (sc->ksp_stat != NULL) 895 kstat_delete(sc->ksp_stat); 896 897 s = &sc->sge; 898 if (s->rxq != NULL) 899 kmem_free(s->rxq, s->nrxq * sizeof (struct sge_rxq)); 900 #ifdef TCP_OFFLOAD_ENABLE 901 if (s->ofld_txq != NULL) 902 kmem_free(s->ofld_txq, s->nofldtxq * sizeof (struct sge_wrq)); 903 if (s->ofld_rxq != NULL) 904 kmem_free(s->ofld_rxq, 905 s->nofldrxq * sizeof (struct sge_ofld_rxq)); 906 if (s->ctrlq != NULL) 907 kmem_free(s->ctrlq, 908 sc->params.nports * sizeof (struct sge_wrq)); 909 #endif 910 if (s->txq != NULL) 911 kmem_free(s->txq, s->ntxq * sizeof (struct sge_txq)); 912 if (s->iqmap != NULL) 913 kmem_free(s->iqmap, s->niq * sizeof (struct sge_iq *)); 914 if (s->eqmap != NULL) 915 kmem_free(s->eqmap, s->neq * sizeof (struct sge_eq *)); 916 917 if (s->rxbuf_cache != NULL) 918 rxbuf_cache_destroy(s->rxbuf_cache); 919 920 if (sc->flags & INTR_ALLOCATED) { 921 for (i = 0; i < sc->intr_count; i++) { 922 (void) ddi_intr_remove_handler(sc->intr_handle[i]); 923 (void) ddi_intr_free(sc->intr_handle[i]); 924 } 925 sc->flags &= ~INTR_ALLOCATED; 926 } 927 928 if (sc->intr_handle != NULL) { 929 kmem_free(sc->intr_handle, 930 sc->intr_count * sizeof (*sc->intr_handle)); 931 } 932 933 for_each_port(sc, i) { 934 pi = sc->port[i]; 935 if (pi != NULL) { 936 mutex_destroy(&pi->lock); 937 kmem_free(pi, sizeof (*pi)); 938 clrbit(&sc->registered_device_map, i); 939 } 940 } 941 942 if (sc->flags & FW_OK) 943 (void) t4_fw_bye(sc, sc->mbox); 944 945 if (sc->reg1h != NULL) 946 ddi_regs_map_free(&sc->reg1h); 947 948 if (sc->regh != NULL) 949 ddi_regs_map_free(&sc->regh); 950 951 if (sc->pci_regh != NULL) 952 pci_config_teardown(&sc->pci_regh); 953 954 mutex_enter(&t4_adapter_list_lock); 955 SLIST_REMOVE_HEAD(&t4_adapter_list, link); 956 mutex_exit(&t4_adapter_list_lock); 957 958 mutex_destroy(&sc->lock); 959 cv_destroy(&sc->cv); 960 mutex_destroy(&sc->sfl_lock); 961 962 #ifdef DEBUG 963 bzero(sc, sizeof (*sc)); 964 #endif 965 ddi_soft_state_free(t4_list, instance); 966 967 return (DDI_SUCCESS); 968 } 969 970 static int 971 t4_devo_quiesce(dev_info_t *dip) 972 { 973 int instance; 974 struct adapter *sc; 975 976 instance = ddi_get_instance(dip); 977 sc = ddi_get_soft_state(t4_list, instance); 978 if (sc == NULL) 979 return (DDI_SUCCESS); 980 981 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 982 t4_intr_disable(sc); 983 t4_write_reg(sc, A_PL_RST, F_PIORSTMODE | F_PIORST); 984 985 return (DDI_SUCCESS); 986 } 987 988 static int 989 t4_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t op, void *arg, 990 void *result) 991 { 992 char s[4]; 993 struct port_info *pi; 994 dev_info_t *child = (dev_info_t *)arg; 995 996 switch (op) { 997 case DDI_CTLOPS_REPORTDEV: 998 pi = ddi_get_parent_data(rdip); 999 pi->instance = ddi_get_instance(dip); 1000 pi->child_inst = ddi_get_instance(rdip); 1001 cmn_err(CE_CONT, "?%s%d is port %s on %s%d\n", 1002 ddi_node_name(rdip), ddi_get_instance(rdip), 1003 ddi_get_name_addr(rdip), ddi_driver_name(dip), 1004 ddi_get_instance(dip)); 1005 return (DDI_SUCCESS); 1006 1007 case DDI_CTLOPS_INITCHILD: 1008 pi = ddi_get_parent_data(child); 1009 if (pi == NULL) 1010 return (DDI_NOT_WELL_FORMED); 1011 (void) snprintf(s, sizeof (s), "%d", pi->port_id); 1012 ddi_set_name_addr(child, s); 1013 return (DDI_SUCCESS); 1014 1015 case DDI_CTLOPS_UNINITCHILD: 1016 ddi_set_name_addr(child, NULL); 1017 return (DDI_SUCCESS); 1018 1019 case DDI_CTLOPS_ATTACH: 1020 case DDI_CTLOPS_DETACH: 1021 return (DDI_SUCCESS); 1022 1023 default: 1024 return (ddi_ctlops(dip, rdip, op, arg, result)); 1025 } 1026 } 1027 1028 static int 1029 t4_bus_config(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, void *arg, 1030 dev_info_t **cdipp) 1031 { 1032 int instance, i; 1033 struct adapter *sc; 1034 1035 instance = ddi_get_instance(dip); 1036 sc = ddi_get_soft_state(t4_list, instance); 1037 1038 if (op == BUS_CONFIG_ONE) { 1039 char *c; 1040 1041 /* 1042 * arg is something like "cxgb@0" where 0 is the port_id hanging 1043 * off this nexus. 1044 */ 1045 1046 c = arg; 1047 while (*(c + 1)) 1048 c++; 1049 1050 /* There should be exactly 1 digit after '@' */ 1051 if (*(c - 1) != '@') 1052 return (NDI_FAILURE); 1053 1054 i = *c - '0'; 1055 1056 if (add_child_node(sc, i) != 0) 1057 return (NDI_FAILURE); 1058 1059 flags |= NDI_ONLINE_ATTACH; 1060 1061 } else if (op == BUS_CONFIG_ALL || op == BUS_CONFIG_DRIVER) { 1062 /* Allocate and bind all child device nodes */ 1063 for_each_port(sc, i) 1064 (void) add_child_node(sc, i); 1065 flags |= NDI_ONLINE_ATTACH; 1066 } 1067 1068 return (ndi_busop_bus_config(dip, flags, op, arg, cdipp, 0)); 1069 } 1070 1071 static int 1072 t4_bus_unconfig(dev_info_t *dip, uint_t flags, ddi_bus_config_op_t op, 1073 void *arg) 1074 { 1075 int instance, i, rc; 1076 struct adapter *sc; 1077 1078 instance = ddi_get_instance(dip); 1079 sc = ddi_get_soft_state(t4_list, instance); 1080 1081 if (op == BUS_CONFIG_ONE || op == BUS_UNCONFIG_ALL || 1082 op == BUS_UNCONFIG_DRIVER) 1083 flags |= NDI_UNCONFIG; 1084 1085 rc = ndi_busop_bus_unconfig(dip, flags, op, arg); 1086 if (rc != 0) 1087 return (rc); 1088 1089 if (op == BUS_UNCONFIG_ONE) { 1090 char *c; 1091 1092 c = arg; 1093 while (*(c + 1)) 1094 c++; 1095 1096 if (*(c - 1) != '@') 1097 return (NDI_SUCCESS); 1098 1099 i = *c - '0'; 1100 1101 rc = remove_child_node(sc, i); 1102 1103 } else if (op == BUS_UNCONFIG_ALL || op == BUS_UNCONFIG_DRIVER) { 1104 1105 for_each_port(sc, i) 1106 (void) remove_child_node(sc, i); 1107 } 1108 1109 return (rc); 1110 } 1111 1112 /* ARGSUSED */ 1113 static int 1114 t4_cb_open(dev_t *devp, int flag, int otyp, cred_t *credp) 1115 { 1116 struct adapter *sc; 1117 1118 if (otyp != OTYP_CHR) 1119 return (EINVAL); 1120 1121 sc = ddi_get_soft_state(t4_list, getminor(*devp)); 1122 if (sc == NULL) 1123 return (ENXIO); 1124 1125 return (atomic_cas_uint(&sc->open, 0, EBUSY)); 1126 } 1127 1128 /* ARGSUSED */ 1129 static int 1130 t4_cb_close(dev_t dev, int flag, int otyp, cred_t *credp) 1131 { 1132 struct adapter *sc; 1133 1134 sc = ddi_get_soft_state(t4_list, getminor(dev)); 1135 if (sc == NULL) 1136 return (EINVAL); 1137 1138 (void) atomic_swap_uint(&sc->open, 0); 1139 return (0); 1140 } 1141 1142 /* ARGSUSED */ 1143 static int 1144 t4_cb_ioctl(dev_t dev, int cmd, intptr_t d, int mode, cred_t *credp, int *rp) 1145 { 1146 int instance; 1147 struct adapter *sc; 1148 void *data = (void *)d; 1149 1150 if (crgetuid(credp) != 0) 1151 return (EPERM); 1152 1153 instance = getminor(dev); 1154 sc = ddi_get_soft_state(t4_list, instance); 1155 if (sc == NULL) 1156 return (EINVAL); 1157 1158 return (t4_ioctl(sc, cmd, data, mode)); 1159 } 1160 1161 static unsigned int 1162 getpf(struct adapter *sc) 1163 { 1164 int rc, *data; 1165 uint_t n, pf; 1166 1167 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1168 DDI_PROP_DONTPASS, "reg", &data, &n); 1169 if (rc != DDI_SUCCESS) { 1170 cxgb_printf(sc->dip, CE_WARN, 1171 "failed to lookup \"reg\" property: %d", rc); 1172 return (0xff); 1173 } 1174 1175 pf = PCI_REG_FUNC_G(data[0]); 1176 ddi_prop_free(data); 1177 1178 return (pf); 1179 } 1180 1181 1182 static struct fw_info * 1183 find_fw_info(int chip) 1184 { 1185 u32 i; 1186 1187 fi[0].chip = CHELSIO_T4; 1188 fi[0].fw_hdr.chip = FW_HDR_CHIP_T4; 1189 fi[0].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T4)); 1190 fi[0].fw_hdr.intfver_nic = FW_INTFVER(T4, NIC); 1191 fi[0].fw_hdr.intfver_vnic = FW_INTFVER(T4, VNIC); 1192 fi[0].fw_hdr.intfver_ofld = FW_INTFVER(T4, OFLD); 1193 fi[0].fw_hdr.intfver_ri = FW_INTFVER(T4, RI); 1194 fi[0].fw_hdr.intfver_iscsipdu = FW_INTFVER(T4, ISCSIPDU); 1195 fi[0].fw_hdr.intfver_iscsi = FW_INTFVER(T4, ISCSI); 1196 fi[0].fw_hdr.intfver_fcoepdu = FW_INTFVER(T4, FCOEPDU); 1197 fi[0].fw_hdr.intfver_fcoe = FW_INTFVER(T4, FCOE); 1198 1199 fi[1].chip = CHELSIO_T5; 1200 fi[1].fw_hdr.chip = FW_HDR_CHIP_T5; 1201 fi[1].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T5)); 1202 fi[1].fw_hdr.intfver_nic = FW_INTFVER(T5, NIC); 1203 fi[1].fw_hdr.intfver_vnic = FW_INTFVER(T5, VNIC); 1204 fi[1].fw_hdr.intfver_ofld = FW_INTFVER(T5, OFLD); 1205 fi[1].fw_hdr.intfver_ri = FW_INTFVER(T5, RI); 1206 fi[1].fw_hdr.intfver_iscsipdu = FW_INTFVER(T5, ISCSIPDU); 1207 fi[1].fw_hdr.intfver_iscsi = FW_INTFVER(T5, ISCSI); 1208 fi[1].fw_hdr.intfver_fcoepdu = FW_INTFVER(T5, FCOEPDU); 1209 fi[1].fw_hdr.intfver_fcoe = FW_INTFVER(T5, FCOE); 1210 1211 fi[2].chip = CHELSIO_T6; 1212 fi[2].fw_hdr.chip = FW_HDR_CHIP_T6; 1213 fi[2].fw_hdr.fw_ver = cpu_to_be32(FW_VERSION(T6)); 1214 fi[2].fw_hdr.intfver_nic = FW_INTFVER(T6, NIC); 1215 fi[2].fw_hdr.intfver_vnic = FW_INTFVER(T6, VNIC); 1216 fi[2].fw_hdr.intfver_ofld = FW_INTFVER(T6, OFLD); 1217 fi[2].fw_hdr.intfver_ri = FW_INTFVER(T6, RI); 1218 fi[2].fw_hdr.intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU); 1219 fi[2].fw_hdr.intfver_iscsi = FW_INTFVER(T6, ISCSI); 1220 fi[2].fw_hdr.intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU); 1221 fi[2].fw_hdr.intfver_fcoe = FW_INTFVER(T6, FCOE); 1222 1223 for (i = 0; i < ARRAY_SIZE(fi); i++) { 1224 if (fi[i].chip == chip) 1225 return &fi[i]; 1226 } 1227 1228 return NULL; 1229 } 1230 1231 /* 1232 * Install a compatible firmware (if required), establish contact with it, 1233 * become the master, and reset the device. 1234 */ 1235 static int 1236 prep_firmware(struct adapter *sc) 1237 { 1238 int rc; 1239 int fw_size; 1240 int reset = 1; 1241 enum dev_state state; 1242 unsigned char *fw_data; 1243 struct fw_info *fw_info; 1244 struct fw_hdr *card_fw; 1245 1246 struct driver_properties *p = &sc->props; 1247 1248 /* Contact firmware, request master */ 1249 rc = t4_fw_hello(sc, sc->mbox, sc->mbox, MASTER_MUST, &state); 1250 if (rc < 0) { 1251 rc = -rc; 1252 cxgb_printf(sc->dip, CE_WARN, 1253 "failed to connect to the firmware: %d.", rc); 1254 return (rc); 1255 } 1256 1257 if (rc == sc->mbox) 1258 sc->flags |= MASTER_PF; 1259 1260 /* We may need FW version info for later reporting */ 1261 t4_get_version_info(sc); 1262 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(sc->params.chip)); 1263 /* allocate memory to read the header of the firmware on the 1264 * card 1265 */ 1266 if (!fw_info) { 1267 cxgb_printf(sc->dip, CE_WARN, 1268 "unable to look up firmware information for chip %d.\n", 1269 CHELSIO_CHIP_VERSION(sc->params.chip)); 1270 return EINVAL; 1271 } 1272 card_fw = kmem_zalloc(sizeof(*card_fw), KM_SLEEP); 1273 if(!card_fw) { 1274 cxgb_printf(sc->dip, CE_WARN, 1275 "Memory allocation for card FW header failed\n"); 1276 return ENOMEM; 1277 } 1278 switch(CHELSIO_CHIP_VERSION(sc->params.chip)) { 1279 case CHELSIO_T4: 1280 fw_data = t4fw_data; 1281 fw_size = t4fw_size; 1282 break; 1283 case CHELSIO_T5: 1284 fw_data = t5fw_data; 1285 fw_size = t5fw_size; 1286 break; 1287 case CHELSIO_T6: 1288 fw_data = t6fw_data; 1289 fw_size = t6fw_size; 1290 break; 1291 default: 1292 cxgb_printf(sc->dip, CE_WARN, "Adapter type not supported\n"); 1293 kmem_free(card_fw, sizeof(*card_fw)); 1294 return EINVAL; 1295 } 1296 1297 rc = -t4_prep_fw(sc, fw_info, fw_data, fw_size, card_fw, 1298 p->t4_fw_install, state, &reset); 1299 1300 kmem_free(card_fw, sizeof(*card_fw)); 1301 1302 if (rc != 0) { 1303 cxgb_printf(sc->dip, CE_WARN, 1304 "failed to install firmware: %d", rc); 1305 return (rc); 1306 } else { 1307 /* refresh */ 1308 (void) t4_check_fw_version(sc); 1309 } 1310 1311 /* Reset device */ 1312 rc = -t4_fw_reset(sc, sc->mbox, F_PIORSTMODE | F_PIORST); 1313 if (rc != 0) { 1314 cxgb_printf(sc->dip, CE_WARN, 1315 "firmware reset failed: %d.", rc); 1316 if (rc != ETIMEDOUT && rc != EIO) 1317 (void) t4_fw_bye(sc, sc->mbox); 1318 return (rc); 1319 } 1320 1321 /* Partition adapter resources as specified in the config file. */ 1322 if (sc->flags & MASTER_PF) { 1323 /* Handle default vs special T4 config file */ 1324 1325 rc = partition_resources(sc); 1326 if (rc != 0) 1327 goto err; /* error message displayed already */ 1328 } 1329 1330 sc->flags |= FW_OK; 1331 return (0); 1332 err: 1333 return (rc); 1334 1335 } 1336 1337 static const struct memwin t4_memwin[] = { 1338 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1339 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1340 { MEMWIN2_BASE, MEMWIN2_APERTURE } 1341 }; 1342 1343 static const struct memwin t5_memwin[] = { 1344 { MEMWIN0_BASE, MEMWIN0_APERTURE }, 1345 { MEMWIN1_BASE, MEMWIN1_APERTURE }, 1346 { MEMWIN2_BASE_T5, MEMWIN2_APERTURE_T5 }, 1347 }; 1348 1349 #define FW_PARAM_DEV(param) \ 1350 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \ 1351 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param)) 1352 #define FW_PARAM_PFVF(param) \ 1353 (V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \ 1354 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param)) 1355 1356 /* 1357 * Verify that the memory range specified by the memtype/offset/len pair is 1358 * valid and lies entirely within the memtype specified. The global address of 1359 * the start of the range is returned in addr. 1360 */ 1361 int 1362 validate_mt_off_len(struct adapter *sc, int mtype, uint32_t off, int len, 1363 uint32_t *addr) 1364 { 1365 uint32_t em, addr_len, maddr, mlen; 1366 1367 /* Memory can only be accessed in naturally aligned 4 byte units */ 1368 if (off & 3 || len & 3 || len == 0) 1369 return (EINVAL); 1370 1371 em = t4_read_reg(sc, A_MA_TARGET_MEM_ENABLE); 1372 switch (mtype) { 1373 case MEM_EDC0: 1374 if (!(em & F_EDRAM0_ENABLE)) 1375 return (EINVAL); 1376 addr_len = t4_read_reg(sc, A_MA_EDRAM0_BAR); 1377 maddr = G_EDRAM0_BASE(addr_len) << 20; 1378 mlen = G_EDRAM0_SIZE(addr_len) << 20; 1379 break; 1380 case MEM_EDC1: 1381 if (!(em & F_EDRAM1_ENABLE)) 1382 return (EINVAL); 1383 addr_len = t4_read_reg(sc, A_MA_EDRAM1_BAR); 1384 maddr = G_EDRAM1_BASE(addr_len) << 20; 1385 mlen = G_EDRAM1_SIZE(addr_len) << 20; 1386 break; 1387 case MEM_MC: 1388 if (!(em & F_EXT_MEM_ENABLE)) 1389 return (EINVAL); 1390 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY_BAR); 1391 maddr = G_EXT_MEM_BASE(addr_len) << 20; 1392 mlen = G_EXT_MEM_SIZE(addr_len) << 20; 1393 break; 1394 case MEM_MC1: 1395 if (is_t4(sc->params.chip) || !(em & F_EXT_MEM1_ENABLE)) 1396 return (EINVAL); 1397 addr_len = t4_read_reg(sc, A_MA_EXT_MEMORY1_BAR); 1398 maddr = G_EXT_MEM1_BASE(addr_len) << 20; 1399 mlen = G_EXT_MEM1_SIZE(addr_len) << 20; 1400 break; 1401 default: 1402 return (EINVAL); 1403 } 1404 1405 if (mlen > 0 && off < mlen && off + len <= mlen) { 1406 *addr = maddr + off; /* global address */ 1407 return (0); 1408 } 1409 1410 return (EFAULT); 1411 } 1412 1413 void 1414 memwin_info(struct adapter *sc, int win, uint32_t *base, uint32_t *aperture) 1415 { 1416 const struct memwin *mw; 1417 1418 if (is_t4(sc->params.chip)) { 1419 mw = &t4_memwin[win]; 1420 } else { 1421 mw = &t5_memwin[win]; 1422 } 1423 1424 if (base != NULL) 1425 *base = mw->base; 1426 if (aperture != NULL) 1427 *aperture = mw->aperture; 1428 } 1429 1430 /* 1431 * Upload configuration file to card's memory. 1432 */ 1433 static int 1434 upload_config_file(struct adapter *sc, uint32_t *mt, uint32_t *ma) 1435 { 1436 int rc = 0, cflen; 1437 u_int i, n; 1438 uint32_t param, val, addr, mtype, maddr; 1439 uint32_t off, mw_base, mw_aperture; 1440 const uint32_t *cfdata; 1441 1442 /* Figure out where the firmware wants us to upload it. */ 1443 param = FW_PARAM_DEV(CF); 1444 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1445 if (rc != 0) { 1446 /* Firmwares without config file support will fail this way */ 1447 cxgb_printf(sc->dip, CE_WARN, 1448 "failed to query config file location: %d.\n", rc); 1449 return (rc); 1450 } 1451 *mt = mtype = G_FW_PARAMS_PARAM_Y(val); 1452 *ma = maddr = G_FW_PARAMS_PARAM_Z(val) << 16; 1453 1454 switch (CHELSIO_CHIP_VERSION(sc->params.chip)) { 1455 case CHELSIO_T4: 1456 cflen = t4cfg_size & ~3; 1457 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1458 cfdata = (const uint32_t *)t4cfg_data; 1459 break; 1460 case CHELSIO_T5: 1461 cflen = t5cfg_size & ~3; 1462 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1463 cfdata = (const uint32_t *)t5cfg_data; 1464 break; 1465 case CHELSIO_T6: 1466 cflen = t6cfg_size & ~3; 1467 /* LINTED: E_BAD_PTR_CAST_ALIGN */ 1468 cfdata = (const uint32_t *)t6cfg_data; 1469 break; 1470 default: 1471 cxgb_printf(sc->dip, CE_WARN, 1472 "Invalid Adapter detected\n"); 1473 return EINVAL; 1474 } 1475 1476 if (cflen > FLASH_CFG_MAX_SIZE) { 1477 cxgb_printf(sc->dip, CE_WARN, 1478 "config file too long (%d, max allowed is %d). ", 1479 cflen, FLASH_CFG_MAX_SIZE); 1480 return (EFBIG); 1481 } 1482 1483 rc = validate_mt_off_len(sc, mtype, maddr, cflen, &addr); 1484 if (rc != 0) { 1485 1486 cxgb_printf(sc->dip, CE_WARN, 1487 "%s: addr (%d/0x%x) or len %d is not valid: %d. " 1488 "Will try to use the config on the card, if any.\n", 1489 __func__, mtype, maddr, cflen, rc); 1490 return (EFAULT); 1491 } 1492 1493 memwin_info(sc, 2, &mw_base, &mw_aperture); 1494 while (cflen) { 1495 off = position_memwin(sc, 2, addr); 1496 n = min(cflen, mw_aperture - off); 1497 for (i = 0; i < n; i += 4) 1498 t4_write_reg(sc, mw_base + off + i, *cfdata++); 1499 cflen -= n; 1500 addr += n; 1501 } 1502 1503 return (rc); 1504 } 1505 1506 /* 1507 * Partition chip resources for use between various PFs, VFs, etc. This is done 1508 * by uploading the firmware configuration file to the adapter and instructing 1509 * the firmware to process it. 1510 */ 1511 static int 1512 partition_resources(struct adapter *sc) 1513 { 1514 int rc; 1515 struct fw_caps_config_cmd caps; 1516 uint32_t mtype, maddr, finicsum, cfcsum; 1517 1518 rc = upload_config_file(sc, &mtype, &maddr); 1519 if (rc != 0) { 1520 mtype = FW_MEMTYPE_CF_FLASH; 1521 maddr = t4_flash_cfg_addr(sc); 1522 } 1523 1524 bzero(&caps, sizeof (caps)); 1525 caps.op_to_write = BE_32(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1526 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1527 caps.cfvalid_to_len16 = BE_32(F_FW_CAPS_CONFIG_CMD_CFVALID | 1528 V_FW_CAPS_CONFIG_CMD_MEMTYPE_CF(mtype) | 1529 V_FW_CAPS_CONFIG_CMD_MEMADDR64K_CF(maddr >> 16) | FW_LEN16(caps)); 1530 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1531 if (rc != 0) { 1532 cxgb_printf(sc->dip, CE_WARN, 1533 "failed to pre-process config file: %d.\n", rc); 1534 return (rc); 1535 } 1536 1537 finicsum = ntohl(caps.finicsum); 1538 cfcsum = ntohl(caps.cfcsum); 1539 if (finicsum != cfcsum) { 1540 cxgb_printf(sc->dip, CE_WARN, 1541 "WARNING: config file checksum mismatch: %08x %08x\n", 1542 finicsum, cfcsum); 1543 } 1544 sc->cfcsum = cfcsum; 1545 1546 /* TODO: Need to configure this correctly */ 1547 caps.toecaps = htons(FW_CAPS_CONFIG_TOE); 1548 caps.iscsicaps = 0; 1549 caps.rdmacaps = 0; 1550 caps.fcoecaps = 0; 1551 /* TODO: Disable VNIC cap for now */ 1552 caps.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 1553 1554 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1555 F_FW_CMD_REQUEST | F_FW_CMD_WRITE); 1556 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1557 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), NULL); 1558 if (rc != 0) { 1559 cxgb_printf(sc->dip, CE_WARN, 1560 "failed to process config file: %d.\n", rc); 1561 return (rc); 1562 } 1563 1564 return (0); 1565 } 1566 1567 /* 1568 * Tweak configuration based on module parameters, etc. Most of these have 1569 * defaults assigned to them by Firmware Configuration Files (if we're using 1570 * them) but need to be explicitly set if we're using hard-coded 1571 * initialization. But even in the case of using Firmware Configuration 1572 * Files, we'd like to expose the ability to change these via module 1573 * parameters so these are essentially common tweaks/settings for 1574 * Configuration Files and hard-coded initialization ... 1575 */ 1576 static int 1577 adap__pre_init_tweaks(struct adapter *sc) 1578 { 1579 int rx_dma_offset = 2; /* Offset of RX packets into DMA buffers */ 1580 1581 /* 1582 * Fix up various Host-Dependent Parameters like Page Size, Cache 1583 * Line Size, etc. The firmware default is for a 4KB Page Size and 1584 * 64B Cache Line Size ... 1585 */ 1586 (void) t4_fixup_host_params_compat(sc, PAGE_SIZE, CACHE_LINE, T5_LAST_REV); 1587 1588 t4_set_reg_field(sc, A_SGE_CONTROL, 1589 V_PKTSHIFT(M_PKTSHIFT), V_PKTSHIFT(rx_dma_offset)); 1590 1591 return 0; 1592 } 1593 /* 1594 * Retrieve parameters that are needed (or nice to have) prior to calling 1595 * t4_sge_init and t4_fw_initialize. 1596 */ 1597 static int 1598 get_params__pre_init(struct adapter *sc) 1599 { 1600 int rc; 1601 uint32_t param[2], val[2]; 1602 struct fw_devlog_cmd cmd; 1603 struct devlog_params *dlog = &sc->params.devlog; 1604 1605 /* 1606 * Grab the raw VPD parameters. 1607 */ 1608 rc = -t4_get_raw_vpd_params(sc, &sc->params.vpd); 1609 if (rc != 0) { 1610 cxgb_printf(sc->dip, CE_WARN, 1611 "failed to query VPD parameters (pre_init): %d.\n", rc); 1612 return (rc); 1613 } 1614 1615 param[0] = FW_PARAM_DEV(PORTVEC); 1616 param[1] = FW_PARAM_DEV(CCLK); 1617 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 2, param, val); 1618 if (rc != 0) { 1619 cxgb_printf(sc->dip, CE_WARN, 1620 "failed to query parameters (pre_init): %d.\n", rc); 1621 return (rc); 1622 } 1623 1624 sc->params.portvec = val[0]; 1625 sc->params.nports = 0; 1626 while (val[0]) { 1627 sc->params.nports++; 1628 val[0] &= val[0] - 1; 1629 } 1630 1631 sc->params.vpd.cclk = val[1]; 1632 1633 /* Read device log parameters. */ 1634 bzero(&cmd, sizeof (cmd)); 1635 cmd.op_to_write = htonl(V_FW_CMD_OP(FW_DEVLOG_CMD) | 1636 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1637 cmd.retval_len16 = htonl(FW_LEN16(cmd)); 1638 rc = -t4_wr_mbox(sc, sc->mbox, &cmd, sizeof (cmd), &cmd); 1639 if (rc != 0) { 1640 cxgb_printf(sc->dip, CE_WARN, 1641 "failed to get devlog parameters: %d.\n", rc); 1642 bzero(dlog, sizeof (*dlog)); 1643 rc = 0; /* devlog isn't critical for device operation */ 1644 } else { 1645 val[0] = ntohl(cmd.memtype_devlog_memaddr16_devlog); 1646 dlog->memtype = G_FW_DEVLOG_CMD_MEMTYPE_DEVLOG(val[0]); 1647 dlog->start = G_FW_DEVLOG_CMD_MEMADDR16_DEVLOG(val[0]) << 4; 1648 dlog->size = ntohl(cmd.memsize_devlog); 1649 } 1650 1651 return (rc); 1652 } 1653 1654 /* 1655 * Retrieve various parameters that are of interest to the driver. The device 1656 * has been initialized by the firmware at this point. 1657 */ 1658 static int 1659 get_params__post_init(struct adapter *sc) 1660 { 1661 int rc; 1662 uint32_t param[7], val[7]; 1663 struct fw_caps_config_cmd caps; 1664 1665 param[0] = FW_PARAM_PFVF(IQFLINT_START); 1666 param[1] = FW_PARAM_PFVF(EQ_START); 1667 param[2] = FW_PARAM_PFVF(FILTER_START); 1668 param[3] = FW_PARAM_PFVF(FILTER_END); 1669 param[4] = FW_PARAM_PFVF(L2T_START); 1670 param[5] = FW_PARAM_PFVF(L2T_END); 1671 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1672 if (rc != 0) { 1673 cxgb_printf(sc->dip, CE_WARN, 1674 "failed to query parameters (post_init): %d.\n", rc); 1675 return (rc); 1676 } 1677 1678 /* LINTED: E_ASSIGN_NARROW_CONV */ 1679 sc->sge.iq_start = val[0]; 1680 sc->sge.eq_start = val[1]; 1681 sc->tids.ftid_base = val[2]; 1682 sc->tids.nftids = val[3] - val[2] + 1; 1683 sc->vres.l2t.start = val[4]; 1684 sc->vres.l2t.size = val[5] - val[4] + 1; 1685 1686 /* get capabilites */ 1687 bzero(&caps, sizeof (caps)); 1688 caps.op_to_write = htonl(V_FW_CMD_OP(FW_CAPS_CONFIG_CMD) | 1689 F_FW_CMD_REQUEST | F_FW_CMD_READ); 1690 caps.cfvalid_to_len16 = htonl(FW_LEN16(caps)); 1691 rc = -t4_wr_mbox(sc, sc->mbox, &caps, sizeof (caps), &caps); 1692 if (rc != 0) { 1693 cxgb_printf(sc->dip, CE_WARN, 1694 "failed to get card capabilities: %d.\n", rc); 1695 return (rc); 1696 } 1697 1698 if (caps.toecaps != 0) { 1699 /* query offload-related parameters */ 1700 param[0] = FW_PARAM_DEV(NTID); 1701 param[1] = FW_PARAM_PFVF(SERVER_START); 1702 param[2] = FW_PARAM_PFVF(SERVER_END); 1703 param[3] = FW_PARAM_PFVF(TDDP_START); 1704 param[4] = FW_PARAM_PFVF(TDDP_END); 1705 param[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 1706 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 6, param, val); 1707 if (rc != 0) { 1708 cxgb_printf(sc->dip, CE_WARN, 1709 "failed to query TOE parameters: %d.\n", rc); 1710 return (rc); 1711 } 1712 sc->tids.ntids = val[0]; 1713 sc->tids.natids = min(sc->tids.ntids / 2, MAX_ATIDS); 1714 sc->tids.stid_base = val[1]; 1715 sc->tids.nstids = val[2] - val[1] + 1; 1716 sc->vres.ddp.start = val[3]; 1717 sc->vres.ddp.size = val[4] - val[3] + 1; 1718 sc->params.ofldq_wr_cred = val[5]; 1719 sc->params.offload = 1; 1720 } 1721 1722 /* These are finalized by FW initialization, load their values now */ 1723 val[0] = t4_read_reg(sc, A_TP_TIMER_RESOLUTION); 1724 sc->params.tp.tre = G_TIMERRESOLUTION(val[0]); 1725 sc->params.tp.dack_re = G_DELAYEDACKRESOLUTION(val[0]); 1726 t4_read_mtu_tbl(sc, sc->params.mtus, NULL); 1727 1728 return (rc); 1729 } 1730 1731 static int 1732 set_params__post_init(struct adapter *sc) 1733 { 1734 uint32_t param, val; 1735 1736 /* ask for encapsulated CPLs */ 1737 param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 1738 val = 1; 1739 (void)t4_set_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 1740 1741 return (0); 1742 } 1743 1744 /* TODO: verify */ 1745 static void 1746 setup_memwin(struct adapter *sc) 1747 { 1748 pci_regspec_t *data; 1749 int rc; 1750 uint_t n; 1751 uintptr_t bar0; 1752 uintptr_t mem_win0_base, mem_win1_base, mem_win2_base; 1753 uintptr_t mem_win2_aperture; 1754 1755 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, sc->dip, 1756 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&data, &n); 1757 if (rc != DDI_SUCCESS) { 1758 cxgb_printf(sc->dip, CE_WARN, 1759 "failed to lookup \"assigned-addresses\" property: %d", rc); 1760 return; 1761 } 1762 n /= sizeof (*data); 1763 1764 bar0 = ((uint64_t)data[0].pci_phys_mid << 32) | data[0].pci_phys_low; 1765 ddi_prop_free(data); 1766 1767 if (is_t4(sc->params.chip)) { 1768 mem_win0_base = bar0 + MEMWIN0_BASE; 1769 mem_win1_base = bar0 + MEMWIN1_BASE; 1770 mem_win2_base = bar0 + MEMWIN2_BASE; 1771 mem_win2_aperture = MEMWIN2_APERTURE; 1772 } else { 1773 /* For T5, only relative offset inside the PCIe BAR is passed */ 1774 mem_win0_base = MEMWIN0_BASE; 1775 mem_win1_base = MEMWIN1_BASE; 1776 mem_win2_base = MEMWIN2_BASE_T5; 1777 mem_win2_aperture = MEMWIN2_APERTURE_T5; 1778 } 1779 1780 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 0), 1781 mem_win0_base | V_BIR(0) | 1782 V_WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 1783 1784 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 1), 1785 mem_win1_base | V_BIR(0) | 1786 V_WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 1787 1788 t4_write_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2), 1789 mem_win2_base | V_BIR(0) | 1790 V_WINDOW(ilog2(mem_win2_aperture) - 10)); 1791 1792 /* flush */ 1793 (void)t4_read_reg(sc, PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_BASE_WIN, 2)); 1794 } 1795 1796 /* 1797 * Positions the memory window such that it can be used to access the specified 1798 * address in the chip's address space. The return value is the offset of addr 1799 * from the start of the window. 1800 */ 1801 uint32_t 1802 position_memwin(struct adapter *sc, int n, uint32_t addr) 1803 { 1804 uint32_t start, pf; 1805 uint32_t reg; 1806 1807 if (addr & 3) { 1808 cxgb_printf(sc->dip, CE_WARN, 1809 "addr (0x%x) is not at a 4B boundary.\n", addr); 1810 return (EFAULT); 1811 } 1812 1813 if (is_t4(sc->params.chip)) { 1814 pf = 0; 1815 start = addr & ~0xf; /* start must be 16B aligned */ 1816 } else { 1817 pf = V_PFNUM(sc->pf); 1818 start = addr & ~0x7f; /* start must be 128B aligned */ 1819 } 1820 reg = PCIE_MEM_ACCESS_REG(A_PCIE_MEM_ACCESS_OFFSET, n); 1821 1822 t4_write_reg(sc, reg, start | pf); 1823 (void) t4_read_reg(sc, reg); 1824 1825 return (addr - start); 1826 } 1827 1828 1829 /* 1830 * Reads the named property and fills up the "data" array (which has at least 1831 * "count" elements). We first try and lookup the property for our dev_t and 1832 * then retry with DDI_DEV_T_ANY if it's not found. 1833 * 1834 * Returns non-zero if the property was found and "data" has been updated. 1835 */ 1836 static int 1837 prop_lookup_int_array(struct adapter *sc, char *name, int *data, uint_t count) 1838 { 1839 dev_info_t *dip = sc->dip; 1840 dev_t dev = sc->dev; 1841 int rc, *d; 1842 uint_t i, n; 1843 1844 rc = ddi_prop_lookup_int_array(dev, dip, DDI_PROP_DONTPASS, 1845 name, &d, &n); 1846 if (rc == DDI_PROP_SUCCESS) 1847 goto found; 1848 1849 if (rc != DDI_PROP_NOT_FOUND) { 1850 cxgb_printf(dip, CE_WARN, 1851 "failed to lookup property %s for minor %d: %d.", 1852 name, getminor(dev), rc); 1853 return (0); 1854 } 1855 1856 rc = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 1857 name, &d, &n); 1858 if (rc == DDI_PROP_SUCCESS) 1859 goto found; 1860 1861 if (rc != DDI_PROP_NOT_FOUND) { 1862 cxgb_printf(dip, CE_WARN, 1863 "failed to lookup property %s: %d.", name, rc); 1864 return (0); 1865 } 1866 1867 return (0); 1868 1869 found: 1870 if (n > count) { 1871 cxgb_printf(dip, CE_NOTE, 1872 "property %s has too many elements (%d), ignoring extras", 1873 name, n); 1874 } 1875 1876 for (i = 0; i < n && i < count; i++) 1877 data[i] = d[i]; 1878 ddi_prop_free(d); 1879 1880 return (1); 1881 } 1882 1883 static int 1884 prop_lookup_int(struct adapter *sc, char *name, int defval) 1885 { 1886 int rc; 1887 1888 rc = ddi_prop_get_int(sc->dev, sc->dip, DDI_PROP_DONTPASS, name, -1); 1889 if (rc != -1) 1890 return (rc); 1891 1892 return (ddi_prop_get_int(DDI_DEV_T_ANY, sc->dip, DDI_PROP_DONTPASS, 1893 name, defval)); 1894 } 1895 1896 static int 1897 init_driver_props(struct adapter *sc, struct driver_properties *p) 1898 { 1899 dev_t dev = sc->dev; 1900 dev_info_t *dip = sc->dip; 1901 int i, *data; 1902 uint_t tmr[SGE_NTIMERS] = {5, 10, 20, 50, 100, 200}; 1903 uint_t cnt[SGE_NCOUNTERS] = {1, 8, 16, 32}; /* 63 max */ 1904 1905 /* 1906 * Holdoff timer 1907 */ 1908 data = &p->timer_val[0]; 1909 for (i = 0; i < SGE_NTIMERS; i++) 1910 data[i] = tmr[i]; 1911 (void) prop_lookup_int_array(sc, "holdoff-timer-values", data, 1912 SGE_NTIMERS); 1913 for (i = 0; i < SGE_NTIMERS; i++) { 1914 int limit = 200U; 1915 if (data[i] > limit) { 1916 cxgb_printf(dip, CE_WARN, 1917 "holdoff timer %d is too high (%d), lowered to %d.", 1918 i, data[i], limit); 1919 data[i] = limit; 1920 } 1921 } 1922 (void) ddi_prop_update_int_array(dev, dip, "holdoff-timer-values", 1923 data, SGE_NTIMERS); 1924 1925 /* 1926 * Holdoff packet counter 1927 */ 1928 data = &p->counter_val[0]; 1929 for (i = 0; i < SGE_NCOUNTERS; i++) 1930 data[i] = cnt[i]; 1931 (void) prop_lookup_int_array(sc, "holdoff-pkt-counter-values", data, 1932 SGE_NCOUNTERS); 1933 for (i = 0; i < SGE_NCOUNTERS; i++) { 1934 int limit = M_THRESHOLD_0; 1935 if (data[i] > limit) { 1936 cxgb_printf(dip, CE_WARN, 1937 "holdoff pkt-counter %d is too high (%d), " 1938 "lowered to %d.", i, data[i], limit); 1939 data[i] = limit; 1940 } 1941 } 1942 (void) ddi_prop_update_int_array(dev, dip, "holdoff-pkt-counter-values", 1943 data, SGE_NCOUNTERS); 1944 1945 /* 1946 * Maximum # of tx and rx queues to use for each 1947 * 100G, 40G, 25G, 10G and 1G port. 1948 */ 1949 p->max_ntxq_10g = prop_lookup_int(sc, "max-ntxq-10G-port", 8); 1950 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1951 p->max_ntxq_10g); 1952 1953 p->max_nrxq_10g = prop_lookup_int(sc, "max-nrxq-10G-port", 8); 1954 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1955 p->max_nrxq_10g); 1956 1957 p->max_ntxq_1g = prop_lookup_int(sc, "max-ntxq-1G-port", 2); 1958 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1959 p->max_ntxq_1g); 1960 1961 p->max_nrxq_1g = prop_lookup_int(sc, "max-nrxq-1G-port", 2); 1962 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1963 p->max_nrxq_1g); 1964 1965 #ifdef TCP_OFFLOAD_ENABLE 1966 p->max_nofldtxq_10g = prop_lookup_int(sc, "max-nofldtxq-10G-port", 8); 1967 (void) ddi_prop_update_int(dev, dip, "max-ntxq-10G-port", 1968 p->max_nofldtxq_10g); 1969 1970 p->max_nofldrxq_10g = prop_lookup_int(sc, "max-nofldrxq-10G-port", 2); 1971 (void) ddi_prop_update_int(dev, dip, "max-nrxq-10G-port", 1972 p->max_nofldrxq_10g); 1973 1974 p->max_nofldtxq_1g = prop_lookup_int(sc, "max-nofldtxq-1G-port", 2); 1975 (void) ddi_prop_update_int(dev, dip, "max-ntxq-1G-port", 1976 p->max_nofldtxq_1g); 1977 1978 p->max_nofldrxq_1g = prop_lookup_int(sc, "max-nofldrxq-1G-port", 1); 1979 (void) ddi_prop_update_int(dev, dip, "max-nrxq-1G-port", 1980 p->max_nofldrxq_1g); 1981 #endif 1982 1983 /* 1984 * Holdoff parameters for 10G and 1G ports. 1985 */ 1986 p->tmr_idx_10g = prop_lookup_int(sc, "holdoff-timer-idx-10G", 0); 1987 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-10G", 1988 p->tmr_idx_10g); 1989 1990 p->pktc_idx_10g = prop_lookup_int(sc, "holdoff-pktc-idx-10G", 2); 1991 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-10G", 1992 p->pktc_idx_10g); 1993 1994 p->tmr_idx_1g = prop_lookup_int(sc, "holdoff-timer-idx-1G", 0); 1995 (void) ddi_prop_update_int(dev, dip, "holdoff-timer-idx-1G", 1996 p->tmr_idx_1g); 1997 1998 p->pktc_idx_1g = prop_lookup_int(sc, "holdoff-pktc-idx-1G", 2); 1999 (void) ddi_prop_update_int(dev, dip, "holdoff-pktc-idx-1G", 2000 p->pktc_idx_1g); 2001 2002 /* 2003 * Size (number of entries) of each tx and rx queue. 2004 */ 2005 i = prop_lookup_int(sc, "qsize-txq", TX_EQ_QSIZE); 2006 p->qsize_txq = max(i, 128); 2007 if (p->qsize_txq != i) { 2008 cxgb_printf(dip, CE_WARN, 2009 "using %d instead of %d as the tx queue size", 2010 p->qsize_txq, i); 2011 } 2012 (void) ddi_prop_update_int(dev, dip, "qsize-txq", p->qsize_txq); 2013 2014 i = prop_lookup_int(sc, "qsize-rxq", RX_IQ_QSIZE); 2015 p->qsize_rxq = max(i, 128); 2016 while (p->qsize_rxq & 7) 2017 p->qsize_rxq--; 2018 if (p->qsize_rxq != i) { 2019 cxgb_printf(dip, CE_WARN, 2020 "using %d instead of %d as the rx queue size", 2021 p->qsize_rxq, i); 2022 } 2023 (void) ddi_prop_update_int(dev, dip, "qsize-rxq", p->qsize_rxq); 2024 2025 /* 2026 * Interrupt types allowed. 2027 * Bits 0, 1, 2 = INTx, MSI, MSI-X respectively. See sys/ddi_intr.h 2028 */ 2029 p->intr_types = prop_lookup_int(sc, "interrupt-types", 2030 DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_FIXED); 2031 (void) ddi_prop_update_int(dev, dip, "interrupt-types", p->intr_types); 2032 2033 /* 2034 * Forwarded interrupt queues. Create this property to force the driver 2035 * to use forwarded interrupt queues. 2036 */ 2037 if (ddi_prop_exists(dev, dip, DDI_PROP_DONTPASS, 2038 "interrupt-forwarding") != 0 || 2039 ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 2040 "interrupt-forwarding") != 0) { 2041 UNIMPLEMENTED(); 2042 (void) ddi_prop_create(dev, dip, DDI_PROP_CANSLEEP, 2043 "interrupt-forwarding", NULL, 0); 2044 } 2045 2046 /* 2047 * Write combining 2048 * 0 to disable, 1 to enable 2049 */ 2050 p->wc = prop_lookup_int(sc, "write-combine", 1); 2051 cxgb_printf(dip, CE_WARN, "write-combine: using of %d", p->wc); 2052 if (p->wc != 0 && p->wc != 1) { 2053 cxgb_printf(dip, CE_WARN, 2054 "write-combine: using 1 instead of %d", p->wc); 2055 p->wc = 1; 2056 } 2057 (void) ddi_prop_update_int(dev, dip, "write-combine", p->wc); 2058 2059 p->t4_fw_install = prop_lookup_int(sc, "t4_fw_install", 1); 2060 if (p->t4_fw_install != 0 && p->t4_fw_install != 2) 2061 p->t4_fw_install = 1; 2062 (void) ddi_prop_update_int(dev, dip, "t4_fw_install", p->t4_fw_install); 2063 2064 /* Multiple Rings */ 2065 p->multi_rings = prop_lookup_int(sc, "multi-rings", 1); 2066 if (p->multi_rings != 0 && p->multi_rings != 1) { 2067 cxgb_printf(dip, CE_NOTE, 2068 "multi-rings: using value 1 instead of %d", p->multi_rings); 2069 p->multi_rings = 1; 2070 } 2071 2072 (void) ddi_prop_update_int(dev, dip, "multi-rings", p->multi_rings); 2073 2074 return (0); 2075 } 2076 2077 static int 2078 remove_extra_props(struct adapter *sc, int n10g, int n1g) 2079 { 2080 if (n10g == 0) { 2081 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-10G-port"); 2082 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-10G-port"); 2083 (void) ddi_prop_remove(sc->dev, sc->dip, 2084 "holdoff-timer-idx-10G"); 2085 (void) ddi_prop_remove(sc->dev, sc->dip, 2086 "holdoff-pktc-idx-10G"); 2087 } 2088 2089 if (n1g == 0) { 2090 (void) ddi_prop_remove(sc->dev, sc->dip, "max-ntxq-1G-port"); 2091 (void) ddi_prop_remove(sc->dev, sc->dip, "max-nrxq-1G-port"); 2092 (void) ddi_prop_remove(sc->dev, sc->dip, 2093 "holdoff-timer-idx-1G"); 2094 (void) ddi_prop_remove(sc->dev, sc->dip, "holdoff-pktc-idx-1G"); 2095 } 2096 2097 return (0); 2098 } 2099 2100 static int 2101 cfg_itype_and_nqueues(struct adapter *sc, int n10g, int n1g, 2102 struct intrs_and_queues *iaq) 2103 { 2104 struct driver_properties *p = &sc->props; 2105 int rc, itype, itypes, navail, nc, nrxq10g, nrxq1g, n; 2106 int nofldrxq10g = 0, nofldrxq1g = 0; 2107 2108 bzero(iaq, sizeof (*iaq)); 2109 nc = ncpus; /* our snapshot of the number of CPUs */ 2110 iaq->ntxq10g = min(nc, p->max_ntxq_10g); 2111 iaq->ntxq1g = min(nc, p->max_ntxq_1g); 2112 iaq->nrxq10g = nrxq10g = min(nc, p->max_nrxq_10g); 2113 iaq->nrxq1g = nrxq1g = min(nc, p->max_nrxq_1g); 2114 #ifdef TCP_OFFLOAD_ENABLE 2115 iaq->nofldtxq10g = min(nc, p->max_nofldtxq_10g); 2116 iaq->nofldtxq1g = min(nc, p->max_nofldtxq_1g); 2117 iaq->nofldrxq10g = nofldrxq10g = min(nc, p->max_nofldrxq_10g); 2118 iaq->nofldrxq1g = nofldrxq1g = min(nc, p->max_nofldrxq_1g); 2119 #endif 2120 2121 rc = ddi_intr_get_supported_types(sc->dip, &itypes); 2122 if (rc != DDI_SUCCESS) { 2123 cxgb_printf(sc->dip, CE_WARN, 2124 "failed to determine supported interrupt types: %d", rc); 2125 return (rc); 2126 } 2127 2128 for (itype = DDI_INTR_TYPE_MSIX; itype; itype >>= 1) { 2129 ASSERT(itype == DDI_INTR_TYPE_MSIX || 2130 itype == DDI_INTR_TYPE_MSI || 2131 itype == DDI_INTR_TYPE_FIXED); 2132 2133 if ((itype & itypes & p->intr_types) == 0) 2134 continue; /* not supported or not allowed */ 2135 2136 navail = 0; 2137 rc = ddi_intr_get_navail(sc->dip, itype, &navail); 2138 if (rc != DDI_SUCCESS || navail == 0) { 2139 cxgb_printf(sc->dip, CE_WARN, 2140 "failed to get # of interrupts for type %d: %d", 2141 itype, rc); 2142 continue; /* carry on */ 2143 } 2144 2145 iaq->intr_type = itype; 2146 if (navail == 0) 2147 continue; 2148 2149 /* 2150 * Best option: an interrupt vector for errors, one for the 2151 * firmware event queue, and one each for each rxq (NIC as well 2152 * as offload). 2153 */ 2154 iaq->nirq = T4_EXTRA_INTR; 2155 iaq->nirq += n10g * (nrxq10g + nofldrxq10g); 2156 iaq->nirq += n1g * (nrxq1g + nofldrxq1g); 2157 2158 if (iaq->nirq <= navail && 2159 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2160 iaq->intr_fwd = 0; 2161 goto allocate; 2162 } 2163 2164 /* 2165 * Second best option: an interrupt vector for errors, one for 2166 * the firmware event queue, and one each for either NIC or 2167 * offload rxq's. 2168 */ 2169 iaq->nirq = T4_EXTRA_INTR; 2170 iaq->nirq += n10g * max(nrxq10g, nofldrxq10g); 2171 iaq->nirq += n1g * max(nrxq1g, nofldrxq1g); 2172 if (iaq->nirq <= navail && 2173 (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq))) { 2174 iaq->intr_fwd = 1; 2175 goto allocate; 2176 } 2177 2178 /* 2179 * Next best option: an interrupt vector for errors, one for the 2180 * firmware event queue, and at least one per port. At this 2181 * point we know we'll have to downsize nrxq or nofldrxq to fit 2182 * what's available to us. 2183 */ 2184 iaq->nirq = T4_EXTRA_INTR; 2185 iaq->nirq += n10g + n1g; 2186 if (iaq->nirq <= navail) { 2187 int leftover = navail - iaq->nirq; 2188 2189 if (n10g > 0) { 2190 int target = max(nrxq10g, nofldrxq10g); 2191 2192 n = 1; 2193 while (n < target && leftover >= n10g) { 2194 leftover -= n10g; 2195 iaq->nirq += n10g; 2196 n++; 2197 } 2198 iaq->nrxq10g = min(n, nrxq10g); 2199 #ifdef TCP_OFFLOAD_ENABLE 2200 iaq->nofldrxq10g = min(n, nofldrxq10g); 2201 #endif 2202 } 2203 2204 if (n1g > 0) { 2205 int target = max(nrxq1g, nofldrxq1g); 2206 2207 n = 1; 2208 while (n < target && leftover >= n1g) { 2209 leftover -= n1g; 2210 iaq->nirq += n1g; 2211 n++; 2212 } 2213 iaq->nrxq1g = min(n, nrxq1g); 2214 #ifdef TCP_OFFLOAD_ENABLE 2215 iaq->nofldrxq1g = min(n, nofldrxq1g); 2216 #endif 2217 } 2218 2219 /* We have arrived at a minimum value required to enable 2220 * per queue irq(either NIC or offload). Thus for non- 2221 * offload case, we will get a vector per queue, while 2222 * offload case, we will get a vector per offload/NIC q. 2223 * Hence enable Interrupt forwarding only for offload 2224 * case. 2225 */ 2226 #ifdef TCP_OFFLOAD_ENABLE 2227 if (itype != DDI_INTR_TYPE_MSI || ISP2(iaq->nirq)) { 2228 iaq->intr_fwd = 1; 2229 #else 2230 if (itype != DDI_INTR_TYPE_MSI) { 2231 #endif 2232 goto allocate; 2233 } 2234 } 2235 2236 /* 2237 * Least desirable option: one interrupt vector for everything. 2238 */ 2239 iaq->nirq = iaq->nrxq10g = iaq->nrxq1g = 1; 2240 #ifdef TCP_OFFLOAD_ENABLE 2241 iaq->nofldrxq10g = iaq->nofldrxq1g = 1; 2242 #endif 2243 iaq->intr_fwd = 1; 2244 2245 allocate: 2246 return (0); 2247 } 2248 2249 cxgb_printf(sc->dip, CE_WARN, 2250 "failed to find a usable interrupt type. supported=%d, allowed=%d", 2251 itypes, p->intr_types); 2252 return (DDI_FAILURE); 2253 } 2254 2255 static int 2256 add_child_node(struct adapter *sc, int idx) 2257 { 2258 int rc; 2259 struct port_info *pi; 2260 2261 if (idx < 0 || idx >= sc->params.nports) 2262 return (EINVAL); 2263 2264 pi = sc->port[idx]; 2265 if (pi == NULL) 2266 return (ENODEV); /* t4_port_init failed earlier */ 2267 2268 PORT_LOCK(pi); 2269 if (pi->dip != NULL) { 2270 rc = 0; /* EEXIST really, but then bus_config fails */ 2271 goto done; 2272 } 2273 2274 rc = ndi_devi_alloc(sc->dip, T4_PORT_NAME, DEVI_SID_NODEID, &pi->dip); 2275 if (rc != DDI_SUCCESS || pi->dip == NULL) { 2276 rc = ENOMEM; 2277 goto done; 2278 } 2279 2280 (void) ddi_set_parent_data(pi->dip, pi); 2281 (void) ndi_devi_bind_driver(pi->dip, 0); 2282 rc = 0; 2283 done: 2284 PORT_UNLOCK(pi); 2285 return (rc); 2286 } 2287 2288 static int 2289 remove_child_node(struct adapter *sc, int idx) 2290 { 2291 int rc; 2292 struct port_info *pi; 2293 2294 if (idx < 0 || idx >= sc->params.nports) 2295 return (EINVAL); 2296 2297 pi = sc->port[idx]; 2298 if (pi == NULL) 2299 return (ENODEV); 2300 2301 PORT_LOCK(pi); 2302 if (pi->dip == NULL) { 2303 rc = ENODEV; 2304 goto done; 2305 } 2306 2307 rc = ndi_devi_free(pi->dip); 2308 if (rc == 0) 2309 pi->dip = NULL; 2310 done: 2311 PORT_UNLOCK(pi); 2312 return (rc); 2313 } 2314 2315 #define KS_UINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_ULONG) 2316 #define KS_CINIT(x) kstat_named_init(&kstatp->x, #x, KSTAT_DATA_CHAR) 2317 #define KS_U_SET(x, y) kstatp->x.value.ul = (y) 2318 #define KS_C_SET(x, ...) \ 2319 (void) snprintf(kstatp->x.value.c, 16, __VA_ARGS__) 2320 2321 /* 2322 * t4nex:X:config 2323 */ 2324 struct t4_kstats { 2325 kstat_named_t chip_ver; 2326 kstat_named_t fw_vers; 2327 kstat_named_t tp_vers; 2328 kstat_named_t driver_version; 2329 kstat_named_t serial_number; 2330 kstat_named_t ec_level; 2331 kstat_named_t id; 2332 kstat_named_t bus_type; 2333 kstat_named_t bus_width; 2334 kstat_named_t bus_speed; 2335 kstat_named_t core_clock; 2336 kstat_named_t port_cnt; 2337 kstat_named_t port_type; 2338 kstat_named_t pci_vendor_id; 2339 kstat_named_t pci_device_id; 2340 }; 2341 static kstat_t * 2342 setup_kstats(struct adapter *sc) 2343 { 2344 kstat_t *ksp; 2345 struct t4_kstats *kstatp; 2346 int ndata; 2347 struct pci_params *p = &sc->params.pci; 2348 struct vpd_params *v = &sc->params.vpd; 2349 uint16_t pci_vendor, pci_device; 2350 2351 ndata = sizeof (struct t4_kstats) / sizeof (kstat_named_t); 2352 2353 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "config", 2354 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2355 if (ksp == NULL) { 2356 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2357 return (NULL); 2358 } 2359 2360 kstatp = (struct t4_kstats *)ksp->ks_data; 2361 2362 KS_UINIT(chip_ver); 2363 KS_CINIT(fw_vers); 2364 KS_CINIT(tp_vers); 2365 KS_CINIT(driver_version); 2366 KS_CINIT(serial_number); 2367 KS_CINIT(ec_level); 2368 KS_CINIT(id); 2369 KS_CINIT(bus_type); 2370 KS_CINIT(bus_width); 2371 KS_CINIT(bus_speed); 2372 KS_UINIT(core_clock); 2373 KS_UINIT(port_cnt); 2374 KS_CINIT(port_type); 2375 KS_CINIT(pci_vendor_id); 2376 KS_CINIT(pci_device_id); 2377 2378 KS_U_SET(chip_ver, sc->params.chip); 2379 KS_C_SET(fw_vers, "%d.%d.%d.%d", 2380 G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers), 2381 G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers), 2382 G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers), 2383 G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers)); 2384 KS_C_SET(tp_vers, "%d.%d.%d.%d", 2385 G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers), 2386 G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers), 2387 G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers), 2388 G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers)); 2389 KS_C_SET(driver_version, DRV_VERSION); 2390 KS_C_SET(serial_number, "%s", v->sn); 2391 KS_C_SET(ec_level, "%s", v->ec); 2392 KS_C_SET(id, "%s", v->id); 2393 KS_C_SET(bus_type, "pci-express"); 2394 KS_C_SET(bus_width, "x%d lanes", p->width); 2395 KS_C_SET(bus_speed, "%d", p->speed); 2396 KS_U_SET(core_clock, v->cclk); 2397 KS_U_SET(port_cnt, sc->params.nports); 2398 2399 t4_os_pci_read_cfg2(sc, PCI_CONF_VENID, &pci_vendor); 2400 KS_C_SET(pci_vendor_id, "0x%x", pci_vendor); 2401 2402 t4_os_pci_read_cfg2(sc, PCI_CONF_DEVID, &pci_device); 2403 KS_C_SET(pci_device_id, "0x%x", pci_device); 2404 2405 KS_C_SET(port_type, "%s/%s/%s/%s", 2406 print_port_speed(sc->port[0]), 2407 print_port_speed(sc->port[1]), 2408 print_port_speed(sc->port[2]), 2409 print_port_speed(sc->port[3])); 2410 2411 /* Do NOT set ksp->ks_update. These kstats do not change. */ 2412 2413 /* Install the kstat */ 2414 ksp->ks_private = (void *)sc; 2415 kstat_install(ksp); 2416 2417 return (ksp); 2418 } 2419 2420 /* 2421 * t4nex:X:stat 2422 */ 2423 struct t4_wc_kstats { 2424 kstat_named_t write_coal_success; 2425 kstat_named_t write_coal_failure; 2426 }; 2427 static kstat_t * 2428 setup_wc_kstats(struct adapter *sc) 2429 { 2430 kstat_t *ksp; 2431 struct t4_wc_kstats *kstatp; 2432 int ndata; 2433 2434 ndata = sizeof(struct t4_wc_kstats) / sizeof(kstat_named_t); 2435 ksp = kstat_create(T4_NEXUS_NAME, ddi_get_instance(sc->dip), "stats", 2436 "nexus", KSTAT_TYPE_NAMED, ndata, 0); 2437 if (ksp == NULL) { 2438 cxgb_printf(sc->dip, CE_WARN, "failed to initialize kstats."); 2439 return (NULL); 2440 } 2441 2442 kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2443 2444 KS_UINIT(write_coal_success); 2445 KS_UINIT(write_coal_failure); 2446 2447 ksp->ks_update = update_wc_kstats; 2448 /* Install the kstat */ 2449 ksp->ks_private = (void *)sc; 2450 kstat_install(ksp); 2451 2452 return (ksp); 2453 } 2454 2455 static int 2456 update_wc_kstats(kstat_t *ksp, int rw) 2457 { 2458 struct t4_wc_kstats *kstatp = (struct t4_wc_kstats *)ksp->ks_data; 2459 struct adapter *sc = ksp->ks_private; 2460 uint32_t wc_total, wc_success, wc_failure; 2461 2462 if (rw == KSTAT_WRITE) 2463 return (0); 2464 2465 if (is_t5(sc->params.chip)) { 2466 wc_total = t4_read_reg(sc, A_SGE_STAT_TOTAL); 2467 wc_failure = t4_read_reg(sc, A_SGE_STAT_MATCH); 2468 wc_success = wc_total - wc_failure; 2469 } else { 2470 wc_success = 0; 2471 wc_failure = 0; 2472 } 2473 2474 KS_U_SET(write_coal_success, wc_success); 2475 KS_U_SET(write_coal_failure, wc_failure); 2476 2477 return (0); 2478 } 2479 2480 int 2481 adapter_full_init(struct adapter *sc) 2482 { 2483 int i, rc = 0; 2484 2485 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2486 2487 rc = t4_setup_adapter_queues(sc); 2488 if (rc != 0) 2489 goto done; 2490 2491 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2492 (void) ddi_intr_block_enable(sc->intr_handle, sc->intr_count); 2493 else { 2494 for (i = 0; i < sc->intr_count; i++) 2495 (void) ddi_intr_enable(sc->intr_handle[i]); 2496 } 2497 t4_intr_enable(sc); 2498 sc->flags |= FULL_INIT_DONE; 2499 2500 #ifdef TCP_OFFLOAD_ENABLE 2501 /* TODO: wrong place to enable TOE capability */ 2502 if (is_offload(sc) != 0) { 2503 for_each_port(sc, i) { 2504 struct port_info *pi = sc->port[i]; 2505 rc = toe_capability(pi, 1); 2506 if (rc != 0) { 2507 cxgb_printf(pi->dip, CE_WARN, 2508 "Failed to activate toe capability: %d", 2509 rc); 2510 rc = 0; /* not a fatal error */ 2511 } 2512 } 2513 } 2514 #endif 2515 2516 done: 2517 if (rc != 0) 2518 (void) adapter_full_uninit(sc); 2519 2520 return (rc); 2521 } 2522 2523 int 2524 adapter_full_uninit(struct adapter *sc) 2525 { 2526 int i, rc = 0; 2527 2528 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2529 2530 if (sc->intr_cap & DDI_INTR_FLAG_BLOCK) 2531 (void) ddi_intr_block_disable(sc->intr_handle, sc->intr_count); 2532 else { 2533 for (i = 0; i < sc->intr_count; i++) 2534 (void) ddi_intr_disable(sc->intr_handle[i]); 2535 } 2536 2537 rc = t4_teardown_adapter_queues(sc); 2538 if (rc != 0) 2539 return (rc); 2540 2541 sc->flags &= ~FULL_INIT_DONE; 2542 2543 return (0); 2544 } 2545 2546 int 2547 port_full_init(struct port_info *pi) 2548 { 2549 struct adapter *sc = pi->adapter; 2550 uint16_t *rss; 2551 struct sge_rxq *rxq; 2552 int rc, i; 2553 2554 ADAPTER_LOCK_ASSERT_NOTOWNED(sc); 2555 ASSERT((pi->flags & PORT_INIT_DONE) == 0); 2556 2557 /* 2558 * Allocate tx/rx/fl queues for this port. 2559 */ 2560 rc = t4_setup_port_queues(pi); 2561 if (rc != 0) 2562 goto done; /* error message displayed already */ 2563 2564 /* 2565 * Setup RSS for this port. 2566 */ 2567 rss = kmem_zalloc(pi->nrxq * sizeof (*rss), KM_SLEEP); 2568 for_each_rxq(pi, i, rxq) { 2569 rss[i] = rxq->iq.abs_id; 2570 } 2571 rc = -t4_config_rss_range(sc, sc->mbox, pi->viid, 0, 2572 pi->rss_size, rss, pi->nrxq); 2573 kmem_free(rss, pi->nrxq * sizeof (*rss)); 2574 if (rc != 0) { 2575 cxgb_printf(pi->dip, CE_WARN, "rss_config failed: %d", rc); 2576 goto done; 2577 } 2578 2579 pi->flags |= PORT_INIT_DONE; 2580 done: 2581 if (rc != 0) 2582 (void) port_full_uninit(pi); 2583 2584 return (rc); 2585 } 2586 2587 /* 2588 * Idempotent. 2589 */ 2590 int 2591 port_full_uninit(struct port_info *pi) 2592 { 2593 2594 ASSERT(pi->flags & PORT_INIT_DONE); 2595 2596 (void) t4_teardown_port_queues(pi); 2597 pi->flags &= ~PORT_INIT_DONE; 2598 2599 return (0); 2600 } 2601 2602 void 2603 enable_port_queues(struct port_info *pi) 2604 { 2605 struct adapter *sc = pi->adapter; 2606 int i; 2607 struct sge_iq *iq; 2608 struct sge_rxq *rxq; 2609 #ifdef TCP_OFFLOAD_ENABLE 2610 struct sge_ofld_rxq *ofld_rxq; 2611 #endif 2612 2613 ASSERT(pi->flags & PORT_INIT_DONE); 2614 2615 /* 2616 * TODO: whatever was queued up after we set iq->state to IQS_DISABLED 2617 * back in disable_port_queues will be processed now, after an unbounded 2618 * delay. This can't be good. 2619 */ 2620 2621 #ifdef TCP_OFFLOAD_ENABLE 2622 for_each_ofld_rxq(pi, i, ofld_rxq) { 2623 iq = &ofld_rxq->iq; 2624 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2625 IQS_DISABLED) 2626 panic("%s: iq %p wasn't disabled", __func__, 2627 (void *)iq); 2628 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2629 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2630 } 2631 #endif 2632 2633 for_each_rxq(pi, i, rxq) { 2634 iq = &rxq->iq; 2635 if (atomic_cas_uint(&iq->state, IQS_DISABLED, IQS_IDLE) != 2636 IQS_DISABLED) 2637 panic("%s: iq %p wasn't disabled", __func__, 2638 (void *) iq); 2639 t4_write_reg(sc, MYPF_REG(A_SGE_PF_GTS), 2640 V_SEINTARM(iq->intr_params) | V_INGRESSQID(iq->cntxt_id)); 2641 } 2642 } 2643 2644 void 2645 disable_port_queues(struct port_info *pi) 2646 { 2647 int i; 2648 struct adapter *sc = pi->adapter; 2649 struct sge_rxq *rxq; 2650 #ifdef TCP_OFFLOAD_ENABLE 2651 struct sge_ofld_rxq *ofld_rxq; 2652 #endif 2653 2654 ASSERT(pi->flags & PORT_INIT_DONE); 2655 2656 /* 2657 * TODO: need proper implementation for all tx queues (ctrl, eth, ofld). 2658 */ 2659 2660 #ifdef TCP_OFFLOAD_ENABLE 2661 for_each_ofld_rxq(pi, i, ofld_rxq) { 2662 while (atomic_cas_uint(&ofld_rxq->iq.state, IQS_IDLE, 2663 IQS_DISABLED) != IQS_IDLE) 2664 msleep(1); 2665 } 2666 #endif 2667 2668 for_each_rxq(pi, i, rxq) { 2669 while (atomic_cas_uint(&rxq->iq.state, IQS_IDLE, 2670 IQS_DISABLED) != IQS_IDLE) 2671 msleep(1); 2672 } 2673 2674 mutex_enter(&sc->sfl_lock); 2675 #ifdef TCP_OFFLOAD_ENABLE 2676 for_each_ofld_rxq(pi, i, ofld_rxq) 2677 ofld_rxq->fl.flags |= FL_DOOMED; 2678 #endif 2679 for_each_rxq(pi, i, rxq) 2680 rxq->fl.flags |= FL_DOOMED; 2681 mutex_exit(&sc->sfl_lock); 2682 /* TODO: need to wait for all fl's to be removed from sc->sfl */ 2683 } 2684 2685 void 2686 t4_fatal_err(struct adapter *sc) 2687 { 2688 t4_set_reg_field(sc, A_SGE_CONTROL, F_GLOBALENABLE, 0); 2689 t4_intr_disable(sc); 2690 cxgb_printf(sc->dip, CE_WARN, 2691 "encountered fatal error, adapter stopped."); 2692 } 2693 2694 int 2695 t4_os_find_pci_capability(struct adapter *sc, int cap) 2696 { 2697 uint16_t stat; 2698 uint8_t cap_ptr, cap_id; 2699 2700 t4_os_pci_read_cfg2(sc, PCI_CONF_STAT, &stat); 2701 if ((stat & PCI_STAT_CAP) == 0) 2702 return (0); /* does not implement capabilities */ 2703 2704 t4_os_pci_read_cfg1(sc, PCI_CONF_CAP_PTR, &cap_ptr); 2705 while (cap_ptr) { 2706 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_ID, &cap_id); 2707 if (cap_id == cap) 2708 return (cap_ptr); /* found */ 2709 t4_os_pci_read_cfg1(sc, cap_ptr + PCI_CAP_NEXT_PTR, &cap_ptr); 2710 } 2711 2712 return (0); /* not found */ 2713 } 2714 2715 void 2716 t4_os_portmod_changed(const struct adapter *sc, int idx) 2717 { 2718 static const char *mod_str[] = { 2719 NULL, "LR", "SR", "ER", "TWINAX", "active TWINAX", "LRM" 2720 }; 2721 const struct port_info *pi = sc->port[idx]; 2722 2723 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 2724 cxgb_printf(pi->dip, CE_NOTE, "transceiver unplugged."); 2725 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 2726 cxgb_printf(pi->dip, CE_NOTE, 2727 "unknown transceiver inserted.\n"); 2728 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 2729 cxgb_printf(pi->dip, CE_NOTE, 2730 "unsupported transceiver inserted.\n"); 2731 else if (pi->mod_type > 0 && pi->mod_type < ARRAY_SIZE(mod_str)) 2732 cxgb_printf(pi->dip, CE_NOTE, "%s transceiver inserted.\n", 2733 mod_str[pi->mod_type]); 2734 else 2735 cxgb_printf(pi->dip, CE_NOTE, "transceiver (type %d) inserted.", 2736 pi->mod_type); 2737 } 2738 2739 /* ARGSUSED */ 2740 static int 2741 cpl_not_handled(struct sge_iq *iq, const struct rss_header *rss, mblk_t *m) 2742 { 2743 if (m != NULL) 2744 freemsg(m); 2745 return (0); 2746 } 2747 2748 int 2749 t4_register_cpl_handler(struct adapter *sc, int opcode, cpl_handler_t h) 2750 { 2751 uint_t *loc, new; 2752 2753 if (opcode >= ARRAY_SIZE(sc->cpl_handler)) 2754 return (EINVAL); 2755 2756 new = (uint_t)(unsigned long) (h ? h : cpl_not_handled); 2757 loc = (uint_t *)&sc->cpl_handler[opcode]; 2758 (void) atomic_swap_uint(loc, new); 2759 2760 return (0); 2761 } 2762 2763 static int 2764 fw_msg_not_handled(struct adapter *sc, const __be64 *data) 2765 { 2766 struct cpl_fw6_msg *cpl; 2767 2768 cpl = __containerof((void *)data, struct cpl_fw6_msg, data); 2769 2770 cxgb_printf(sc->dip, CE_WARN, "%s fw_msg type %d", __func__, cpl->type); 2771 return (0); 2772 } 2773 2774 int 2775 t4_register_fw_msg_handler(struct adapter *sc, int type, fw_msg_handler_t h) 2776 { 2777 fw_msg_handler_t *loc, new; 2778 2779 if (type >= ARRAY_SIZE(sc->fw_msg_handler)) 2780 return (EINVAL); 2781 2782 /* 2783 * These are dispatched by the handler for FW{4|6}_CPL_MSG using the CPL 2784 * handler dispatch table. Reject any attempt to install a handler for 2785 * this subtype. 2786 */ 2787 if (type == FW_TYPE_RSSCPL || type == FW6_TYPE_RSSCPL) 2788 return (EINVAL); 2789 2790 new = h ? h : fw_msg_not_handled; 2791 loc = &sc->fw_msg_handler[type]; 2792 (void)atomic_swap_ptr(loc, (void *)new); 2793 2794 return (0); 2795 } 2796 2797 #ifdef TCP_OFFLOAD_ENABLE 2798 static int 2799 toe_capability(struct port_info *pi, int enable) 2800 { 2801 int rc; 2802 struct adapter *sc = pi->adapter; 2803 2804 if (!is_offload(sc)) 2805 return (ENODEV); 2806 2807 if (enable != 0) { 2808 if (isset(&sc->offload_map, pi->port_id) != 0) 2809 return (0); 2810 2811 if (sc->offload_map == 0) { 2812 rc = activate_uld(sc, ULD_TOM, &sc->tom); 2813 if (rc != 0) 2814 return (rc); 2815 } 2816 2817 setbit(&sc->offload_map, pi->port_id); 2818 } else { 2819 if (!isset(&sc->offload_map, pi->port_id)) 2820 return (0); 2821 2822 clrbit(&sc->offload_map, pi->port_id); 2823 2824 if (sc->offload_map == 0) { 2825 rc = deactivate_uld(&sc->tom); 2826 if (rc != 0) { 2827 setbit(&sc->offload_map, pi->port_id); 2828 return (rc); 2829 } 2830 } 2831 } 2832 2833 return (0); 2834 } 2835 2836 /* 2837 * Add an upper layer driver to the global list. 2838 */ 2839 int 2840 t4_register_uld(struct uld_info *ui) 2841 { 2842 int rc = 0; 2843 struct uld_info *u; 2844 2845 mutex_enter(&t4_uld_list_lock); 2846 SLIST_FOREACH(u, &t4_uld_list, link) { 2847 if (u->uld_id == ui->uld_id) { 2848 rc = EEXIST; 2849 goto done; 2850 } 2851 } 2852 2853 SLIST_INSERT_HEAD(&t4_uld_list, ui, link); 2854 ui->refcount = 0; 2855 done: 2856 mutex_exit(&t4_uld_list_lock); 2857 return (rc); 2858 } 2859 2860 int 2861 t4_unregister_uld(struct uld_info *ui) 2862 { 2863 int rc = EINVAL; 2864 struct uld_info *u; 2865 2866 mutex_enter(&t4_uld_list_lock); 2867 2868 SLIST_FOREACH(u, &t4_uld_list, link) { 2869 if (u == ui) { 2870 if (ui->refcount > 0) { 2871 rc = EBUSY; 2872 goto done; 2873 } 2874 2875 SLIST_REMOVE(&t4_uld_list, ui, uld_info, link); 2876 rc = 0; 2877 goto done; 2878 } 2879 } 2880 done: 2881 mutex_exit(&t4_uld_list_lock); 2882 return (rc); 2883 } 2884 2885 static int 2886 activate_uld(struct adapter *sc, int id, struct uld_softc *usc) 2887 { 2888 int rc = EAGAIN; 2889 struct uld_info *ui; 2890 2891 mutex_enter(&t4_uld_list_lock); 2892 2893 SLIST_FOREACH(ui, &t4_uld_list, link) { 2894 if (ui->uld_id == id) { 2895 rc = ui->attach(sc, &usc->softc); 2896 if (rc == 0) { 2897 ASSERT(usc->softc != NULL); 2898 ui->refcount++; 2899 usc->uld = ui; 2900 } 2901 goto done; 2902 } 2903 } 2904 done: 2905 mutex_exit(&t4_uld_list_lock); 2906 2907 return (rc); 2908 } 2909 2910 static int 2911 deactivate_uld(struct uld_softc *usc) 2912 { 2913 int rc; 2914 2915 mutex_enter(&t4_uld_list_lock); 2916 2917 if (usc->uld == NULL || usc->softc == NULL) { 2918 rc = EINVAL; 2919 goto done; 2920 } 2921 2922 rc = usc->uld->detach(usc->softc); 2923 if (rc == 0) { 2924 ASSERT(usc->uld->refcount > 0); 2925 usc->uld->refcount--; 2926 usc->uld = NULL; 2927 usc->softc = NULL; 2928 } 2929 done: 2930 mutex_exit(&t4_uld_list_lock); 2931 2932 return (rc); 2933 } 2934 2935 void 2936 t4_iterate(void (*func)(int, void *), void *arg) 2937 { 2938 struct adapter *sc; 2939 2940 mutex_enter(&t4_adapter_list_lock); 2941 SLIST_FOREACH(sc, &t4_adapter_list, link) { 2942 /* 2943 * func should not make any assumptions about what state sc is 2944 * in - the only guarantee is that sc->sc_lock is a valid lock. 2945 */ 2946 func(ddi_get_instance(sc->dip), arg); 2947 } 2948 mutex_exit(&t4_adapter_list_lock); 2949 } 2950 2951 #endif 2952 2953 static int 2954 t4_sensor_read(struct adapter *sc, uint32_t diag, uint32_t *valp) 2955 { 2956 int rc; 2957 struct port_info *pi = sc->port[0]; 2958 uint32_t param, val; 2959 2960 rc = begin_synchronized_op(pi, 1, 1); 2961 if (rc != 0) { 2962 return (rc); 2963 } 2964 param = V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | 2965 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_DIAG) | 2966 V_FW_PARAMS_PARAM_Y(diag); 2967 rc = -t4_query_params(sc, sc->mbox, sc->pf, 0, 1, ¶m, &val); 2968 end_synchronized_op(pi, 1); 2969 2970 if (rc != 0) { 2971 return (rc); 2972 } 2973 2974 if (val == 0) { 2975 return (EIO); 2976 } 2977 2978 *valp = val; 2979 return (0); 2980 } 2981 2982 static int 2983 t4_temperature_read(void *arg, sensor_ioctl_scalar_t *scalar) 2984 { 2985 int ret; 2986 struct adapter *sc = arg; 2987 uint32_t val; 2988 2989 ret = t4_sensor_read(sc, FW_PARAM_DEV_DIAG_TMP, &val); 2990 if (ret != 0) { 2991 return (ret); 2992 } 2993 2994 /* 2995 * The device measures temperature in units of 1 degree Celsius. We 2996 * don't know its precision. 2997 */ 2998 scalar->sis_unit = SENSOR_UNIT_CELSIUS; 2999 scalar->sis_gran = 1; 3000 scalar->sis_prec = 0; 3001 scalar->sis_value = val; 3002 3003 return (0); 3004 } 3005 3006 static int 3007 t4_voltage_read(void *arg, sensor_ioctl_scalar_t *scalar) 3008 { 3009 int ret; 3010 struct adapter *sc = arg; 3011 uint32_t val; 3012 3013 ret = t4_sensor_read(sc, FW_PARAM_DEV_DIAG_VDD, &val); 3014 if (ret != 0) { 3015 return (ret); 3016 } 3017 3018 scalar->sis_unit = SENSOR_UNIT_VOLTS; 3019 scalar->sis_gran = 1000; 3020 scalar->sis_prec = 0; 3021 scalar->sis_value = val; 3022 3023 return (0); 3024 } 3025