1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 25 * Use is subject to license terms. 26 */ 27 28 #include "ixgbe_sw.h" 29 30 static char ident[] = "Intel 10Gb Ethernet 1.1.0"; 31 32 /* 33 * Local function protoypes 34 */ 35 static int ixgbe_register_mac(ixgbe_t *); 36 static int ixgbe_identify_hardware(ixgbe_t *); 37 static int ixgbe_regs_map(ixgbe_t *); 38 static void ixgbe_init_properties(ixgbe_t *); 39 static int ixgbe_init_driver_settings(ixgbe_t *); 40 static void ixgbe_init_locks(ixgbe_t *); 41 static void ixgbe_destroy_locks(ixgbe_t *); 42 static int ixgbe_init(ixgbe_t *); 43 static int ixgbe_chip_start(ixgbe_t *); 44 static void ixgbe_chip_stop(ixgbe_t *); 45 static int ixgbe_reset(ixgbe_t *); 46 static void ixgbe_tx_clean(ixgbe_t *); 47 static boolean_t ixgbe_tx_drain(ixgbe_t *); 48 static boolean_t ixgbe_rx_drain(ixgbe_t *); 49 static int ixgbe_alloc_rings(ixgbe_t *); 50 static void ixgbe_free_rings(ixgbe_t *); 51 static int ixgbe_alloc_rx_data(ixgbe_t *); 52 static void ixgbe_free_rx_data(ixgbe_t *); 53 static void ixgbe_setup_rings(ixgbe_t *); 54 static void ixgbe_setup_rx(ixgbe_t *); 55 static void ixgbe_setup_tx(ixgbe_t *); 56 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 57 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 58 static void ixgbe_setup_rss(ixgbe_t *); 59 static void ixgbe_init_unicst(ixgbe_t *); 60 static int ixgbe_unicst_set(ixgbe_t *, const uint8_t *, int); 61 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 62 static void ixgbe_setup_multicst(ixgbe_t *); 63 static void ixgbe_get_hw_state(ixgbe_t *); 64 static void ixgbe_get_conf(ixgbe_t *); 65 static void ixgbe_init_params(ixgbe_t *); 66 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 67 static void ixgbe_driver_link_check(void *); 68 static void ixgbe_sfp_check(void *); 69 static void ixgbe_local_timer(void *); 70 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 71 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 72 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 73 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 74 static boolean_t is_valid_mac_addr(uint8_t *); 75 static boolean_t ixgbe_stall_check(ixgbe_t *); 76 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 77 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 78 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 79 static int ixgbe_alloc_intrs(ixgbe_t *); 80 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 81 static int ixgbe_add_intr_handlers(ixgbe_t *); 82 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 83 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 84 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 85 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 86 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 87 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 88 static void ixgbe_setup_adapter_vector(ixgbe_t *); 89 static void ixgbe_rem_intr_handlers(ixgbe_t *); 90 static void ixgbe_rem_intrs(ixgbe_t *); 91 static int ixgbe_enable_intrs(ixgbe_t *); 92 static int ixgbe_disable_intrs(ixgbe_t *); 93 static uint_t ixgbe_intr_legacy(void *, void *); 94 static uint_t ixgbe_intr_msi(void *, void *); 95 static uint_t ixgbe_intr_msix(void *, void *); 96 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 97 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 98 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 99 static void ixgbe_get_driver_control(struct ixgbe_hw *); 100 static int ixgbe_addmac(void *, const uint8_t *); 101 static int ixgbe_remmac(void *, const uint8_t *); 102 static void ixgbe_release_driver_control(struct ixgbe_hw *); 103 104 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 105 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 106 static int ixgbe_resume(dev_info_t *); 107 static int ixgbe_suspend(dev_info_t *); 108 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 109 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 110 111 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 112 const void *impl_data); 113 static void ixgbe_fm_init(ixgbe_t *); 114 static void ixgbe_fm_fini(ixgbe_t *); 115 116 mac_priv_prop_t ixgbe_priv_props[] = { 117 {"_tx_copy_thresh", MAC_PROP_PERM_RW}, 118 {"_tx_recycle_thresh", MAC_PROP_PERM_RW}, 119 {"_tx_overload_thresh", MAC_PROP_PERM_RW}, 120 {"_tx_resched_thresh", MAC_PROP_PERM_RW}, 121 {"_rx_copy_thresh", MAC_PROP_PERM_RW}, 122 {"_rx_limit_per_intr", MAC_PROP_PERM_RW}, 123 {"_intr_throttling", MAC_PROP_PERM_RW}, 124 {"_adv_pause_cap", MAC_PROP_PERM_READ}, 125 {"_adv_asym_pause_cap", MAC_PROP_PERM_READ} 126 }; 127 128 #define IXGBE_MAX_PRIV_PROPS \ 129 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 130 131 static struct cb_ops ixgbe_cb_ops = { 132 nulldev, /* cb_open */ 133 nulldev, /* cb_close */ 134 nodev, /* cb_strategy */ 135 nodev, /* cb_print */ 136 nodev, /* cb_dump */ 137 nodev, /* cb_read */ 138 nodev, /* cb_write */ 139 nodev, /* cb_ioctl */ 140 nodev, /* cb_devmap */ 141 nodev, /* cb_mmap */ 142 nodev, /* cb_segmap */ 143 nochpoll, /* cb_chpoll */ 144 ddi_prop_op, /* cb_prop_op */ 145 NULL, /* cb_stream */ 146 D_MP | D_HOTPLUG, /* cb_flag */ 147 CB_REV, /* cb_rev */ 148 nodev, /* cb_aread */ 149 nodev /* cb_awrite */ 150 }; 151 152 static struct dev_ops ixgbe_dev_ops = { 153 DEVO_REV, /* devo_rev */ 154 0, /* devo_refcnt */ 155 NULL, /* devo_getinfo */ 156 nulldev, /* devo_identify */ 157 nulldev, /* devo_probe */ 158 ixgbe_attach, /* devo_attach */ 159 ixgbe_detach, /* devo_detach */ 160 nodev, /* devo_reset */ 161 &ixgbe_cb_ops, /* devo_cb_ops */ 162 NULL, /* devo_bus_ops */ 163 ddi_power, /* devo_power */ 164 ddi_quiesce_not_supported, /* devo_quiesce */ 165 }; 166 167 static struct modldrv ixgbe_modldrv = { 168 &mod_driverops, /* Type of module. This one is a driver */ 169 ident, /* Discription string */ 170 &ixgbe_dev_ops /* driver ops */ 171 }; 172 173 static struct modlinkage ixgbe_modlinkage = { 174 MODREV_1, &ixgbe_modldrv, NULL 175 }; 176 177 /* 178 * Access attributes for register mapping 179 */ 180 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 181 DDI_DEVICE_ATTR_V0, 182 DDI_STRUCTURE_LE_ACC, 183 DDI_STRICTORDER_ACC, 184 DDI_FLAGERR_ACC 185 }; 186 187 /* 188 * Loopback property 189 */ 190 static lb_property_t lb_normal = { 191 normal, "normal", IXGBE_LB_NONE 192 }; 193 194 static lb_property_t lb_mac = { 195 internal, "MAC", IXGBE_LB_INTERNAL_MAC 196 }; 197 198 #define IXGBE_M_CALLBACK_FLAGS \ 199 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP) 200 201 static mac_callbacks_t ixgbe_m_callbacks = { 202 IXGBE_M_CALLBACK_FLAGS, 203 ixgbe_m_stat, 204 ixgbe_m_start, 205 ixgbe_m_stop, 206 ixgbe_m_promisc, 207 ixgbe_m_multicst, 208 NULL, 209 NULL, 210 ixgbe_m_ioctl, 211 ixgbe_m_getcapab, 212 NULL, 213 NULL, 214 ixgbe_m_setprop, 215 ixgbe_m_getprop 216 }; 217 218 /* 219 * Initialize capabilities of each supported adapter type 220 */ 221 static adapter_info_t ixgbe_82598eb_cap = { 222 64, /* maximum number of rx queues */ 223 1, /* minimum number of rx queues */ 224 8, /* default number of rx queues */ 225 32, /* maximum number of tx queues */ 226 1, /* minimum number of tx queues */ 227 8, /* default number of tx queues */ 228 0xFFFF, /* maximum interrupt throttle rate */ 229 0, /* minimum interrupt throttle rate */ 230 200, /* default interrupt throttle rate */ 231 18, /* maximum total msix vectors */ 232 16, /* maximum number of ring vectors */ 233 2, /* maximum number of other vectors */ 234 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 235 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 236 | IXGBE_FLAG_RSS_CAPABLE 237 | IXGBE_FLAG_VMDQ_CAPABLE) 238 }; 239 240 static adapter_info_t ixgbe_82599eb_cap = { 241 128, /* maximum number of rx queues */ 242 1, /* minimum number of rx queues */ 243 8, /* default number of rx queues */ 244 128, /* maximum number of tx queues */ 245 1, /* minimum number of tx queues */ 246 8, /* default number of tx queues */ 247 0xFF8, /* maximum interrupt throttle rate */ 248 0, /* minimum interrupt throttle rate */ 249 200, /* default interrupt throttle rate */ 250 64, /* maximum total msix vectors */ 251 16, /* maximum number of ring vectors */ 252 2, /* maximum number of other vectors */ 253 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 254 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 255 | IXGBE_FLAG_RSS_CAPABLE 256 | IXGBE_FLAG_VMDQ_CAPABLE) 257 }; 258 259 /* 260 * Module Initialization Functions. 261 */ 262 263 int 264 _init(void) 265 { 266 int status; 267 268 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 269 270 status = mod_install(&ixgbe_modlinkage); 271 272 if (status != DDI_SUCCESS) { 273 mac_fini_ops(&ixgbe_dev_ops); 274 } 275 276 return (status); 277 } 278 279 int 280 _fini(void) 281 { 282 int status; 283 284 status = mod_remove(&ixgbe_modlinkage); 285 286 if (status == DDI_SUCCESS) { 287 mac_fini_ops(&ixgbe_dev_ops); 288 } 289 290 return (status); 291 } 292 293 int 294 _info(struct modinfo *modinfop) 295 { 296 int status; 297 298 status = mod_info(&ixgbe_modlinkage, modinfop); 299 300 return (status); 301 } 302 303 /* 304 * ixgbe_attach - Driver attach. 305 * 306 * This function is the device specific initialization entry 307 * point. This entry point is required and must be written. 308 * The DDI_ATTACH command must be provided in the attach entry 309 * point. When attach() is called with cmd set to DDI_ATTACH, 310 * all normal kernel services (such as kmem_alloc(9F)) are 311 * available for use by the driver. 312 * 313 * The attach() function will be called once for each instance 314 * of the device on the system with cmd set to DDI_ATTACH. 315 * Until attach() succeeds, the only driver entry points which 316 * may be called are open(9E) and getinfo(9E). 317 */ 318 static int 319 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 320 { 321 ixgbe_t *ixgbe; 322 struct ixgbe_osdep *osdep; 323 struct ixgbe_hw *hw; 324 int instance; 325 char taskqname[32]; 326 327 /* 328 * Check the command and perform corresponding operations 329 */ 330 switch (cmd) { 331 default: 332 return (DDI_FAILURE); 333 334 case DDI_RESUME: 335 return (ixgbe_resume(devinfo)); 336 337 case DDI_ATTACH: 338 break; 339 } 340 341 /* Get the device instance */ 342 instance = ddi_get_instance(devinfo); 343 344 /* Allocate memory for the instance data structure */ 345 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 346 347 ixgbe->dip = devinfo; 348 ixgbe->instance = instance; 349 350 hw = &ixgbe->hw; 351 osdep = &ixgbe->osdep; 352 hw->back = osdep; 353 osdep->ixgbe = ixgbe; 354 355 /* Attach the instance pointer to the dev_info data structure */ 356 ddi_set_driver_private(devinfo, ixgbe); 357 358 /* 359 * Initialize for fma support 360 */ 361 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 362 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 363 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 364 ixgbe_fm_init(ixgbe); 365 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 366 367 /* 368 * Map PCI config space registers 369 */ 370 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 371 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 372 goto attach_fail; 373 } 374 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 375 376 /* 377 * Identify the chipset family 378 */ 379 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 380 ixgbe_error(ixgbe, "Failed to identify hardware"); 381 goto attach_fail; 382 } 383 384 /* 385 * Map device registers 386 */ 387 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 388 ixgbe_error(ixgbe, "Failed to map device registers"); 389 goto attach_fail; 390 } 391 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 392 393 /* 394 * Initialize driver parameters 395 */ 396 ixgbe_init_properties(ixgbe); 397 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 398 399 /* 400 * Allocate interrupts 401 */ 402 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 403 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 404 goto attach_fail; 405 } 406 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 407 408 /* 409 * Allocate rx/tx rings based on the ring numbers. 410 * The actual numbers of rx/tx rings are decided by the number of 411 * allocated interrupt vectors, so we should allocate the rings after 412 * interrupts are allocated. 413 */ 414 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 415 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 416 goto attach_fail; 417 } 418 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 419 420 /* 421 * Map rings to interrupt vectors 422 */ 423 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 424 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 425 goto attach_fail; 426 } 427 428 /* 429 * Add interrupt handlers 430 */ 431 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 432 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 433 goto attach_fail; 434 } 435 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 436 437 /* 438 * Create a taskq for link-status-change 439 */ 440 (void) sprintf(taskqname, "ixgbe%d_taskq", instance); 441 if ((ixgbe->lsc_taskq = ddi_taskq_create(devinfo, taskqname, 442 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 443 ixgbe_error(ixgbe, "taskq_create failed"); 444 goto attach_fail; 445 } 446 ixgbe->attach_progress |= ATTACH_PROGRESS_LSC_TASKQ; 447 448 /* 449 * Initialize driver parameters 450 */ 451 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 452 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 453 goto attach_fail; 454 } 455 456 /* 457 * Initialize mutexes for this device. 458 * Do this before enabling the interrupt handler and 459 * register the softint to avoid the condition where 460 * interrupt handler can try using uninitialized mutex. 461 */ 462 ixgbe_init_locks(ixgbe); 463 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 464 465 /* 466 * Initialize chipset hardware 467 */ 468 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 469 ixgbe_error(ixgbe, "Failed to initialize adapter"); 470 goto attach_fail; 471 } 472 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 473 474 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 475 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 476 goto attach_fail; 477 } 478 479 /* 480 * Initialize statistics 481 */ 482 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 483 ixgbe_error(ixgbe, "Failed to initialize statistics"); 484 goto attach_fail; 485 } 486 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 487 488 /* 489 * Register the driver to the MAC 490 */ 491 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 492 ixgbe_error(ixgbe, "Failed to register MAC"); 493 goto attach_fail; 494 } 495 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 496 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 497 498 /* 499 * Now that mutex locks are initialized, and the chip is also 500 * initialized, enable interrupts. 501 */ 502 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 503 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 504 goto attach_fail; 505 } 506 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 507 508 ixgbe->ixgbe_state |= IXGBE_INITIALIZED; 509 510 return (DDI_SUCCESS); 511 512 attach_fail: 513 ixgbe_unconfigure(devinfo, ixgbe); 514 return (DDI_FAILURE); 515 } 516 517 /* 518 * ixgbe_detach - Driver detach. 519 * 520 * The detach() function is the complement of the attach routine. 521 * If cmd is set to DDI_DETACH, detach() is used to remove the 522 * state associated with a given instance of a device node 523 * prior to the removal of that instance from the system. 524 * 525 * The detach() function will be called once for each instance 526 * of the device for which there has been a successful attach() 527 * once there are no longer any opens on the device. 528 * 529 * Interrupts routine are disabled, All memory allocated by this 530 * driver are freed. 531 */ 532 static int 533 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 534 { 535 ixgbe_t *ixgbe; 536 537 /* 538 * Check detach command 539 */ 540 switch (cmd) { 541 default: 542 return (DDI_FAILURE); 543 544 case DDI_SUSPEND: 545 return (ixgbe_suspend(devinfo)); 546 547 case DDI_DETACH: 548 break; 549 } 550 551 552 /* 553 * Get the pointer to the driver private data structure 554 */ 555 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 556 if (ixgbe == NULL) 557 return (DDI_FAILURE); 558 559 /* 560 * Unregister MAC. If failed, we have to fail the detach 561 */ 562 if (mac_unregister(ixgbe->mac_hdl) != 0) { 563 ixgbe_error(ixgbe, "Failed to unregister MAC"); 564 return (DDI_FAILURE); 565 } 566 ixgbe->attach_progress &= ~ATTACH_PROGRESS_MAC; 567 568 /* 569 * If the device is still running, it needs to be stopped first. 570 * This check is necessary because under some specific circumstances, 571 * the detach routine can be called without stopping the interface 572 * first. 573 */ 574 mutex_enter(&ixgbe->gen_lock); 575 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 576 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 577 ixgbe_stop(ixgbe, B_TRUE); 578 mutex_exit(&ixgbe->gen_lock); 579 /* Disable and stop the watchdog timer */ 580 ixgbe_disable_watchdog_timer(ixgbe); 581 } else 582 mutex_exit(&ixgbe->gen_lock); 583 584 /* 585 * Check if there are still rx buffers held by the upper layer. 586 * If so, fail the detach. 587 */ 588 if (!ixgbe_rx_drain(ixgbe)) 589 return (DDI_FAILURE); 590 591 /* 592 * Do the remaining unconfigure routines 593 */ 594 ixgbe_unconfigure(devinfo, ixgbe); 595 596 return (DDI_SUCCESS); 597 } 598 599 static void 600 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 601 { 602 /* 603 * Disable interrupt 604 */ 605 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 606 (void) ixgbe_disable_intrs(ixgbe); 607 } 608 609 /* 610 * Unregister MAC 611 */ 612 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 613 (void) mac_unregister(ixgbe->mac_hdl); 614 } 615 616 /* 617 * Free statistics 618 */ 619 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 620 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 621 } 622 623 /* 624 * Remove interrupt handlers 625 */ 626 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 627 ixgbe_rem_intr_handlers(ixgbe); 628 } 629 630 /* 631 * Remove taskq for link-status-change 632 */ 633 if (ixgbe->attach_progress & ATTACH_PROGRESS_LSC_TASKQ) { 634 ddi_taskq_destroy(ixgbe->lsc_taskq); 635 } 636 637 /* 638 * Remove interrupts 639 */ 640 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 641 ixgbe_rem_intrs(ixgbe); 642 } 643 644 /* 645 * Remove driver properties 646 */ 647 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 648 (void) ddi_prop_remove_all(devinfo); 649 } 650 651 /* 652 * Stop the chipset 653 */ 654 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 655 mutex_enter(&ixgbe->gen_lock); 656 ixgbe_chip_stop(ixgbe); 657 mutex_exit(&ixgbe->gen_lock); 658 } 659 660 /* 661 * Free register handle 662 */ 663 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 664 if (ixgbe->osdep.reg_handle != NULL) 665 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 666 } 667 668 /* 669 * Free PCI config handle 670 */ 671 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 672 if (ixgbe->osdep.cfg_handle != NULL) 673 pci_config_teardown(&ixgbe->osdep.cfg_handle); 674 } 675 676 /* 677 * Free locks 678 */ 679 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 680 ixgbe_destroy_locks(ixgbe); 681 } 682 683 /* 684 * Free the rx/tx rings 685 */ 686 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 687 ixgbe_free_rings(ixgbe); 688 } 689 690 /* 691 * Unregister FMA capabilities 692 */ 693 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 694 ixgbe_fm_fini(ixgbe); 695 } 696 697 /* 698 * Free the driver data structure 699 */ 700 kmem_free(ixgbe, sizeof (ixgbe_t)); 701 702 ddi_set_driver_private(devinfo, NULL); 703 } 704 705 /* 706 * ixgbe_register_mac - Register the driver and its function pointers with 707 * the GLD interface. 708 */ 709 static int 710 ixgbe_register_mac(ixgbe_t *ixgbe) 711 { 712 struct ixgbe_hw *hw = &ixgbe->hw; 713 mac_register_t *mac; 714 int status; 715 716 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 717 return (IXGBE_FAILURE); 718 719 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 720 mac->m_driver = ixgbe; 721 mac->m_dip = ixgbe->dip; 722 mac->m_src_addr = hw->mac.addr; 723 mac->m_callbacks = &ixgbe_m_callbacks; 724 mac->m_min_sdu = 0; 725 mac->m_max_sdu = ixgbe->default_mtu; 726 mac->m_margin = VLAN_TAGSZ; 727 mac->m_priv_props = ixgbe_priv_props; 728 mac->m_priv_prop_count = IXGBE_MAX_PRIV_PROPS; 729 mac->m_v12n = MAC_VIRT_LEVEL1; 730 731 status = mac_register(mac, &ixgbe->mac_hdl); 732 733 mac_free(mac); 734 735 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 736 } 737 738 /* 739 * ixgbe_identify_hardware - Identify the type of the chipset. 740 */ 741 static int 742 ixgbe_identify_hardware(ixgbe_t *ixgbe) 743 { 744 struct ixgbe_hw *hw = &ixgbe->hw; 745 struct ixgbe_osdep *osdep = &ixgbe->osdep; 746 747 /* 748 * Get the device id 749 */ 750 hw->vendor_id = 751 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 752 hw->device_id = 753 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 754 hw->revision_id = 755 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 756 hw->subsystem_device_id = 757 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 758 hw->subsystem_vendor_id = 759 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 760 761 /* 762 * Set the mac type of the adapter based on the device id 763 */ 764 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 765 return (IXGBE_FAILURE); 766 } 767 768 /* 769 * Install adapter capabilities 770 */ 771 switch (hw->mac.type) { 772 case ixgbe_mac_82598EB: 773 ixgbe_log(ixgbe, "identify 82598 adapter\n"); 774 ixgbe->capab = &ixgbe_82598eb_cap; 775 776 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 777 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 778 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 779 } 780 ixgbe->capab->other_intr |= IXGBE_EICR_LSC; 781 782 break; 783 case ixgbe_mac_82599EB: 784 ixgbe_log(ixgbe, "identify 82599 adapter\n"); 785 ixgbe->capab = &ixgbe_82599eb_cap; 786 787 ixgbe->capab->other_intr = (IXGBE_EICR_GPI_SDP1 | 788 IXGBE_EICR_GPI_SDP2 | IXGBE_EICR_LSC); 789 790 break; 791 default: 792 ixgbe_log(ixgbe, 793 "adapter not supported in ixgbe_identify_hardware(): %d\n", 794 hw->mac.type); 795 return (IXGBE_FAILURE); 796 } 797 798 return (IXGBE_SUCCESS); 799 } 800 801 /* 802 * ixgbe_regs_map - Map the device registers. 803 * 804 */ 805 static int 806 ixgbe_regs_map(ixgbe_t *ixgbe) 807 { 808 dev_info_t *devinfo = ixgbe->dip; 809 struct ixgbe_hw *hw = &ixgbe->hw; 810 struct ixgbe_osdep *osdep = &ixgbe->osdep; 811 off_t mem_size; 812 813 /* 814 * First get the size of device registers to be mapped. 815 */ 816 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 817 != DDI_SUCCESS) { 818 return (IXGBE_FAILURE); 819 } 820 821 /* 822 * Call ddi_regs_map_setup() to map registers 823 */ 824 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 825 (caddr_t *)&hw->hw_addr, 0, 826 mem_size, &ixgbe_regs_acc_attr, 827 &osdep->reg_handle)) != DDI_SUCCESS) { 828 return (IXGBE_FAILURE); 829 } 830 831 return (IXGBE_SUCCESS); 832 } 833 834 /* 835 * ixgbe_init_properties - Initialize driver properties. 836 */ 837 static void 838 ixgbe_init_properties(ixgbe_t *ixgbe) 839 { 840 /* 841 * Get conf file properties, including link settings 842 * jumbo frames, ring number, descriptor number, etc. 843 */ 844 ixgbe_get_conf(ixgbe); 845 846 ixgbe_init_params(ixgbe); 847 } 848 849 /* 850 * ixgbe_init_driver_settings - Initialize driver settings. 851 * 852 * The settings include hardware function pointers, bus information, 853 * rx/tx rings settings, link state, and any other parameters that 854 * need to be setup during driver initialization. 855 */ 856 static int 857 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 858 { 859 struct ixgbe_hw *hw = &ixgbe->hw; 860 dev_info_t *devinfo = ixgbe->dip; 861 ixgbe_rx_ring_t *rx_ring; 862 ixgbe_tx_ring_t *tx_ring; 863 uint32_t rx_size; 864 uint32_t tx_size; 865 int i; 866 867 /* 868 * Initialize chipset specific hardware function pointers 869 */ 870 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 871 return (IXGBE_FAILURE); 872 } 873 874 /* 875 * Get the system page size 876 */ 877 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 878 879 /* 880 * Set rx buffer size 881 * 882 * The IP header alignment room is counted in the calculation. 883 * The rx buffer size is in unit of 1K that is required by the 884 * chipset hardware. 885 */ 886 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 887 ixgbe->rx_buf_size = ((rx_size >> 10) + 888 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 889 890 /* 891 * Set tx buffer size 892 */ 893 tx_size = ixgbe->max_frame_size; 894 ixgbe->tx_buf_size = ((tx_size >> 10) + 895 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 896 897 /* 898 * Initialize rx/tx rings parameters 899 */ 900 for (i = 0; i < ixgbe->num_rx_rings; i++) { 901 rx_ring = &ixgbe->rx_rings[i]; 902 rx_ring->index = i; 903 rx_ring->ixgbe = ixgbe; 904 } 905 906 for (i = 0; i < ixgbe->num_tx_rings; i++) { 907 tx_ring = &ixgbe->tx_rings[i]; 908 tx_ring->index = i; 909 tx_ring->ixgbe = ixgbe; 910 if (ixgbe->tx_head_wb_enable) 911 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 912 else 913 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 914 915 tx_ring->ring_size = ixgbe->tx_ring_size; 916 tx_ring->free_list_size = ixgbe->tx_ring_size + 917 (ixgbe->tx_ring_size >> 1); 918 } 919 920 /* 921 * Initialize values of interrupt throttling rate 922 */ 923 for (i = 1; i < MAX_INTR_VECTOR; i++) 924 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 925 926 /* 927 * The initial link state should be "unknown" 928 */ 929 ixgbe->link_state = LINK_STATE_UNKNOWN; 930 931 return (IXGBE_SUCCESS); 932 } 933 934 /* 935 * ixgbe_init_locks - Initialize locks. 936 */ 937 static void 938 ixgbe_init_locks(ixgbe_t *ixgbe) 939 { 940 ixgbe_rx_ring_t *rx_ring; 941 ixgbe_tx_ring_t *tx_ring; 942 int i; 943 944 for (i = 0; i < ixgbe->num_rx_rings; i++) { 945 rx_ring = &ixgbe->rx_rings[i]; 946 mutex_init(&rx_ring->rx_lock, NULL, 947 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 948 } 949 950 for (i = 0; i < ixgbe->num_tx_rings; i++) { 951 tx_ring = &ixgbe->tx_rings[i]; 952 mutex_init(&tx_ring->tx_lock, NULL, 953 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 954 mutex_init(&tx_ring->recycle_lock, NULL, 955 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 956 mutex_init(&tx_ring->tcb_head_lock, NULL, 957 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 958 mutex_init(&tx_ring->tcb_tail_lock, NULL, 959 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 960 } 961 962 mutex_init(&ixgbe->gen_lock, NULL, 963 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 964 965 mutex_init(&ixgbe->watchdog_lock, NULL, 966 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 967 } 968 969 /* 970 * ixgbe_destroy_locks - Destroy locks. 971 */ 972 static void 973 ixgbe_destroy_locks(ixgbe_t *ixgbe) 974 { 975 ixgbe_rx_ring_t *rx_ring; 976 ixgbe_tx_ring_t *tx_ring; 977 int i; 978 979 for (i = 0; i < ixgbe->num_rx_rings; i++) { 980 rx_ring = &ixgbe->rx_rings[i]; 981 mutex_destroy(&rx_ring->rx_lock); 982 } 983 984 for (i = 0; i < ixgbe->num_tx_rings; i++) { 985 tx_ring = &ixgbe->tx_rings[i]; 986 mutex_destroy(&tx_ring->tx_lock); 987 mutex_destroy(&tx_ring->recycle_lock); 988 mutex_destroy(&tx_ring->tcb_head_lock); 989 mutex_destroy(&tx_ring->tcb_tail_lock); 990 } 991 992 mutex_destroy(&ixgbe->gen_lock); 993 mutex_destroy(&ixgbe->watchdog_lock); 994 } 995 996 static int 997 ixgbe_resume(dev_info_t *devinfo) 998 { 999 ixgbe_t *ixgbe; 1000 1001 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1002 if (ixgbe == NULL) 1003 return (DDI_FAILURE); 1004 1005 mutex_enter(&ixgbe->gen_lock); 1006 1007 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1008 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1009 mutex_exit(&ixgbe->gen_lock); 1010 return (DDI_FAILURE); 1011 } 1012 1013 /* 1014 * Enable and start the watchdog timer 1015 */ 1016 ixgbe_enable_watchdog_timer(ixgbe); 1017 } 1018 1019 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1020 1021 mutex_exit(&ixgbe->gen_lock); 1022 1023 return (DDI_SUCCESS); 1024 } 1025 1026 static int 1027 ixgbe_suspend(dev_info_t *devinfo) 1028 { 1029 ixgbe_t *ixgbe; 1030 1031 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1032 if (ixgbe == NULL) 1033 return (DDI_FAILURE); 1034 1035 mutex_enter(&ixgbe->gen_lock); 1036 1037 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1038 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1039 mutex_exit(&ixgbe->gen_lock); 1040 return (DDI_SUCCESS); 1041 } 1042 ixgbe_stop(ixgbe, B_FALSE); 1043 1044 mutex_exit(&ixgbe->gen_lock); 1045 1046 /* 1047 * Disable and stop the watchdog timer 1048 */ 1049 ixgbe_disable_watchdog_timer(ixgbe); 1050 1051 return (DDI_SUCCESS); 1052 } 1053 1054 /* 1055 * ixgbe_init - Initialize the device. 1056 */ 1057 static int 1058 ixgbe_init(ixgbe_t *ixgbe) 1059 { 1060 struct ixgbe_hw *hw = &ixgbe->hw; 1061 1062 mutex_enter(&ixgbe->gen_lock); 1063 1064 /* 1065 * Reset chipset to put the hardware in a known state 1066 * before we try to do anything with the eeprom. 1067 */ 1068 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1069 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1070 goto init_fail; 1071 } 1072 1073 /* 1074 * Need to init eeprom before validating the checksum. 1075 */ 1076 if (ixgbe_init_eeprom_params(hw) < 0) { 1077 ixgbe_error(ixgbe, 1078 "Unable to intitialize the eeprom interface."); 1079 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1080 goto init_fail; 1081 } 1082 1083 /* 1084 * NVM validation 1085 */ 1086 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1087 /* 1088 * Some PCI-E parts fail the first check due to 1089 * the link being in sleep state. Call it again, 1090 * if it fails a second time it's a real issue. 1091 */ 1092 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1093 ixgbe_error(ixgbe, 1094 "Invalid NVM checksum. Please contact " 1095 "the vendor to update the NVM."); 1096 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1097 goto init_fail; 1098 } 1099 } 1100 1101 /* 1102 * Setup default flow control thresholds - enable/disable 1103 * & flow control type is controlled by ixgbe.conf 1104 */ 1105 hw->fc.high_water = DEFAULT_FCRTH; 1106 hw->fc.low_water = DEFAULT_FCRTL; 1107 hw->fc.pause_time = DEFAULT_FCPAUSE; 1108 hw->fc.send_xon = B_TRUE; 1109 1110 /* 1111 * Don't wait for auto-negotiation to complete 1112 */ 1113 hw->phy.autoneg_wait_to_complete = B_FALSE; 1114 1115 /* 1116 * Initialize link settings 1117 */ 1118 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1119 1120 /* 1121 * Initialize the chipset hardware 1122 */ 1123 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1124 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1125 goto init_fail; 1126 } 1127 1128 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 1129 goto init_fail; 1130 } 1131 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1132 goto init_fail; 1133 } 1134 1135 mutex_exit(&ixgbe->gen_lock); 1136 return (IXGBE_SUCCESS); 1137 1138 init_fail: 1139 /* 1140 * Reset PHY 1141 */ 1142 (void) ixgbe_reset_phy(hw); 1143 1144 mutex_exit(&ixgbe->gen_lock); 1145 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1146 return (IXGBE_FAILURE); 1147 } 1148 1149 /* 1150 * ixgbe_chip_start - Initialize and start the chipset hardware. 1151 */ 1152 static int 1153 ixgbe_chip_start(ixgbe_t *ixgbe) 1154 { 1155 struct ixgbe_hw *hw = &ixgbe->hw; 1156 int ret_val, i; 1157 1158 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1159 1160 /* 1161 * Get the mac address 1162 * This function should handle SPARC case correctly. 1163 */ 1164 if (!ixgbe_find_mac_address(ixgbe)) { 1165 ixgbe_error(ixgbe, "Failed to get the mac address"); 1166 return (IXGBE_FAILURE); 1167 } 1168 1169 /* 1170 * Validate the mac address 1171 */ 1172 (void) ixgbe_init_rx_addrs(hw); 1173 if (!is_valid_mac_addr(hw->mac.addr)) { 1174 ixgbe_error(ixgbe, "Invalid mac address"); 1175 return (IXGBE_FAILURE); 1176 } 1177 1178 /* 1179 * Configure/Initialize hardware 1180 */ 1181 ret_val = ixgbe_init_hw(hw); 1182 if (ret_val != IXGBE_SUCCESS) { 1183 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1184 ixgbe_error(ixgbe, 1185 "This 82599 device is pre-release and contains" 1186 " outdated firmware, please contact your hardware" 1187 " vendor for a replacement."); 1188 } else { 1189 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1190 return (IXGBE_FAILURE); 1191 } 1192 } 1193 1194 /* 1195 * Setup adapter interrupt vectors 1196 */ 1197 ixgbe_setup_adapter_vector(ixgbe); 1198 1199 /* 1200 * Initialize unicast addresses. 1201 */ 1202 ixgbe_init_unicst(ixgbe); 1203 1204 /* 1205 * Setup and initialize the mctable structures. 1206 */ 1207 ixgbe_setup_multicst(ixgbe); 1208 1209 /* 1210 * Set interrupt throttling rate 1211 */ 1212 for (i = 0; i < ixgbe->intr_cnt; i++) { 1213 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1214 } 1215 1216 /* 1217 * Save the state of the phy 1218 */ 1219 ixgbe_get_hw_state(ixgbe); 1220 1221 /* 1222 * Make sure driver has control 1223 */ 1224 ixgbe_get_driver_control(hw); 1225 1226 return (IXGBE_SUCCESS); 1227 } 1228 1229 /* 1230 * ixgbe_chip_stop - Stop the chipset hardware 1231 */ 1232 static void 1233 ixgbe_chip_stop(ixgbe_t *ixgbe) 1234 { 1235 struct ixgbe_hw *hw = &ixgbe->hw; 1236 1237 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1238 1239 /* 1240 * Tell firmware driver is no longer in control 1241 */ 1242 ixgbe_release_driver_control(hw); 1243 1244 /* 1245 * Reset the chipset 1246 */ 1247 (void) ixgbe_reset_hw(hw); 1248 1249 /* 1250 * Reset PHY 1251 */ 1252 (void) ixgbe_reset_phy(hw); 1253 } 1254 1255 /* 1256 * ixgbe_reset - Reset the chipset and re-start the driver. 1257 * 1258 * It involves stopping and re-starting the chipset, 1259 * and re-configuring the rx/tx rings. 1260 */ 1261 static int 1262 ixgbe_reset(ixgbe_t *ixgbe) 1263 { 1264 /* 1265 * Disable and stop the watchdog timer 1266 */ 1267 ixgbe_disable_watchdog_timer(ixgbe); 1268 1269 mutex_enter(&ixgbe->gen_lock); 1270 1271 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1272 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1273 1274 ixgbe_stop(ixgbe, B_FALSE); 1275 1276 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1277 mutex_exit(&ixgbe->gen_lock); 1278 return (IXGBE_FAILURE); 1279 } 1280 1281 ixgbe->ixgbe_state |= IXGBE_STARTED; 1282 mutex_exit(&ixgbe->gen_lock); 1283 1284 /* 1285 * Enable and start the watchdog timer 1286 */ 1287 ixgbe_enable_watchdog_timer(ixgbe); 1288 1289 return (IXGBE_SUCCESS); 1290 } 1291 1292 /* 1293 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1294 */ 1295 static void 1296 ixgbe_tx_clean(ixgbe_t *ixgbe) 1297 { 1298 ixgbe_tx_ring_t *tx_ring; 1299 tx_control_block_t *tcb; 1300 link_list_t pending_list; 1301 uint32_t desc_num; 1302 int i, j; 1303 1304 LINK_LIST_INIT(&pending_list); 1305 1306 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1307 tx_ring = &ixgbe->tx_rings[i]; 1308 1309 mutex_enter(&tx_ring->recycle_lock); 1310 1311 /* 1312 * Clean the pending tx data - the pending packets in the 1313 * work_list that have no chances to be transmitted again. 1314 * 1315 * We must ensure the chipset is stopped or the link is down 1316 * before cleaning the transmit packets. 1317 */ 1318 desc_num = 0; 1319 for (j = 0; j < tx_ring->ring_size; j++) { 1320 tcb = tx_ring->work_list[j]; 1321 if (tcb != NULL) { 1322 desc_num += tcb->desc_num; 1323 1324 tx_ring->work_list[j] = NULL; 1325 1326 ixgbe_free_tcb(tcb); 1327 1328 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1329 } 1330 } 1331 1332 if (desc_num > 0) { 1333 atomic_add_32(&tx_ring->tbd_free, desc_num); 1334 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1335 1336 /* 1337 * Reset the head and tail pointers of the tbd ring; 1338 * Reset the writeback head if it's enable. 1339 */ 1340 tx_ring->tbd_head = 0; 1341 tx_ring->tbd_tail = 0; 1342 if (ixgbe->tx_head_wb_enable) 1343 *tx_ring->tbd_head_wb = 0; 1344 1345 IXGBE_WRITE_REG(&ixgbe->hw, 1346 IXGBE_TDH(tx_ring->index), 0); 1347 IXGBE_WRITE_REG(&ixgbe->hw, 1348 IXGBE_TDT(tx_ring->index), 0); 1349 } 1350 1351 mutex_exit(&tx_ring->recycle_lock); 1352 1353 /* 1354 * Add the tx control blocks in the pending list to 1355 * the free list. 1356 */ 1357 ixgbe_put_free_list(tx_ring, &pending_list); 1358 } 1359 } 1360 1361 /* 1362 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1363 * transmitted. 1364 */ 1365 static boolean_t 1366 ixgbe_tx_drain(ixgbe_t *ixgbe) 1367 { 1368 ixgbe_tx_ring_t *tx_ring; 1369 boolean_t done; 1370 int i, j; 1371 1372 /* 1373 * Wait for a specific time to allow pending tx packets 1374 * to be transmitted. 1375 * 1376 * Check the counter tbd_free to see if transmission is done. 1377 * No lock protection is needed here. 1378 * 1379 * Return B_TRUE if all pending packets have been transmitted; 1380 * Otherwise return B_FALSE; 1381 */ 1382 for (i = 0; i < TX_DRAIN_TIME; i++) { 1383 1384 done = B_TRUE; 1385 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1386 tx_ring = &ixgbe->tx_rings[j]; 1387 done = done && 1388 (tx_ring->tbd_free == tx_ring->ring_size); 1389 } 1390 1391 if (done) 1392 break; 1393 1394 msec_delay(1); 1395 } 1396 1397 return (done); 1398 } 1399 1400 /* 1401 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1402 */ 1403 static boolean_t 1404 ixgbe_rx_drain(ixgbe_t *ixgbe) 1405 { 1406 boolean_t done = B_TRUE; 1407 int i; 1408 1409 /* 1410 * Polling the rx free list to check if those rx buffers held by 1411 * the upper layer are released. 1412 * 1413 * Check the counter rcb_free to see if all pending buffers are 1414 * released. No lock protection is needed here. 1415 * 1416 * Return B_TRUE if all pending buffers have been released; 1417 * Otherwise return B_FALSE; 1418 */ 1419 for (i = 0; i < RX_DRAIN_TIME; i++) { 1420 done = (ixgbe->rcb_pending == 0); 1421 1422 if (done) 1423 break; 1424 1425 msec_delay(1); 1426 } 1427 1428 return (done); 1429 } 1430 1431 /* 1432 * ixgbe_start - Start the driver/chipset. 1433 */ 1434 int 1435 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1436 { 1437 int i; 1438 1439 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1440 1441 if (alloc_buffer) { 1442 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1443 ixgbe_error(ixgbe, 1444 "Failed to allocate software receive rings"); 1445 return (IXGBE_FAILURE); 1446 } 1447 1448 /* Allocate buffers for all the rx/tx rings */ 1449 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1450 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1451 return (IXGBE_FAILURE); 1452 } 1453 1454 ixgbe->tx_ring_init = B_TRUE; 1455 } else { 1456 ixgbe->tx_ring_init = B_FALSE; 1457 } 1458 1459 for (i = 0; i < ixgbe->num_rx_rings; i++) 1460 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1461 for (i = 0; i < ixgbe->num_tx_rings; i++) 1462 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1463 1464 /* 1465 * Start the chipset hardware 1466 */ 1467 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1468 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1469 goto start_failure; 1470 } 1471 1472 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1473 goto start_failure; 1474 } 1475 1476 /* 1477 * Setup the rx/tx rings 1478 */ 1479 ixgbe_setup_rings(ixgbe); 1480 1481 /* 1482 * Enable adapter interrupts 1483 * The interrupts must be enabled after the driver state is START 1484 */ 1485 ixgbe_enable_adapter_interrupts(ixgbe); 1486 1487 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1488 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1489 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1490 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1491 1492 return (IXGBE_SUCCESS); 1493 1494 start_failure: 1495 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1496 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1497 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1498 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1499 1500 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1501 1502 return (IXGBE_FAILURE); 1503 } 1504 1505 /* 1506 * ixgbe_stop - Stop the driver/chipset. 1507 */ 1508 void 1509 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1510 { 1511 int i; 1512 1513 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1514 1515 /* 1516 * Disable the adapter interrupts 1517 */ 1518 ixgbe_disable_adapter_interrupts(ixgbe); 1519 1520 /* 1521 * Drain the pending tx packets 1522 */ 1523 (void) ixgbe_tx_drain(ixgbe); 1524 1525 for (i = 0; i < ixgbe->num_rx_rings; i++) 1526 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1527 for (i = 0; i < ixgbe->num_tx_rings; i++) 1528 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1529 1530 /* 1531 * Stop the chipset hardware 1532 */ 1533 ixgbe_chip_stop(ixgbe); 1534 1535 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1536 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1537 } 1538 1539 /* 1540 * Clean the pending tx data/resources 1541 */ 1542 ixgbe_tx_clean(ixgbe); 1543 1544 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1545 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1546 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1547 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1548 1549 if (ixgbe->link_state == LINK_STATE_UP) { 1550 ixgbe->link_state = LINK_STATE_UNKNOWN; 1551 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1552 } 1553 1554 if (free_buffer) { 1555 /* 1556 * Release the DMA/memory resources of rx/tx rings 1557 */ 1558 ixgbe_free_dma(ixgbe); 1559 ixgbe_free_rx_data(ixgbe); 1560 } 1561 } 1562 1563 /* 1564 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 1565 */ 1566 static int 1567 ixgbe_alloc_rings(ixgbe_t *ixgbe) 1568 { 1569 /* 1570 * Allocate memory space for rx rings 1571 */ 1572 ixgbe->rx_rings = kmem_zalloc( 1573 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 1574 KM_NOSLEEP); 1575 1576 if (ixgbe->rx_rings == NULL) { 1577 return (IXGBE_FAILURE); 1578 } 1579 1580 /* 1581 * Allocate memory space for tx rings 1582 */ 1583 ixgbe->tx_rings = kmem_zalloc( 1584 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 1585 KM_NOSLEEP); 1586 1587 if (ixgbe->tx_rings == NULL) { 1588 kmem_free(ixgbe->rx_rings, 1589 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1590 ixgbe->rx_rings = NULL; 1591 return (IXGBE_FAILURE); 1592 } 1593 1594 /* 1595 * Allocate memory space for rx ring groups 1596 */ 1597 ixgbe->rx_groups = kmem_zalloc( 1598 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 1599 KM_NOSLEEP); 1600 1601 if (ixgbe->rx_groups == NULL) { 1602 kmem_free(ixgbe->rx_rings, 1603 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1604 kmem_free(ixgbe->tx_rings, 1605 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1606 ixgbe->rx_rings = NULL; 1607 ixgbe->tx_rings = NULL; 1608 return (IXGBE_FAILURE); 1609 } 1610 1611 return (IXGBE_SUCCESS); 1612 } 1613 1614 /* 1615 * ixgbe_free_rings - Free the memory space of rx/tx rings. 1616 */ 1617 static void 1618 ixgbe_free_rings(ixgbe_t *ixgbe) 1619 { 1620 if (ixgbe->rx_rings != NULL) { 1621 kmem_free(ixgbe->rx_rings, 1622 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1623 ixgbe->rx_rings = NULL; 1624 } 1625 1626 if (ixgbe->tx_rings != NULL) { 1627 kmem_free(ixgbe->tx_rings, 1628 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1629 ixgbe->tx_rings = NULL; 1630 } 1631 1632 if (ixgbe->rx_groups != NULL) { 1633 kmem_free(ixgbe->rx_groups, 1634 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 1635 ixgbe->rx_groups = NULL; 1636 } 1637 } 1638 1639 static int 1640 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 1641 { 1642 ixgbe_rx_ring_t *rx_ring; 1643 int i; 1644 1645 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1646 rx_ring = &ixgbe->rx_rings[i]; 1647 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 1648 goto alloc_rx_rings_failure; 1649 } 1650 return (IXGBE_SUCCESS); 1651 1652 alloc_rx_rings_failure: 1653 ixgbe_free_rx_data(ixgbe); 1654 return (IXGBE_FAILURE); 1655 } 1656 1657 static void 1658 ixgbe_free_rx_data(ixgbe_t *ixgbe) 1659 { 1660 ixgbe_rx_ring_t *rx_ring; 1661 ixgbe_rx_data_t *rx_data; 1662 int i; 1663 1664 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1665 rx_ring = &ixgbe->rx_rings[i]; 1666 1667 mutex_enter(&ixgbe->rx_pending_lock); 1668 rx_data = rx_ring->rx_data; 1669 1670 if (rx_data != NULL) { 1671 rx_data->flag |= IXGBE_RX_STOPPED; 1672 1673 if (rx_data->rcb_pending == 0) { 1674 ixgbe_free_rx_ring_data(rx_data); 1675 rx_ring->rx_data = NULL; 1676 } 1677 } 1678 1679 mutex_exit(&ixgbe->rx_pending_lock); 1680 } 1681 } 1682 1683 /* 1684 * ixgbe_setup_rings - Setup rx/tx rings. 1685 */ 1686 static void 1687 ixgbe_setup_rings(ixgbe_t *ixgbe) 1688 { 1689 /* 1690 * Setup the rx/tx rings, including the following: 1691 * 1692 * 1. Setup the descriptor ring and the control block buffers; 1693 * 2. Initialize necessary registers for receive/transmit; 1694 * 3. Initialize software pointers/parameters for receive/transmit; 1695 */ 1696 ixgbe_setup_rx(ixgbe); 1697 1698 ixgbe_setup_tx(ixgbe); 1699 } 1700 1701 static void 1702 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 1703 { 1704 ixgbe_t *ixgbe = rx_ring->ixgbe; 1705 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 1706 struct ixgbe_hw *hw = &ixgbe->hw; 1707 rx_control_block_t *rcb; 1708 union ixgbe_adv_rx_desc *rbd; 1709 uint32_t size; 1710 uint32_t buf_low; 1711 uint32_t buf_high; 1712 uint32_t reg_val; 1713 int i; 1714 1715 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1716 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1717 1718 for (i = 0; i < ixgbe->rx_ring_size; i++) { 1719 rcb = rx_data->work_list[i]; 1720 rbd = &rx_data->rbd_ring[i]; 1721 1722 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1723 rbd->read.hdr_addr = NULL; 1724 } 1725 1726 /* 1727 * Initialize the length register 1728 */ 1729 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 1730 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->index), size); 1731 1732 /* 1733 * Initialize the base address registers 1734 */ 1735 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 1736 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 1737 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->index), buf_high); 1738 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->index), buf_low); 1739 1740 /* 1741 * Setup head & tail pointers 1742 */ 1743 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->index), rx_data->ring_size - 1); 1744 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->index), 0); 1745 1746 rx_data->rbd_next = 0; 1747 1748 /* 1749 * Setup the Receive Descriptor Control Register (RXDCTL) 1750 * PTHRESH=32 descriptors (half the internal cache) 1751 * HTHRESH=0 descriptors (to minimize latency on fetch) 1752 * WTHRESH defaults to 1 (writeback each descriptor) 1753 */ 1754 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->index)); 1755 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 1756 1757 /* Not a valid value for 82599 */ 1758 if (hw->mac.type < ixgbe_mac_82599EB) { 1759 reg_val |= 0x0020; /* pthresh */ 1760 } 1761 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->index), reg_val); 1762 1763 if (hw->mac.type == ixgbe_mac_82599EB) { 1764 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 1765 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 1766 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 1767 } 1768 1769 /* 1770 * Setup the Split and Replication Receive Control Register. 1771 * Set the rx buffer size and the advanced descriptor type. 1772 */ 1773 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 1774 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 1775 reg_val |= IXGBE_SRRCTL_DROP_EN; 1776 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->index), reg_val); 1777 } 1778 1779 static void 1780 ixgbe_setup_rx(ixgbe_t *ixgbe) 1781 { 1782 ixgbe_rx_ring_t *rx_ring; 1783 struct ixgbe_hw *hw = &ixgbe->hw; 1784 ixgbe_rx_group_t *rx_group; 1785 uint32_t reg_val; 1786 uint32_t ring_mapping; 1787 int i; 1788 1789 /* PSRTYPE must be configured for 82599 */ 1790 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 1791 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 1792 #define IXGBE_PSRTYPE_L2_PKT 0x00001000 1793 reg_val |= IXGBE_PSRTYPE_L2_PKT; 1794 reg_val |= 0xE0000000; 1795 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 1796 1797 /* 1798 * Set filter control in FCTRL to accept broadcast packets and do 1799 * not pass pause frames to host. Flow control settings are already 1800 * in this register, so preserve them. 1801 */ 1802 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 1803 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 1804 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 1805 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 1806 1807 /* 1808 * Enable the receive unit. This must be done after filter 1809 * control is set in FCTRL. 1810 */ 1811 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 1812 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 1813 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 1814 1815 /* 1816 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 1817 */ 1818 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1819 rx_ring = &ixgbe->rx_rings[i]; 1820 ixgbe_setup_rx_ring(rx_ring); 1821 } 1822 1823 /* 1824 * Setup rx groups. 1825 */ 1826 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1827 rx_group = &ixgbe->rx_groups[i]; 1828 rx_group->index = i; 1829 rx_group->ixgbe = ixgbe; 1830 } 1831 1832 /* 1833 * Setup the per-ring statistics mapping. 1834 */ 1835 ring_mapping = 0; 1836 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1837 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 1838 if ((i & 0x3) == 0x3) { 1839 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i >> 2), ring_mapping); 1840 ring_mapping = 0; 1841 } 1842 } 1843 if ((i & 0x3) != 0x3) 1844 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(i >> 2), ring_mapping); 1845 1846 /* 1847 * The Max Frame Size in MHADD/MAXFRS will be internally increased 1848 * by four bytes if the packet has a VLAN field, so includes MTU, 1849 * ethernet header and frame check sequence. 1850 * Register is MAXFRS in 82599. 1851 */ 1852 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 1853 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 1854 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 1855 1856 /* 1857 * Setup Jumbo Frame enable bit 1858 */ 1859 if (ixgbe->default_mtu > ETHERMTU) { 1860 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 1861 reg_val |= IXGBE_HLREG0_JUMBOEN; 1862 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 1863 } 1864 1865 /* 1866 * Hardware checksum settings 1867 */ 1868 if (ixgbe->rx_hcksum_enable) { 1869 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 1870 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 1871 } 1872 1873 /* 1874 * Setup RSS for multiple receive queues 1875 */ 1876 if (ixgbe->num_rx_rings > 1) 1877 ixgbe_setup_rss(ixgbe); 1878 } 1879 1880 static void 1881 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 1882 { 1883 ixgbe_t *ixgbe = tx_ring->ixgbe; 1884 struct ixgbe_hw *hw = &ixgbe->hw; 1885 uint32_t size; 1886 uint32_t buf_low; 1887 uint32_t buf_high; 1888 uint32_t reg_val; 1889 1890 ASSERT(mutex_owned(&tx_ring->tx_lock)); 1891 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1892 1893 /* 1894 * Initialize the length register 1895 */ 1896 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 1897 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 1898 1899 /* 1900 * Initialize the base address registers 1901 */ 1902 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 1903 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 1904 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 1905 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 1906 1907 /* 1908 * Setup head & tail pointers 1909 */ 1910 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 1911 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 1912 1913 /* 1914 * Setup head write-back 1915 */ 1916 if (ixgbe->tx_head_wb_enable) { 1917 /* 1918 * The memory of the head write-back is allocated using 1919 * the extra tbd beyond the tail of the tbd ring. 1920 */ 1921 tx_ring->tbd_head_wb = (uint32_t *) 1922 ((uintptr_t)tx_ring->tbd_area.address + size); 1923 *tx_ring->tbd_head_wb = 0; 1924 1925 buf_low = (uint32_t) 1926 (tx_ring->tbd_area.dma_address + size); 1927 buf_high = (uint32_t) 1928 ((tx_ring->tbd_area.dma_address + size) >> 32); 1929 1930 /* Set the head write-back enable bit */ 1931 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 1932 1933 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 1934 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 1935 1936 /* 1937 * Turn off relaxed ordering for head write back or it will 1938 * cause problems with the tx recycling 1939 */ 1940 reg_val = IXGBE_READ_REG(hw, 1941 IXGBE_DCA_TXCTRL(tx_ring->index)); 1942 reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 1943 IXGBE_WRITE_REG(hw, 1944 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 1945 } else { 1946 tx_ring->tbd_head_wb = NULL; 1947 } 1948 1949 tx_ring->tbd_head = 0; 1950 tx_ring->tbd_tail = 0; 1951 tx_ring->tbd_free = tx_ring->ring_size; 1952 1953 if (ixgbe->tx_ring_init == B_TRUE) { 1954 tx_ring->tcb_head = 0; 1955 tx_ring->tcb_tail = 0; 1956 tx_ring->tcb_free = tx_ring->free_list_size; 1957 } 1958 1959 /* 1960 * Initialize the s/w context structure 1961 */ 1962 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 1963 } 1964 1965 static void 1966 ixgbe_setup_tx(ixgbe_t *ixgbe) 1967 { 1968 struct ixgbe_hw *hw = &ixgbe->hw; 1969 ixgbe_tx_ring_t *tx_ring; 1970 uint32_t reg_val; 1971 uint32_t ring_mapping; 1972 int i; 1973 1974 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1975 tx_ring = &ixgbe->tx_rings[i]; 1976 ixgbe_setup_tx_ring(tx_ring); 1977 } 1978 1979 /* 1980 * Setup the per-ring statistics mapping. 1981 */ 1982 ring_mapping = 0; 1983 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1984 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 1985 if ((i & 0x3) == 0x3) { 1986 if (hw->mac.type >= ixgbe_mac_82599EB) { 1987 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 1988 ring_mapping); 1989 } else { 1990 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 1991 ring_mapping); 1992 } 1993 ring_mapping = 0; 1994 } 1995 } 1996 if ((i & 0x3) != 0x3) 1997 if (hw->mac.type >= ixgbe_mac_82599EB) { 1998 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 1999 } else { 2000 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2001 } 2002 2003 /* 2004 * Enable CRC appending and TX padding (for short tx frames) 2005 */ 2006 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2007 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2008 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2009 2010 /* 2011 * enable DMA for 82599 parts 2012 */ 2013 if (hw->mac.type == ixgbe_mac_82599EB) { 2014 /* DMATXCTL.TE must be set after all Tx config is complete */ 2015 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2016 reg_val |= IXGBE_DMATXCTL_TE; 2017 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2018 } 2019 2020 /* 2021 * Enabling tx queues .. 2022 * For 82599 must be done after DMATXCTL.TE is set 2023 */ 2024 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2025 tx_ring = &ixgbe->tx_rings[i]; 2026 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2027 reg_val |= IXGBE_TXDCTL_ENABLE; 2028 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2029 } 2030 } 2031 2032 /* 2033 * ixgbe_setup_rss - Setup receive-side scaling feature. 2034 */ 2035 static void 2036 ixgbe_setup_rss(ixgbe_t *ixgbe) 2037 { 2038 struct ixgbe_hw *hw = &ixgbe->hw; 2039 uint32_t i, mrqc, rxcsum; 2040 uint32_t random; 2041 uint32_t reta; 2042 2043 /* 2044 * Fill out redirection table 2045 */ 2046 reta = 0; 2047 for (i = 0; i < 128; i++) { 2048 reta = (reta << 8) | (i % ixgbe->num_rx_rings); 2049 if ((i & 3) == 3) 2050 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2051 } 2052 2053 /* 2054 * Fill out hash function seeds with a random constant 2055 */ 2056 for (i = 0; i < 10; i++) { 2057 (void) random_get_pseudo_bytes((uint8_t *)&random, 2058 sizeof (uint32_t)); 2059 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2060 } 2061 2062 /* 2063 * Enable RSS & perform hash on these packet types 2064 */ 2065 mrqc = IXGBE_MRQC_RSSEN | 2066 IXGBE_MRQC_RSS_FIELD_IPV4 | 2067 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2068 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2069 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2070 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2071 IXGBE_MRQC_RSS_FIELD_IPV6 | 2072 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2073 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2074 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2075 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2076 2077 /* 2078 * Disable Packet Checksum to enable RSS for multiple receive queues. 2079 * It is an adapter hardware limitation that Packet Checksum is 2080 * mutually exclusive with RSS. 2081 */ 2082 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2083 rxcsum |= IXGBE_RXCSUM_PCSD; 2084 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2085 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2086 } 2087 2088 /* 2089 * ixgbe_init_unicst - Initialize the unicast addresses. 2090 */ 2091 static void 2092 ixgbe_init_unicst(ixgbe_t *ixgbe) 2093 { 2094 struct ixgbe_hw *hw = &ixgbe->hw; 2095 uint8_t *mac_addr; 2096 int slot; 2097 /* 2098 * Here we should consider two situations: 2099 * 2100 * 1. Chipset is initialized at the first time, 2101 * Clear all the multiple unicast addresses. 2102 * 2103 * 2. Chipset is reset 2104 * Recover the multiple unicast addresses from the 2105 * software data structure to the RAR registers. 2106 */ 2107 if (!ixgbe->unicst_init) { 2108 /* 2109 * Initialize the multiple unicast addresses 2110 */ 2111 ixgbe->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2112 ixgbe->unicst_avail = ixgbe->unicst_total; 2113 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2114 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2115 bzero(mac_addr, ETHERADDRL); 2116 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2117 ixgbe->unicst_addr[slot].mac.set = 0; 2118 } 2119 ixgbe->unicst_init = B_TRUE; 2120 } else { 2121 /* Re-configure the RAR registers */ 2122 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2123 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2124 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2125 (void) ixgbe_set_rar(hw, slot, mac_addr, 2126 NULL, IXGBE_RAH_AV); 2127 } else { 2128 bzero(mac_addr, ETHERADDRL); 2129 (void) ixgbe_set_rar(hw, slot, mac_addr, 2130 NULL, NULL); 2131 } 2132 } 2133 } 2134 } 2135 2136 /* 2137 * ixgbe_unicst_set - Set the unicast address to the specified slot. 2138 */ 2139 int 2140 ixgbe_unicst_set(ixgbe_t *ixgbe, const uint8_t *mac_addr, 2141 int slot) 2142 { 2143 struct ixgbe_hw *hw = &ixgbe->hw; 2144 2145 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2146 2147 /* 2148 * Save the unicast address in the software data structure 2149 */ 2150 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 2151 2152 /* 2153 * Set the unicast address to the RAR register 2154 */ 2155 (void) ixgbe_set_rar(hw, slot, (uint8_t *)mac_addr, NULL, IXGBE_RAH_AV); 2156 2157 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2158 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2159 return (EIO); 2160 } 2161 2162 return (0); 2163 } 2164 2165 /* 2166 * ixgbe_unicst_find - Find the slot for the specified unicast address 2167 */ 2168 int 2169 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2170 { 2171 int slot; 2172 2173 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2174 2175 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2176 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2177 mac_addr, ETHERADDRL) == 0) 2178 return (slot); 2179 } 2180 2181 return (-1); 2182 } 2183 2184 /* 2185 * ixgbe_multicst_add - Add a multicst address. 2186 */ 2187 int 2188 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2189 { 2190 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2191 2192 if ((multiaddr[0] & 01) == 0) { 2193 return (EINVAL); 2194 } 2195 2196 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2197 return (ENOENT); 2198 } 2199 2200 bcopy(multiaddr, 2201 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2202 ixgbe->mcast_count++; 2203 2204 /* 2205 * Update the multicast table in the hardware 2206 */ 2207 ixgbe_setup_multicst(ixgbe); 2208 2209 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2210 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2211 return (EIO); 2212 } 2213 2214 return (0); 2215 } 2216 2217 /* 2218 * ixgbe_multicst_remove - Remove a multicst address. 2219 */ 2220 int 2221 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2222 { 2223 int i; 2224 2225 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2226 2227 for (i = 0; i < ixgbe->mcast_count; i++) { 2228 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2229 ETHERADDRL) == 0) { 2230 for (i++; i < ixgbe->mcast_count; i++) { 2231 ixgbe->mcast_table[i - 1] = 2232 ixgbe->mcast_table[i]; 2233 } 2234 ixgbe->mcast_count--; 2235 break; 2236 } 2237 } 2238 2239 /* 2240 * Update the multicast table in the hardware 2241 */ 2242 ixgbe_setup_multicst(ixgbe); 2243 2244 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2245 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2246 return (EIO); 2247 } 2248 2249 return (0); 2250 } 2251 2252 /* 2253 * ixgbe_setup_multicast - Setup multicast data structures. 2254 * 2255 * This routine initializes all of the multicast related structures 2256 * and save them in the hardware registers. 2257 */ 2258 static void 2259 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2260 { 2261 uint8_t *mc_addr_list; 2262 uint32_t mc_addr_count; 2263 struct ixgbe_hw *hw = &ixgbe->hw; 2264 2265 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2266 2267 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2268 2269 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2270 mc_addr_count = ixgbe->mcast_count; 2271 2272 /* 2273 * Update the multicast addresses to the MTA registers 2274 */ 2275 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2276 ixgbe_mc_table_itr); 2277 } 2278 2279 /* 2280 * ixgbe_get_conf - Get driver configurations set in driver.conf. 2281 * 2282 * This routine gets user-configured values out of the configuration 2283 * file ixgbe.conf. 2284 * 2285 * For each configurable value, there is a minimum, a maximum, and a 2286 * default. 2287 * If user does not configure a value, use the default. 2288 * If user configures below the minimum, use the minumum. 2289 * If user configures above the maximum, use the maxumum. 2290 */ 2291 static void 2292 ixgbe_get_conf(ixgbe_t *ixgbe) 2293 { 2294 struct ixgbe_hw *hw = &ixgbe->hw; 2295 uint32_t flow_control; 2296 2297 /* 2298 * ixgbe driver supports the following user configurations: 2299 * 2300 * Jumbo frame configuration: 2301 * default_mtu 2302 * 2303 * Ethernet flow control configuration: 2304 * flow_control 2305 * 2306 * Multiple rings configurations: 2307 * tx_queue_number 2308 * tx_ring_size 2309 * rx_queue_number 2310 * rx_ring_size 2311 * 2312 * Call ixgbe_get_prop() to get the value for a specific 2313 * configuration parameter. 2314 */ 2315 2316 /* 2317 * Jumbo frame configuration - max_frame_size controls host buffer 2318 * allocation, so includes MTU, ethernet header, vlan tag and 2319 * frame check sequence. 2320 */ 2321 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 2322 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2323 2324 ixgbe->max_frame_size = ixgbe->default_mtu + 2325 sizeof (struct ether_vlan_header) + ETHERFCSL; 2326 2327 /* 2328 * Ethernet flow control configuration 2329 */ 2330 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 2331 ixgbe_fc_none, 3, ixgbe_fc_none); 2332 if (flow_control == 3) 2333 flow_control = ixgbe_fc_default; 2334 2335 /* 2336 * fc.requested mode is what the user requests. After autoneg, 2337 * fc.current_mode will be the flow_control mode that was negotiated. 2338 */ 2339 hw->fc.requested_mode = flow_control; 2340 2341 /* 2342 * Multiple rings configurations 2343 */ 2344 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 2345 ixgbe->capab->min_tx_que_num, 2346 ixgbe->capab->max_tx_que_num, 2347 ixgbe->capab->def_tx_que_num); 2348 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 2349 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2350 2351 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 2352 ixgbe->capab->min_rx_que_num, 2353 ixgbe->capab->max_rx_que_num, 2354 ixgbe->capab->def_rx_que_num); 2355 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 2356 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2357 2358 /* 2359 * Multiple groups configuration 2360 */ 2361 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 2362 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2363 2364 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 2365 0, 1, DEFAULT_MR_ENABLE); 2366 2367 if (ixgbe->mr_enable == B_FALSE) { 2368 ixgbe->num_tx_rings = 1; 2369 ixgbe->num_rx_rings = 1; 2370 ixgbe->num_rx_groups = 1; 2371 } 2372 2373 /* 2374 * Tunable used to force an interrupt type. The only use is 2375 * for testing of the lesser interrupt types. 2376 * 0 = don't force interrupt type 2377 * 1 = force interrupt type MSI-X 2378 * 2 = force interrupt type MSI 2379 * 3 = force interrupt type Legacy 2380 */ 2381 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 2382 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 2383 2384 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 2385 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 2386 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 2387 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 2388 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 2389 0, 1, DEFAULT_LSO_ENABLE); 2390 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 2391 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 2392 2393 /* Head Write Back not recommended for 82599 */ 2394 if (hw->mac.type >= ixgbe_mac_82599EB) { 2395 ixgbe->tx_head_wb_enable = B_FALSE; 2396 } 2397 2398 /* 2399 * ixgbe LSO needs the tx h/w checksum support. 2400 * LSO will be disabled if tx h/w checksum is not 2401 * enabled. 2402 */ 2403 if (ixgbe->tx_hcksum_enable == B_FALSE) { 2404 ixgbe->lso_enable = B_FALSE; 2405 } 2406 2407 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 2408 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2409 DEFAULT_TX_COPY_THRESHOLD); 2410 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 2411 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 2412 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 2413 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 2414 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 2415 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 2416 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 2417 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 2418 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 2419 2420 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 2421 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2422 DEFAULT_RX_COPY_THRESHOLD); 2423 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 2424 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2425 DEFAULT_RX_LIMIT_PER_INTR); 2426 2427 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 2428 ixgbe->capab->min_intr_throttle, 2429 ixgbe->capab->max_intr_throttle, 2430 ixgbe->capab->def_intr_throttle); 2431 /* 2432 * 82599 requires the interupt throttling rate is 2433 * a multiple of 8. This is enforced by the register 2434 * definiton. 2435 */ 2436 if (hw->mac.type == ixgbe_mac_82599EB) 2437 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 2438 } 2439 2440 static void 2441 ixgbe_init_params(ixgbe_t *ixgbe) 2442 { 2443 ixgbe->param_en_10000fdx_cap = 1; 2444 ixgbe->param_en_1000fdx_cap = 1; 2445 ixgbe->param_en_100fdx_cap = 1; 2446 ixgbe->param_adv_10000fdx_cap = 1; 2447 ixgbe->param_adv_1000fdx_cap = 1; 2448 ixgbe->param_adv_100fdx_cap = 1; 2449 2450 ixgbe->param_pause_cap = 1; 2451 ixgbe->param_asym_pause_cap = 1; 2452 ixgbe->param_rem_fault = 0; 2453 2454 ixgbe->param_adv_autoneg_cap = 1; 2455 ixgbe->param_adv_pause_cap = 1; 2456 ixgbe->param_adv_asym_pause_cap = 1; 2457 ixgbe->param_adv_rem_fault = 0; 2458 2459 ixgbe->param_lp_10000fdx_cap = 0; 2460 ixgbe->param_lp_1000fdx_cap = 0; 2461 ixgbe->param_lp_100fdx_cap = 0; 2462 ixgbe->param_lp_autoneg_cap = 0; 2463 ixgbe->param_lp_pause_cap = 0; 2464 ixgbe->param_lp_asym_pause_cap = 0; 2465 ixgbe->param_lp_rem_fault = 0; 2466 } 2467 2468 /* 2469 * ixgbe_get_prop - Get a property value out of the configuration file 2470 * ixgbe.conf. 2471 * 2472 * Caller provides the name of the property, a default value, a minimum 2473 * value, and a maximum value. 2474 * 2475 * Return configured value of the property, with default, minimum and 2476 * maximum properly applied. 2477 */ 2478 static int 2479 ixgbe_get_prop(ixgbe_t *ixgbe, 2480 char *propname, /* name of the property */ 2481 int minval, /* minimum acceptable value */ 2482 int maxval, /* maximim acceptable value */ 2483 int defval) /* default value */ 2484 { 2485 int value; 2486 2487 /* 2488 * Call ddi_prop_get_int() to read the conf settings 2489 */ 2490 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 2491 DDI_PROP_DONTPASS, propname, defval); 2492 if (value > maxval) 2493 value = maxval; 2494 2495 if (value < minval) 2496 value = minval; 2497 2498 return (value); 2499 } 2500 2501 /* 2502 * ixgbe_driver_setup_link - Using the link properties to setup the link. 2503 */ 2504 int 2505 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 2506 { 2507 struct ixgbe_mac_info *mac; 2508 struct ixgbe_phy_info *phy; 2509 boolean_t invalid; 2510 2511 mac = &ixgbe->hw.mac; 2512 phy = &ixgbe->hw.phy; 2513 invalid = B_FALSE; 2514 2515 if (ixgbe->param_adv_autoneg_cap == 1) { 2516 mac->autoneg = B_TRUE; 2517 phy->autoneg_advertised = 0; 2518 2519 /* 2520 * No half duplex support with 10Gb parts 2521 */ 2522 if (ixgbe->param_adv_10000fdx_cap == 1) 2523 phy->autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 2524 2525 if (ixgbe->param_adv_1000fdx_cap == 1) 2526 phy->autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 2527 2528 if (ixgbe->param_adv_100fdx_cap == 1) 2529 phy->autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 2530 2531 if (phy->autoneg_advertised == 0) 2532 invalid = B_TRUE; 2533 } else { 2534 ixgbe->hw.mac.autoneg = B_FALSE; 2535 } 2536 2537 if (invalid) { 2538 ixgbe_notice(ixgbe, "Invalid link settings. Setup link to " 2539 "autonegotiation with full link capabilities."); 2540 ixgbe->hw.mac.autoneg = B_TRUE; 2541 } 2542 2543 if (setup_hw) { 2544 if (ixgbe_setup_link(&ixgbe->hw) != IXGBE_SUCCESS) { 2545 ixgbe_notice(ixgbe, "Setup link failed on this " 2546 "device."); 2547 return (IXGBE_FAILURE); 2548 } 2549 } 2550 2551 return (IXGBE_SUCCESS); 2552 } 2553 2554 /* 2555 * ixgbe_driver_link_check - Link status processing done in taskq. 2556 */ 2557 static void 2558 ixgbe_driver_link_check(void *arg) 2559 { 2560 ixgbe_t *ixgbe = (ixgbe_t *)arg; 2561 struct ixgbe_hw *hw = &ixgbe->hw; 2562 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 2563 boolean_t link_up = B_FALSE; 2564 boolean_t link_changed = B_FALSE; 2565 2566 mutex_enter(&ixgbe->gen_lock); 2567 2568 /* check for link, wait the full time */ 2569 (void) ixgbe_check_link(hw, &speed, &link_up, true); 2570 if (link_up) { 2571 /* Link is up, enable flow control settings */ 2572 (void) ixgbe_fc_enable(hw, 0); 2573 2574 /* 2575 * The Link is up, check whether it was marked as down earlier 2576 */ 2577 if (ixgbe->link_state != LINK_STATE_UP) { 2578 switch (speed) { 2579 case IXGBE_LINK_SPEED_10GB_FULL: 2580 ixgbe->link_speed = SPEED_10GB; 2581 break; 2582 case IXGBE_LINK_SPEED_1GB_FULL: 2583 ixgbe->link_speed = SPEED_1GB; 2584 break; 2585 case IXGBE_LINK_SPEED_100_FULL: 2586 ixgbe->link_speed = SPEED_100; 2587 } 2588 ixgbe->link_duplex = LINK_DUPLEX_FULL; 2589 ixgbe->link_state = LINK_STATE_UP; 2590 ixgbe->link_down_timeout = 0; 2591 link_changed = B_TRUE; 2592 } 2593 } else { 2594 if (ixgbe->link_state != LINK_STATE_DOWN) { 2595 ixgbe->link_speed = 0; 2596 ixgbe->link_duplex = 0; 2597 ixgbe->link_state = LINK_STATE_DOWN; 2598 link_changed = B_TRUE; 2599 } 2600 2601 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 2602 if (ixgbe->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) { 2603 ixgbe->link_down_timeout++; 2604 } else if (ixgbe->link_down_timeout == 2605 MAX_LINK_DOWN_TIMEOUT) { 2606 ixgbe_tx_clean(ixgbe); 2607 ixgbe->link_down_timeout++; 2608 } 2609 } 2610 } 2611 2612 /* 2613 * this is only reached after a link-status-change interrupt 2614 * so always get new phy state 2615 */ 2616 ixgbe_get_hw_state(ixgbe); 2617 2618 /* re-enable the interrupt, which was automasked */ 2619 ixgbe->eims |= IXGBE_EICR_LSC; 2620 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 2621 2622 mutex_exit(&ixgbe->gen_lock); 2623 2624 /* outside the gen_lock */ 2625 if (link_changed) { 2626 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 2627 } 2628 } 2629 2630 /* 2631 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 2632 */ 2633 static void 2634 ixgbe_sfp_check(void *arg) 2635 { 2636 ixgbe_t *ixgbe = (ixgbe_t *)arg; 2637 uint32_t eicr = ixgbe->eicr; 2638 struct ixgbe_hw *hw = &ixgbe->hw; 2639 uint32_t autoneg; 2640 2641 if (eicr & IXGBE_EICR_GPI_SDP1) { 2642 /* clear the interrupt */ 2643 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 2644 2645 /* if link up, do multispeed fiber setup */ 2646 (void) ixgbe_get_link_capabilities(hw, &autoneg, 2647 &hw->mac.autoneg); 2648 (void) ixgbe_setup_link_speed(hw, autoneg, B_TRUE, B_TRUE); 2649 ixgbe_driver_link_check(ixgbe); 2650 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 2651 /* clear the interrupt */ 2652 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 2653 2654 /* if link up, do sfp module setup */ 2655 (void) hw->mac.ops.setup_sfp(hw); 2656 2657 /* do multispeed fiber setup */ 2658 (void) ixgbe_get_link_capabilities(hw, &autoneg, 2659 &hw->mac.autoneg); 2660 (void) ixgbe_setup_link_speed(hw, autoneg, B_TRUE, B_TRUE); 2661 ixgbe_driver_link_check(ixgbe); 2662 } 2663 } 2664 2665 /* 2666 * ixgbe_local_timer - Driver watchdog function. 2667 * 2668 * This function will handle the transmit stall check, link status check and 2669 * other routines. 2670 */ 2671 static void 2672 ixgbe_local_timer(void *arg) 2673 { 2674 ixgbe_t *ixgbe = (ixgbe_t *)arg; 2675 2676 if (ixgbe_stall_check(ixgbe)) { 2677 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2678 ixgbe->reset_count++; 2679 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 2680 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 2681 } 2682 2683 ixgbe_restart_watchdog_timer(ixgbe); 2684 } 2685 2686 /* 2687 * ixgbe_stall_check - Check for transmit stall. 2688 * 2689 * This function checks if the adapter is stalled (in transmit). 2690 * 2691 * It is called each time the watchdog timeout is invoked. 2692 * If the transmit descriptor reclaim continuously fails, 2693 * the watchdog value will increment by 1. If the watchdog 2694 * value exceeds the threshold, the ixgbe is assumed to 2695 * have stalled and need to be reset. 2696 */ 2697 static boolean_t 2698 ixgbe_stall_check(ixgbe_t *ixgbe) 2699 { 2700 ixgbe_tx_ring_t *tx_ring; 2701 boolean_t result; 2702 int i; 2703 2704 if (ixgbe->link_state != LINK_STATE_UP) 2705 return (B_FALSE); 2706 2707 /* 2708 * If any tx ring is stalled, we'll reset the chipset 2709 */ 2710 result = B_FALSE; 2711 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2712 tx_ring = &ixgbe->tx_rings[i]; 2713 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 2714 tx_ring->tx_recycle(tx_ring); 2715 } 2716 2717 if (tx_ring->recycle_fail > 0) 2718 tx_ring->stall_watchdog++; 2719 else 2720 tx_ring->stall_watchdog = 0; 2721 2722 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 2723 result = B_TRUE; 2724 break; 2725 } 2726 } 2727 2728 if (result) { 2729 tx_ring->stall_watchdog = 0; 2730 tx_ring->recycle_fail = 0; 2731 } 2732 2733 return (result); 2734 } 2735 2736 2737 /* 2738 * is_valid_mac_addr - Check if the mac address is valid. 2739 */ 2740 static boolean_t 2741 is_valid_mac_addr(uint8_t *mac_addr) 2742 { 2743 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 2744 const uint8_t addr_test2[6] = 2745 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 2746 2747 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 2748 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 2749 return (B_FALSE); 2750 2751 return (B_TRUE); 2752 } 2753 2754 static boolean_t 2755 ixgbe_find_mac_address(ixgbe_t *ixgbe) 2756 { 2757 #ifdef __sparc 2758 struct ixgbe_hw *hw = &ixgbe->hw; 2759 uchar_t *bytes; 2760 struct ether_addr sysaddr; 2761 uint_t nelts; 2762 int err; 2763 boolean_t found = B_FALSE; 2764 2765 /* 2766 * The "vendor's factory-set address" may already have 2767 * been extracted from the chip, but if the property 2768 * "local-mac-address" is set we use that instead. 2769 * 2770 * We check whether it looks like an array of 6 2771 * bytes (which it should, if OBP set it). If we can't 2772 * make sense of it this way, we'll ignore it. 2773 */ 2774 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 2775 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 2776 if (err == DDI_PROP_SUCCESS) { 2777 if (nelts == ETHERADDRL) { 2778 while (nelts--) 2779 hw->mac.addr[nelts] = bytes[nelts]; 2780 found = B_TRUE; 2781 } 2782 ddi_prop_free(bytes); 2783 } 2784 2785 /* 2786 * Look up the OBP property "local-mac-address?". If the user has set 2787 * 'local-mac-address? = false', use "the system address" instead. 2788 */ 2789 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 2790 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 2791 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 2792 if (localetheraddr(NULL, &sysaddr) != 0) { 2793 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 2794 found = B_TRUE; 2795 } 2796 } 2797 ddi_prop_free(bytes); 2798 } 2799 2800 /* 2801 * Finally(!), if there's a valid "mac-address" property (created 2802 * if we netbooted from this interface), we must use this instead 2803 * of any of the above to ensure that the NFS/install server doesn't 2804 * get confused by the address changing as Solaris takes over! 2805 */ 2806 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 2807 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 2808 if (err == DDI_PROP_SUCCESS) { 2809 if (nelts == ETHERADDRL) { 2810 while (nelts--) 2811 hw->mac.addr[nelts] = bytes[nelts]; 2812 found = B_TRUE; 2813 } 2814 ddi_prop_free(bytes); 2815 } 2816 2817 if (found) { 2818 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 2819 return (B_TRUE); 2820 } 2821 #else 2822 _NOTE(ARGUNUSED(ixgbe)); 2823 #endif 2824 2825 return (B_TRUE); 2826 } 2827 2828 #pragma inline(ixgbe_arm_watchdog_timer) 2829 static void 2830 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 2831 { 2832 /* 2833 * Fire a watchdog timer 2834 */ 2835 ixgbe->watchdog_tid = 2836 timeout(ixgbe_local_timer, 2837 (void *)ixgbe, 1 * drv_usectohz(1000000)); 2838 2839 } 2840 2841 /* 2842 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 2843 */ 2844 void 2845 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 2846 { 2847 mutex_enter(&ixgbe->watchdog_lock); 2848 2849 if (!ixgbe->watchdog_enable) { 2850 ixgbe->watchdog_enable = B_TRUE; 2851 ixgbe->watchdog_start = B_TRUE; 2852 ixgbe_arm_watchdog_timer(ixgbe); 2853 } 2854 2855 mutex_exit(&ixgbe->watchdog_lock); 2856 } 2857 2858 /* 2859 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 2860 */ 2861 void 2862 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 2863 { 2864 timeout_id_t tid; 2865 2866 mutex_enter(&ixgbe->watchdog_lock); 2867 2868 ixgbe->watchdog_enable = B_FALSE; 2869 ixgbe->watchdog_start = B_FALSE; 2870 tid = ixgbe->watchdog_tid; 2871 ixgbe->watchdog_tid = 0; 2872 2873 mutex_exit(&ixgbe->watchdog_lock); 2874 2875 if (tid != 0) 2876 (void) untimeout(tid); 2877 } 2878 2879 /* 2880 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 2881 */ 2882 void 2883 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 2884 { 2885 mutex_enter(&ixgbe->watchdog_lock); 2886 2887 if (ixgbe->watchdog_enable) { 2888 if (!ixgbe->watchdog_start) { 2889 ixgbe->watchdog_start = B_TRUE; 2890 ixgbe_arm_watchdog_timer(ixgbe); 2891 } 2892 } 2893 2894 mutex_exit(&ixgbe->watchdog_lock); 2895 } 2896 2897 /* 2898 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 2899 */ 2900 static void 2901 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 2902 { 2903 mutex_enter(&ixgbe->watchdog_lock); 2904 2905 if (ixgbe->watchdog_start) 2906 ixgbe_arm_watchdog_timer(ixgbe); 2907 2908 mutex_exit(&ixgbe->watchdog_lock); 2909 } 2910 2911 /* 2912 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 2913 */ 2914 void 2915 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 2916 { 2917 timeout_id_t tid; 2918 2919 mutex_enter(&ixgbe->watchdog_lock); 2920 2921 ixgbe->watchdog_start = B_FALSE; 2922 tid = ixgbe->watchdog_tid; 2923 ixgbe->watchdog_tid = 0; 2924 2925 mutex_exit(&ixgbe->watchdog_lock); 2926 2927 if (tid != 0) 2928 (void) untimeout(tid); 2929 } 2930 2931 /* 2932 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 2933 */ 2934 static void 2935 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 2936 { 2937 struct ixgbe_hw *hw = &ixgbe->hw; 2938 2939 /* 2940 * mask all interrupts off 2941 */ 2942 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 2943 2944 /* 2945 * for MSI-X, also disable autoclear 2946 */ 2947 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 2948 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 2949 } 2950 2951 IXGBE_WRITE_FLUSH(hw); 2952 } 2953 2954 /* 2955 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 2956 */ 2957 static void 2958 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 2959 { 2960 struct ixgbe_hw *hw = &ixgbe->hw; 2961 uint32_t eiac, eiam; 2962 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 2963 2964 /* interrupt types to enable */ 2965 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 2966 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 2967 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 2968 2969 /* enable automask on "other" causes that this adapter can generate */ 2970 eiam = ixgbe->capab->other_intr; 2971 2972 /* 2973 * msi-x mode 2974 */ 2975 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 2976 /* enable autoclear but not on bits 29:20 */ 2977 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 2978 2979 /* general purpose interrupt enable */ 2980 gpie |= (IXGBE_GPIE_MSIX_MODE 2981 | IXGBE_GPIE_PBA_SUPPORT 2982 | IXGBE_GPIE_OCD 2983 | IXGBE_GPIE_EIAME); 2984 /* 2985 * non-msi-x mode 2986 */ 2987 } else { 2988 2989 /* disable autoclear, leave gpie at default */ 2990 eiac = 0; 2991 2992 /* 2993 * General purpose interrupt enable. 2994 * For 82599, extended interrupt automask enable 2995 * only in MSI or MSI-X mode 2996 */ 2997 if ((hw->mac.type < ixgbe_mac_82599EB) || 2998 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 2999 gpie |= IXGBE_GPIE_EIAME; 3000 } 3001 } 3002 /* Enable specific interrupts for 82599 */ 3003 if (hw->mac.type == ixgbe_mac_82599EB) { 3004 gpie |= IXGBE_SDP2_GPIEN; /* pluggable optics intr */ 3005 gpie |= IXGBE_SDP1_GPIEN; /* LSC interrupt */ 3006 } 3007 3008 /* write to interrupt control registers */ 3009 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3010 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3011 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3012 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3013 IXGBE_WRITE_FLUSH(hw); 3014 } 3015 3016 /* 3017 * ixgbe_loopback_ioctl - Loopback support. 3018 */ 3019 enum ioc_reply 3020 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3021 { 3022 lb_info_sz_t *lbsp; 3023 lb_property_t *lbpp; 3024 uint32_t *lbmp; 3025 uint32_t size; 3026 uint32_t value; 3027 3028 if (mp->b_cont == NULL) 3029 return (IOC_INVAL); 3030 3031 switch (iocp->ioc_cmd) { 3032 default: 3033 return (IOC_INVAL); 3034 3035 case LB_GET_INFO_SIZE: 3036 size = sizeof (lb_info_sz_t); 3037 if (iocp->ioc_count != size) 3038 return (IOC_INVAL); 3039 3040 value = sizeof (lb_normal); 3041 value += sizeof (lb_mac); 3042 3043 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3044 *lbsp = value; 3045 break; 3046 3047 case LB_GET_INFO: 3048 value = sizeof (lb_normal); 3049 value += sizeof (lb_mac); 3050 3051 size = value; 3052 if (iocp->ioc_count != size) 3053 return (IOC_INVAL); 3054 3055 value = 0; 3056 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3057 3058 lbpp[value++] = lb_normal; 3059 lbpp[value++] = lb_mac; 3060 break; 3061 3062 case LB_GET_MODE: 3063 size = sizeof (uint32_t); 3064 if (iocp->ioc_count != size) 3065 return (IOC_INVAL); 3066 3067 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3068 *lbmp = ixgbe->loopback_mode; 3069 break; 3070 3071 case LB_SET_MODE: 3072 size = 0; 3073 if (iocp->ioc_count != sizeof (uint32_t)) 3074 return (IOC_INVAL); 3075 3076 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3077 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3078 return (IOC_INVAL); 3079 break; 3080 } 3081 3082 iocp->ioc_count = size; 3083 iocp->ioc_error = 0; 3084 3085 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3086 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3087 return (IOC_INVAL); 3088 } 3089 3090 return (IOC_REPLY); 3091 } 3092 3093 /* 3094 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3095 */ 3096 static boolean_t 3097 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3098 { 3099 struct ixgbe_hw *hw; 3100 3101 if (mode == ixgbe->loopback_mode) 3102 return (B_TRUE); 3103 3104 hw = &ixgbe->hw; 3105 3106 ixgbe->loopback_mode = mode; 3107 3108 if (mode == IXGBE_LB_NONE) { 3109 /* 3110 * Reset the chip 3111 */ 3112 hw->phy.autoneg_wait_to_complete = B_TRUE; 3113 (void) ixgbe_reset(ixgbe); 3114 hw->phy.autoneg_wait_to_complete = B_FALSE; 3115 return (B_TRUE); 3116 } 3117 3118 mutex_enter(&ixgbe->gen_lock); 3119 3120 switch (mode) { 3121 default: 3122 mutex_exit(&ixgbe->gen_lock); 3123 return (B_FALSE); 3124 3125 case IXGBE_LB_INTERNAL_MAC: 3126 ixgbe_set_internal_mac_loopback(ixgbe); 3127 break; 3128 } 3129 3130 mutex_exit(&ixgbe->gen_lock); 3131 3132 return (B_TRUE); 3133 } 3134 3135 /* 3136 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 3137 */ 3138 static void 3139 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 3140 { 3141 struct ixgbe_hw *hw; 3142 uint32_t reg; 3143 uint8_t atlas; 3144 3145 hw = &ixgbe->hw; 3146 3147 /* 3148 * Setup MAC loopback 3149 */ 3150 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 3151 reg |= IXGBE_HLREG0_LPBK; 3152 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 3153 3154 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 3155 reg &= ~IXGBE_AUTOC_LMS_MASK; 3156 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 3157 3158 /* 3159 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 3160 */ 3161 if (hw->mac.type == ixgbe_mac_82598EB) { 3162 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3163 &atlas); 3164 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 3165 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3166 atlas); 3167 3168 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3169 &atlas); 3170 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 3171 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3172 atlas); 3173 3174 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3175 &atlas); 3176 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 3177 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3178 atlas); 3179 3180 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3181 &atlas); 3182 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 3183 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3184 atlas); 3185 } 3186 } 3187 3188 #pragma inline(ixgbe_intr_rx_work) 3189 /* 3190 * ixgbe_intr_rx_work - RX processing of ISR. 3191 */ 3192 static void 3193 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 3194 { 3195 mblk_t *mp; 3196 3197 mutex_enter(&rx_ring->rx_lock); 3198 3199 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 3200 mutex_exit(&rx_ring->rx_lock); 3201 3202 if (mp != NULL) 3203 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 3204 rx_ring->ring_gen_num); 3205 } 3206 3207 #pragma inline(ixgbe_intr_tx_work) 3208 /* 3209 * ixgbe_intr_tx_work - TX processing of ISR. 3210 */ 3211 static void 3212 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 3213 { 3214 ixgbe_t *ixgbe = tx_ring->ixgbe; 3215 3216 /* 3217 * Recycle the tx descriptors 3218 */ 3219 tx_ring->tx_recycle(tx_ring); 3220 3221 /* 3222 * Schedule the re-transmit 3223 */ 3224 if (tx_ring->reschedule && 3225 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 3226 tx_ring->reschedule = B_FALSE; 3227 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 3228 tx_ring->ring_handle); 3229 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 3230 } 3231 } 3232 3233 #pragma inline(ixgbe_intr_other_work) 3234 /* 3235 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 3236 */ 3237 static void 3238 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 3239 { 3240 struct ixgbe_hw *hw = &ixgbe->hw; 3241 /* 3242 * dispatch taskq to handle link status change 3243 */ 3244 if (eicr & IXGBE_EICR_LSC) { 3245 if ((ddi_taskq_dispatch(ixgbe->lsc_taskq, 3246 ixgbe_driver_link_check, (void *)ixgbe, DDI_NOSLEEP)) 3247 != DDI_SUCCESS) { 3248 ixgbe_log(ixgbe, "Fail to dispatch taskq"); 3249 } 3250 } 3251 3252 /* 3253 * check for fan failure on adapters with fans 3254 */ 3255 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 3256 (eicr & IXGBE_EICR_GPI_SDP1)) { 3257 if (hw->mac.type < ixgbe_mac_82599EB) { 3258 ixgbe_log(ixgbe, 3259 "Fan has stopped, replace the adapter\n"); 3260 3261 /* re-enable the interrupt, which was automasked */ 3262 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 3263 } 3264 } 3265 3266 /* 3267 * Do SFP check for 82599 3268 */ 3269 if (hw->mac.type == ixgbe_mac_82599EB) { 3270 if ((ddi_taskq_dispatch(ixgbe->lsc_taskq, 3271 ixgbe_sfp_check, (void *)ixgbe, 3272 DDI_NOSLEEP)) != DDI_SUCCESS) { 3273 ixgbe_log(ixgbe, "No memory available to dispatch " 3274 "taskq for SFP check"); 3275 } 3276 } 3277 } 3278 3279 /* 3280 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 3281 */ 3282 static uint_t 3283 ixgbe_intr_legacy(void *arg1, void *arg2) 3284 { 3285 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 3286 struct ixgbe_hw *hw = &ixgbe->hw; 3287 ixgbe_tx_ring_t *tx_ring; 3288 ixgbe_rx_ring_t *rx_ring; 3289 uint32_t eicr; 3290 mblk_t *mp; 3291 boolean_t tx_reschedule; 3292 uint_t result; 3293 3294 _NOTE(ARGUNUSED(arg2)); 3295 3296 mutex_enter(&ixgbe->gen_lock); 3297 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 3298 mutex_exit(&ixgbe->gen_lock); 3299 return (DDI_INTR_UNCLAIMED); 3300 } 3301 3302 mp = NULL; 3303 tx_reschedule = B_FALSE; 3304 3305 /* 3306 * Any bit set in eicr: claim this interrupt 3307 */ 3308 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 3309 if (eicr) { 3310 /* 3311 * For legacy interrupt, we have only one interrupt, 3312 * so we have only one rx ring and one tx ring enabled. 3313 */ 3314 ASSERT(ixgbe->num_rx_rings == 1); 3315 ASSERT(ixgbe->num_tx_rings == 1); 3316 3317 /* 3318 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 3319 */ 3320 if (eicr & 0x1) { 3321 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 3322 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 3323 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 3324 /* 3325 * Clean the rx descriptors 3326 */ 3327 rx_ring = &ixgbe->rx_rings[0]; 3328 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 3329 } 3330 3331 /* 3332 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 3333 */ 3334 if (eicr & 0x2) { 3335 /* 3336 * Recycle the tx descriptors 3337 */ 3338 tx_ring = &ixgbe->tx_rings[0]; 3339 tx_ring->tx_recycle(tx_ring); 3340 3341 /* 3342 * Schedule the re-transmit 3343 */ 3344 tx_reschedule = (tx_ring->reschedule && 3345 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 3346 } 3347 3348 /* any interrupt type other than tx/rx */ 3349 if (eicr & ixgbe->capab->other_intr) { 3350 if (hw->mac.type < ixgbe_mac_82599EB) { 3351 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3352 } 3353 if (hw->mac.type == ixgbe_mac_82599EB) { 3354 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 3355 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 3356 } 3357 ixgbe_intr_other_work(ixgbe, eicr); 3358 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3359 } 3360 3361 mutex_exit(&ixgbe->gen_lock); 3362 3363 result = DDI_INTR_CLAIMED; 3364 } else { 3365 mutex_exit(&ixgbe->gen_lock); 3366 3367 /* 3368 * No interrupt cause bits set: don't claim this interrupt. 3369 */ 3370 result = DDI_INTR_UNCLAIMED; 3371 } 3372 3373 /* re-enable the interrupts which were automasked */ 3374 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3375 3376 /* 3377 * Do the following work outside of the gen_lock 3378 */ 3379 if (mp != NULL) { 3380 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 3381 rx_ring->ring_gen_num); 3382 } 3383 3384 if (tx_reschedule) { 3385 tx_ring->reschedule = B_FALSE; 3386 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 3387 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 3388 } 3389 3390 return (result); 3391 } 3392 3393 /* 3394 * ixgbe_intr_msi - Interrupt handler for MSI. 3395 */ 3396 static uint_t 3397 ixgbe_intr_msi(void *arg1, void *arg2) 3398 { 3399 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 3400 struct ixgbe_hw *hw = &ixgbe->hw; 3401 uint32_t eicr; 3402 3403 _NOTE(ARGUNUSED(arg2)); 3404 3405 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 3406 3407 /* 3408 * For MSI interrupt, we have only one vector, 3409 * so we have only one rx ring and one tx ring enabled. 3410 */ 3411 ASSERT(ixgbe->num_rx_rings == 1); 3412 ASSERT(ixgbe->num_tx_rings == 1); 3413 3414 /* 3415 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 3416 */ 3417 if (eicr & 0x1) { 3418 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 3419 } 3420 3421 /* 3422 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 3423 */ 3424 if (eicr & 0x2) { 3425 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 3426 } 3427 3428 /* any interrupt type other than tx/rx */ 3429 if (eicr & ixgbe->capab->other_intr) { 3430 mutex_enter(&ixgbe->gen_lock); 3431 if (hw->mac.type < ixgbe_mac_82599EB) { 3432 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3433 } 3434 if (hw->mac.type == ixgbe_mac_82599EB) { 3435 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 3436 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 3437 } 3438 ixgbe_intr_other_work(ixgbe, eicr); 3439 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3440 mutex_exit(&ixgbe->gen_lock); 3441 } 3442 3443 /* re-enable the interrupts which were automasked */ 3444 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3445 3446 return (DDI_INTR_CLAIMED); 3447 } 3448 3449 /* 3450 * ixgbe_intr_msix - Interrupt handler for MSI-X. 3451 */ 3452 static uint_t 3453 ixgbe_intr_msix(void *arg1, void *arg2) 3454 { 3455 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 3456 ixgbe_t *ixgbe = vect->ixgbe; 3457 struct ixgbe_hw *hw = &ixgbe->hw; 3458 uint32_t eicr; 3459 int r_idx = 0; 3460 3461 _NOTE(ARGUNUSED(arg2)); 3462 3463 /* 3464 * Clean each rx ring that has its bit set in the map 3465 */ 3466 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 3467 while (r_idx >= 0) { 3468 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 3469 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 3470 (ixgbe->num_rx_rings - 1)); 3471 } 3472 3473 /* 3474 * Clean each tx ring that has its bit set in the map 3475 */ 3476 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 3477 while (r_idx >= 0) { 3478 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 3479 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 3480 (ixgbe->num_tx_rings - 1)); 3481 } 3482 3483 3484 /* 3485 * Clean other interrupt (link change) that has its bit set in the map 3486 */ 3487 if (BT_TEST(vect->other_map, 0) == 1) { 3488 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 3489 3490 /* 3491 * Need check cause bits and only other causes will 3492 * be processed 3493 */ 3494 /* any interrupt type other than tx/rx */ 3495 if (eicr & ixgbe->capab->other_intr) { 3496 if (hw->mac.type < ixgbe_mac_82599EB) { 3497 mutex_enter(&ixgbe->gen_lock); 3498 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3499 ixgbe_intr_other_work(ixgbe, eicr); 3500 mutex_exit(&ixgbe->gen_lock); 3501 } else { 3502 if (hw->mac.type == ixgbe_mac_82599EB) { 3503 mutex_enter(&ixgbe->gen_lock); 3504 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 3505 ixgbe_intr_other_work(ixgbe, eicr); 3506 mutex_exit(&ixgbe->gen_lock); 3507 } 3508 } 3509 } 3510 3511 /* re-enable the interrupts which were automasked */ 3512 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3513 } 3514 3515 return (DDI_INTR_CLAIMED); 3516 } 3517 3518 /* 3519 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 3520 * 3521 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 3522 * if not successful, try Legacy. 3523 * ixgbe->intr_force can be used to force sequence to start with 3524 * any of the 3 types. 3525 * If MSI-X is not used, number of tx/rx rings is forced to 1. 3526 */ 3527 static int 3528 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 3529 { 3530 dev_info_t *devinfo; 3531 int intr_types; 3532 int rc; 3533 3534 devinfo = ixgbe->dip; 3535 3536 /* 3537 * Get supported interrupt types 3538 */ 3539 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 3540 3541 if (rc != DDI_SUCCESS) { 3542 ixgbe_log(ixgbe, 3543 "Get supported interrupt types failed: %d", rc); 3544 return (IXGBE_FAILURE); 3545 } 3546 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 3547 3548 ixgbe->intr_type = 0; 3549 3550 /* 3551 * Install MSI-X interrupts 3552 */ 3553 if ((intr_types & DDI_INTR_TYPE_MSIX) && 3554 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 3555 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 3556 if (rc == IXGBE_SUCCESS) 3557 return (IXGBE_SUCCESS); 3558 3559 ixgbe_log(ixgbe, 3560 "Allocate MSI-X failed, trying MSI interrupts..."); 3561 } 3562 3563 /* 3564 * MSI-X not used, force rings and groups to 1 3565 */ 3566 ixgbe->num_rx_rings = 1; 3567 ixgbe->num_rx_groups = 1; 3568 ixgbe->num_tx_rings = 1; 3569 ixgbe_log(ixgbe, 3570 "MSI-X not used, force rings and groups number to 1"); 3571 3572 /* 3573 * Install MSI interrupts 3574 */ 3575 if ((intr_types & DDI_INTR_TYPE_MSI) && 3576 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 3577 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 3578 if (rc == IXGBE_SUCCESS) 3579 return (IXGBE_SUCCESS); 3580 3581 ixgbe_log(ixgbe, 3582 "Allocate MSI failed, trying Legacy interrupts..."); 3583 } 3584 3585 /* 3586 * Install legacy interrupts 3587 */ 3588 if (intr_types & DDI_INTR_TYPE_FIXED) { 3589 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 3590 if (rc == IXGBE_SUCCESS) 3591 return (IXGBE_SUCCESS); 3592 3593 ixgbe_log(ixgbe, 3594 "Allocate Legacy interrupts failed"); 3595 } 3596 3597 /* 3598 * If none of the 3 types succeeded, return failure 3599 */ 3600 return (IXGBE_FAILURE); 3601 } 3602 3603 /* 3604 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 3605 * 3606 * For legacy and MSI, only 1 handle is needed. For MSI-X, 3607 * if fewer than 2 handles are available, return failure. 3608 * Upon success, this maps the vectors to rx and tx rings for 3609 * interrupts. 3610 */ 3611 static int 3612 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 3613 { 3614 dev_info_t *devinfo; 3615 int request, count, avail, actual; 3616 int minimum; 3617 int rc; 3618 3619 devinfo = ixgbe->dip; 3620 3621 switch (intr_type) { 3622 case DDI_INTR_TYPE_FIXED: 3623 request = 1; /* Request 1 legacy interrupt handle */ 3624 minimum = 1; 3625 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 3626 break; 3627 3628 case DDI_INTR_TYPE_MSI: 3629 request = 1; /* Request 1 MSI interrupt handle */ 3630 minimum = 1; 3631 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 3632 break; 3633 3634 case DDI_INTR_TYPE_MSIX: 3635 /* 3636 * Best number of vectors for the adapter is 3637 * # rx rings + # tx rings. 3638 */ 3639 request = ixgbe->num_rx_rings + ixgbe->num_tx_rings; 3640 if (request > ixgbe->capab->max_ring_vect) 3641 request = ixgbe->capab->max_ring_vect; 3642 minimum = 2; 3643 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 3644 break; 3645 3646 default: 3647 ixgbe_log(ixgbe, 3648 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 3649 intr_type); 3650 return (IXGBE_FAILURE); 3651 } 3652 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 3653 request, minimum); 3654 3655 /* 3656 * Get number of supported interrupts 3657 */ 3658 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 3659 if ((rc != DDI_SUCCESS) || (count < minimum)) { 3660 ixgbe_log(ixgbe, 3661 "Get interrupt number failed. Return: %d, count: %d", 3662 rc, count); 3663 return (IXGBE_FAILURE); 3664 } 3665 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 3666 3667 /* 3668 * Get number of available interrupts 3669 */ 3670 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 3671 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 3672 ixgbe_log(ixgbe, 3673 "Get interrupt available number failed. " 3674 "Return: %d, available: %d", rc, avail); 3675 return (IXGBE_FAILURE); 3676 } 3677 IXGBE_DEBUGLOG_1(ixgbe, "interrupts available: %d", avail); 3678 3679 if (avail < request) { 3680 ixgbe_log(ixgbe, "Request %d handles, %d available", 3681 request, avail); 3682 request = avail; 3683 } 3684 3685 actual = 0; 3686 ixgbe->intr_cnt = 0; 3687 3688 /* 3689 * Allocate an array of interrupt handles 3690 */ 3691 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 3692 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 3693 3694 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 3695 request, &actual, DDI_INTR_ALLOC_NORMAL); 3696 if (rc != DDI_SUCCESS) { 3697 ixgbe_log(ixgbe, "Allocate interrupts failed. " 3698 "return: %d, request: %d, actual: %d", 3699 rc, request, actual); 3700 goto alloc_handle_fail; 3701 } 3702 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 3703 3704 ixgbe->intr_cnt = actual; 3705 3706 /* 3707 * Now we know the actual number of vectors. Here we map the vector 3708 * to other, rx rings and tx ring. 3709 */ 3710 if (actual < minimum) { 3711 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 3712 actual); 3713 goto alloc_handle_fail; 3714 } 3715 3716 /* 3717 * Get priority for first vector, assume remaining are all the same 3718 */ 3719 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 3720 if (rc != DDI_SUCCESS) { 3721 ixgbe_log(ixgbe, 3722 "Get interrupt priority failed: %d", rc); 3723 goto alloc_handle_fail; 3724 } 3725 3726 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 3727 if (rc != DDI_SUCCESS) { 3728 ixgbe_log(ixgbe, 3729 "Get interrupt cap failed: %d", rc); 3730 goto alloc_handle_fail; 3731 } 3732 3733 ixgbe->intr_type = intr_type; 3734 3735 return (IXGBE_SUCCESS); 3736 3737 alloc_handle_fail: 3738 ixgbe_rem_intrs(ixgbe); 3739 3740 return (IXGBE_FAILURE); 3741 } 3742 3743 /* 3744 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 3745 * 3746 * Before adding the interrupt handlers, the interrupt vectors have 3747 * been allocated, and the rx/tx rings have also been allocated. 3748 */ 3749 static int 3750 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 3751 { 3752 int vector = 0; 3753 int rc; 3754 3755 switch (ixgbe->intr_type) { 3756 case DDI_INTR_TYPE_MSIX: 3757 /* 3758 * Add interrupt handler for all vectors 3759 */ 3760 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 3761 /* 3762 * install pointer to vect_map[vector] 3763 */ 3764 rc = ddi_intr_add_handler(ixgbe->htable[vector], 3765 (ddi_intr_handler_t *)ixgbe_intr_msix, 3766 (void *)&ixgbe->vect_map[vector], NULL); 3767 3768 if (rc != DDI_SUCCESS) { 3769 ixgbe_log(ixgbe, 3770 "Add rx interrupt handler failed. " 3771 "return: %d, vector: %d", rc, vector); 3772 for (vector--; vector >= 0; vector--) { 3773 (void) ddi_intr_remove_handler( 3774 ixgbe->htable[vector]); 3775 } 3776 return (IXGBE_FAILURE); 3777 } 3778 } 3779 3780 break; 3781 3782 case DDI_INTR_TYPE_MSI: 3783 /* 3784 * Add interrupt handlers for the only vector 3785 */ 3786 rc = ddi_intr_add_handler(ixgbe->htable[vector], 3787 (ddi_intr_handler_t *)ixgbe_intr_msi, 3788 (void *)ixgbe, NULL); 3789 3790 if (rc != DDI_SUCCESS) { 3791 ixgbe_log(ixgbe, 3792 "Add MSI interrupt handler failed: %d", rc); 3793 return (IXGBE_FAILURE); 3794 } 3795 3796 break; 3797 3798 case DDI_INTR_TYPE_FIXED: 3799 /* 3800 * Add interrupt handlers for the only vector 3801 */ 3802 rc = ddi_intr_add_handler(ixgbe->htable[vector], 3803 (ddi_intr_handler_t *)ixgbe_intr_legacy, 3804 (void *)ixgbe, NULL); 3805 3806 if (rc != DDI_SUCCESS) { 3807 ixgbe_log(ixgbe, 3808 "Add legacy interrupt handler failed: %d", rc); 3809 return (IXGBE_FAILURE); 3810 } 3811 3812 break; 3813 3814 default: 3815 return (IXGBE_FAILURE); 3816 } 3817 3818 return (IXGBE_SUCCESS); 3819 } 3820 3821 #pragma inline(ixgbe_map_rxring_to_vector) 3822 /* 3823 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 3824 */ 3825 static void 3826 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 3827 { 3828 /* 3829 * Set bit in map 3830 */ 3831 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 3832 3833 /* 3834 * Count bits set 3835 */ 3836 ixgbe->vect_map[v_idx].rxr_cnt++; 3837 3838 /* 3839 * Remember bit position 3840 */ 3841 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 3842 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 3843 } 3844 3845 #pragma inline(ixgbe_map_txring_to_vector) 3846 /* 3847 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 3848 */ 3849 static void 3850 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 3851 { 3852 /* 3853 * Set bit in map 3854 */ 3855 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 3856 3857 /* 3858 * Count bits set 3859 */ 3860 ixgbe->vect_map[v_idx].txr_cnt++; 3861 3862 /* 3863 * Remember bit position 3864 */ 3865 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 3866 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 3867 } 3868 3869 /* 3870 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 3871 * allocation register (IVAR). 3872 * cause: 3873 * -1 : other cause 3874 * 0 : rx 3875 * 1 : tx 3876 */ 3877 static void 3878 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 3879 int8_t cause) 3880 { 3881 struct ixgbe_hw *hw = &ixgbe->hw; 3882 u32 ivar, index; 3883 3884 switch (hw->mac.type) { 3885 case ixgbe_mac_82598EB: 3886 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 3887 if (cause == -1) { 3888 cause = 0; 3889 } 3890 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 3891 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3892 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 3893 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 3894 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3895 break; 3896 case ixgbe_mac_82599EB: 3897 if (cause == -1) { 3898 /* other causes */ 3899 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 3900 index = (intr_alloc_entry & 1) * 8; 3901 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3902 ivar &= ~(0xFF << index); 3903 ivar |= (msix_vector << index); 3904 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3905 } else { 3906 /* tx or rx causes */ 3907 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 3908 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 3909 ivar = IXGBE_READ_REG(hw, 3910 IXGBE_IVAR(intr_alloc_entry >> 1)); 3911 ivar &= ~(0xFF << index); 3912 ivar |= (msix_vector << index); 3913 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 3914 ivar); 3915 } 3916 break; 3917 default: 3918 break; 3919 } 3920 } 3921 3922 /* 3923 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 3924 * given interrupt vector allocation register (IVAR). 3925 * cause: 3926 * -1 : other cause 3927 * 0 : rx 3928 * 1 : tx 3929 */ 3930 static void 3931 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 3932 { 3933 struct ixgbe_hw *hw = &ixgbe->hw; 3934 u32 ivar, index; 3935 3936 switch (hw->mac.type) { 3937 case ixgbe_mac_82598EB: 3938 if (cause == -1) { 3939 cause = 0; 3940 } 3941 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 3942 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3943 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 3944 (intr_alloc_entry & 0x3))); 3945 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3946 break; 3947 case ixgbe_mac_82599EB: 3948 if (cause == -1) { 3949 /* other causes */ 3950 index = (intr_alloc_entry & 1) * 8; 3951 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3952 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 3953 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 3954 } else { 3955 /* tx or rx causes */ 3956 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 3957 ivar = IXGBE_READ_REG(hw, 3958 IXGBE_IVAR(intr_alloc_entry >> 1)); 3959 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 3960 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 3961 ivar); 3962 } 3963 break; 3964 default: 3965 break; 3966 } 3967 } 3968 3969 /* 3970 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 3971 * given interrupt vector allocation register (IVAR). 3972 * cause: 3973 * -1 : other cause 3974 * 0 : rx 3975 * 1 : tx 3976 */ 3977 static void 3978 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 3979 { 3980 struct ixgbe_hw *hw = &ixgbe->hw; 3981 u32 ivar, index; 3982 3983 switch (hw->mac.type) { 3984 case ixgbe_mac_82598EB: 3985 if (cause == -1) { 3986 cause = 0; 3987 } 3988 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 3989 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 3990 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 3991 (intr_alloc_entry & 0x3))); 3992 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 3993 break; 3994 case ixgbe_mac_82599EB: 3995 if (cause == -1) { 3996 /* other causes */ 3997 index = (intr_alloc_entry & 1) * 8; 3998 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 3999 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4000 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4001 } else { 4002 /* tx or rx causes */ 4003 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4004 ivar = IXGBE_READ_REG(hw, 4005 IXGBE_IVAR(intr_alloc_entry >> 1)); 4006 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4007 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4008 ivar); 4009 } 4010 break; 4011 default: 4012 break; 4013 } 4014 } 4015 4016 /* 4017 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 4018 * 4019 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 4020 * to vector[0 - (intr_cnt -1)]. 4021 */ 4022 static int 4023 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 4024 { 4025 int i, vector = 0; 4026 4027 /* initialize vector map */ 4028 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 4029 for (i = 0; i < ixgbe->intr_cnt; i++) { 4030 ixgbe->vect_map[i].ixgbe = ixgbe; 4031 } 4032 4033 /* 4034 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 4035 * tx rings[0] on RTxQ[1]. 4036 */ 4037 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 4038 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 4039 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 4040 return (IXGBE_SUCCESS); 4041 } 4042 4043 /* 4044 * Interrupts/vectors mapping for MSI-X 4045 */ 4046 4047 /* 4048 * Map other interrupt to vector 0, 4049 * Set bit in map and count the bits set. 4050 */ 4051 BT_SET(ixgbe->vect_map[vector].other_map, 0); 4052 ixgbe->vect_map[vector].other_cnt++; 4053 vector++; 4054 4055 /* 4056 * Map rx ring interrupts to vectors 4057 */ 4058 for (i = 0; i < ixgbe->num_rx_rings; i++) { 4059 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 4060 vector = (vector +1) % ixgbe->intr_cnt; 4061 } 4062 4063 /* 4064 * Map tx ring interrupts to vectors 4065 */ 4066 for (i = 0; i < ixgbe->num_tx_rings; i++) { 4067 ixgbe_map_txring_to_vector(ixgbe, i, vector); 4068 vector = (vector +1) % ixgbe->intr_cnt; 4069 } 4070 4071 return (IXGBE_SUCCESS); 4072 } 4073 4074 /* 4075 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 4076 * 4077 * This relies on ring/vector mapping already set up in the 4078 * vect_map[] structures 4079 */ 4080 static void 4081 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 4082 { 4083 struct ixgbe_hw *hw = &ixgbe->hw; 4084 ixgbe_intr_vector_t *vect; /* vector bitmap */ 4085 int r_idx; /* ring index */ 4086 int v_idx; /* vector index */ 4087 4088 /* 4089 * Clear any previous entries 4090 */ 4091 switch (hw->mac.type) { 4092 case ixgbe_mac_82598EB: 4093 for (v_idx = 0; v_idx < 25; v_idx++) 4094 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4095 4096 break; 4097 case ixgbe_mac_82599EB: 4098 for (v_idx = 0; v_idx < 64; v_idx++) 4099 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4100 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 4101 4102 break; 4103 default: 4104 break; 4105 } 4106 4107 /* 4108 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 4109 * tx rings[0] will use RTxQ[1]. 4110 */ 4111 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 4112 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 4113 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 4114 return; 4115 } 4116 4117 /* 4118 * For MSI-X interrupt, "Other" is always on vector[0]. 4119 */ 4120 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 4121 4122 /* 4123 * For each interrupt vector, populate the IVAR table 4124 */ 4125 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 4126 vect = &ixgbe->vect_map[v_idx]; 4127 4128 /* 4129 * For each rx ring bit set 4130 */ 4131 r_idx = bt_getlowbit(vect->rx_map, 0, 4132 (ixgbe->num_rx_rings - 1)); 4133 4134 while (r_idx >= 0) { 4135 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 0); 4136 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4137 (ixgbe->num_rx_rings - 1)); 4138 } 4139 4140 /* 4141 * For each tx ring bit set 4142 */ 4143 r_idx = bt_getlowbit(vect->tx_map, 0, 4144 (ixgbe->num_tx_rings - 1)); 4145 4146 while (r_idx >= 0) { 4147 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 4148 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4149 (ixgbe->num_tx_rings - 1)); 4150 } 4151 } 4152 } 4153 4154 /* 4155 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 4156 */ 4157 static void 4158 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 4159 { 4160 int i; 4161 int rc; 4162 4163 for (i = 0; i < ixgbe->intr_cnt; i++) { 4164 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 4165 if (rc != DDI_SUCCESS) { 4166 IXGBE_DEBUGLOG_1(ixgbe, 4167 "Remove intr handler failed: %d", rc); 4168 } 4169 } 4170 } 4171 4172 /* 4173 * ixgbe_rem_intrs - Remove the allocated interrupts. 4174 */ 4175 static void 4176 ixgbe_rem_intrs(ixgbe_t *ixgbe) 4177 { 4178 int i; 4179 int rc; 4180 4181 for (i = 0; i < ixgbe->intr_cnt; i++) { 4182 rc = ddi_intr_free(ixgbe->htable[i]); 4183 if (rc != DDI_SUCCESS) { 4184 IXGBE_DEBUGLOG_1(ixgbe, 4185 "Free intr failed: %d", rc); 4186 } 4187 } 4188 4189 kmem_free(ixgbe->htable, ixgbe->intr_size); 4190 ixgbe->htable = NULL; 4191 } 4192 4193 /* 4194 * ixgbe_enable_intrs - Enable all the ddi interrupts. 4195 */ 4196 static int 4197 ixgbe_enable_intrs(ixgbe_t *ixgbe) 4198 { 4199 int i; 4200 int rc; 4201 4202 /* 4203 * Enable interrupts 4204 */ 4205 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 4206 /* 4207 * Call ddi_intr_block_enable() for MSI 4208 */ 4209 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 4210 if (rc != DDI_SUCCESS) { 4211 ixgbe_log(ixgbe, 4212 "Enable block intr failed: %d", rc); 4213 return (IXGBE_FAILURE); 4214 } 4215 } else { 4216 /* 4217 * Call ddi_intr_enable() for Legacy/MSI non block enable 4218 */ 4219 for (i = 0; i < ixgbe->intr_cnt; i++) { 4220 rc = ddi_intr_enable(ixgbe->htable[i]); 4221 if (rc != DDI_SUCCESS) { 4222 ixgbe_log(ixgbe, 4223 "Enable intr failed: %d", rc); 4224 return (IXGBE_FAILURE); 4225 } 4226 } 4227 } 4228 4229 return (IXGBE_SUCCESS); 4230 } 4231 4232 /* 4233 * ixgbe_disable_intrs - Disable all the interrupts. 4234 */ 4235 static int 4236 ixgbe_disable_intrs(ixgbe_t *ixgbe) 4237 { 4238 int i; 4239 int rc; 4240 4241 /* 4242 * Disable all interrupts 4243 */ 4244 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 4245 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 4246 if (rc != DDI_SUCCESS) { 4247 ixgbe_log(ixgbe, 4248 "Disable block intr failed: %d", rc); 4249 return (IXGBE_FAILURE); 4250 } 4251 } else { 4252 for (i = 0; i < ixgbe->intr_cnt; i++) { 4253 rc = ddi_intr_disable(ixgbe->htable[i]); 4254 if (rc != DDI_SUCCESS) { 4255 ixgbe_log(ixgbe, 4256 "Disable intr failed: %d", rc); 4257 return (IXGBE_FAILURE); 4258 } 4259 } 4260 } 4261 4262 return (IXGBE_SUCCESS); 4263 } 4264 4265 /* 4266 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 4267 */ 4268 static void 4269 ixgbe_get_hw_state(ixgbe_t *ixgbe) 4270 { 4271 struct ixgbe_hw *hw = &ixgbe->hw; 4272 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 4273 boolean_t link_up = B_FALSE; 4274 uint32_t pcs1g_anlp = 0; 4275 uint32_t pcs1g_ana = 0; 4276 4277 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4278 ixgbe->param_lp_1000fdx_cap = 0; 4279 ixgbe->param_lp_100fdx_cap = 0; 4280 4281 /* check for link, don't wait */ 4282 (void) ixgbe_check_link(hw, &speed, &link_up, false); 4283 if (link_up) { 4284 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 4285 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 4286 4287 ixgbe->param_lp_1000fdx_cap = 4288 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 4289 ixgbe->param_lp_100fdx_cap = 4290 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 4291 } 4292 4293 ixgbe->param_adv_1000fdx_cap = 4294 (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; 4295 ixgbe->param_adv_100fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; 4296 } 4297 4298 /* 4299 * ixgbe_get_driver_control - Notify that driver is in control of device. 4300 */ 4301 static void 4302 ixgbe_get_driver_control(struct ixgbe_hw *hw) 4303 { 4304 uint32_t ctrl_ext; 4305 4306 /* 4307 * Notify firmware that driver is in control of device 4308 */ 4309 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4310 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 4311 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4312 } 4313 4314 /* 4315 * ixgbe_release_driver_control - Notify that driver is no longer in control 4316 * of device. 4317 */ 4318 static void 4319 ixgbe_release_driver_control(struct ixgbe_hw *hw) 4320 { 4321 uint32_t ctrl_ext; 4322 4323 /* 4324 * Notify firmware that driver is no longer in control of device 4325 */ 4326 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4327 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 4328 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4329 } 4330 4331 /* 4332 * ixgbe_atomic_reserve - Atomic decrease operation. 4333 */ 4334 int 4335 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 4336 { 4337 uint32_t oldval; 4338 uint32_t newval; 4339 4340 /* 4341 * ATOMICALLY 4342 */ 4343 do { 4344 oldval = *count_p; 4345 if (oldval < n) 4346 return (-1); 4347 newval = oldval - n; 4348 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 4349 4350 return (newval); 4351 } 4352 4353 /* 4354 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 4355 */ 4356 static uint8_t * 4357 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 4358 { 4359 uint8_t *addr = *upd_ptr; 4360 uint8_t *new_ptr; 4361 4362 _NOTE(ARGUNUSED(hw)); 4363 _NOTE(ARGUNUSED(vmdq)); 4364 4365 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 4366 *upd_ptr = new_ptr; 4367 return (addr); 4368 } 4369 4370 /* 4371 * FMA support 4372 */ 4373 int 4374 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 4375 { 4376 ddi_fm_error_t de; 4377 4378 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 4379 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 4380 return (de.fme_status); 4381 } 4382 4383 int 4384 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 4385 { 4386 ddi_fm_error_t de; 4387 4388 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 4389 return (de.fme_status); 4390 } 4391 4392 /* 4393 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 4394 */ 4395 static int 4396 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 4397 { 4398 _NOTE(ARGUNUSED(impl_data)); 4399 /* 4400 * as the driver can always deal with an error in any dma or 4401 * access handle, we can just return the fme_status value. 4402 */ 4403 pci_ereport_post(dip, err, NULL); 4404 return (err->fme_status); 4405 } 4406 4407 static void 4408 ixgbe_fm_init(ixgbe_t *ixgbe) 4409 { 4410 ddi_iblock_cookie_t iblk; 4411 int fma_acc_flag, fma_dma_flag; 4412 4413 /* 4414 * Only register with IO Fault Services if we have some capability 4415 */ 4416 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 4417 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 4418 fma_acc_flag = 1; 4419 } else { 4420 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 4421 fma_acc_flag = 0; 4422 } 4423 4424 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 4425 fma_dma_flag = 1; 4426 } else { 4427 fma_dma_flag = 0; 4428 } 4429 4430 ixgbe_set_fma_flags(fma_acc_flag, fma_dma_flag); 4431 4432 if (ixgbe->fm_capabilities) { 4433 4434 /* 4435 * Register capabilities with IO Fault Services 4436 */ 4437 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 4438 4439 /* 4440 * Initialize pci ereport capabilities if ereport capable 4441 */ 4442 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 4443 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 4444 pci_ereport_setup(ixgbe->dip); 4445 4446 /* 4447 * Register error callback if error callback capable 4448 */ 4449 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 4450 ddi_fm_handler_register(ixgbe->dip, 4451 ixgbe_fm_error_cb, (void*) ixgbe); 4452 } 4453 } 4454 4455 static void 4456 ixgbe_fm_fini(ixgbe_t *ixgbe) 4457 { 4458 /* 4459 * Only unregister FMA capabilities if they are registered 4460 */ 4461 if (ixgbe->fm_capabilities) { 4462 4463 /* 4464 * Release any resources allocated by pci_ereport_setup() 4465 */ 4466 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 4467 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 4468 pci_ereport_teardown(ixgbe->dip); 4469 4470 /* 4471 * Un-register error callback if error callback capable 4472 */ 4473 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 4474 ddi_fm_handler_unregister(ixgbe->dip); 4475 4476 /* 4477 * Unregister from IO Fault Service 4478 */ 4479 ddi_fm_fini(ixgbe->dip); 4480 } 4481 } 4482 4483 void 4484 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 4485 { 4486 uint64_t ena; 4487 char buf[FM_MAX_CLASS]; 4488 4489 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 4490 ena = fm_ena_generate(0, FM_ENA_FMT1); 4491 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 4492 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 4493 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 4494 } 4495 } 4496 4497 static int 4498 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 4499 { 4500 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 4501 4502 mutex_enter(&rx_ring->rx_lock); 4503 rx_ring->ring_gen_num = mr_gen_num; 4504 mutex_exit(&rx_ring->rx_lock); 4505 return (0); 4506 } 4507 4508 /* 4509 * Callback funtion for MAC layer to register all rings. 4510 */ 4511 /* ARGSUSED */ 4512 void 4513 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int rg_index, 4514 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 4515 { 4516 ixgbe_t *ixgbe = (ixgbe_t *)arg; 4517 mac_intr_t *mintr = &infop->mri_intr; 4518 4519 switch (rtype) { 4520 case MAC_RING_TYPE_RX: { 4521 ASSERT(rg_index == 0); 4522 ASSERT(ring_index < ixgbe->num_rx_rings); 4523 4524 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[ring_index]; 4525 rx_ring->ring_handle = rh; 4526 4527 infop->mri_driver = (mac_ring_driver_t)rx_ring; 4528 infop->mri_start = ixgbe_ring_start; 4529 infop->mri_stop = NULL; 4530 infop->mri_poll = ixgbe_ring_rx_poll; 4531 4532 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 4533 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 4534 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 4535 4536 break; 4537 } 4538 case MAC_RING_TYPE_TX: { 4539 ASSERT(rg_index == -1); 4540 ASSERT(ring_index < ixgbe->num_tx_rings); 4541 4542 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 4543 tx_ring->ring_handle = rh; 4544 4545 infop->mri_driver = (mac_ring_driver_t)tx_ring; 4546 infop->mri_start = NULL; 4547 infop->mri_stop = NULL; 4548 infop->mri_tx = ixgbe_ring_tx; 4549 4550 break; 4551 } 4552 default: 4553 break; 4554 } 4555 } 4556 4557 /* 4558 * Callback funtion for MAC layer to register all groups. 4559 */ 4560 void 4561 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 4562 mac_group_info_t *infop, mac_group_handle_t gh) 4563 { 4564 ixgbe_t *ixgbe = (ixgbe_t *)arg; 4565 4566 switch (rtype) { 4567 case MAC_RING_TYPE_RX: { 4568 ixgbe_rx_group_t *rx_group; 4569 4570 rx_group = &ixgbe->rx_groups[index]; 4571 rx_group->group_handle = gh; 4572 4573 infop->mgi_driver = (mac_group_driver_t)rx_group; 4574 infop->mgi_start = NULL; 4575 infop->mgi_stop = NULL; 4576 infop->mgi_addmac = ixgbe_addmac; 4577 infop->mgi_remmac = ixgbe_remmac; 4578 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 4579 4580 break; 4581 } 4582 case MAC_RING_TYPE_TX: 4583 break; 4584 default: 4585 break; 4586 } 4587 } 4588 4589 /* 4590 * Enable interrupt on the specificed rx ring. 4591 */ 4592 int 4593 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 4594 { 4595 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 4596 ixgbe_t *ixgbe = rx_ring->ixgbe; 4597 int r_idx = rx_ring->index; 4598 int v_idx = rx_ring->intr_vector; 4599 4600 mutex_enter(&ixgbe->gen_lock); 4601 ASSERT(BT_TEST(ixgbe->vect_map[v_idx].rx_map, r_idx) == 0); 4602 4603 /* 4604 * To enable interrupt by setting the VAL bit of given interrupt 4605 * vector allocation register (IVAR). 4606 */ 4607 ixgbe_enable_ivar(ixgbe, r_idx, 0); 4608 4609 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4610 4611 /* 4612 * To trigger a Rx interrupt to on this ring 4613 */ 4614 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 4615 IXGBE_WRITE_FLUSH(&ixgbe->hw); 4616 4617 mutex_exit(&ixgbe->gen_lock); 4618 4619 return (0); 4620 } 4621 4622 /* 4623 * Disable interrupt on the specificed rx ring. 4624 */ 4625 int 4626 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 4627 { 4628 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 4629 ixgbe_t *ixgbe = rx_ring->ixgbe; 4630 int r_idx = rx_ring->index; 4631 int v_idx = rx_ring->intr_vector; 4632 4633 mutex_enter(&ixgbe->gen_lock); 4634 ASSERT(BT_TEST(ixgbe->vect_map[v_idx].rx_map, r_idx) == 1); 4635 4636 /* 4637 * To disable interrupt by clearing the VAL bit of given interrupt 4638 * vector allocation register (IVAR). 4639 */ 4640 ixgbe_disable_ivar(ixgbe, r_idx, 0); 4641 4642 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 4643 4644 mutex_exit(&ixgbe->gen_lock); 4645 4646 return (0); 4647 } 4648 4649 /* 4650 * Add a mac address. 4651 */ 4652 static int 4653 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 4654 { 4655 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 4656 ixgbe_t *ixgbe = rx_group->ixgbe; 4657 int slot; 4658 int err; 4659 4660 mutex_enter(&ixgbe->gen_lock); 4661 4662 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4663 mutex_exit(&ixgbe->gen_lock); 4664 return (ECANCELED); 4665 } 4666 4667 if (ixgbe->unicst_avail == 0) { 4668 /* no slots available */ 4669 mutex_exit(&ixgbe->gen_lock); 4670 return (ENOSPC); 4671 } 4672 4673 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 4674 if (ixgbe->unicst_addr[slot].mac.set == 0) 4675 break; 4676 } 4677 4678 ASSERT((slot >= 0) && (slot < ixgbe->unicst_total)); 4679 4680 if ((err = ixgbe_unicst_set(ixgbe, mac_addr, slot)) == 0) { 4681 ixgbe->unicst_addr[slot].mac.set = 1; 4682 ixgbe->unicst_avail--; 4683 } 4684 4685 mutex_exit(&ixgbe->gen_lock); 4686 4687 return (err); 4688 } 4689 4690 /* 4691 * Remove a mac address. 4692 */ 4693 static int 4694 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 4695 { 4696 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 4697 ixgbe_t *ixgbe = rx_group->ixgbe; 4698 int slot; 4699 int err; 4700 4701 mutex_enter(&ixgbe->gen_lock); 4702 4703 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4704 mutex_exit(&ixgbe->gen_lock); 4705 return (ECANCELED); 4706 } 4707 4708 slot = ixgbe_unicst_find(ixgbe, mac_addr); 4709 if (slot == -1) { 4710 mutex_exit(&ixgbe->gen_lock); 4711 return (EINVAL); 4712 } 4713 4714 if (ixgbe->unicst_addr[slot].mac.set == 0) { 4715 mutex_exit(&ixgbe->gen_lock); 4716 return (EINVAL); 4717 } 4718 4719 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 4720 if ((err = ixgbe_unicst_set(ixgbe, 4721 ixgbe->unicst_addr[slot].mac.addr, slot)) == 0) { 4722 ixgbe->unicst_addr[slot].mac.set = 0; 4723 ixgbe->unicst_avail++; 4724 } 4725 4726 mutex_exit(&ixgbe->gen_lock); 4727 4728 return (err); 4729 } 4730