1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2007-2012 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 */ 29 30 #include "igb_sw.h" 31 32 static char ident[] = "Intel 1Gb Ethernet"; 33 static char igb_version[] = "igb 1.1.18"; 34 35 /* 36 * Local function protoypes 37 */ 38 static int igb_register_mac(igb_t *); 39 static int igb_identify_hardware(igb_t *); 40 static int igb_regs_map(igb_t *); 41 static void igb_init_properties(igb_t *); 42 static int igb_init_driver_settings(igb_t *); 43 static void igb_init_locks(igb_t *); 44 static void igb_destroy_locks(igb_t *); 45 static int igb_init_mac_address(igb_t *); 46 static int igb_init(igb_t *); 47 static int igb_init_adapter(igb_t *); 48 static void igb_stop_adapter(igb_t *); 49 static int igb_reset(igb_t *); 50 static void igb_tx_clean(igb_t *); 51 static boolean_t igb_tx_drain(igb_t *); 52 static boolean_t igb_rx_drain(igb_t *); 53 static int igb_alloc_rings(igb_t *); 54 static int igb_alloc_rx_data(igb_t *); 55 static void igb_free_rx_data(igb_t *); 56 static void igb_free_rings(igb_t *); 57 static void igb_setup_rings(igb_t *); 58 static void igb_setup_rx(igb_t *); 59 static void igb_setup_tx(igb_t *); 60 static void igb_setup_rx_ring(igb_rx_ring_t *); 61 static void igb_setup_tx_ring(igb_tx_ring_t *); 62 static void igb_setup_rss(igb_t *); 63 static void igb_setup_mac_rss_classify(igb_t *); 64 static void igb_setup_mac_classify(igb_t *); 65 static void igb_init_unicst(igb_t *); 66 static void igb_setup_multicst(igb_t *); 67 static void igb_get_phy_state(igb_t *); 68 static void igb_param_sync(igb_t *); 69 static void igb_get_conf(igb_t *); 70 static int igb_get_prop(igb_t *, char *, int, int, int); 71 static boolean_t igb_is_link_up(igb_t *); 72 static boolean_t igb_link_check(igb_t *); 73 static void igb_local_timer(void *); 74 static void igb_link_timer(void *); 75 static void igb_arm_watchdog_timer(igb_t *); 76 static void igb_start_watchdog_timer(igb_t *); 77 static void igb_restart_watchdog_timer(igb_t *); 78 static void igb_stop_watchdog_timer(igb_t *); 79 static void igb_start_link_timer(igb_t *); 80 static void igb_stop_link_timer(igb_t *); 81 static void igb_disable_adapter_interrupts(igb_t *); 82 static void igb_enable_adapter_interrupts_82575(igb_t *); 83 static void igb_enable_adapter_interrupts_82576(igb_t *); 84 static void igb_enable_adapter_interrupts_82580(igb_t *); 85 static boolean_t is_valid_mac_addr(uint8_t *); 86 static boolean_t igb_stall_check(igb_t *); 87 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t); 88 static void igb_set_external_loopback(igb_t *); 89 static void igb_set_internal_phy_loopback(igb_t *); 90 static void igb_set_internal_serdes_loopback(igb_t *); 91 static boolean_t igb_find_mac_address(igb_t *); 92 static int igb_alloc_intrs(igb_t *); 93 static int igb_alloc_intr_handles(igb_t *, int); 94 static int igb_add_intr_handlers(igb_t *); 95 static void igb_rem_intr_handlers(igb_t *); 96 static void igb_rem_intrs(igb_t *); 97 static int igb_enable_intrs(igb_t *); 98 static int igb_disable_intrs(igb_t *); 99 static void igb_setup_msix_82575(igb_t *); 100 static void igb_setup_msix_82576(igb_t *); 101 static void igb_setup_msix_82580(igb_t *); 102 static uint_t igb_intr_legacy(void *, void *); 103 static uint_t igb_intr_msi(void *, void *); 104 static uint_t igb_intr_rx(void *, void *); 105 static uint_t igb_intr_tx(void *, void *); 106 static uint_t igb_intr_tx_other(void *, void *); 107 static void igb_intr_rx_work(igb_rx_ring_t *); 108 static void igb_intr_tx_work(igb_tx_ring_t *); 109 static void igb_intr_link_work(igb_t *); 110 static void igb_get_driver_control(struct e1000_hw *); 111 static void igb_release_driver_control(struct e1000_hw *); 112 113 static int igb_attach(dev_info_t *, ddi_attach_cmd_t); 114 static int igb_detach(dev_info_t *, ddi_detach_cmd_t); 115 static int igb_resume(dev_info_t *); 116 static int igb_suspend(dev_info_t *); 117 static int igb_quiesce(dev_info_t *); 118 static void igb_unconfigure(dev_info_t *, igb_t *); 119 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *, 120 const void *); 121 static void igb_fm_init(igb_t *); 122 static void igb_fm_fini(igb_t *); 123 static void igb_release_multicast(igb_t *); 124 125 char *igb_priv_props[] = { 126 "_tx_copy_thresh", 127 "_tx_recycle_thresh", 128 "_tx_overload_thresh", 129 "_tx_resched_thresh", 130 "_rx_copy_thresh", 131 "_rx_limit_per_intr", 132 "_intr_throttling", 133 "_adv_pause_cap", 134 "_adv_asym_pause_cap", 135 NULL 136 }; 137 138 static struct cb_ops igb_cb_ops = { 139 nulldev, /* cb_open */ 140 nulldev, /* cb_close */ 141 nodev, /* cb_strategy */ 142 nodev, /* cb_print */ 143 nodev, /* cb_dump */ 144 nodev, /* cb_read */ 145 nodev, /* cb_write */ 146 nodev, /* cb_ioctl */ 147 nodev, /* cb_devmap */ 148 nodev, /* cb_mmap */ 149 nodev, /* cb_segmap */ 150 nochpoll, /* cb_chpoll */ 151 ddi_prop_op, /* cb_prop_op */ 152 NULL, /* cb_stream */ 153 D_MP | D_HOTPLUG, /* cb_flag */ 154 CB_REV, /* cb_rev */ 155 nodev, /* cb_aread */ 156 nodev /* cb_awrite */ 157 }; 158 159 static struct dev_ops igb_dev_ops = { 160 DEVO_REV, /* devo_rev */ 161 0, /* devo_refcnt */ 162 NULL, /* devo_getinfo */ 163 nulldev, /* devo_identify */ 164 nulldev, /* devo_probe */ 165 igb_attach, /* devo_attach */ 166 igb_detach, /* devo_detach */ 167 nodev, /* devo_reset */ 168 &igb_cb_ops, /* devo_cb_ops */ 169 NULL, /* devo_bus_ops */ 170 ddi_power, /* devo_power */ 171 igb_quiesce, /* devo_quiesce */ 172 }; 173 174 static struct modldrv igb_modldrv = { 175 &mod_driverops, /* Type of module. This one is a driver */ 176 ident, /* Discription string */ 177 &igb_dev_ops, /* driver ops */ 178 }; 179 180 static struct modlinkage igb_modlinkage = { 181 MODREV_1, &igb_modldrv, NULL 182 }; 183 184 /* Access attributes for register mapping */ 185 ddi_device_acc_attr_t igb_regs_acc_attr = { 186 DDI_DEVICE_ATTR_V1, 187 DDI_STRUCTURE_LE_ACC, 188 DDI_STRICTORDER_ACC, 189 DDI_FLAGERR_ACC 190 }; 191 192 #define IGB_M_CALLBACK_FLAGS \ 193 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 194 195 static mac_callbacks_t igb_m_callbacks = { 196 IGB_M_CALLBACK_FLAGS, 197 igb_m_stat, 198 igb_m_start, 199 igb_m_stop, 200 igb_m_promisc, 201 igb_m_multicst, 202 NULL, 203 NULL, 204 NULL, 205 igb_m_ioctl, 206 igb_m_getcapab, 207 NULL, 208 NULL, 209 igb_m_setprop, 210 igb_m_getprop, 211 igb_m_propinfo 212 }; 213 214 /* 215 * Initialize capabilities of each supported adapter type 216 */ 217 static adapter_info_t igb_82575_cap = { 218 /* limits */ 219 4, /* maximum number of rx queues */ 220 1, /* minimum number of rx queues */ 221 4, /* default number of rx queues */ 222 4, /* maximum number of tx queues */ 223 1, /* minimum number of tx queues */ 224 4, /* default number of tx queues */ 225 65535, /* maximum interrupt throttle rate */ 226 0, /* minimum interrupt throttle rate */ 227 200, /* default interrupt throttle rate */ 228 229 /* function pointers */ 230 igb_enable_adapter_interrupts_82575, 231 igb_setup_msix_82575, 232 233 /* capabilities */ 234 (IGB_FLAG_HAS_DCA | /* capability flags */ 235 IGB_FLAG_VMDQ_POOL), 236 237 0xffc00000 /* mask for RXDCTL register */ 238 }; 239 240 static adapter_info_t igb_82576_cap = { 241 /* limits */ 242 16, /* maximum number of rx queues */ 243 1, /* minimum number of rx queues */ 244 4, /* default number of rx queues */ 245 16, /* maximum number of tx queues */ 246 1, /* minimum number of tx queues */ 247 4, /* default number of tx queues */ 248 65535, /* maximum interrupt throttle rate */ 249 0, /* minimum interrupt throttle rate */ 250 200, /* default interrupt throttle rate */ 251 252 /* function pointers */ 253 igb_enable_adapter_interrupts_82576, 254 igb_setup_msix_82576, 255 256 /* capabilities */ 257 (IGB_FLAG_HAS_DCA | /* capability flags */ 258 IGB_FLAG_VMDQ_POOL | 259 IGB_FLAG_NEED_CTX_IDX), 260 261 0xffe00000 /* mask for RXDCTL register */ 262 }; 263 264 static adapter_info_t igb_82580_cap = { 265 /* limits */ 266 8, /* maximum number of rx queues */ 267 1, /* minimum number of rx queues */ 268 4, /* default number of rx queues */ 269 8, /* maximum number of tx queues */ 270 1, /* minimum number of tx queues */ 271 4, /* default number of tx queues */ 272 65535, /* maximum interrupt throttle rate */ 273 0, /* minimum interrupt throttle rate */ 274 200, /* default interrupt throttle rate */ 275 276 /* function pointers */ 277 igb_enable_adapter_interrupts_82580, 278 igb_setup_msix_82580, 279 280 /* capabilities */ 281 (IGB_FLAG_HAS_DCA | /* capability flags */ 282 IGB_FLAG_VMDQ_POOL | 283 IGB_FLAG_NEED_CTX_IDX), 284 285 0xffe00000 /* mask for RXDCTL register */ 286 }; 287 288 static adapter_info_t igb_i350_cap = { 289 /* limits */ 290 8, /* maximum number of rx queues */ 291 1, /* minimum number of rx queues */ 292 4, /* default number of rx queues */ 293 8, /* maximum number of tx queues */ 294 1, /* minimum number of tx queues */ 295 4, /* default number of tx queues */ 296 65535, /* maximum interrupt throttle rate */ 297 0, /* minimum interrupt throttle rate */ 298 200, /* default interrupt throttle rate */ 299 300 /* function pointers */ 301 igb_enable_adapter_interrupts_82580, 302 igb_setup_msix_82580, 303 304 /* capabilities */ 305 (IGB_FLAG_HAS_DCA | /* capability flags */ 306 IGB_FLAG_VMDQ_POOL | 307 IGB_FLAG_NEED_CTX_IDX), 308 309 0xffe00000 /* mask for RXDCTL register */ 310 }; 311 312 /* 313 * Module Initialization Functions 314 */ 315 316 int 317 _init(void) 318 { 319 int status; 320 321 mac_init_ops(&igb_dev_ops, MODULE_NAME); 322 323 status = mod_install(&igb_modlinkage); 324 325 if (status != DDI_SUCCESS) { 326 mac_fini_ops(&igb_dev_ops); 327 } 328 329 return (status); 330 } 331 332 int 333 _fini(void) 334 { 335 int status; 336 337 status = mod_remove(&igb_modlinkage); 338 339 if (status == DDI_SUCCESS) { 340 mac_fini_ops(&igb_dev_ops); 341 } 342 343 return (status); 344 345 } 346 347 int 348 _info(struct modinfo *modinfop) 349 { 350 int status; 351 352 status = mod_info(&igb_modlinkage, modinfop); 353 354 return (status); 355 } 356 357 /* 358 * igb_attach - driver attach 359 * 360 * This function is the device specific initialization entry 361 * point. This entry point is required and must be written. 362 * The DDI_ATTACH command must be provided in the attach entry 363 * point. When attach() is called with cmd set to DDI_ATTACH, 364 * all normal kernel services (such as kmem_alloc(9F)) are 365 * available for use by the driver. 366 * 367 * The attach() function will be called once for each instance 368 * of the device on the system with cmd set to DDI_ATTACH. 369 * Until attach() succeeds, the only driver entry points which 370 * may be called are open(9E) and getinfo(9E). 371 */ 372 static int 373 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 374 { 375 igb_t *igb; 376 struct igb_osdep *osdep; 377 struct e1000_hw *hw; 378 int instance; 379 380 /* 381 * Check the command and perform corresponding operations 382 */ 383 switch (cmd) { 384 default: 385 return (DDI_FAILURE); 386 387 case DDI_RESUME: 388 return (igb_resume(devinfo)); 389 390 case DDI_ATTACH: 391 break; 392 } 393 394 /* Get the device instance */ 395 instance = ddi_get_instance(devinfo); 396 397 /* Allocate memory for the instance data structure */ 398 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP); 399 400 igb->dip = devinfo; 401 igb->instance = instance; 402 403 hw = &igb->hw; 404 osdep = &igb->osdep; 405 hw->back = osdep; 406 osdep->igb = igb; 407 408 /* Attach the instance pointer to the dev_info data structure */ 409 ddi_set_driver_private(devinfo, igb); 410 411 412 /* Initialize for fma support */ 413 igb->fm_capabilities = igb_get_prop(igb, "fm-capable", 414 0, 0x0f, 415 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 416 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 417 igb_fm_init(igb); 418 igb->attach_progress |= ATTACH_PROGRESS_FMINIT; 419 420 /* 421 * Map PCI config space registers 422 */ 423 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 424 igb_error(igb, "Failed to map PCI configurations"); 425 goto attach_fail; 426 } 427 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 428 429 /* 430 * Identify the chipset family 431 */ 432 if (igb_identify_hardware(igb) != IGB_SUCCESS) { 433 igb_error(igb, "Failed to identify hardware"); 434 goto attach_fail; 435 } 436 437 /* 438 * Map device registers 439 */ 440 if (igb_regs_map(igb) != IGB_SUCCESS) { 441 igb_error(igb, "Failed to map device registers"); 442 goto attach_fail; 443 } 444 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 445 446 /* 447 * Initialize driver parameters 448 */ 449 igb_init_properties(igb); 450 igb->attach_progress |= ATTACH_PROGRESS_PROPS; 451 452 /* 453 * Allocate interrupts 454 */ 455 if (igb_alloc_intrs(igb) != IGB_SUCCESS) { 456 igb_error(igb, "Failed to allocate interrupts"); 457 goto attach_fail; 458 } 459 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 460 461 /* 462 * Allocate rx/tx rings based on the ring numbers. 463 * The actual numbers of rx/tx rings are decided by the number of 464 * allocated interrupt vectors, so we should allocate the rings after 465 * interrupts are allocated. 466 */ 467 if (igb_alloc_rings(igb) != IGB_SUCCESS) { 468 igb_error(igb, "Failed to allocate rx/tx rings or groups"); 469 goto attach_fail; 470 } 471 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 472 473 /* 474 * Add interrupt handlers 475 */ 476 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) { 477 igb_error(igb, "Failed to add interrupt handlers"); 478 goto attach_fail; 479 } 480 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 481 482 /* 483 * Initialize driver parameters 484 */ 485 if (igb_init_driver_settings(igb) != IGB_SUCCESS) { 486 igb_error(igb, "Failed to initialize driver settings"); 487 goto attach_fail; 488 } 489 490 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) { 491 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 492 goto attach_fail; 493 } 494 495 /* 496 * Initialize mutexes for this device. 497 * Do this before enabling the interrupt handler and 498 * register the softint to avoid the condition where 499 * interrupt handler can try using uninitialized mutex 500 */ 501 igb_init_locks(igb); 502 igb->attach_progress |= ATTACH_PROGRESS_LOCKS; 503 504 /* 505 * Initialize the adapter 506 */ 507 if (igb_init(igb) != IGB_SUCCESS) { 508 igb_error(igb, "Failed to initialize adapter"); 509 goto attach_fail; 510 } 511 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 512 513 /* 514 * Initialize statistics 515 */ 516 if (igb_init_stats(igb) != IGB_SUCCESS) { 517 igb_error(igb, "Failed to initialize statistics"); 518 goto attach_fail; 519 } 520 igb->attach_progress |= ATTACH_PROGRESS_STATS; 521 522 /* 523 * Register the driver to the MAC 524 */ 525 if (igb_register_mac(igb) != IGB_SUCCESS) { 526 igb_error(igb, "Failed to register MAC"); 527 goto attach_fail; 528 } 529 igb->attach_progress |= ATTACH_PROGRESS_MAC; 530 531 /* 532 * Now that mutex locks are initialized, and the chip is also 533 * initialized, enable interrupts. 534 */ 535 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 536 igb_error(igb, "Failed to enable DDI interrupts"); 537 goto attach_fail; 538 } 539 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 540 541 igb_log(igb, "%s", igb_version); 542 atomic_or_32(&igb->igb_state, IGB_INITIALIZED); 543 544 /* 545 * Newer models have Energy Efficient Ethernet, let's disable this by 546 * default. 547 */ 548 if (igb->hw.mac.type == e1000_i350) 549 (void) e1000_set_eee_i350(&igb->hw); 550 551 return (DDI_SUCCESS); 552 553 attach_fail: 554 igb_unconfigure(devinfo, igb); 555 return (DDI_FAILURE); 556 } 557 558 /* 559 * igb_detach - driver detach 560 * 561 * The detach() function is the complement of the attach routine. 562 * If cmd is set to DDI_DETACH, detach() is used to remove the 563 * state associated with a given instance of a device node 564 * prior to the removal of that instance from the system. 565 * 566 * The detach() function will be called once for each instance 567 * of the device for which there has been a successful attach() 568 * once there are no longer any opens on the device. 569 * 570 * Interrupts routine are disabled, All memory allocated by this 571 * driver are freed. 572 */ 573 static int 574 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 575 { 576 igb_t *igb; 577 578 /* 579 * Check detach command 580 */ 581 switch (cmd) { 582 default: 583 return (DDI_FAILURE); 584 585 case DDI_SUSPEND: 586 return (igb_suspend(devinfo)); 587 588 case DDI_DETACH: 589 break; 590 } 591 592 593 /* 594 * Get the pointer to the driver private data structure 595 */ 596 igb = (igb_t *)ddi_get_driver_private(devinfo); 597 if (igb == NULL) 598 return (DDI_FAILURE); 599 600 /* 601 * Unregister MAC. If failed, we have to fail the detach 602 */ 603 if (mac_unregister(igb->mac_hdl) != 0) { 604 igb_error(igb, "Failed to unregister MAC"); 605 return (DDI_FAILURE); 606 } 607 igb->attach_progress &= ~ATTACH_PROGRESS_MAC; 608 609 /* 610 * If the device is still running, it needs to be stopped first. 611 * This check is necessary because under some specific circumstances, 612 * the detach routine can be called without stopping the interface 613 * first. 614 */ 615 mutex_enter(&igb->gen_lock); 616 if (igb->igb_state & IGB_STARTED) { 617 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 618 igb_stop(igb, B_TRUE); 619 mutex_exit(&igb->gen_lock); 620 /* Disable and stop the watchdog timer */ 621 igb_disable_watchdog_timer(igb); 622 } else 623 mutex_exit(&igb->gen_lock); 624 625 /* 626 * Check if there are still rx buffers held by the upper layer. 627 * If so, fail the detach. 628 */ 629 if (!igb_rx_drain(igb)) 630 return (DDI_FAILURE); 631 632 /* 633 * Do the remaining unconfigure routines 634 */ 635 igb_unconfigure(devinfo, igb); 636 637 return (DDI_SUCCESS); 638 } 639 640 /* 641 * quiesce(9E) entry point. 642 * 643 * This function is called when the system is single-threaded at high 644 * PIL with preemption disabled. Therefore, this function must not be 645 * blocked. 646 * 647 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 648 * DDI_FAILURE indicates an error condition and should almost never happen. 649 */ 650 static int 651 igb_quiesce(dev_info_t *devinfo) 652 { 653 igb_t *igb; 654 struct e1000_hw *hw; 655 656 igb = (igb_t *)ddi_get_driver_private(devinfo); 657 658 if (igb == NULL) 659 return (DDI_FAILURE); 660 661 hw = &igb->hw; 662 663 /* 664 * Disable the adapter interrupts 665 */ 666 igb_disable_adapter_interrupts(igb); 667 668 /* Tell firmware driver is no longer in control */ 669 igb_release_driver_control(hw); 670 671 /* 672 * Reset the chipset 673 */ 674 (void) e1000_reset_hw(hw); 675 676 /* 677 * Reset PHY if possible 678 */ 679 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 680 (void) e1000_phy_hw_reset(hw); 681 682 return (DDI_SUCCESS); 683 } 684 685 /* 686 * igb_unconfigure - release all resources held by this instance 687 */ 688 static void 689 igb_unconfigure(dev_info_t *devinfo, igb_t *igb) 690 { 691 /* 692 * Disable interrupt 693 */ 694 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 695 (void) igb_disable_intrs(igb); 696 } 697 698 /* 699 * Unregister MAC 700 */ 701 if (igb->attach_progress & ATTACH_PROGRESS_MAC) { 702 (void) mac_unregister(igb->mac_hdl); 703 } 704 705 /* 706 * Free statistics 707 */ 708 if (igb->attach_progress & ATTACH_PROGRESS_STATS) { 709 kstat_delete((kstat_t *)igb->igb_ks); 710 } 711 712 /* 713 * Remove interrupt handlers 714 */ 715 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 716 igb_rem_intr_handlers(igb); 717 } 718 719 /* 720 * Remove interrupts 721 */ 722 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 723 igb_rem_intrs(igb); 724 } 725 726 /* 727 * Remove driver properties 728 */ 729 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) { 730 (void) ddi_prop_remove_all(devinfo); 731 } 732 733 /* 734 * Stop the adapter 735 */ 736 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) { 737 mutex_enter(&igb->gen_lock); 738 igb_stop_adapter(igb); 739 mutex_exit(&igb->gen_lock); 740 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 741 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED); 742 } 743 744 /* 745 * Free multicast table 746 */ 747 igb_release_multicast(igb); 748 749 /* 750 * Free register handle 751 */ 752 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 753 if (igb->osdep.reg_handle != NULL) 754 ddi_regs_map_free(&igb->osdep.reg_handle); 755 } 756 757 /* 758 * Free PCI config handle 759 */ 760 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 761 if (igb->osdep.cfg_handle != NULL) 762 pci_config_teardown(&igb->osdep.cfg_handle); 763 } 764 765 /* 766 * Free locks 767 */ 768 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) { 769 igb_destroy_locks(igb); 770 } 771 772 /* 773 * Free the rx/tx rings 774 */ 775 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 776 igb_free_rings(igb); 777 } 778 779 /* 780 * Remove FMA 781 */ 782 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) { 783 igb_fm_fini(igb); 784 } 785 786 /* 787 * Free the driver data structure 788 */ 789 kmem_free(igb, sizeof (igb_t)); 790 791 ddi_set_driver_private(devinfo, NULL); 792 } 793 794 /* 795 * igb_register_mac - Register the driver and its function pointers with 796 * the GLD interface 797 */ 798 static int 799 igb_register_mac(igb_t *igb) 800 { 801 struct e1000_hw *hw = &igb->hw; 802 mac_register_t *mac; 803 int status; 804 805 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 806 return (IGB_FAILURE); 807 808 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 809 mac->m_driver = igb; 810 mac->m_dip = igb->dip; 811 mac->m_src_addr = hw->mac.addr; 812 mac->m_callbacks = &igb_m_callbacks; 813 mac->m_min_sdu = 0; 814 mac->m_max_sdu = igb->max_frame_size - 815 sizeof (struct ether_vlan_header) - ETHERFCSL; 816 mac->m_margin = VLAN_TAGSZ; 817 mac->m_priv_props = igb_priv_props; 818 mac->m_v12n = MAC_VIRT_LEVEL1; 819 820 status = mac_register(mac, &igb->mac_hdl); 821 822 mac_free(mac); 823 824 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE); 825 } 826 827 /* 828 * igb_identify_hardware - Identify the type of the chipset 829 */ 830 static int 831 igb_identify_hardware(igb_t *igb) 832 { 833 struct e1000_hw *hw = &igb->hw; 834 struct igb_osdep *osdep = &igb->osdep; 835 836 /* 837 * Get the device id 838 */ 839 hw->vendor_id = 840 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 841 hw->device_id = 842 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 843 hw->revision_id = 844 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 845 hw->subsystem_device_id = 846 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 847 hw->subsystem_vendor_id = 848 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 849 850 /* 851 * Set the mac type of the adapter based on the device id 852 */ 853 if (e1000_set_mac_type(hw) != E1000_SUCCESS) { 854 return (IGB_FAILURE); 855 } 856 857 /* 858 * Install adapter capabilities based on mac type 859 */ 860 switch (hw->mac.type) { 861 case e1000_82575: 862 igb->capab = &igb_82575_cap; 863 break; 864 case e1000_82576: 865 igb->capab = &igb_82576_cap; 866 break; 867 case e1000_82580: 868 igb->capab = &igb_82580_cap; 869 break; 870 case e1000_i350: 871 igb->capab = &igb_i350_cap; 872 break; 873 default: 874 return (IGB_FAILURE); 875 } 876 877 return (IGB_SUCCESS); 878 } 879 880 /* 881 * igb_regs_map - Map the device registers 882 */ 883 static int 884 igb_regs_map(igb_t *igb) 885 { 886 dev_info_t *devinfo = igb->dip; 887 struct e1000_hw *hw = &igb->hw; 888 struct igb_osdep *osdep = &igb->osdep; 889 off_t mem_size; 890 891 /* 892 * First get the size of device registers to be mapped. 893 */ 894 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) != 895 DDI_SUCCESS) { 896 return (IGB_FAILURE); 897 } 898 899 /* 900 * Call ddi_regs_map_setup() to map registers 901 */ 902 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET, 903 (caddr_t *)&hw->hw_addr, 0, 904 mem_size, &igb_regs_acc_attr, 905 &osdep->reg_handle)) != DDI_SUCCESS) { 906 return (IGB_FAILURE); 907 } 908 909 return (IGB_SUCCESS); 910 } 911 912 /* 913 * igb_init_properties - Initialize driver properties 914 */ 915 static void 916 igb_init_properties(igb_t *igb) 917 { 918 /* 919 * Get conf file properties, including link settings 920 * jumbo frames, ring number, descriptor number, etc. 921 */ 922 igb_get_conf(igb); 923 } 924 925 /* 926 * igb_init_driver_settings - Initialize driver settings 927 * 928 * The settings include hardware function pointers, bus information, 929 * rx/tx rings settings, link state, and any other parameters that 930 * need to be setup during driver initialization. 931 */ 932 static int 933 igb_init_driver_settings(igb_t *igb) 934 { 935 struct e1000_hw *hw = &igb->hw; 936 igb_rx_ring_t *rx_ring; 937 igb_tx_ring_t *tx_ring; 938 uint32_t rx_size; 939 uint32_t tx_size; 940 int i; 941 942 /* 943 * Initialize chipset specific hardware function pointers 944 */ 945 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) { 946 return (IGB_FAILURE); 947 } 948 949 /* 950 * Get bus information 951 */ 952 if (e1000_get_bus_info(hw) != E1000_SUCCESS) { 953 return (IGB_FAILURE); 954 } 955 956 /* 957 * Get the system page size 958 */ 959 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1); 960 961 /* 962 * Set rx buffer size 963 * The IP header alignment room is counted in the calculation. 964 * The rx buffer size is in unit of 1K that is required by the 965 * chipset hardware. 966 */ 967 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM; 968 igb->rx_buf_size = ((rx_size >> 10) + 969 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 970 971 /* 972 * Set tx buffer size 973 */ 974 tx_size = igb->max_frame_size; 975 igb->tx_buf_size = ((tx_size >> 10) + 976 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 977 978 /* 979 * Initialize rx/tx rings parameters 980 */ 981 for (i = 0; i < igb->num_rx_rings; i++) { 982 rx_ring = &igb->rx_rings[i]; 983 rx_ring->index = i; 984 rx_ring->igb = igb; 985 } 986 987 for (i = 0; i < igb->num_tx_rings; i++) { 988 tx_ring = &igb->tx_rings[i]; 989 tx_ring->index = i; 990 tx_ring->igb = igb; 991 if (igb->tx_head_wb_enable) 992 tx_ring->tx_recycle = igb_tx_recycle_head_wb; 993 else 994 tx_ring->tx_recycle = igb_tx_recycle_legacy; 995 996 tx_ring->ring_size = igb->tx_ring_size; 997 tx_ring->free_list_size = igb->tx_ring_size + 998 (igb->tx_ring_size >> 1); 999 } 1000 1001 /* 1002 * Initialize values of interrupt throttling rates 1003 */ 1004 for (i = 1; i < MAX_NUM_EITR; i++) 1005 igb->intr_throttling[i] = igb->intr_throttling[0]; 1006 1007 /* 1008 * The initial link state should be "unknown" 1009 */ 1010 igb->link_state = LINK_STATE_UNKNOWN; 1011 1012 return (IGB_SUCCESS); 1013 } 1014 1015 /* 1016 * igb_init_locks - Initialize locks 1017 */ 1018 static void 1019 igb_init_locks(igb_t *igb) 1020 { 1021 igb_rx_ring_t *rx_ring; 1022 igb_tx_ring_t *tx_ring; 1023 int i; 1024 1025 for (i = 0; i < igb->num_rx_rings; i++) { 1026 rx_ring = &igb->rx_rings[i]; 1027 mutex_init(&rx_ring->rx_lock, NULL, 1028 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1029 } 1030 1031 for (i = 0; i < igb->num_tx_rings; i++) { 1032 tx_ring = &igb->tx_rings[i]; 1033 mutex_init(&tx_ring->tx_lock, NULL, 1034 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1035 mutex_init(&tx_ring->recycle_lock, NULL, 1036 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1037 mutex_init(&tx_ring->tcb_head_lock, NULL, 1038 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1039 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1040 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1041 } 1042 1043 mutex_init(&igb->gen_lock, NULL, 1044 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1045 1046 mutex_init(&igb->watchdog_lock, NULL, 1047 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1048 1049 mutex_init(&igb->link_lock, NULL, 1050 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1051 } 1052 1053 /* 1054 * igb_destroy_locks - Destroy locks 1055 */ 1056 static void 1057 igb_destroy_locks(igb_t *igb) 1058 { 1059 igb_rx_ring_t *rx_ring; 1060 igb_tx_ring_t *tx_ring; 1061 int i; 1062 1063 for (i = 0; i < igb->num_rx_rings; i++) { 1064 rx_ring = &igb->rx_rings[i]; 1065 mutex_destroy(&rx_ring->rx_lock); 1066 } 1067 1068 for (i = 0; i < igb->num_tx_rings; i++) { 1069 tx_ring = &igb->tx_rings[i]; 1070 mutex_destroy(&tx_ring->tx_lock); 1071 mutex_destroy(&tx_ring->recycle_lock); 1072 mutex_destroy(&tx_ring->tcb_head_lock); 1073 mutex_destroy(&tx_ring->tcb_tail_lock); 1074 } 1075 1076 mutex_destroy(&igb->gen_lock); 1077 mutex_destroy(&igb->watchdog_lock); 1078 mutex_destroy(&igb->link_lock); 1079 } 1080 1081 static int 1082 igb_resume(dev_info_t *devinfo) 1083 { 1084 igb_t *igb; 1085 1086 igb = (igb_t *)ddi_get_driver_private(devinfo); 1087 if (igb == NULL) 1088 return (DDI_FAILURE); 1089 1090 mutex_enter(&igb->gen_lock); 1091 1092 /* 1093 * Enable interrupts 1094 */ 1095 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1096 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 1097 igb_error(igb, "Failed to enable DDI interrupts"); 1098 mutex_exit(&igb->gen_lock); 1099 return (DDI_FAILURE); 1100 } 1101 } 1102 1103 if (igb->igb_state & IGB_STARTED) { 1104 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) { 1105 mutex_exit(&igb->gen_lock); 1106 return (DDI_FAILURE); 1107 } 1108 1109 /* 1110 * Enable and start the watchdog timer 1111 */ 1112 igb_enable_watchdog_timer(igb); 1113 } 1114 1115 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED); 1116 1117 mutex_exit(&igb->gen_lock); 1118 1119 return (DDI_SUCCESS); 1120 } 1121 1122 static int 1123 igb_suspend(dev_info_t *devinfo) 1124 { 1125 igb_t *igb; 1126 1127 igb = (igb_t *)ddi_get_driver_private(devinfo); 1128 if (igb == NULL) 1129 return (DDI_FAILURE); 1130 1131 mutex_enter(&igb->gen_lock); 1132 1133 atomic_or_32(&igb->igb_state, IGB_SUSPENDED); 1134 1135 /* 1136 * Disable interrupts 1137 */ 1138 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1139 (void) igb_disable_intrs(igb); 1140 } 1141 1142 if (!(igb->igb_state & IGB_STARTED)) { 1143 mutex_exit(&igb->gen_lock); 1144 return (DDI_SUCCESS); 1145 } 1146 1147 igb_stop(igb, B_FALSE); 1148 1149 mutex_exit(&igb->gen_lock); 1150 1151 /* 1152 * Disable and stop the watchdog timer 1153 */ 1154 igb_disable_watchdog_timer(igb); 1155 1156 return (DDI_SUCCESS); 1157 } 1158 1159 static int 1160 igb_init(igb_t *igb) 1161 { 1162 mutex_enter(&igb->gen_lock); 1163 1164 /* 1165 * Initilize the adapter 1166 */ 1167 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1168 mutex_exit(&igb->gen_lock); 1169 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1170 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1171 return (IGB_FAILURE); 1172 } 1173 1174 mutex_exit(&igb->gen_lock); 1175 1176 return (IGB_SUCCESS); 1177 } 1178 1179 /* 1180 * igb_init_mac_address - Initialize the default MAC address 1181 * 1182 * On success, the MAC address is entered in the igb->hw.mac.addr 1183 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive 1184 * address register. 1185 * 1186 * Important side effects: 1187 * 1. adapter is reset - this is required to put it in a known state. 1188 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where 1189 * MAC address and all default settings are stored, so a valid checksum 1190 * is required. 1191 */ 1192 static int 1193 igb_init_mac_address(igb_t *igb) 1194 { 1195 struct e1000_hw *hw = &igb->hw; 1196 1197 ASSERT(mutex_owned(&igb->gen_lock)); 1198 1199 /* 1200 * Reset chipset to put the hardware in a known state 1201 * before we try to get MAC address from NVM. 1202 */ 1203 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1204 igb_error(igb, "Adapter reset failed."); 1205 goto init_mac_fail; 1206 } 1207 1208 /* 1209 * NVM validation 1210 */ 1211 if (e1000_validate_nvm_checksum(hw) < 0) { 1212 /* 1213 * Some PCI-E parts fail the first check due to 1214 * the link being in sleep state. Call it again, 1215 * if it fails a second time its a real issue. 1216 */ 1217 if (e1000_validate_nvm_checksum(hw) < 0) { 1218 igb_error(igb, 1219 "Invalid NVM checksum. Please contact " 1220 "the vendor to update the NVM."); 1221 goto init_mac_fail; 1222 } 1223 } 1224 1225 /* 1226 * Get the mac address 1227 * This function should handle SPARC case correctly. 1228 */ 1229 if (!igb_find_mac_address(igb)) { 1230 igb_error(igb, "Failed to get the mac address"); 1231 goto init_mac_fail; 1232 } 1233 1234 /* Validate mac address */ 1235 if (!is_valid_mac_addr(hw->mac.addr)) { 1236 igb_error(igb, "Invalid mac address"); 1237 goto init_mac_fail; 1238 } 1239 1240 return (IGB_SUCCESS); 1241 1242 init_mac_fail: 1243 return (IGB_FAILURE); 1244 } 1245 1246 /* 1247 * igb_init_adapter - Initialize the adapter 1248 */ 1249 static int 1250 igb_init_adapter(igb_t *igb) 1251 { 1252 struct e1000_hw *hw = &igb->hw; 1253 uint32_t pba; 1254 uint32_t high_water; 1255 int i; 1256 1257 ASSERT(mutex_owned(&igb->gen_lock)); 1258 1259 /* 1260 * In order to obtain the default MAC address, this will reset the 1261 * adapter and validate the NVM that the address and many other 1262 * default settings come from. 1263 */ 1264 if (igb_init_mac_address(igb) != IGB_SUCCESS) { 1265 igb_error(igb, "Failed to initialize MAC address"); 1266 goto init_adapter_fail; 1267 } 1268 1269 /* 1270 * Setup flow control 1271 * 1272 * These parameters set thresholds for the adapter's generation(Tx) 1273 * and response(Rx) to Ethernet PAUSE frames. These are just threshold 1274 * settings. Flow control is enabled or disabled in the configuration 1275 * file. 1276 * High-water mark is set down from the top of the rx fifo (not 1277 * sensitive to max_frame_size) and low-water is set just below 1278 * high-water mark. 1279 * The high water mark must be low enough to fit one full frame above 1280 * it in the rx FIFO. Should be the lower of: 1281 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full 1282 * frame. 1283 */ 1284 /* 1285 * The default setting of PBA is correct for 82575 and other supported 1286 * adapters do not have the E1000_PBA register, so PBA value is only 1287 * used for calculation here and is never written to the adapter. 1288 */ 1289 if (hw->mac.type == e1000_82575) { 1290 pba = E1000_PBA_34K; 1291 } else { 1292 pba = E1000_PBA_64K; 1293 } 1294 1295 high_water = min(((pba << 10) * 9 / 10), 1296 ((pba << 10) - igb->max_frame_size)); 1297 1298 if (hw->mac.type == e1000_82575) { 1299 /* 8-byte granularity */ 1300 hw->fc.high_water = high_water & 0xFFF8; 1301 hw->fc.low_water = hw->fc.high_water - 8; 1302 } else { 1303 /* 16-byte granularity */ 1304 hw->fc.high_water = high_water & 0xFFF0; 1305 hw->fc.low_water = hw->fc.high_water - 16; 1306 } 1307 1308 hw->fc.pause_time = E1000_FC_PAUSE_TIME; 1309 hw->fc.send_xon = B_TRUE; 1310 1311 (void) e1000_validate_mdi_setting(hw); 1312 1313 /* 1314 * Reset the chipset hardware the second time to put PBA settings 1315 * into effect. 1316 */ 1317 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1318 igb_error(igb, "Second reset failed"); 1319 goto init_adapter_fail; 1320 } 1321 1322 /* 1323 * Don't wait for auto-negotiation to complete 1324 */ 1325 hw->phy.autoneg_wait_to_complete = B_FALSE; 1326 1327 /* 1328 * Copper options 1329 */ 1330 if (hw->phy.media_type == e1000_media_type_copper) { 1331 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 1332 hw->phy.disable_polarity_correction = B_FALSE; 1333 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */ 1334 } 1335 1336 /* 1337 * Initialize link settings 1338 */ 1339 (void) igb_setup_link(igb, B_FALSE); 1340 1341 /* 1342 * Configure/Initialize hardware 1343 */ 1344 if (e1000_init_hw(hw) != E1000_SUCCESS) { 1345 igb_error(igb, "Failed to initialize hardware"); 1346 goto init_adapter_fail; 1347 } 1348 1349 /* 1350 * Start the link setup timer 1351 */ 1352 igb_start_link_timer(igb); 1353 1354 /* 1355 * Disable wakeup control by default 1356 */ 1357 E1000_WRITE_REG(hw, E1000_WUC, 0); 1358 1359 /* 1360 * Record phy info in hw struct 1361 */ 1362 (void) e1000_get_phy_info(hw); 1363 1364 /* 1365 * Make sure driver has control 1366 */ 1367 igb_get_driver_control(hw); 1368 1369 /* 1370 * Restore LED settings to the default from EEPROM 1371 * to meet the standard for Sun platforms. 1372 */ 1373 (void) e1000_cleanup_led(hw); 1374 1375 /* 1376 * Setup MSI-X interrupts 1377 */ 1378 if (igb->intr_type == DDI_INTR_TYPE_MSIX) 1379 igb->capab->setup_msix(igb); 1380 1381 /* 1382 * Initialize unicast addresses. 1383 */ 1384 igb_init_unicst(igb); 1385 1386 /* 1387 * Setup and initialize the mctable structures. 1388 */ 1389 igb_setup_multicst(igb); 1390 1391 /* 1392 * Set interrupt throttling rate 1393 */ 1394 for (i = 0; i < igb->intr_cnt; i++) 1395 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]); 1396 1397 /* 1398 * Save the state of the phy 1399 */ 1400 igb_get_phy_state(igb); 1401 1402 igb_param_sync(igb); 1403 1404 return (IGB_SUCCESS); 1405 1406 init_adapter_fail: 1407 /* 1408 * Reset PHY if possible 1409 */ 1410 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 1411 (void) e1000_phy_hw_reset(hw); 1412 1413 return (IGB_FAILURE); 1414 } 1415 1416 /* 1417 * igb_stop_adapter - Stop the adapter 1418 */ 1419 static void 1420 igb_stop_adapter(igb_t *igb) 1421 { 1422 struct e1000_hw *hw = &igb->hw; 1423 1424 ASSERT(mutex_owned(&igb->gen_lock)); 1425 1426 /* Stop the link setup timer */ 1427 igb_stop_link_timer(igb); 1428 1429 /* Tell firmware driver is no longer in control */ 1430 igb_release_driver_control(hw); 1431 1432 /* 1433 * Reset the chipset 1434 */ 1435 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1436 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1437 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1438 } 1439 1440 /* 1441 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient 1442 */ 1443 } 1444 1445 /* 1446 * igb_reset - Reset the chipset and restart the driver. 1447 * 1448 * It involves stopping and re-starting the chipset, 1449 * and re-configuring the rx/tx rings. 1450 */ 1451 static int 1452 igb_reset(igb_t *igb) 1453 { 1454 int i; 1455 1456 mutex_enter(&igb->gen_lock); 1457 1458 ASSERT(igb->igb_state & IGB_STARTED); 1459 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 1460 1461 /* 1462 * Disable the adapter interrupts to stop any rx/tx activities 1463 * before draining pending data and resetting hardware. 1464 */ 1465 igb_disable_adapter_interrupts(igb); 1466 1467 /* 1468 * Drain the pending transmit packets 1469 */ 1470 (void) igb_tx_drain(igb); 1471 1472 for (i = 0; i < igb->num_rx_rings; i++) 1473 mutex_enter(&igb->rx_rings[i].rx_lock); 1474 for (i = 0; i < igb->num_tx_rings; i++) 1475 mutex_enter(&igb->tx_rings[i].tx_lock); 1476 1477 /* 1478 * Stop the adapter 1479 */ 1480 igb_stop_adapter(igb); 1481 1482 /* 1483 * Clean the pending tx data/resources 1484 */ 1485 igb_tx_clean(igb); 1486 1487 /* 1488 * Start the adapter 1489 */ 1490 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1491 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1492 goto reset_failure; 1493 } 1494 1495 /* 1496 * Setup the rx/tx rings 1497 */ 1498 igb->tx_ring_init = B_FALSE; 1499 igb_setup_rings(igb); 1500 1501 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL)); 1502 1503 /* 1504 * Enable adapter interrupts 1505 * The interrupts must be enabled after the driver state is START 1506 */ 1507 igb->capab->enable_intr(igb); 1508 1509 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1510 goto reset_failure; 1511 1512 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1513 goto reset_failure; 1514 1515 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1516 mutex_exit(&igb->tx_rings[i].tx_lock); 1517 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1518 mutex_exit(&igb->rx_rings[i].rx_lock); 1519 1520 atomic_or_32(&igb->igb_state, IGB_STARTED); 1521 1522 mutex_exit(&igb->gen_lock); 1523 1524 return (IGB_SUCCESS); 1525 1526 reset_failure: 1527 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1528 mutex_exit(&igb->tx_rings[i].tx_lock); 1529 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1530 mutex_exit(&igb->rx_rings[i].rx_lock); 1531 1532 mutex_exit(&igb->gen_lock); 1533 1534 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1535 1536 return (IGB_FAILURE); 1537 } 1538 1539 /* 1540 * igb_tx_clean - Clean the pending transmit packets and DMA resources 1541 */ 1542 static void 1543 igb_tx_clean(igb_t *igb) 1544 { 1545 igb_tx_ring_t *tx_ring; 1546 tx_control_block_t *tcb; 1547 link_list_t pending_list; 1548 uint32_t desc_num; 1549 int i, j; 1550 1551 LINK_LIST_INIT(&pending_list); 1552 1553 for (i = 0; i < igb->num_tx_rings; i++) { 1554 tx_ring = &igb->tx_rings[i]; 1555 1556 mutex_enter(&tx_ring->recycle_lock); 1557 1558 /* 1559 * Clean the pending tx data - the pending packets in the 1560 * work_list that have no chances to be transmitted again. 1561 * 1562 * We must ensure the chipset is stopped or the link is down 1563 * before cleaning the transmit packets. 1564 */ 1565 desc_num = 0; 1566 for (j = 0; j < tx_ring->ring_size; j++) { 1567 tcb = tx_ring->work_list[j]; 1568 if (tcb != NULL) { 1569 desc_num += tcb->desc_num; 1570 1571 tx_ring->work_list[j] = NULL; 1572 1573 igb_free_tcb(tcb); 1574 1575 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1576 } 1577 } 1578 1579 if (desc_num > 0) { 1580 atomic_add_32(&tx_ring->tbd_free, desc_num); 1581 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1582 1583 /* 1584 * Reset the head and tail pointers of the tbd ring; 1585 * Reset the head write-back if it is enabled. 1586 */ 1587 tx_ring->tbd_head = 0; 1588 tx_ring->tbd_tail = 0; 1589 if (igb->tx_head_wb_enable) 1590 *tx_ring->tbd_head_wb = 0; 1591 1592 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0); 1593 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0); 1594 } 1595 1596 mutex_exit(&tx_ring->recycle_lock); 1597 1598 /* 1599 * Add the tx control blocks in the pending list to 1600 * the free list. 1601 */ 1602 igb_put_free_list(tx_ring, &pending_list); 1603 } 1604 } 1605 1606 /* 1607 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted 1608 */ 1609 static boolean_t 1610 igb_tx_drain(igb_t *igb) 1611 { 1612 igb_tx_ring_t *tx_ring; 1613 boolean_t done; 1614 int i, j; 1615 1616 /* 1617 * Wait for a specific time to allow pending tx packets 1618 * to be transmitted. 1619 * 1620 * Check the counter tbd_free to see if transmission is done. 1621 * No lock protection is needed here. 1622 * 1623 * Return B_TRUE if all pending packets have been transmitted; 1624 * Otherwise return B_FALSE; 1625 */ 1626 for (i = 0; i < TX_DRAIN_TIME; i++) { 1627 1628 done = B_TRUE; 1629 for (j = 0; j < igb->num_tx_rings; j++) { 1630 tx_ring = &igb->tx_rings[j]; 1631 done = done && 1632 (tx_ring->tbd_free == tx_ring->ring_size); 1633 } 1634 1635 if (done) 1636 break; 1637 1638 msec_delay(1); 1639 } 1640 1641 return (done); 1642 } 1643 1644 /* 1645 * igb_rx_drain - Wait for all rx buffers to be released by upper layer 1646 */ 1647 static boolean_t 1648 igb_rx_drain(igb_t *igb) 1649 { 1650 boolean_t done; 1651 int i; 1652 1653 /* 1654 * Polling the rx free list to check if those rx buffers held by 1655 * the upper layer are released. 1656 * 1657 * Check the counter rcb_free to see if all pending buffers are 1658 * released. No lock protection is needed here. 1659 * 1660 * Return B_TRUE if all pending buffers have been released; 1661 * Otherwise return B_FALSE; 1662 */ 1663 for (i = 0; i < RX_DRAIN_TIME; i++) { 1664 done = (igb->rcb_pending == 0); 1665 1666 if (done) 1667 break; 1668 1669 msec_delay(1); 1670 } 1671 1672 return (done); 1673 } 1674 1675 /* 1676 * igb_start - Start the driver/chipset 1677 */ 1678 int 1679 igb_start(igb_t *igb, boolean_t alloc_buffer) 1680 { 1681 int i; 1682 1683 ASSERT(mutex_owned(&igb->gen_lock)); 1684 1685 if (alloc_buffer) { 1686 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) { 1687 igb_error(igb, 1688 "Failed to allocate software receive rings"); 1689 return (IGB_FAILURE); 1690 } 1691 1692 /* Allocate buffers for all the rx/tx rings */ 1693 if (igb_alloc_dma(igb) != IGB_SUCCESS) { 1694 igb_error(igb, "Failed to allocate DMA resource"); 1695 return (IGB_FAILURE); 1696 } 1697 1698 igb->tx_ring_init = B_TRUE; 1699 } else { 1700 igb->tx_ring_init = B_FALSE; 1701 } 1702 1703 for (i = 0; i < igb->num_rx_rings; i++) 1704 mutex_enter(&igb->rx_rings[i].rx_lock); 1705 for (i = 0; i < igb->num_tx_rings; i++) 1706 mutex_enter(&igb->tx_rings[i].tx_lock); 1707 1708 /* 1709 * Start the adapter 1710 */ 1711 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) { 1712 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1713 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1714 goto start_failure; 1715 } 1716 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 1717 } 1718 1719 /* 1720 * Setup the rx/tx rings 1721 */ 1722 igb_setup_rings(igb); 1723 1724 /* 1725 * Enable adapter interrupts 1726 * The interrupts must be enabled after the driver state is START 1727 */ 1728 igb->capab->enable_intr(igb); 1729 1730 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1731 goto start_failure; 1732 1733 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1734 goto start_failure; 1735 1736 if (igb->hw.mac.type == e1000_i350) 1737 (void) e1000_set_eee_i350(&igb->hw); 1738 1739 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1740 mutex_exit(&igb->tx_rings[i].tx_lock); 1741 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1742 mutex_exit(&igb->rx_rings[i].rx_lock); 1743 1744 return (IGB_SUCCESS); 1745 1746 start_failure: 1747 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1748 mutex_exit(&igb->tx_rings[i].tx_lock); 1749 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1750 mutex_exit(&igb->rx_rings[i].rx_lock); 1751 1752 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1753 1754 return (IGB_FAILURE); 1755 } 1756 1757 /* 1758 * igb_stop - Stop the driver/chipset 1759 */ 1760 void 1761 igb_stop(igb_t *igb, boolean_t free_buffer) 1762 { 1763 int i; 1764 1765 ASSERT(mutex_owned(&igb->gen_lock)); 1766 1767 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER; 1768 1769 /* 1770 * Disable the adapter interrupts 1771 */ 1772 igb_disable_adapter_interrupts(igb); 1773 1774 /* 1775 * Drain the pending tx packets 1776 */ 1777 (void) igb_tx_drain(igb); 1778 1779 for (i = 0; i < igb->num_rx_rings; i++) 1780 mutex_enter(&igb->rx_rings[i].rx_lock); 1781 for (i = 0; i < igb->num_tx_rings; i++) 1782 mutex_enter(&igb->tx_rings[i].tx_lock); 1783 1784 /* 1785 * Stop the adapter 1786 */ 1787 igb_stop_adapter(igb); 1788 1789 /* 1790 * Clean the pending tx data/resources 1791 */ 1792 igb_tx_clean(igb); 1793 1794 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1795 mutex_exit(&igb->tx_rings[i].tx_lock); 1796 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1797 mutex_exit(&igb->rx_rings[i].rx_lock); 1798 1799 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1800 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1801 1802 if (igb->link_state == LINK_STATE_UP) { 1803 igb->link_state = LINK_STATE_UNKNOWN; 1804 mac_link_update(igb->mac_hdl, igb->link_state); 1805 } 1806 1807 if (free_buffer) { 1808 /* 1809 * Release the DMA/memory resources of rx/tx rings 1810 */ 1811 igb_free_dma(igb); 1812 igb_free_rx_data(igb); 1813 } 1814 } 1815 1816 /* 1817 * igb_alloc_rings - Allocate memory space for rx/tx rings 1818 */ 1819 static int 1820 igb_alloc_rings(igb_t *igb) 1821 { 1822 /* 1823 * Allocate memory space for rx rings 1824 */ 1825 igb->rx_rings = kmem_zalloc( 1826 sizeof (igb_rx_ring_t) * igb->num_rx_rings, 1827 KM_NOSLEEP); 1828 1829 if (igb->rx_rings == NULL) { 1830 return (IGB_FAILURE); 1831 } 1832 1833 /* 1834 * Allocate memory space for tx rings 1835 */ 1836 igb->tx_rings = kmem_zalloc( 1837 sizeof (igb_tx_ring_t) * igb->num_tx_rings, 1838 KM_NOSLEEP); 1839 1840 if (igb->tx_rings == NULL) { 1841 kmem_free(igb->rx_rings, 1842 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1843 igb->rx_rings = NULL; 1844 return (IGB_FAILURE); 1845 } 1846 1847 /* 1848 * Allocate memory space for rx ring groups 1849 */ 1850 igb->rx_groups = kmem_zalloc( 1851 sizeof (igb_rx_group_t) * igb->num_rx_groups, 1852 KM_NOSLEEP); 1853 1854 if (igb->rx_groups == NULL) { 1855 kmem_free(igb->rx_rings, 1856 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1857 kmem_free(igb->tx_rings, 1858 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1859 igb->rx_rings = NULL; 1860 igb->tx_rings = NULL; 1861 return (IGB_FAILURE); 1862 } 1863 1864 return (IGB_SUCCESS); 1865 } 1866 1867 /* 1868 * igb_free_rings - Free the memory space of rx/tx rings. 1869 */ 1870 static void 1871 igb_free_rings(igb_t *igb) 1872 { 1873 if (igb->rx_rings != NULL) { 1874 kmem_free(igb->rx_rings, 1875 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1876 igb->rx_rings = NULL; 1877 } 1878 1879 if (igb->tx_rings != NULL) { 1880 kmem_free(igb->tx_rings, 1881 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1882 igb->tx_rings = NULL; 1883 } 1884 1885 if (igb->rx_groups != NULL) { 1886 kmem_free(igb->rx_groups, 1887 sizeof (igb_rx_group_t) * igb->num_rx_groups); 1888 igb->rx_groups = NULL; 1889 } 1890 } 1891 1892 static int 1893 igb_alloc_rx_data(igb_t *igb) 1894 { 1895 igb_rx_ring_t *rx_ring; 1896 int i; 1897 1898 for (i = 0; i < igb->num_rx_rings; i++) { 1899 rx_ring = &igb->rx_rings[i]; 1900 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS) 1901 goto alloc_rx_rings_failure; 1902 } 1903 return (IGB_SUCCESS); 1904 1905 alloc_rx_rings_failure: 1906 igb_free_rx_data(igb); 1907 return (IGB_FAILURE); 1908 } 1909 1910 static void 1911 igb_free_rx_data(igb_t *igb) 1912 { 1913 igb_rx_ring_t *rx_ring; 1914 igb_rx_data_t *rx_data; 1915 int i; 1916 1917 for (i = 0; i < igb->num_rx_rings; i++) { 1918 rx_ring = &igb->rx_rings[i]; 1919 1920 mutex_enter(&igb->rx_pending_lock); 1921 rx_data = rx_ring->rx_data; 1922 1923 if (rx_data != NULL) { 1924 rx_data->flag |= IGB_RX_STOPPED; 1925 1926 if (rx_data->rcb_pending == 0) { 1927 igb_free_rx_ring_data(rx_data); 1928 rx_ring->rx_data = NULL; 1929 } 1930 } 1931 1932 mutex_exit(&igb->rx_pending_lock); 1933 } 1934 } 1935 1936 /* 1937 * igb_setup_rings - Setup rx/tx rings 1938 */ 1939 static void 1940 igb_setup_rings(igb_t *igb) 1941 { 1942 /* 1943 * Setup the rx/tx rings, including the following: 1944 * 1945 * 1. Setup the descriptor ring and the control block buffers; 1946 * 2. Initialize necessary registers for receive/transmit; 1947 * 3. Initialize software pointers/parameters for receive/transmit; 1948 */ 1949 igb_setup_rx(igb); 1950 1951 igb_setup_tx(igb); 1952 } 1953 1954 static void 1955 igb_setup_rx_ring(igb_rx_ring_t *rx_ring) 1956 { 1957 igb_t *igb = rx_ring->igb; 1958 igb_rx_data_t *rx_data = rx_ring->rx_data; 1959 struct e1000_hw *hw = &igb->hw; 1960 rx_control_block_t *rcb; 1961 union e1000_adv_rx_desc *rbd; 1962 uint32_t size; 1963 uint32_t buf_low; 1964 uint32_t buf_high; 1965 uint32_t rxdctl; 1966 int i; 1967 1968 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1969 ASSERT(mutex_owned(&igb->gen_lock)); 1970 1971 /* 1972 * Initialize descriptor ring with buffer addresses 1973 */ 1974 for (i = 0; i < igb->rx_ring_size; i++) { 1975 rcb = rx_data->work_list[i]; 1976 rbd = &rx_data->rbd_ring[i]; 1977 1978 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1979 rbd->read.hdr_addr = NULL; 1980 } 1981 1982 /* 1983 * Initialize the base address registers 1984 */ 1985 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 1986 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 1987 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high); 1988 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low); 1989 1990 /* 1991 * Initialize the length register 1992 */ 1993 size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc); 1994 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size); 1995 1996 /* 1997 * Initialize buffer size & descriptor type 1998 */ 1999 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index), 2000 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) | 2001 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF)); 2002 2003 /* 2004 * Setup the Receive Descriptor Control Register (RXDCTL) 2005 */ 2006 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index)); 2007 rxdctl &= igb->capab->rxdctl_mask; 2008 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 2009 rxdctl |= 16; /* pthresh */ 2010 rxdctl |= 8 << 8; /* hthresh */ 2011 rxdctl |= 1 << 16; /* wthresh */ 2012 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl); 2013 2014 rx_data->rbd_next = 0; 2015 } 2016 2017 static void 2018 igb_setup_rx(igb_t *igb) 2019 { 2020 igb_rx_ring_t *rx_ring; 2021 igb_rx_data_t *rx_data; 2022 igb_rx_group_t *rx_group; 2023 struct e1000_hw *hw = &igb->hw; 2024 uint32_t rctl, rxcsum; 2025 uint32_t ring_per_group; 2026 int i; 2027 2028 /* 2029 * Setup the Receive Control Register (RCTL), and enable the 2030 * receiver. The initial configuration is to: enable the receiver, 2031 * accept broadcasts, discard bad packets, accept long packets, 2032 * disable VLAN filter checking, and set receive buffer size to 2033 * 2k. For 82575, also set the receive descriptor minimum 2034 * threshold size to 1/2 the ring. 2035 */ 2036 rctl = E1000_READ_REG(hw, E1000_RCTL); 2037 2038 /* 2039 * Clear the field used for wakeup control. This driver doesn't do 2040 * wakeup but leave this here for completeness. 2041 */ 2042 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 2043 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 2044 2045 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */ 2046 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 2047 E1000_RCTL_LPE | /* Large Packet Enable */ 2048 /* Multicast filter offset */ 2049 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 2050 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */ 2051 E1000_RCTL_SECRC); /* Strip Ethernet CRC */ 2052 2053 for (i = 0; i < igb->num_rx_groups; i++) { 2054 rx_group = &igb->rx_groups[i]; 2055 rx_group->index = i; 2056 rx_group->igb = igb; 2057 } 2058 2059 /* 2060 * Set up all rx descriptor rings - must be called before receive unit 2061 * enabled. 2062 */ 2063 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2064 for (i = 0; i < igb->num_rx_rings; i++) { 2065 rx_ring = &igb->rx_rings[i]; 2066 igb_setup_rx_ring(rx_ring); 2067 2068 /* 2069 * Map a ring to a group by assigning a group index 2070 */ 2071 rx_ring->group_index = i / ring_per_group; 2072 } 2073 2074 /* 2075 * Setup the Rx Long Packet Max Length register 2076 */ 2077 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size); 2078 2079 /* 2080 * Hardware checksum settings 2081 */ 2082 if (igb->rx_hcksum_enable) { 2083 rxcsum = 2084 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */ 2085 E1000_RXCSUM_IPOFL; /* IP checksum */ 2086 2087 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2088 } 2089 2090 /* 2091 * Setup classify and RSS for multiple receive queues 2092 */ 2093 switch (igb->vmdq_mode) { 2094 case E1000_VMDQ_OFF: 2095 /* 2096 * One ring group, only RSS is needed when more than 2097 * one ring enabled. 2098 */ 2099 if (igb->num_rx_rings > 1) 2100 igb_setup_rss(igb); 2101 break; 2102 case E1000_VMDQ_MAC: 2103 /* 2104 * Multiple groups, each group has one ring, 2105 * only the MAC classification is needed. 2106 */ 2107 igb_setup_mac_classify(igb); 2108 break; 2109 case E1000_VMDQ_MAC_RSS: 2110 /* 2111 * Multiple groups and multiple rings, both 2112 * MAC classification and RSS are needed. 2113 */ 2114 igb_setup_mac_rss_classify(igb); 2115 break; 2116 } 2117 2118 /* 2119 * Enable the receive unit - must be done after all 2120 * the rx setup above. 2121 */ 2122 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2123 2124 /* 2125 * Initialize all adapter ring head & tail pointers - must 2126 * be done after receive unit is enabled 2127 */ 2128 for (i = 0; i < igb->num_rx_rings; i++) { 2129 rx_ring = &igb->rx_rings[i]; 2130 rx_data = rx_ring->rx_data; 2131 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 2132 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1); 2133 } 2134 2135 /* 2136 * 82575 with manageability enabled needs a special flush to make 2137 * sure the fifos start clean. 2138 */ 2139 if ((hw->mac.type == e1000_82575) && 2140 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) { 2141 e1000_rx_fifo_flush_82575(hw); 2142 } 2143 } 2144 2145 static void 2146 igb_setup_tx_ring(igb_tx_ring_t *tx_ring) 2147 { 2148 igb_t *igb = tx_ring->igb; 2149 struct e1000_hw *hw = &igb->hw; 2150 uint32_t size; 2151 uint32_t buf_low; 2152 uint32_t buf_high; 2153 uint32_t reg_val; 2154 2155 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2156 ASSERT(mutex_owned(&igb->gen_lock)); 2157 2158 2159 /* 2160 * Initialize the length register 2161 */ 2162 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc); 2163 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size); 2164 2165 /* 2166 * Initialize the base address registers 2167 */ 2168 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2169 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2170 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low); 2171 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high); 2172 2173 /* 2174 * Setup head & tail pointers 2175 */ 2176 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0); 2177 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0); 2178 2179 /* 2180 * Setup head write-back 2181 */ 2182 if (igb->tx_head_wb_enable) { 2183 /* 2184 * The memory of the head write-back is allocated using 2185 * the extra tbd beyond the tail of the tbd ring. 2186 */ 2187 tx_ring->tbd_head_wb = (uint32_t *) 2188 ((uintptr_t)tx_ring->tbd_area.address + size); 2189 *tx_ring->tbd_head_wb = 0; 2190 2191 buf_low = (uint32_t) 2192 (tx_ring->tbd_area.dma_address + size); 2193 buf_high = (uint32_t) 2194 ((tx_ring->tbd_area.dma_address + size) >> 32); 2195 2196 /* Set the head write-back enable bit */ 2197 buf_low |= E1000_TX_HEAD_WB_ENABLE; 2198 2199 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low); 2200 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high); 2201 2202 /* 2203 * Turn off relaxed ordering for head write back or it will 2204 * cause problems with the tx recycling 2205 */ 2206 reg_val = E1000_READ_REG(hw, 2207 E1000_DCA_TXCTRL(tx_ring->index)); 2208 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN; 2209 E1000_WRITE_REG(hw, 2210 E1000_DCA_TXCTRL(tx_ring->index), reg_val); 2211 } else { 2212 tx_ring->tbd_head_wb = NULL; 2213 } 2214 2215 tx_ring->tbd_head = 0; 2216 tx_ring->tbd_tail = 0; 2217 tx_ring->tbd_free = tx_ring->ring_size; 2218 2219 if (igb->tx_ring_init == B_TRUE) { 2220 tx_ring->tcb_head = 0; 2221 tx_ring->tcb_tail = 0; 2222 tx_ring->tcb_free = tx_ring->free_list_size; 2223 } 2224 2225 /* 2226 * Enable TXDCTL per queue 2227 */ 2228 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index)); 2229 reg_val |= E1000_TXDCTL_QUEUE_ENABLE; 2230 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val); 2231 2232 /* 2233 * Initialize hardware checksum offload settings 2234 */ 2235 bzero(&tx_ring->tx_context, sizeof (tx_context_t)); 2236 } 2237 2238 static void 2239 igb_setup_tx(igb_t *igb) 2240 { 2241 igb_tx_ring_t *tx_ring; 2242 struct e1000_hw *hw = &igb->hw; 2243 uint32_t reg_val; 2244 int i; 2245 2246 for (i = 0; i < igb->num_tx_rings; i++) { 2247 tx_ring = &igb->tx_rings[i]; 2248 igb_setup_tx_ring(tx_ring); 2249 } 2250 2251 /* 2252 * Setup the Transmit Control Register (TCTL) 2253 */ 2254 reg_val = E1000_READ_REG(hw, E1000_TCTL); 2255 reg_val &= ~E1000_TCTL_CT; 2256 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2257 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2258 2259 /* Enable transmits */ 2260 reg_val |= E1000_TCTL_EN; 2261 2262 E1000_WRITE_REG(hw, E1000_TCTL, reg_val); 2263 } 2264 2265 /* 2266 * igb_setup_rss - Setup receive-side scaling feature 2267 */ 2268 static void 2269 igb_setup_rss(igb_t *igb) 2270 { 2271 struct e1000_hw *hw = &igb->hw; 2272 uint32_t i, mrqc, rxcsum; 2273 int shift = 0; 2274 uint32_t random; 2275 union e1000_reta { 2276 uint32_t dword; 2277 uint8_t bytes[4]; 2278 } reta; 2279 2280 /* Setup the Redirection Table */ 2281 if (hw->mac.type == e1000_82576) { 2282 shift = 3; 2283 } else if (hw->mac.type == e1000_82575) { 2284 shift = 6; 2285 } 2286 for (i = 0; i < (32 * 4); i++) { 2287 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift; 2288 if ((i & 3) == 3) { 2289 E1000_WRITE_REG(hw, 2290 (E1000_RETA(0) + (i & ~3)), reta.dword); 2291 } 2292 } 2293 2294 /* Fill out hash function seeds */ 2295 for (i = 0; i < 10; i++) { 2296 (void) random_get_pseudo_bytes((uint8_t *)&random, 2297 sizeof (uint32_t)); 2298 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2299 } 2300 2301 /* Setup the Multiple Receive Queue Control register */ 2302 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2303 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2304 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2305 E1000_MRQC_RSS_FIELD_IPV6 | 2306 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2307 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2308 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2309 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2310 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2311 2312 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2313 2314 /* 2315 * Disable Packet Checksum to enable RSS for multiple receive queues. 2316 * 2317 * The Packet Checksum is not ethernet CRC. It is another kind of 2318 * checksum offloading provided by the 82575 chipset besides the IP 2319 * header checksum offloading and the TCP/UDP checksum offloading. 2320 * The Packet Checksum is by default computed over the entire packet 2321 * from the first byte of the DA through the last byte of the CRC, 2322 * including the Ethernet and IP headers. 2323 * 2324 * It is a hardware limitation that Packet Checksum is mutually 2325 * exclusive with RSS. 2326 */ 2327 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2328 rxcsum |= E1000_RXCSUM_PCSD; 2329 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2330 } 2331 2332 /* 2333 * igb_setup_mac_rss_classify - Setup MAC classification and rss 2334 */ 2335 static void 2336 igb_setup_mac_rss_classify(igb_t *igb) 2337 { 2338 struct e1000_hw *hw = &igb->hw; 2339 uint32_t i, mrqc, vmdctl, rxcsum; 2340 uint32_t ring_per_group; 2341 int shift_group0, shift_group1; 2342 uint32_t random; 2343 union e1000_reta { 2344 uint32_t dword; 2345 uint8_t bytes[4]; 2346 } reta; 2347 2348 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2349 2350 /* Setup the Redirection Table, it is shared between two groups */ 2351 shift_group0 = 2; 2352 shift_group1 = 6; 2353 for (i = 0; i < (32 * 4); i++) { 2354 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) | 2355 ((ring_per_group + (i % ring_per_group)) << shift_group1); 2356 if ((i & 3) == 3) { 2357 E1000_WRITE_REG(hw, 2358 (E1000_RETA(0) + (i & ~3)), reta.dword); 2359 } 2360 } 2361 2362 /* Fill out hash function seeds */ 2363 for (i = 0; i < 10; i++) { 2364 (void) random_get_pseudo_bytes((uint8_t *)&random, 2365 sizeof (uint32_t)); 2366 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2367 } 2368 2369 /* 2370 * Setup the Multiple Receive Queue Control register, 2371 * enable VMDq based on packet destination MAC address and RSS. 2372 */ 2373 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP; 2374 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2375 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2376 E1000_MRQC_RSS_FIELD_IPV6 | 2377 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2378 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2379 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2380 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2381 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2382 2383 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2384 2385 2386 /* Define the default group and default queues */ 2387 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE; 2388 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl); 2389 2390 /* 2391 * Disable Packet Checksum to enable RSS for multiple receive queues. 2392 * 2393 * The Packet Checksum is not ethernet CRC. It is another kind of 2394 * checksum offloading provided by the 82575 chipset besides the IP 2395 * header checksum offloading and the TCP/UDP checksum offloading. 2396 * The Packet Checksum is by default computed over the entire packet 2397 * from the first byte of the DA through the last byte of the CRC, 2398 * including the Ethernet and IP headers. 2399 * 2400 * It is a hardware limitation that Packet Checksum is mutually 2401 * exclusive with RSS. 2402 */ 2403 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2404 rxcsum |= E1000_RXCSUM_PCSD; 2405 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2406 } 2407 2408 /* 2409 * igb_setup_mac_classify - Setup MAC classification feature 2410 */ 2411 static void 2412 igb_setup_mac_classify(igb_t *igb) 2413 { 2414 struct e1000_hw *hw = &igb->hw; 2415 uint32_t mrqc, rxcsum; 2416 2417 /* 2418 * Setup the Multiple Receive Queue Control register, 2419 * enable VMDq based on packet destination MAC address. 2420 */ 2421 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP; 2422 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2423 2424 /* 2425 * Disable Packet Checksum to enable RSS for multiple receive queues. 2426 * 2427 * The Packet Checksum is not ethernet CRC. It is another kind of 2428 * checksum offloading provided by the 82575 chipset besides the IP 2429 * header checksum offloading and the TCP/UDP checksum offloading. 2430 * The Packet Checksum is by default computed over the entire packet 2431 * from the first byte of the DA through the last byte of the CRC, 2432 * including the Ethernet and IP headers. 2433 * 2434 * It is a hardware limitation that Packet Checksum is mutually 2435 * exclusive with RSS. 2436 */ 2437 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2438 rxcsum |= E1000_RXCSUM_PCSD; 2439 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2440 2441 } 2442 2443 /* 2444 * igb_init_unicst - Initialize the unicast addresses 2445 */ 2446 static void 2447 igb_init_unicst(igb_t *igb) 2448 { 2449 struct e1000_hw *hw = &igb->hw; 2450 int slot; 2451 2452 /* 2453 * Here we should consider two situations: 2454 * 2455 * 1. Chipset is initialized the first time 2456 * Initialize the multiple unicast addresses, and 2457 * save the default MAC address. 2458 * 2459 * 2. Chipset is reset 2460 * Recover the multiple unicast addresses from the 2461 * software data structure to the RAR registers. 2462 */ 2463 2464 /* 2465 * Clear the default MAC address in the RAR0 rgister, 2466 * which is loaded from EEPROM when system boot or chipreset, 2467 * this will cause the conficts with add_mac/rem_mac entry 2468 * points when VMDq is enabled. For this reason, the RAR0 2469 * must be cleared for both cases mentioned above. 2470 */ 2471 e1000_rar_clear(hw, 0); 2472 2473 if (!igb->unicst_init) { 2474 2475 /* Initialize the multiple unicast addresses */ 2476 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2477 igb->unicst_avail = igb->unicst_total; 2478 2479 for (slot = 0; slot < igb->unicst_total; slot++) 2480 igb->unicst_addr[slot].mac.set = 0; 2481 2482 igb->unicst_init = B_TRUE; 2483 } else { 2484 /* Re-configure the RAR registers */ 2485 for (slot = 0; slot < igb->unicst_total; slot++) { 2486 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr, 2487 slot, igb->vmdq_mode, 2488 igb->unicst_addr[slot].mac.group_index); 2489 } 2490 } 2491 } 2492 2493 /* 2494 * igb_unicst_find - Find the slot for the specified unicast address 2495 */ 2496 int 2497 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr) 2498 { 2499 int slot; 2500 2501 ASSERT(mutex_owned(&igb->gen_lock)); 2502 2503 for (slot = 0; slot < igb->unicst_total; slot++) { 2504 if (bcmp(igb->unicst_addr[slot].mac.addr, 2505 mac_addr, ETHERADDRL) == 0) 2506 return (slot); 2507 } 2508 2509 return (-1); 2510 } 2511 2512 /* 2513 * igb_unicst_set - Set the unicast address to the specified slot 2514 */ 2515 int 2516 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr, 2517 int slot) 2518 { 2519 struct e1000_hw *hw = &igb->hw; 2520 2521 ASSERT(mutex_owned(&igb->gen_lock)); 2522 2523 /* 2524 * Save the unicast address in the software data structure 2525 */ 2526 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL); 2527 2528 /* 2529 * Set the unicast address to the RAR register 2530 */ 2531 e1000_rar_set(hw, (uint8_t *)mac_addr, slot); 2532 2533 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2534 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2535 return (EIO); 2536 } 2537 2538 return (0); 2539 } 2540 2541 /* 2542 * igb_multicst_add - Add a multicst address 2543 */ 2544 int 2545 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr) 2546 { 2547 struct ether_addr *new_table; 2548 size_t new_len; 2549 size_t old_len; 2550 2551 ASSERT(mutex_owned(&igb->gen_lock)); 2552 2553 if ((multiaddr[0] & 01) == 0) { 2554 igb_error(igb, "Illegal multicast address"); 2555 return (EINVAL); 2556 } 2557 2558 if (igb->mcast_count >= igb->mcast_max_num) { 2559 igb_error(igb, "Adapter requested more than %d mcast addresses", 2560 igb->mcast_max_num); 2561 return (ENOENT); 2562 } 2563 2564 if (igb->mcast_count == igb->mcast_alloc_count) { 2565 old_len = igb->mcast_alloc_count * 2566 sizeof (struct ether_addr); 2567 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) * 2568 sizeof (struct ether_addr); 2569 2570 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2571 if (new_table == NULL) { 2572 igb_error(igb, 2573 "Not enough memory to alloc mcast table"); 2574 return (ENOMEM); 2575 } 2576 2577 if (igb->mcast_table != NULL) { 2578 bcopy(igb->mcast_table, new_table, old_len); 2579 kmem_free(igb->mcast_table, old_len); 2580 } 2581 igb->mcast_alloc_count += MCAST_ALLOC_COUNT; 2582 igb->mcast_table = new_table; 2583 } 2584 2585 bcopy(multiaddr, 2586 &igb->mcast_table[igb->mcast_count], ETHERADDRL); 2587 igb->mcast_count++; 2588 2589 /* 2590 * Update the multicast table in the hardware 2591 */ 2592 igb_setup_multicst(igb); 2593 2594 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2595 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2596 return (EIO); 2597 } 2598 2599 return (0); 2600 } 2601 2602 /* 2603 * igb_multicst_remove - Remove a multicst address 2604 */ 2605 int 2606 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr) 2607 { 2608 struct ether_addr *new_table; 2609 size_t new_len; 2610 size_t old_len; 2611 int i; 2612 2613 ASSERT(mutex_owned(&igb->gen_lock)); 2614 2615 for (i = 0; i < igb->mcast_count; i++) { 2616 if (bcmp(multiaddr, &igb->mcast_table[i], 2617 ETHERADDRL) == 0) { 2618 for (i++; i < igb->mcast_count; i++) { 2619 igb->mcast_table[i - 1] = 2620 igb->mcast_table[i]; 2621 } 2622 igb->mcast_count--; 2623 break; 2624 } 2625 } 2626 2627 if ((igb->mcast_alloc_count - igb->mcast_count) > 2628 MCAST_ALLOC_COUNT) { 2629 old_len = igb->mcast_alloc_count * 2630 sizeof (struct ether_addr); 2631 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) * 2632 sizeof (struct ether_addr); 2633 2634 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2635 if (new_table != NULL) { 2636 bcopy(igb->mcast_table, new_table, new_len); 2637 kmem_free(igb->mcast_table, old_len); 2638 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT; 2639 igb->mcast_table = new_table; 2640 } 2641 } 2642 2643 /* 2644 * Update the multicast table in the hardware 2645 */ 2646 igb_setup_multicst(igb); 2647 2648 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2649 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2650 return (EIO); 2651 } 2652 2653 return (0); 2654 } 2655 2656 static void 2657 igb_release_multicast(igb_t *igb) 2658 { 2659 if (igb->mcast_table != NULL) { 2660 kmem_free(igb->mcast_table, 2661 igb->mcast_alloc_count * sizeof (struct ether_addr)); 2662 igb->mcast_table = NULL; 2663 } 2664 } 2665 2666 /* 2667 * igb_setup_multicast - setup multicast data structures 2668 * 2669 * This routine initializes all of the multicast related structures 2670 * and save them in the hardware registers. 2671 */ 2672 static void 2673 igb_setup_multicst(igb_t *igb) 2674 { 2675 uint8_t *mc_addr_list; 2676 uint32_t mc_addr_count; 2677 struct e1000_hw *hw = &igb->hw; 2678 2679 ASSERT(mutex_owned(&igb->gen_lock)); 2680 ASSERT(igb->mcast_count <= igb->mcast_max_num); 2681 2682 mc_addr_list = (uint8_t *)igb->mcast_table; 2683 mc_addr_count = igb->mcast_count; 2684 2685 /* 2686 * Update the multicase addresses to the MTA registers 2687 */ 2688 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count); 2689 } 2690 2691 /* 2692 * igb_get_conf - Get driver configurations set in driver.conf 2693 * 2694 * This routine gets user-configured values out of the configuration 2695 * file igb.conf. 2696 * 2697 * For each configurable value, there is a minimum, a maximum, and a 2698 * default. 2699 * If user does not configure a value, use the default. 2700 * If user configures below the minimum, use the minumum. 2701 * If user configures above the maximum, use the maxumum. 2702 */ 2703 static void 2704 igb_get_conf(igb_t *igb) 2705 { 2706 struct e1000_hw *hw = &igb->hw; 2707 uint32_t default_mtu; 2708 uint32_t flow_control; 2709 uint32_t ring_per_group; 2710 int i; 2711 2712 /* 2713 * igb driver supports the following user configurations: 2714 * 2715 * Link configurations: 2716 * adv_autoneg_cap 2717 * adv_1000fdx_cap 2718 * adv_100fdx_cap 2719 * adv_100hdx_cap 2720 * adv_10fdx_cap 2721 * adv_10hdx_cap 2722 * Note: 1000hdx is not supported. 2723 * 2724 * Jumbo frame configuration: 2725 * default_mtu 2726 * 2727 * Ethernet flow control configuration: 2728 * flow_control 2729 * 2730 * Multiple rings configurations: 2731 * tx_queue_number 2732 * tx_ring_size 2733 * rx_queue_number 2734 * rx_ring_size 2735 * 2736 * Call igb_get_prop() to get the value for a specific 2737 * configuration parameter. 2738 */ 2739 2740 /* 2741 * Link configurations 2742 */ 2743 igb->param_adv_autoneg_cap = igb_get_prop(igb, 2744 PROP_ADV_AUTONEG_CAP, 0, 1, 1); 2745 igb->param_adv_1000fdx_cap = igb_get_prop(igb, 2746 PROP_ADV_1000FDX_CAP, 0, 1, 1); 2747 igb->param_adv_100fdx_cap = igb_get_prop(igb, 2748 PROP_ADV_100FDX_CAP, 0, 1, 1); 2749 igb->param_adv_100hdx_cap = igb_get_prop(igb, 2750 PROP_ADV_100HDX_CAP, 0, 1, 1); 2751 igb->param_adv_10fdx_cap = igb_get_prop(igb, 2752 PROP_ADV_10FDX_CAP, 0, 1, 1); 2753 igb->param_adv_10hdx_cap = igb_get_prop(igb, 2754 PROP_ADV_10HDX_CAP, 0, 1, 1); 2755 2756 /* 2757 * Jumbo frame configurations 2758 */ 2759 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU, 2760 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2761 2762 igb->max_frame_size = default_mtu + 2763 sizeof (struct ether_vlan_header) + ETHERFCSL; 2764 2765 /* 2766 * Ethernet flow control configuration 2767 */ 2768 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL, 2769 e1000_fc_none, 4, e1000_fc_full); 2770 if (flow_control == 4) 2771 flow_control = e1000_fc_default; 2772 2773 hw->fc.requested_mode = flow_control; 2774 2775 /* 2776 * Multiple rings configurations 2777 */ 2778 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE, 2779 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2780 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE, 2781 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2782 2783 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0); 2784 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM, 2785 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2786 /* 2787 * Currently we do not support VMDq for 82576 and 82580. 2788 * If it is e1000_82576, set num_rx_groups to 1. 2789 */ 2790 if (hw->mac.type >= e1000_82576) 2791 igb->num_rx_groups = 1; 2792 2793 if (igb->mr_enable) { 2794 igb->num_tx_rings = igb->capab->def_tx_que_num; 2795 igb->num_rx_rings = igb->capab->def_rx_que_num; 2796 } else { 2797 igb->num_tx_rings = 1; 2798 igb->num_rx_rings = 1; 2799 2800 if (igb->num_rx_groups > 1) { 2801 igb_error(igb, 2802 "Invalid rx groups number. Please enable multiple " 2803 "rings first"); 2804 igb->num_rx_groups = 1; 2805 } 2806 } 2807 2808 /* 2809 * Check the divisibility between rx rings and rx groups. 2810 */ 2811 for (i = igb->num_rx_groups; i > 0; i--) { 2812 if ((igb->num_rx_rings % i) == 0) 2813 break; 2814 } 2815 if (i != igb->num_rx_groups) { 2816 igb_error(igb, 2817 "Invalid rx groups number. Downgrade the rx group " 2818 "number to %d.", i); 2819 igb->num_rx_groups = i; 2820 } 2821 2822 /* 2823 * Get the ring number per group. 2824 */ 2825 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2826 2827 if (igb->num_rx_groups == 1) { 2828 /* 2829 * One rx ring group, the rx ring number is num_rx_rings. 2830 */ 2831 igb->vmdq_mode = E1000_VMDQ_OFF; 2832 } else if (ring_per_group == 1) { 2833 /* 2834 * Multiple rx groups, each group has one rx ring. 2835 */ 2836 igb->vmdq_mode = E1000_VMDQ_MAC; 2837 } else { 2838 /* 2839 * Multiple groups and multiple rings. 2840 */ 2841 igb->vmdq_mode = E1000_VMDQ_MAC_RSS; 2842 } 2843 2844 /* 2845 * Tunable used to force an interrupt type. The only use is 2846 * for testing of the lesser interrupt types. 2847 * 0 = don't force interrupt type 2848 * 1 = force interrupt type MSIX 2849 * 2 = force interrupt type MSI 2850 * 3 = force interrupt type Legacy 2851 */ 2852 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE, 2853 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE); 2854 2855 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE, 2856 0, 1, 1); 2857 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE, 2858 0, 1, 1); 2859 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE, 2860 0, 1, 1); 2861 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE, 2862 0, 1, 1); 2863 2864 /* 2865 * igb LSO needs the tx h/w checksum support. 2866 * Here LSO will be disabled if tx h/w checksum has been disabled. 2867 */ 2868 if (igb->tx_hcksum_enable == B_FALSE) 2869 igb->lso_enable = B_FALSE; 2870 2871 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD, 2872 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2873 DEFAULT_TX_COPY_THRESHOLD); 2874 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD, 2875 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, 2876 DEFAULT_TX_RECYCLE_THRESHOLD); 2877 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD, 2878 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, 2879 DEFAULT_TX_OVERLOAD_THRESHOLD); 2880 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD, 2881 MIN_TX_RESCHED_THRESHOLD, 2882 MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD), 2883 igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ? 2884 DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW); 2885 2886 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD, 2887 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2888 DEFAULT_RX_COPY_THRESHOLD); 2889 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR, 2890 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2891 DEFAULT_RX_LIMIT_PER_INTR); 2892 2893 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING, 2894 igb->capab->min_intr_throttle, 2895 igb->capab->max_intr_throttle, 2896 igb->capab->def_intr_throttle); 2897 2898 /* 2899 * Max number of multicast addresses 2900 */ 2901 igb->mcast_max_num = 2902 igb_get_prop(igb, PROP_MCAST_MAX_NUM, 2903 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM); 2904 } 2905 2906 /* 2907 * igb_get_prop - Get a property value out of the configuration file igb.conf 2908 * 2909 * Caller provides the name of the property, a default value, a minimum 2910 * value, and a maximum value. 2911 * 2912 * Return configured value of the property, with default, minimum and 2913 * maximum properly applied. 2914 */ 2915 static int 2916 igb_get_prop(igb_t *igb, 2917 char *propname, /* name of the property */ 2918 int minval, /* minimum acceptable value */ 2919 int maxval, /* maximim acceptable value */ 2920 int defval) /* default value */ 2921 { 2922 int value; 2923 2924 /* 2925 * Call ddi_prop_get_int() to read the conf settings 2926 */ 2927 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip, 2928 DDI_PROP_DONTPASS, propname, defval); 2929 2930 if (value > maxval) 2931 value = maxval; 2932 2933 if (value < minval) 2934 value = minval; 2935 2936 return (value); 2937 } 2938 2939 /* 2940 * igb_setup_link - Using the link properties to setup the link 2941 */ 2942 int 2943 igb_setup_link(igb_t *igb, boolean_t setup_hw) 2944 { 2945 struct e1000_mac_info *mac; 2946 struct e1000_phy_info *phy; 2947 boolean_t invalid; 2948 2949 mac = &igb->hw.mac; 2950 phy = &igb->hw.phy; 2951 invalid = B_FALSE; 2952 2953 if (igb->param_adv_autoneg_cap == 1) { 2954 mac->autoneg = B_TRUE; 2955 phy->autoneg_advertised = 0; 2956 2957 /* 2958 * 1000hdx is not supported for autonegotiation 2959 */ 2960 if (igb->param_adv_1000fdx_cap == 1) 2961 phy->autoneg_advertised |= ADVERTISE_1000_FULL; 2962 2963 if (igb->param_adv_100fdx_cap == 1) 2964 phy->autoneg_advertised |= ADVERTISE_100_FULL; 2965 2966 if (igb->param_adv_100hdx_cap == 1) 2967 phy->autoneg_advertised |= ADVERTISE_100_HALF; 2968 2969 if (igb->param_adv_10fdx_cap == 1) 2970 phy->autoneg_advertised |= ADVERTISE_10_FULL; 2971 2972 if (igb->param_adv_10hdx_cap == 1) 2973 phy->autoneg_advertised |= ADVERTISE_10_HALF; 2974 2975 if (phy->autoneg_advertised == 0) 2976 invalid = B_TRUE; 2977 } else { 2978 mac->autoneg = B_FALSE; 2979 2980 /* 2981 * 1000fdx and 1000hdx are not supported for forced link 2982 */ 2983 if (igb->param_adv_100fdx_cap == 1) 2984 mac->forced_speed_duplex = ADVERTISE_100_FULL; 2985 else if (igb->param_adv_100hdx_cap == 1) 2986 mac->forced_speed_duplex = ADVERTISE_100_HALF; 2987 else if (igb->param_adv_10fdx_cap == 1) 2988 mac->forced_speed_duplex = ADVERTISE_10_FULL; 2989 else if (igb->param_adv_10hdx_cap == 1) 2990 mac->forced_speed_duplex = ADVERTISE_10_HALF; 2991 else 2992 invalid = B_TRUE; 2993 } 2994 2995 if (invalid) { 2996 igb_notice(igb, "Invalid link settings. Setup link to " 2997 "autonegotiation with full link capabilities."); 2998 mac->autoneg = B_TRUE; 2999 phy->autoneg_advertised = ADVERTISE_1000_FULL | 3000 ADVERTISE_100_FULL | ADVERTISE_100_HALF | 3001 ADVERTISE_10_FULL | ADVERTISE_10_HALF; 3002 } 3003 3004 if (setup_hw) { 3005 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS) 3006 return (IGB_FAILURE); 3007 } 3008 3009 return (IGB_SUCCESS); 3010 } 3011 3012 3013 /* 3014 * igb_is_link_up - Check if the link is up 3015 */ 3016 static boolean_t 3017 igb_is_link_up(igb_t *igb) 3018 { 3019 struct e1000_hw *hw = &igb->hw; 3020 boolean_t link_up = B_FALSE; 3021 3022 ASSERT(mutex_owned(&igb->gen_lock)); 3023 3024 /* 3025 * get_link_status is set in the interrupt handler on link-status-change 3026 * or rx sequence error interrupt. get_link_status will stay 3027 * false until the e1000_check_for_link establishes link only 3028 * for copper adapters. 3029 */ 3030 switch (hw->phy.media_type) { 3031 case e1000_media_type_copper: 3032 if (hw->mac.get_link_status) { 3033 (void) e1000_check_for_link(hw); 3034 link_up = !hw->mac.get_link_status; 3035 } else { 3036 link_up = B_TRUE; 3037 } 3038 break; 3039 case e1000_media_type_fiber: 3040 (void) e1000_check_for_link(hw); 3041 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 3042 break; 3043 case e1000_media_type_internal_serdes: 3044 (void) e1000_check_for_link(hw); 3045 link_up = hw->mac.serdes_has_link; 3046 break; 3047 } 3048 3049 return (link_up); 3050 } 3051 3052 /* 3053 * igb_link_check - Link status processing 3054 */ 3055 static boolean_t 3056 igb_link_check(igb_t *igb) 3057 { 3058 struct e1000_hw *hw = &igb->hw; 3059 uint16_t speed = 0, duplex = 0; 3060 boolean_t link_changed = B_FALSE; 3061 3062 ASSERT(mutex_owned(&igb->gen_lock)); 3063 3064 if (igb_is_link_up(igb)) { 3065 /* 3066 * The Link is up, check whether it was marked as down earlier 3067 */ 3068 if (igb->link_state != LINK_STATE_UP) { 3069 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex); 3070 igb->link_speed = speed; 3071 igb->link_duplex = duplex; 3072 igb->link_state = LINK_STATE_UP; 3073 link_changed = B_TRUE; 3074 if (!igb->link_complete) 3075 igb_stop_link_timer(igb); 3076 } 3077 } else if (igb->link_complete) { 3078 if (igb->link_state != LINK_STATE_DOWN) { 3079 igb->link_speed = 0; 3080 igb->link_duplex = 0; 3081 igb->link_state = LINK_STATE_DOWN; 3082 link_changed = B_TRUE; 3083 } 3084 } 3085 3086 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3087 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3088 return (B_FALSE); 3089 } 3090 3091 return (link_changed); 3092 } 3093 3094 /* 3095 * igb_local_timer - driver watchdog function 3096 * 3097 * This function will handle the hardware stall check, link status 3098 * check and other routines. 3099 */ 3100 static void 3101 igb_local_timer(void *arg) 3102 { 3103 igb_t *igb = (igb_t *)arg; 3104 boolean_t link_changed = B_FALSE; 3105 3106 if (igb->igb_state & IGB_ERROR) { 3107 igb->reset_count++; 3108 if (igb_reset(igb) == IGB_SUCCESS) 3109 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3110 3111 igb_restart_watchdog_timer(igb); 3112 return; 3113 } 3114 3115 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) { 3116 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL); 3117 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 3118 igb->reset_count++; 3119 if (igb_reset(igb) == IGB_SUCCESS) 3120 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3121 3122 igb_restart_watchdog_timer(igb); 3123 return; 3124 } 3125 3126 mutex_enter(&igb->gen_lock); 3127 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED)) 3128 link_changed = igb_link_check(igb); 3129 mutex_exit(&igb->gen_lock); 3130 3131 if (link_changed) 3132 mac_link_update(igb->mac_hdl, igb->link_state); 3133 3134 igb_restart_watchdog_timer(igb); 3135 } 3136 3137 /* 3138 * igb_link_timer - link setup timer function 3139 * 3140 * It is called when the timer for link setup is expired, which indicates 3141 * the completion of the link setup. The link state will not be updated 3142 * until the link setup is completed. And the link state will not be sent 3143 * to the upper layer through mac_link_update() in this function. It will 3144 * be updated in the local timer routine or the interrupts service routine 3145 * after the interface is started (plumbed). 3146 */ 3147 static void 3148 igb_link_timer(void *arg) 3149 { 3150 igb_t *igb = (igb_t *)arg; 3151 3152 mutex_enter(&igb->link_lock); 3153 igb->link_complete = B_TRUE; 3154 igb->link_tid = 0; 3155 mutex_exit(&igb->link_lock); 3156 } 3157 /* 3158 * igb_stall_check - check for transmit stall 3159 * 3160 * This function checks if the adapter is stalled (in transmit). 3161 * 3162 * It is called each time the watchdog timeout is invoked. 3163 * If the transmit descriptor reclaim continuously fails, 3164 * the watchdog value will increment by 1. If the watchdog 3165 * value exceeds the threshold, the igb is assumed to 3166 * have stalled and need to be reset. 3167 */ 3168 static boolean_t 3169 igb_stall_check(igb_t *igb) 3170 { 3171 igb_tx_ring_t *tx_ring; 3172 struct e1000_hw *hw = &igb->hw; 3173 boolean_t result; 3174 int i; 3175 3176 if (igb->link_state != LINK_STATE_UP) 3177 return (B_FALSE); 3178 3179 /* 3180 * If any tx ring is stalled, we'll reset the chipset 3181 */ 3182 result = B_FALSE; 3183 for (i = 0; i < igb->num_tx_rings; i++) { 3184 tx_ring = &igb->tx_rings[i]; 3185 3186 if (tx_ring->recycle_fail > 0) 3187 tx_ring->stall_watchdog++; 3188 else 3189 tx_ring->stall_watchdog = 0; 3190 3191 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3192 result = B_TRUE; 3193 if (hw->mac.type == e1000_82580) { 3194 hw->dev_spec._82575.global_device_reset 3195 = B_TRUE; 3196 } 3197 break; 3198 } 3199 } 3200 3201 if (result) { 3202 tx_ring->stall_watchdog = 0; 3203 tx_ring->recycle_fail = 0; 3204 } 3205 3206 return (result); 3207 } 3208 3209 3210 /* 3211 * is_valid_mac_addr - Check if the mac address is valid 3212 */ 3213 static boolean_t 3214 is_valid_mac_addr(uint8_t *mac_addr) 3215 { 3216 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3217 const uint8_t addr_test2[6] = 3218 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3219 3220 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3221 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3222 return (B_FALSE); 3223 3224 return (B_TRUE); 3225 } 3226 3227 static boolean_t 3228 igb_find_mac_address(igb_t *igb) 3229 { 3230 struct e1000_hw *hw = &igb->hw; 3231 #ifdef __sparc 3232 uchar_t *bytes; 3233 struct ether_addr sysaddr; 3234 uint_t nelts; 3235 int err; 3236 boolean_t found = B_FALSE; 3237 3238 /* 3239 * The "vendor's factory-set address" may already have 3240 * been extracted from the chip, but if the property 3241 * "local-mac-address" is set we use that instead. 3242 * 3243 * We check whether it looks like an array of 6 3244 * bytes (which it should, if OBP set it). If we can't 3245 * make sense of it this way, we'll ignore it. 3246 */ 3247 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3248 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3249 if (err == DDI_PROP_SUCCESS) { 3250 if (nelts == ETHERADDRL) { 3251 while (nelts--) 3252 hw->mac.addr[nelts] = bytes[nelts]; 3253 found = B_TRUE; 3254 } 3255 ddi_prop_free(bytes); 3256 } 3257 3258 /* 3259 * Look up the OBP property "local-mac-address?". If the user has set 3260 * 'local-mac-address? = false', use "the system address" instead. 3261 */ 3262 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0, 3263 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3264 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3265 if (localetheraddr(NULL, &sysaddr) != 0) { 3266 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3267 found = B_TRUE; 3268 } 3269 } 3270 ddi_prop_free(bytes); 3271 } 3272 3273 /* 3274 * Finally(!), if there's a valid "mac-address" property (created 3275 * if we netbooted from this interface), we must use this instead 3276 * of any of the above to ensure that the NFS/install server doesn't 3277 * get confused by the address changing as Solaris takes over! 3278 */ 3279 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3280 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3281 if (err == DDI_PROP_SUCCESS) { 3282 if (nelts == ETHERADDRL) { 3283 while (nelts--) 3284 hw->mac.addr[nelts] = bytes[nelts]; 3285 found = B_TRUE; 3286 } 3287 ddi_prop_free(bytes); 3288 } 3289 3290 if (found) { 3291 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3292 return (B_TRUE); 3293 } 3294 #endif 3295 3296 /* 3297 * Read the device MAC address from the EEPROM 3298 */ 3299 if (e1000_read_mac_addr(hw) != E1000_SUCCESS) 3300 return (B_FALSE); 3301 3302 return (B_TRUE); 3303 } 3304 3305 #pragma inline(igb_arm_watchdog_timer) 3306 3307 static void 3308 igb_arm_watchdog_timer(igb_t *igb) 3309 { 3310 /* 3311 * Fire a watchdog timer 3312 */ 3313 igb->watchdog_tid = 3314 timeout(igb_local_timer, 3315 (void *)igb, 1 * drv_usectohz(1000000)); 3316 3317 } 3318 3319 /* 3320 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer 3321 */ 3322 void 3323 igb_enable_watchdog_timer(igb_t *igb) 3324 { 3325 mutex_enter(&igb->watchdog_lock); 3326 3327 if (!igb->watchdog_enable) { 3328 igb->watchdog_enable = B_TRUE; 3329 igb->watchdog_start = B_TRUE; 3330 igb_arm_watchdog_timer(igb); 3331 } 3332 3333 mutex_exit(&igb->watchdog_lock); 3334 3335 } 3336 3337 /* 3338 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer 3339 */ 3340 void 3341 igb_disable_watchdog_timer(igb_t *igb) 3342 { 3343 timeout_id_t tid; 3344 3345 mutex_enter(&igb->watchdog_lock); 3346 3347 igb->watchdog_enable = B_FALSE; 3348 igb->watchdog_start = B_FALSE; 3349 tid = igb->watchdog_tid; 3350 igb->watchdog_tid = 0; 3351 3352 mutex_exit(&igb->watchdog_lock); 3353 3354 if (tid != 0) 3355 (void) untimeout(tid); 3356 3357 } 3358 3359 /* 3360 * igb_start_watchdog_timer - Start the driver watchdog timer 3361 */ 3362 static void 3363 igb_start_watchdog_timer(igb_t *igb) 3364 { 3365 mutex_enter(&igb->watchdog_lock); 3366 3367 if (igb->watchdog_enable) { 3368 if (!igb->watchdog_start) { 3369 igb->watchdog_start = B_TRUE; 3370 igb_arm_watchdog_timer(igb); 3371 } 3372 } 3373 3374 mutex_exit(&igb->watchdog_lock); 3375 } 3376 3377 /* 3378 * igb_restart_watchdog_timer - Restart the driver watchdog timer 3379 */ 3380 static void 3381 igb_restart_watchdog_timer(igb_t *igb) 3382 { 3383 mutex_enter(&igb->watchdog_lock); 3384 3385 if (igb->watchdog_start) 3386 igb_arm_watchdog_timer(igb); 3387 3388 mutex_exit(&igb->watchdog_lock); 3389 } 3390 3391 /* 3392 * igb_stop_watchdog_timer - Stop the driver watchdog timer 3393 */ 3394 static void 3395 igb_stop_watchdog_timer(igb_t *igb) 3396 { 3397 timeout_id_t tid; 3398 3399 mutex_enter(&igb->watchdog_lock); 3400 3401 igb->watchdog_start = B_FALSE; 3402 tid = igb->watchdog_tid; 3403 igb->watchdog_tid = 0; 3404 3405 mutex_exit(&igb->watchdog_lock); 3406 3407 if (tid != 0) 3408 (void) untimeout(tid); 3409 } 3410 3411 /* 3412 * igb_start_link_timer - Start the link setup timer 3413 */ 3414 static void 3415 igb_start_link_timer(struct igb *igb) 3416 { 3417 struct e1000_hw *hw = &igb->hw; 3418 clock_t link_timeout; 3419 3420 if (hw->mac.autoneg) 3421 link_timeout = PHY_AUTO_NEG_LIMIT * 3422 drv_usectohz(100000); 3423 else 3424 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000); 3425 3426 mutex_enter(&igb->link_lock); 3427 if (hw->phy.autoneg_wait_to_complete) { 3428 igb->link_complete = B_TRUE; 3429 } else { 3430 igb->link_complete = B_FALSE; 3431 igb->link_tid = timeout(igb_link_timer, (void *)igb, 3432 link_timeout); 3433 } 3434 mutex_exit(&igb->link_lock); 3435 } 3436 3437 /* 3438 * igb_stop_link_timer - Stop the link setup timer 3439 */ 3440 static void 3441 igb_stop_link_timer(struct igb *igb) 3442 { 3443 timeout_id_t tid; 3444 3445 mutex_enter(&igb->link_lock); 3446 igb->link_complete = B_TRUE; 3447 tid = igb->link_tid; 3448 igb->link_tid = 0; 3449 mutex_exit(&igb->link_lock); 3450 3451 if (tid != 0) 3452 (void) untimeout(tid); 3453 } 3454 3455 /* 3456 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts 3457 */ 3458 static void 3459 igb_disable_adapter_interrupts(igb_t *igb) 3460 { 3461 struct e1000_hw *hw = &igb->hw; 3462 3463 /* 3464 * Set the IMC register to mask all the interrupts, 3465 * including the tx interrupts. 3466 */ 3467 E1000_WRITE_REG(hw, E1000_IMC, ~0); 3468 E1000_WRITE_REG(hw, E1000_IAM, 0); 3469 3470 /* 3471 * Additional disabling for MSI-X 3472 */ 3473 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3474 E1000_WRITE_REG(hw, E1000_EIMC, ~0); 3475 E1000_WRITE_REG(hw, E1000_EIAC, 0); 3476 E1000_WRITE_REG(hw, E1000_EIAM, 0); 3477 } 3478 3479 E1000_WRITE_FLUSH(hw); 3480 } 3481 3482 /* 3483 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580 3484 */ 3485 static void 3486 igb_enable_adapter_interrupts_82580(igb_t *igb) 3487 { 3488 struct e1000_hw *hw = &igb->hw; 3489 3490 /* Clear any pending interrupts */ 3491 (void) E1000_READ_REG(hw, E1000_ICR); 3492 igb->ims_mask |= E1000_IMS_DRSTA; 3493 3494 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3495 3496 /* Interrupt enabling for MSI-X */ 3497 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3498 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3499 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA); 3500 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3501 } else { /* Interrupt enabling for MSI and legacy */ 3502 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3503 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3504 igb->ims_mask |= E1000_IMS_DRSTA; 3505 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3506 } 3507 3508 /* Disable auto-mask for ICR interrupt bits */ 3509 E1000_WRITE_REG(hw, E1000_IAM, 0); 3510 3511 E1000_WRITE_FLUSH(hw); 3512 } 3513 3514 /* 3515 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576 3516 */ 3517 static void 3518 igb_enable_adapter_interrupts_82576(igb_t *igb) 3519 { 3520 struct e1000_hw *hw = &igb->hw; 3521 3522 /* Clear any pending interrupts */ 3523 (void) E1000_READ_REG(hw, E1000_ICR); 3524 3525 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3526 3527 /* Interrupt enabling for MSI-X */ 3528 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3529 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3530 igb->ims_mask = E1000_IMS_LSC; 3531 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3532 } else { 3533 /* Interrupt enabling for MSI and legacy */ 3534 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3535 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3536 E1000_WRITE_REG(hw, E1000_IMS, 3537 (IMS_ENABLE_MASK | E1000_IMS_TXQE)); 3538 } 3539 3540 /* Disable auto-mask for ICR interrupt bits */ 3541 E1000_WRITE_REG(hw, E1000_IAM, 0); 3542 3543 E1000_WRITE_FLUSH(hw); 3544 } 3545 3546 /* 3547 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575 3548 */ 3549 static void 3550 igb_enable_adapter_interrupts_82575(igb_t *igb) 3551 { 3552 struct e1000_hw *hw = &igb->hw; 3553 uint32_t reg; 3554 3555 /* Clear any pending interrupts */ 3556 (void) E1000_READ_REG(hw, E1000_ICR); 3557 3558 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3559 /* Interrupt enabling for MSI-X */ 3560 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3561 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3562 igb->ims_mask = E1000_IMS_LSC; 3563 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3564 3565 /* Enable MSI-X PBA support */ 3566 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 3567 reg |= E1000_CTRL_EXT_PBA_CLR; 3568 3569 /* Non-selective interrupt clear-on-read */ 3570 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */ 3571 3572 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 3573 } else { 3574 /* Interrupt enabling for MSI and legacy */ 3575 igb->ims_mask = IMS_ENABLE_MASK; 3576 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 3577 } 3578 3579 E1000_WRITE_FLUSH(hw); 3580 } 3581 3582 /* 3583 * Loopback Support 3584 */ 3585 static lb_property_t lb_normal = 3586 { normal, "normal", IGB_LB_NONE }; 3587 static lb_property_t lb_external = 3588 { external, "External", IGB_LB_EXTERNAL }; 3589 static lb_property_t lb_phy = 3590 { internal, "PHY", IGB_LB_INTERNAL_PHY }; 3591 static lb_property_t lb_serdes = 3592 { internal, "SerDes", IGB_LB_INTERNAL_SERDES }; 3593 3594 enum ioc_reply 3595 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp) 3596 { 3597 lb_info_sz_t *lbsp; 3598 lb_property_t *lbpp; 3599 struct e1000_hw *hw; 3600 uint32_t *lbmp; 3601 uint32_t size; 3602 uint32_t value; 3603 3604 hw = &igb->hw; 3605 3606 if (mp->b_cont == NULL) 3607 return (IOC_INVAL); 3608 3609 switch (iocp->ioc_cmd) { 3610 default: 3611 return (IOC_INVAL); 3612 3613 case LB_GET_INFO_SIZE: 3614 size = sizeof (lb_info_sz_t); 3615 if (iocp->ioc_count != size) 3616 return (IOC_INVAL); 3617 3618 value = sizeof (lb_normal); 3619 if (hw->phy.media_type == e1000_media_type_copper) 3620 value += sizeof (lb_phy); 3621 else 3622 value += sizeof (lb_serdes); 3623 value += sizeof (lb_external); 3624 3625 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3626 *lbsp = value; 3627 break; 3628 3629 case LB_GET_INFO: 3630 value = sizeof (lb_normal); 3631 if (hw->phy.media_type == e1000_media_type_copper) 3632 value += sizeof (lb_phy); 3633 else 3634 value += sizeof (lb_serdes); 3635 value += sizeof (lb_external); 3636 3637 size = value; 3638 if (iocp->ioc_count != size) 3639 return (IOC_INVAL); 3640 3641 value = 0; 3642 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3643 3644 lbpp[value++] = lb_normal; 3645 if (hw->phy.media_type == e1000_media_type_copper) 3646 lbpp[value++] = lb_phy; 3647 else 3648 lbpp[value++] = lb_serdes; 3649 lbpp[value++] = lb_external; 3650 break; 3651 3652 case LB_GET_MODE: 3653 size = sizeof (uint32_t); 3654 if (iocp->ioc_count != size) 3655 return (IOC_INVAL); 3656 3657 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3658 *lbmp = igb->loopback_mode; 3659 break; 3660 3661 case LB_SET_MODE: 3662 size = 0; 3663 if (iocp->ioc_count != sizeof (uint32_t)) 3664 return (IOC_INVAL); 3665 3666 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3667 if (!igb_set_loopback_mode(igb, *lbmp)) 3668 return (IOC_INVAL); 3669 break; 3670 } 3671 3672 iocp->ioc_count = size; 3673 iocp->ioc_error = 0; 3674 3675 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3676 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3677 return (IOC_INVAL); 3678 } 3679 3680 return (IOC_REPLY); 3681 } 3682 3683 /* 3684 * igb_set_loopback_mode - Setup loopback based on the loopback mode 3685 */ 3686 static boolean_t 3687 igb_set_loopback_mode(igb_t *igb, uint32_t mode) 3688 { 3689 struct e1000_hw *hw; 3690 int i; 3691 3692 if (mode == igb->loopback_mode) 3693 return (B_TRUE); 3694 3695 hw = &igb->hw; 3696 3697 igb->loopback_mode = mode; 3698 3699 if (mode == IGB_LB_NONE) { 3700 /* Reset the chip */ 3701 hw->phy.autoneg_wait_to_complete = B_TRUE; 3702 (void) igb_reset(igb); 3703 hw->phy.autoneg_wait_to_complete = B_FALSE; 3704 return (B_TRUE); 3705 } 3706 3707 mutex_enter(&igb->gen_lock); 3708 3709 switch (mode) { 3710 default: 3711 mutex_exit(&igb->gen_lock); 3712 return (B_FALSE); 3713 3714 case IGB_LB_EXTERNAL: 3715 igb_set_external_loopback(igb); 3716 break; 3717 3718 case IGB_LB_INTERNAL_PHY: 3719 igb_set_internal_phy_loopback(igb); 3720 break; 3721 3722 case IGB_LB_INTERNAL_SERDES: 3723 igb_set_internal_serdes_loopback(igb); 3724 break; 3725 } 3726 3727 mutex_exit(&igb->gen_lock); 3728 3729 /* 3730 * When external loopback is set, wait up to 1000ms to get the link up. 3731 * According to test, 1000ms can work and it's an experimental value. 3732 */ 3733 if (mode == IGB_LB_EXTERNAL) { 3734 for (i = 0; i <= 10; i++) { 3735 mutex_enter(&igb->gen_lock); 3736 (void) igb_link_check(igb); 3737 mutex_exit(&igb->gen_lock); 3738 3739 if (igb->link_state == LINK_STATE_UP) 3740 break; 3741 3742 msec_delay(100); 3743 } 3744 3745 if (igb->link_state != LINK_STATE_UP) { 3746 /* 3747 * Does not support external loopback. 3748 * Reset driver to loopback none. 3749 */ 3750 igb->loopback_mode = IGB_LB_NONE; 3751 3752 /* Reset the chip */ 3753 hw->phy.autoneg_wait_to_complete = B_TRUE; 3754 (void) igb_reset(igb); 3755 hw->phy.autoneg_wait_to_complete = B_FALSE; 3756 3757 IGB_DEBUGLOG_0(igb, "Set external loopback failed, " 3758 "reset to loopback none."); 3759 3760 return (B_FALSE); 3761 } 3762 } 3763 3764 return (B_TRUE); 3765 } 3766 3767 /* 3768 * igb_set_external_loopback - Set the external loopback mode 3769 */ 3770 static void 3771 igb_set_external_loopback(igb_t *igb) 3772 { 3773 struct e1000_hw *hw; 3774 uint32_t ctrl_ext; 3775 3776 hw = &igb->hw; 3777 3778 /* Set link mode to PHY (00b) in the Extended Control register */ 3779 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3780 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3781 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3782 3783 (void) e1000_write_phy_reg(hw, 0x0, 0x0140); 3784 (void) e1000_write_phy_reg(hw, 0x9, 0x1a00); 3785 (void) e1000_write_phy_reg(hw, 0x12, 0x1610); 3786 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c); 3787 } 3788 3789 /* 3790 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode 3791 */ 3792 static void 3793 igb_set_internal_phy_loopback(igb_t *igb) 3794 { 3795 struct e1000_hw *hw; 3796 uint32_t ctrl_ext; 3797 uint16_t phy_ctrl; 3798 uint16_t phy_pconf; 3799 3800 hw = &igb->hw; 3801 3802 /* Set link mode to PHY (00b) in the Extended Control register */ 3803 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3804 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3805 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3806 3807 /* 3808 * Set PHY control register (0x4140): 3809 * Set full duplex mode 3810 * Set loopback bit 3811 * Clear auto-neg enable bit 3812 * Set PHY speed 3813 */ 3814 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK; 3815 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3816 3817 /* Set the link disable bit in the Port Configuration register */ 3818 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf); 3819 phy_pconf |= (uint16_t)1 << 14; 3820 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf); 3821 } 3822 3823 /* 3824 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode 3825 */ 3826 static void 3827 igb_set_internal_serdes_loopback(igb_t *igb) 3828 { 3829 struct e1000_hw *hw; 3830 uint32_t ctrl_ext; 3831 uint32_t ctrl; 3832 uint32_t pcs_lctl; 3833 uint32_t connsw; 3834 3835 hw = &igb->hw; 3836 3837 /* Set link mode to SerDes (11b) in the Extended Control register */ 3838 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3839 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 3840 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3841 3842 /* Configure the SerDes to loopback */ 3843 E1000_WRITE_REG(hw, E1000_SCTL, 0x410); 3844 3845 /* Set Device Control register */ 3846 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3847 ctrl |= (E1000_CTRL_FD | /* Force full duplex */ 3848 E1000_CTRL_SLU); /* Force link up */ 3849 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */ 3850 E1000_CTRL_TFCE | /* Disable transmit flow control */ 3851 E1000_CTRL_LRST); /* Clear link reset */ 3852 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3853 3854 /* Set PCS Link Control register */ 3855 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL); 3856 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK | 3857 E1000_PCS_LCTL_FSD | 3858 E1000_PCS_LCTL_FDV_FULL | 3859 E1000_PCS_LCTL_FLV_LINK_UP); 3860 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE; 3861 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl); 3862 3863 /* Set the Copper/Fiber Switch Control - CONNSW register */ 3864 connsw = E1000_READ_REG(hw, E1000_CONNSW); 3865 connsw &= ~E1000_CONNSW_ENRGSRC; 3866 E1000_WRITE_REG(hw, E1000_CONNSW, connsw); 3867 } 3868 3869 #pragma inline(igb_intr_rx_work) 3870 /* 3871 * igb_intr_rx_work - rx processing of ISR 3872 */ 3873 static void 3874 igb_intr_rx_work(igb_rx_ring_t *rx_ring) 3875 { 3876 mblk_t *mp; 3877 3878 mutex_enter(&rx_ring->rx_lock); 3879 mp = igb_rx(rx_ring, IGB_NO_POLL); 3880 mutex_exit(&rx_ring->rx_lock); 3881 3882 if (mp != NULL) 3883 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp, 3884 rx_ring->ring_gen_num); 3885 } 3886 3887 #pragma inline(igb_intr_tx_work) 3888 /* 3889 * igb_intr_tx_work - tx processing of ISR 3890 */ 3891 static void 3892 igb_intr_tx_work(igb_tx_ring_t *tx_ring) 3893 { 3894 igb_t *igb = tx_ring->igb; 3895 3896 /* Recycle the tx descriptors */ 3897 tx_ring->tx_recycle(tx_ring); 3898 3899 /* Schedule the re-transmit */ 3900 if (tx_ring->reschedule && 3901 (tx_ring->tbd_free >= igb->tx_resched_thresh)) { 3902 tx_ring->reschedule = B_FALSE; 3903 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle); 3904 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3905 } 3906 } 3907 3908 #pragma inline(igb_intr_link_work) 3909 /* 3910 * igb_intr_link_work - link-status-change processing of ISR 3911 */ 3912 static void 3913 igb_intr_link_work(igb_t *igb) 3914 { 3915 boolean_t link_changed; 3916 3917 igb_stop_watchdog_timer(igb); 3918 3919 mutex_enter(&igb->gen_lock); 3920 3921 /* 3922 * Because we got a link-status-change interrupt, force 3923 * e1000_check_for_link() to look at phy 3924 */ 3925 igb->hw.mac.get_link_status = B_TRUE; 3926 3927 /* igb_link_check takes care of link status change */ 3928 link_changed = igb_link_check(igb); 3929 3930 /* Get new phy state */ 3931 igb_get_phy_state(igb); 3932 3933 mutex_exit(&igb->gen_lock); 3934 3935 if (link_changed) 3936 mac_link_update(igb->mac_hdl, igb->link_state); 3937 3938 igb_start_watchdog_timer(igb); 3939 } 3940 3941 /* 3942 * igb_intr_legacy - Interrupt handler for legacy interrupts 3943 */ 3944 static uint_t 3945 igb_intr_legacy(void *arg1, void *arg2) 3946 { 3947 igb_t *igb = (igb_t *)arg1; 3948 igb_tx_ring_t *tx_ring; 3949 uint32_t icr; 3950 mblk_t *mp; 3951 boolean_t tx_reschedule; 3952 boolean_t link_changed; 3953 uint_t result; 3954 3955 _NOTE(ARGUNUSED(arg2)); 3956 3957 mutex_enter(&igb->gen_lock); 3958 3959 if (igb->igb_state & IGB_SUSPENDED) { 3960 mutex_exit(&igb->gen_lock); 3961 return (DDI_INTR_UNCLAIMED); 3962 } 3963 3964 mp = NULL; 3965 tx_reschedule = B_FALSE; 3966 link_changed = B_FALSE; 3967 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3968 3969 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3970 mutex_exit(&igb->gen_lock); 3971 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3972 atomic_or_32(&igb->igb_state, IGB_ERROR); 3973 return (DDI_INTR_UNCLAIMED); 3974 } 3975 3976 if (icr & E1000_ICR_INT_ASSERTED) { 3977 /* 3978 * E1000_ICR_INT_ASSERTED bit was set: 3979 * Read(Clear) the ICR, claim this interrupt, 3980 * look for work to do. 3981 */ 3982 ASSERT(igb->num_rx_rings == 1); 3983 ASSERT(igb->num_tx_rings == 1); 3984 3985 /* Make sure all interrupt causes cleared */ 3986 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3987 3988 if (icr & E1000_ICR_RXT0) { 3989 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL); 3990 } 3991 3992 if (icr & E1000_ICR_TXDW) { 3993 tx_ring = &igb->tx_rings[0]; 3994 3995 /* Recycle the tx descriptors */ 3996 tx_ring->tx_recycle(tx_ring); 3997 3998 /* Schedule the re-transmit */ 3999 tx_reschedule = (tx_ring->reschedule && 4000 (tx_ring->tbd_free >= igb->tx_resched_thresh)); 4001 } 4002 4003 if (icr & E1000_ICR_LSC) { 4004 /* 4005 * Because we got a link-status-change interrupt, force 4006 * e1000_check_for_link() to look at phy 4007 */ 4008 igb->hw.mac.get_link_status = B_TRUE; 4009 4010 /* igb_link_check takes care of link status change */ 4011 link_changed = igb_link_check(igb); 4012 4013 /* Get new phy state */ 4014 igb_get_phy_state(igb); 4015 } 4016 4017 if (icr & E1000_ICR_DRSTA) { 4018 /* 82580 Full Device Reset needed */ 4019 atomic_or_32(&igb->igb_state, IGB_STALL); 4020 } 4021 4022 result = DDI_INTR_CLAIMED; 4023 } else { 4024 /* 4025 * E1000_ICR_INT_ASSERTED bit was not set: 4026 * Don't claim this interrupt. 4027 */ 4028 result = DDI_INTR_UNCLAIMED; 4029 } 4030 4031 mutex_exit(&igb->gen_lock); 4032 4033 /* 4034 * Do the following work outside of the gen_lock 4035 */ 4036 if (mp != NULL) 4037 mac_rx(igb->mac_hdl, NULL, mp); 4038 4039 if (tx_reschedule) { 4040 tx_ring->reschedule = B_FALSE; 4041 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle); 4042 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 4043 } 4044 4045 if (link_changed) 4046 mac_link_update(igb->mac_hdl, igb->link_state); 4047 4048 return (result); 4049 } 4050 4051 /* 4052 * igb_intr_msi - Interrupt handler for MSI 4053 */ 4054 static uint_t 4055 igb_intr_msi(void *arg1, void *arg2) 4056 { 4057 igb_t *igb = (igb_t *)arg1; 4058 uint32_t icr; 4059 4060 _NOTE(ARGUNUSED(arg2)); 4061 4062 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4063 4064 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4065 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4066 atomic_or_32(&igb->igb_state, IGB_ERROR); 4067 return (DDI_INTR_CLAIMED); 4068 } 4069 4070 /* Make sure all interrupt causes cleared */ 4071 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 4072 4073 /* 4074 * For MSI interrupt, we have only one vector, 4075 * so we have only one rx ring and one tx ring enabled. 4076 */ 4077 ASSERT(igb->num_rx_rings == 1); 4078 ASSERT(igb->num_tx_rings == 1); 4079 4080 if (icr & E1000_ICR_RXT0) { 4081 igb_intr_rx_work(&igb->rx_rings[0]); 4082 } 4083 4084 if (icr & E1000_ICR_TXDW) { 4085 igb_intr_tx_work(&igb->tx_rings[0]); 4086 } 4087 4088 if (icr & E1000_ICR_LSC) { 4089 igb_intr_link_work(igb); 4090 } 4091 4092 if (icr & E1000_ICR_DRSTA) { 4093 /* 82580 Full Device Reset needed */ 4094 atomic_or_32(&igb->igb_state, IGB_STALL); 4095 } 4096 4097 return (DDI_INTR_CLAIMED); 4098 } 4099 4100 /* 4101 * igb_intr_rx - Interrupt handler for rx 4102 */ 4103 static uint_t 4104 igb_intr_rx(void *arg1, void *arg2) 4105 { 4106 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1; 4107 4108 _NOTE(ARGUNUSED(arg2)); 4109 4110 /* 4111 * Only used via MSI-X vector so don't check cause bits 4112 * and only clean the given ring. 4113 */ 4114 igb_intr_rx_work(rx_ring); 4115 4116 return (DDI_INTR_CLAIMED); 4117 } 4118 4119 /* 4120 * igb_intr_tx - Interrupt handler for tx 4121 */ 4122 static uint_t 4123 igb_intr_tx(void *arg1, void *arg2) 4124 { 4125 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1; 4126 4127 _NOTE(ARGUNUSED(arg2)); 4128 4129 /* 4130 * Only used via MSI-X vector so don't check cause bits 4131 * and only clean the given ring. 4132 */ 4133 igb_intr_tx_work(tx_ring); 4134 4135 return (DDI_INTR_CLAIMED); 4136 } 4137 4138 /* 4139 * igb_intr_tx_other - Interrupt handler for both tx and other 4140 * 4141 */ 4142 static uint_t 4143 igb_intr_tx_other(void *arg1, void *arg2) 4144 { 4145 igb_t *igb = (igb_t *)arg1; 4146 uint32_t icr; 4147 4148 _NOTE(ARGUNUSED(arg2)); 4149 4150 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4151 4152 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4153 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4154 atomic_or_32(&igb->igb_state, IGB_ERROR); 4155 return (DDI_INTR_CLAIMED); 4156 } 4157 4158 /* 4159 * Look for tx reclaiming work first. Remember, in the 4160 * case of only interrupt sharing, only one tx ring is 4161 * used 4162 */ 4163 igb_intr_tx_work(&igb->tx_rings[0]); 4164 4165 /* 4166 * Check for "other" causes. 4167 */ 4168 if (icr & E1000_ICR_LSC) { 4169 igb_intr_link_work(igb); 4170 } 4171 4172 /* 4173 * The DOUTSYNC bit indicates a tx packet dropped because 4174 * DMA engine gets "out of sync". There isn't a real fix 4175 * for this. The Intel recommendation is to count the number 4176 * of occurrences so user can detect when it is happening. 4177 * The issue is non-fatal and there's no recovery action 4178 * available. 4179 */ 4180 if (icr & E1000_ICR_DOUTSYNC) { 4181 IGB_STAT(igb->dout_sync); 4182 } 4183 4184 if (icr & E1000_ICR_DRSTA) { 4185 /* 82580 Full Device Reset needed */ 4186 atomic_or_32(&igb->igb_state, IGB_STALL); 4187 } 4188 4189 return (DDI_INTR_CLAIMED); 4190 } 4191 4192 /* 4193 * igb_alloc_intrs - Allocate interrupts for the driver 4194 * 4195 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4196 * if not successful, try Legacy. 4197 * igb->intr_force can be used to force sequence to start with 4198 * any of the 3 types. 4199 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4200 */ 4201 static int 4202 igb_alloc_intrs(igb_t *igb) 4203 { 4204 dev_info_t *devinfo; 4205 int intr_types; 4206 int rc; 4207 4208 devinfo = igb->dip; 4209 4210 /* Get supported interrupt types */ 4211 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4212 4213 if (rc != DDI_SUCCESS) { 4214 igb_log(igb, 4215 "Get supported interrupt types failed: %d", rc); 4216 return (IGB_FAILURE); 4217 } 4218 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types); 4219 4220 igb->intr_type = 0; 4221 4222 /* Install MSI-X interrupts */ 4223 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4224 (igb->intr_force <= IGB_INTR_MSIX)) { 4225 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX); 4226 4227 if (rc == IGB_SUCCESS) 4228 return (IGB_SUCCESS); 4229 4230 igb_log(igb, 4231 "Allocate MSI-X failed, trying MSI interrupts..."); 4232 } 4233 4234 /* MSI-X not used, force rings to 1 */ 4235 igb->num_rx_rings = 1; 4236 igb->num_tx_rings = 1; 4237 igb_log(igb, 4238 "MSI-X not used, force rx and tx queue number to 1"); 4239 4240 /* Install MSI interrupts */ 4241 if ((intr_types & DDI_INTR_TYPE_MSI) && 4242 (igb->intr_force <= IGB_INTR_MSI)) { 4243 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI); 4244 4245 if (rc == IGB_SUCCESS) 4246 return (IGB_SUCCESS); 4247 4248 igb_log(igb, 4249 "Allocate MSI failed, trying Legacy interrupts..."); 4250 } 4251 4252 /* Install legacy interrupts */ 4253 if (intr_types & DDI_INTR_TYPE_FIXED) { 4254 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED); 4255 4256 if (rc == IGB_SUCCESS) 4257 return (IGB_SUCCESS); 4258 4259 igb_log(igb, 4260 "Allocate Legacy interrupts failed"); 4261 } 4262 4263 /* If none of the 3 types succeeded, return failure */ 4264 return (IGB_FAILURE); 4265 } 4266 4267 /* 4268 * igb_alloc_intr_handles - Allocate interrupt handles. 4269 * 4270 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4271 * if fewer than 2 handles are available, return failure. 4272 * Upon success, this sets the number of Rx rings to a number that 4273 * matches the handles available for Rx interrupts. 4274 */ 4275 static int 4276 igb_alloc_intr_handles(igb_t *igb, int intr_type) 4277 { 4278 dev_info_t *devinfo; 4279 int orig, request, count, avail, actual; 4280 int diff, minimum; 4281 int rc; 4282 4283 devinfo = igb->dip; 4284 4285 switch (intr_type) { 4286 case DDI_INTR_TYPE_FIXED: 4287 request = 1; /* Request 1 legacy interrupt handle */ 4288 minimum = 1; 4289 IGB_DEBUGLOG_0(igb, "interrupt type: legacy"); 4290 break; 4291 4292 case DDI_INTR_TYPE_MSI: 4293 request = 1; /* Request 1 MSI interrupt handle */ 4294 minimum = 1; 4295 IGB_DEBUGLOG_0(igb, "interrupt type: MSI"); 4296 break; 4297 4298 case DDI_INTR_TYPE_MSIX: 4299 /* 4300 * Number of vectors for the adapter is 4301 * # rx rings + # tx rings 4302 * One of tx vectors is for tx & other 4303 */ 4304 request = igb->num_rx_rings + igb->num_tx_rings; 4305 orig = request; 4306 minimum = 2; 4307 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X"); 4308 break; 4309 4310 default: 4311 igb_log(igb, 4312 "invalid call to igb_alloc_intr_handles(): %d\n", 4313 intr_type); 4314 return (IGB_FAILURE); 4315 } 4316 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d", 4317 request, minimum); 4318 4319 /* 4320 * Get number of supported interrupts 4321 */ 4322 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4323 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4324 igb_log(igb, 4325 "Get supported interrupt number failed. " 4326 "Return: %d, count: %d", rc, count); 4327 return (IGB_FAILURE); 4328 } 4329 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count); 4330 4331 /* 4332 * Get number of available interrupts 4333 */ 4334 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 4335 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 4336 igb_log(igb, 4337 "Get available interrupt number failed. " 4338 "Return: %d, available: %d", rc, avail); 4339 return (IGB_FAILURE); 4340 } 4341 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail); 4342 4343 if (avail < request) { 4344 igb_log(igb, "Request %d handles, %d available", 4345 request, avail); 4346 request = avail; 4347 } 4348 4349 actual = 0; 4350 igb->intr_cnt = 0; 4351 4352 /* 4353 * Allocate an array of interrupt handles 4354 */ 4355 igb->intr_size = request * sizeof (ddi_intr_handle_t); 4356 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP); 4357 4358 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0, 4359 request, &actual, DDI_INTR_ALLOC_NORMAL); 4360 if (rc != DDI_SUCCESS) { 4361 igb_log(igb, "Allocate interrupts failed. " 4362 "return: %d, request: %d, actual: %d", 4363 rc, request, actual); 4364 goto alloc_handle_fail; 4365 } 4366 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual); 4367 4368 igb->intr_cnt = actual; 4369 4370 if (actual < minimum) { 4371 igb_log(igb, "Insufficient interrupt handles allocated: %d", 4372 actual); 4373 goto alloc_handle_fail; 4374 } 4375 4376 /* 4377 * For MSI-X, actual might force us to reduce number of tx & rx rings 4378 */ 4379 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { 4380 diff = orig - actual; 4381 if (diff < igb->num_tx_rings) { 4382 igb_log(igb, 4383 "MSI-X vectors force Tx queue number to %d", 4384 igb->num_tx_rings - diff); 4385 igb->num_tx_rings -= diff; 4386 } else { 4387 igb_log(igb, 4388 "MSI-X vectors force Tx queue number to 1"); 4389 igb->num_tx_rings = 1; 4390 4391 igb_log(igb, 4392 "MSI-X vectors force Rx queue number to %d", 4393 actual - 1); 4394 igb->num_rx_rings = actual - 1; 4395 } 4396 } 4397 4398 /* 4399 * Get priority for first vector, assume remaining are all the same 4400 */ 4401 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri); 4402 if (rc != DDI_SUCCESS) { 4403 igb_log(igb, 4404 "Get interrupt priority failed: %d", rc); 4405 goto alloc_handle_fail; 4406 } 4407 4408 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap); 4409 if (rc != DDI_SUCCESS) { 4410 igb_log(igb, 4411 "Get interrupt cap failed: %d", rc); 4412 goto alloc_handle_fail; 4413 } 4414 4415 igb->intr_type = intr_type; 4416 4417 return (IGB_SUCCESS); 4418 4419 alloc_handle_fail: 4420 igb_rem_intrs(igb); 4421 4422 return (IGB_FAILURE); 4423 } 4424 4425 /* 4426 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type 4427 * 4428 * Before adding the interrupt handlers, the interrupt vectors have 4429 * been allocated, and the rx/tx rings have also been allocated. 4430 */ 4431 static int 4432 igb_add_intr_handlers(igb_t *igb) 4433 { 4434 igb_rx_ring_t *rx_ring; 4435 igb_tx_ring_t *tx_ring; 4436 int vector; 4437 int rc; 4438 int i; 4439 4440 vector = 0; 4441 4442 switch (igb->intr_type) { 4443 case DDI_INTR_TYPE_MSIX: 4444 /* Add interrupt handler for tx + other */ 4445 tx_ring = &igb->tx_rings[0]; 4446 rc = ddi_intr_add_handler(igb->htable[vector], 4447 (ddi_intr_handler_t *)igb_intr_tx_other, 4448 (void *)igb, NULL); 4449 4450 if (rc != DDI_SUCCESS) { 4451 igb_log(igb, 4452 "Add tx/other interrupt handler failed: %d", rc); 4453 return (IGB_FAILURE); 4454 } 4455 tx_ring->intr_vector = vector; 4456 vector++; 4457 4458 /* Add interrupt handler for each rx ring */ 4459 for (i = 0; i < igb->num_rx_rings; i++) { 4460 rx_ring = &igb->rx_rings[i]; 4461 4462 rc = ddi_intr_add_handler(igb->htable[vector], 4463 (ddi_intr_handler_t *)igb_intr_rx, 4464 (void *)rx_ring, NULL); 4465 4466 if (rc != DDI_SUCCESS) { 4467 igb_log(igb, 4468 "Add rx interrupt handler failed. " 4469 "return: %d, rx ring: %d", rc, i); 4470 for (vector--; vector >= 0; vector--) { 4471 (void) ddi_intr_remove_handler( 4472 igb->htable[vector]); 4473 } 4474 return (IGB_FAILURE); 4475 } 4476 4477 rx_ring->intr_vector = vector; 4478 4479 vector++; 4480 } 4481 4482 /* Add interrupt handler for each tx ring from 2nd ring */ 4483 for (i = 1; i < igb->num_tx_rings; i++) { 4484 tx_ring = &igb->tx_rings[i]; 4485 4486 rc = ddi_intr_add_handler(igb->htable[vector], 4487 (ddi_intr_handler_t *)igb_intr_tx, 4488 (void *)tx_ring, NULL); 4489 4490 if (rc != DDI_SUCCESS) { 4491 igb_log(igb, 4492 "Add tx interrupt handler failed. " 4493 "return: %d, tx ring: %d", rc, i); 4494 for (vector--; vector >= 0; vector--) { 4495 (void) ddi_intr_remove_handler( 4496 igb->htable[vector]); 4497 } 4498 return (IGB_FAILURE); 4499 } 4500 4501 tx_ring->intr_vector = vector; 4502 4503 vector++; 4504 } 4505 4506 break; 4507 4508 case DDI_INTR_TYPE_MSI: 4509 /* Add interrupt handlers for the only vector */ 4510 rc = ddi_intr_add_handler(igb->htable[vector], 4511 (ddi_intr_handler_t *)igb_intr_msi, 4512 (void *)igb, NULL); 4513 4514 if (rc != DDI_SUCCESS) { 4515 igb_log(igb, 4516 "Add MSI interrupt handler failed: %d", rc); 4517 return (IGB_FAILURE); 4518 } 4519 4520 rx_ring = &igb->rx_rings[0]; 4521 rx_ring->intr_vector = vector; 4522 4523 vector++; 4524 break; 4525 4526 case DDI_INTR_TYPE_FIXED: 4527 /* Add interrupt handlers for the only vector */ 4528 rc = ddi_intr_add_handler(igb->htable[vector], 4529 (ddi_intr_handler_t *)igb_intr_legacy, 4530 (void *)igb, NULL); 4531 4532 if (rc != DDI_SUCCESS) { 4533 igb_log(igb, 4534 "Add legacy interrupt handler failed: %d", rc); 4535 return (IGB_FAILURE); 4536 } 4537 4538 rx_ring = &igb->rx_rings[0]; 4539 rx_ring->intr_vector = vector; 4540 4541 vector++; 4542 break; 4543 4544 default: 4545 return (IGB_FAILURE); 4546 } 4547 4548 ASSERT(vector == igb->intr_cnt); 4549 4550 return (IGB_SUCCESS); 4551 } 4552 4553 /* 4554 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts 4555 * 4556 * For each vector enabled on the adapter, Set the MSIXBM register accordingly 4557 */ 4558 static void 4559 igb_setup_msix_82575(igb_t *igb) 4560 { 4561 uint32_t eims = 0; 4562 int i, vector; 4563 struct e1000_hw *hw = &igb->hw; 4564 4565 /* 4566 * Set vector for tx ring 0 and other causes. 4567 * NOTE assumption that it is vector 0. 4568 */ 4569 vector = 0; 4570 4571 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER; 4572 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask); 4573 vector++; 4574 4575 for (i = 0; i < igb->num_rx_rings; i++) { 4576 /* 4577 * Set vector for each rx ring 4578 */ 4579 eims = (E1000_EICR_RX_QUEUE0 << i); 4580 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4581 4582 /* 4583 * Accumulate bits to enable in 4584 * igb_enable_adapter_interrupts_82575() 4585 */ 4586 igb->eims_mask |= eims; 4587 4588 vector++; 4589 } 4590 4591 for (i = 1; i < igb->num_tx_rings; i++) { 4592 /* 4593 * Set vector for each tx ring from 2nd tx ring 4594 */ 4595 eims = (E1000_EICR_TX_QUEUE0 << i); 4596 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4597 4598 /* 4599 * Accumulate bits to enable in 4600 * igb_enable_adapter_interrupts_82575() 4601 */ 4602 igb->eims_mask |= eims; 4603 4604 vector++; 4605 } 4606 4607 ASSERT(vector == igb->intr_cnt); 4608 4609 /* 4610 * Disable IAM for ICR interrupt bits 4611 */ 4612 E1000_WRITE_REG(hw, E1000_IAM, 0); 4613 E1000_WRITE_FLUSH(hw); 4614 } 4615 4616 /* 4617 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts 4618 * 4619 * 82576 uses a table based method for assigning vectors. Each queue has a 4620 * single entry in the table to which we write a vector number along with a 4621 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors 4622 * take a different position in the 4-byte register depending on whether 4623 * they are numbered above or below 8. 4624 */ 4625 static void 4626 igb_setup_msix_82576(igb_t *igb) 4627 { 4628 struct e1000_hw *hw = &igb->hw; 4629 uint32_t ivar, index, vector; 4630 int i; 4631 4632 /* must enable msi-x capability before IVAR settings */ 4633 E1000_WRITE_REG(hw, E1000_GPIE, 4634 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR)); 4635 4636 /* 4637 * Set vector for tx ring 0 and other causes. 4638 * NOTE assumption that it is vector 0. 4639 * This is also interdependent with installation of interrupt service 4640 * routines in igb_add_intr_handlers(). 4641 */ 4642 4643 /* assign "other" causes to vector 0 */ 4644 vector = 0; 4645 ivar = ((vector | E1000_IVAR_VALID) << 8); 4646 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4647 4648 /* assign tx ring 0 to vector 0 */ 4649 ivar = ((vector | E1000_IVAR_VALID) << 8); 4650 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4651 4652 /* prepare to enable tx & other interrupt causes */ 4653 igb->eims_mask = (1 << vector); 4654 4655 vector ++; 4656 for (i = 0; i < igb->num_rx_rings; i++) { 4657 /* 4658 * Set vector for each rx ring 4659 */ 4660 index = (i & 0x7); 4661 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4662 4663 if (i < 8) { 4664 /* vector goes into low byte of register */ 4665 ivar = ivar & 0xFFFFFF00; 4666 ivar |= (vector | E1000_IVAR_VALID); 4667 } else { 4668 /* vector goes into third byte of register */ 4669 ivar = ivar & 0xFF00FFFF; 4670 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4671 } 4672 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4673 4674 /* Accumulate interrupt-cause bits to enable */ 4675 igb->eims_mask |= (1 << vector); 4676 4677 vector ++; 4678 } 4679 4680 for (i = 1; i < igb->num_tx_rings; i++) { 4681 /* 4682 * Set vector for each tx ring from 2nd tx ring. 4683 * Note assumption that tx vectors numericall follow rx vectors. 4684 */ 4685 index = (i & 0x7); 4686 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4687 4688 if (i < 8) { 4689 /* vector goes into second byte of register */ 4690 ivar = ivar & 0xFFFF00FF; 4691 ivar |= ((vector | E1000_IVAR_VALID) << 8); 4692 } else { 4693 /* vector goes into fourth byte of register */ 4694 ivar = ivar & 0x00FFFFFF; 4695 ivar |= (vector | E1000_IVAR_VALID) << 24; 4696 } 4697 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4698 4699 /* Accumulate interrupt-cause bits to enable */ 4700 igb->eims_mask |= (1 << vector); 4701 4702 vector ++; 4703 } 4704 4705 ASSERT(vector == igb->intr_cnt); 4706 } 4707 4708 /* 4709 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts 4710 * 4711 * 82580 uses same table approach at 82576 but has fewer entries. Each 4712 * queue has a single entry in the table to which we write a vector number 4713 * along with a "valid" bit. Vectors take a different position in the 4714 * register depending on * whether * they are numbered above or below 4. 4715 */ 4716 static void 4717 igb_setup_msix_82580(igb_t *igb) 4718 { 4719 struct e1000_hw *hw = &igb->hw; 4720 uint32_t ivar, index, vector; 4721 int i; 4722 4723 /* must enable msi-x capability before IVAR settings */ 4724 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE | 4725 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME)); 4726 /* 4727 * Set vector for tx ring 0 and other causes. 4728 * NOTE assumption that it is vector 0. 4729 * This is also interdependent with installation of interrupt service 4730 * routines in igb_add_intr_handlers(). 4731 */ 4732 4733 /* assign "other" causes to vector 0 */ 4734 vector = 0; 4735 ivar = ((vector | E1000_IVAR_VALID) << 8); 4736 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4737 4738 /* assign tx ring 0 to vector 0 */ 4739 ivar = ((vector | E1000_IVAR_VALID) << 8); 4740 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4741 4742 /* prepare to enable tx & other interrupt causes */ 4743 igb->eims_mask = (1 << vector); 4744 4745 vector ++; 4746 4747 for (i = 0; i < igb->num_rx_rings; i++) { 4748 /* 4749 * Set vector for each rx ring 4750 */ 4751 index = (i >> 1); 4752 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4753 4754 if (i & 1) { 4755 /* vector goes into third byte of register */ 4756 ivar = ivar & 0xFF00FFFF; 4757 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4758 } else { 4759 /* vector goes into low byte of register */ 4760 ivar = ivar & 0xFFFFFF00; 4761 ivar |= (vector | E1000_IVAR_VALID); 4762 } 4763 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4764 4765 /* Accumulate interrupt-cause bits to enable */ 4766 igb->eims_mask |= (1 << vector); 4767 4768 vector ++; 4769 } 4770 4771 for (i = 1; i < igb->num_tx_rings; i++) { 4772 /* 4773 * Set vector for each tx ring from 2nd tx ring. 4774 * Note assumption that tx vectors numericall follow rx vectors. 4775 */ 4776 index = (i >> 1); 4777 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4778 4779 if (i & 1) { 4780 /* vector goes into high byte of register */ 4781 ivar = ivar & 0x00FFFFFF; 4782 ivar |= ((vector | E1000_IVAR_VALID) << 24); 4783 } else { 4784 /* vector goes into second byte of register */ 4785 ivar = ivar & 0xFFFF00FF; 4786 ivar |= (vector | E1000_IVAR_VALID) << 8; 4787 } 4788 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4789 4790 /* Accumulate interrupt-cause bits to enable */ 4791 igb->eims_mask |= (1 << vector); 4792 4793 vector ++; 4794 } 4795 ASSERT(vector == igb->intr_cnt); 4796 } 4797 4798 /* 4799 * igb_rem_intr_handlers - remove the interrupt handlers 4800 */ 4801 static void 4802 igb_rem_intr_handlers(igb_t *igb) 4803 { 4804 int i; 4805 int rc; 4806 4807 for (i = 0; i < igb->intr_cnt; i++) { 4808 rc = ddi_intr_remove_handler(igb->htable[i]); 4809 if (rc != DDI_SUCCESS) { 4810 IGB_DEBUGLOG_1(igb, 4811 "Remove intr handler failed: %d", rc); 4812 } 4813 } 4814 } 4815 4816 /* 4817 * igb_rem_intrs - remove the allocated interrupts 4818 */ 4819 static void 4820 igb_rem_intrs(igb_t *igb) 4821 { 4822 int i; 4823 int rc; 4824 4825 for (i = 0; i < igb->intr_cnt; i++) { 4826 rc = ddi_intr_free(igb->htable[i]); 4827 if (rc != DDI_SUCCESS) { 4828 IGB_DEBUGLOG_1(igb, 4829 "Free intr failed: %d", rc); 4830 } 4831 } 4832 4833 kmem_free(igb->htable, igb->intr_size); 4834 igb->htable = NULL; 4835 } 4836 4837 /* 4838 * igb_enable_intrs - enable all the ddi interrupts 4839 */ 4840 static int 4841 igb_enable_intrs(igb_t *igb) 4842 { 4843 int i; 4844 int rc; 4845 4846 /* Enable interrupts */ 4847 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4848 /* Call ddi_intr_block_enable() for MSI */ 4849 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt); 4850 if (rc != DDI_SUCCESS) { 4851 igb_log(igb, 4852 "Enable block intr failed: %d", rc); 4853 return (IGB_FAILURE); 4854 } 4855 } else { 4856 /* Call ddi_intr_enable() for Legacy/MSI non block enable */ 4857 for (i = 0; i < igb->intr_cnt; i++) { 4858 rc = ddi_intr_enable(igb->htable[i]); 4859 if (rc != DDI_SUCCESS) { 4860 igb_log(igb, 4861 "Enable intr failed: %d", rc); 4862 return (IGB_FAILURE); 4863 } 4864 } 4865 } 4866 4867 return (IGB_SUCCESS); 4868 } 4869 4870 /* 4871 * igb_disable_intrs - disable all the ddi interrupts 4872 */ 4873 static int 4874 igb_disable_intrs(igb_t *igb) 4875 { 4876 int i; 4877 int rc; 4878 4879 /* Disable all interrupts */ 4880 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4881 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt); 4882 if (rc != DDI_SUCCESS) { 4883 igb_log(igb, 4884 "Disable block intr failed: %d", rc); 4885 return (IGB_FAILURE); 4886 } 4887 } else { 4888 for (i = 0; i < igb->intr_cnt; i++) { 4889 rc = ddi_intr_disable(igb->htable[i]); 4890 if (rc != DDI_SUCCESS) { 4891 igb_log(igb, 4892 "Disable intr failed: %d", rc); 4893 return (IGB_FAILURE); 4894 } 4895 } 4896 } 4897 4898 return (IGB_SUCCESS); 4899 } 4900 4901 /* 4902 * igb_get_phy_state - Get and save the parameters read from PHY registers 4903 */ 4904 static void 4905 igb_get_phy_state(igb_t *igb) 4906 { 4907 struct e1000_hw *hw = &igb->hw; 4908 uint16_t phy_ctrl; 4909 uint16_t phy_status; 4910 uint16_t phy_an_adv; 4911 uint16_t phy_an_exp; 4912 uint16_t phy_ext_status; 4913 uint16_t phy_1000t_ctrl; 4914 uint16_t phy_1000t_status; 4915 uint16_t phy_lp_able; 4916 4917 ASSERT(mutex_owned(&igb->gen_lock)); 4918 4919 if (hw->phy.media_type == e1000_media_type_copper) { 4920 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 4921 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 4922 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv); 4923 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp); 4924 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status); 4925 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl); 4926 (void) e1000_read_phy_reg(hw, 4927 PHY_1000T_STATUS, &phy_1000t_status); 4928 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able); 4929 4930 igb->param_autoneg_cap = 4931 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0; 4932 igb->param_pause_cap = 4933 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4934 igb->param_asym_pause_cap = 4935 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4936 igb->param_1000fdx_cap = 4937 ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) || 4938 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0; 4939 igb->param_1000hdx_cap = 4940 ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) || 4941 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0; 4942 igb->param_100t4_cap = 4943 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0; 4944 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) || 4945 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0; 4946 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) || 4947 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0; 4948 igb->param_10fdx_cap = 4949 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0; 4950 igb->param_10hdx_cap = 4951 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0; 4952 igb->param_rem_fault = 4953 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0; 4954 4955 igb->param_adv_autoneg_cap = hw->mac.autoneg; 4956 igb->param_adv_pause_cap = 4957 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4958 igb->param_adv_asym_pause_cap = 4959 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4960 igb->param_adv_1000hdx_cap = 4961 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0; 4962 igb->param_adv_100t4_cap = 4963 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0; 4964 igb->param_adv_rem_fault = 4965 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0; 4966 if (igb->param_adv_autoneg_cap == 1) { 4967 igb->param_adv_1000fdx_cap = 4968 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0; 4969 igb->param_adv_100fdx_cap = 4970 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0; 4971 igb->param_adv_100hdx_cap = 4972 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0; 4973 igb->param_adv_10fdx_cap = 4974 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0; 4975 igb->param_adv_10hdx_cap = 4976 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0; 4977 } 4978 4979 igb->param_lp_autoneg_cap = 4980 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0; 4981 igb->param_lp_pause_cap = 4982 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0; 4983 igb->param_lp_asym_pause_cap = 4984 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0; 4985 igb->param_lp_1000fdx_cap = 4986 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0; 4987 igb->param_lp_1000hdx_cap = 4988 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0; 4989 igb->param_lp_100t4_cap = 4990 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0; 4991 igb->param_lp_100fdx_cap = 4992 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0; 4993 igb->param_lp_100hdx_cap = 4994 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0; 4995 igb->param_lp_10fdx_cap = 4996 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0; 4997 igb->param_lp_10hdx_cap = 4998 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0; 4999 igb->param_lp_rem_fault = 5000 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0; 5001 } else { 5002 /* 5003 * 1Gig Fiber adapter only offers 1Gig Full Duplex. 5004 */ 5005 igb->param_autoneg_cap = 0; 5006 igb->param_pause_cap = 1; 5007 igb->param_asym_pause_cap = 1; 5008 igb->param_1000fdx_cap = 1; 5009 igb->param_1000hdx_cap = 0; 5010 igb->param_100t4_cap = 0; 5011 igb->param_100fdx_cap = 0; 5012 igb->param_100hdx_cap = 0; 5013 igb->param_10fdx_cap = 0; 5014 igb->param_10hdx_cap = 0; 5015 5016 igb->param_adv_autoneg_cap = 0; 5017 igb->param_adv_pause_cap = 1; 5018 igb->param_adv_asym_pause_cap = 1; 5019 igb->param_adv_1000fdx_cap = 1; 5020 igb->param_adv_1000hdx_cap = 0; 5021 igb->param_adv_100t4_cap = 0; 5022 igb->param_adv_100fdx_cap = 0; 5023 igb->param_adv_100hdx_cap = 0; 5024 igb->param_adv_10fdx_cap = 0; 5025 igb->param_adv_10hdx_cap = 0; 5026 5027 igb->param_lp_autoneg_cap = 0; 5028 igb->param_lp_pause_cap = 0; 5029 igb->param_lp_asym_pause_cap = 0; 5030 igb->param_lp_1000fdx_cap = 0; 5031 igb->param_lp_1000hdx_cap = 0; 5032 igb->param_lp_100t4_cap = 0; 5033 igb->param_lp_100fdx_cap = 0; 5034 igb->param_lp_100hdx_cap = 0; 5035 igb->param_lp_10fdx_cap = 0; 5036 igb->param_lp_10hdx_cap = 0; 5037 igb->param_lp_rem_fault = 0; 5038 } 5039 } 5040 5041 /* 5042 * synchronize the adv* and en* parameters. 5043 * 5044 * See comments in <sys/dld.h> for details of the *_en_* 5045 * parameters. The usage of ndd for setting adv parameters will 5046 * synchronize all the en parameters with the e1000g parameters, 5047 * implicitly disabling any settings made via dladm. 5048 */ 5049 static void 5050 igb_param_sync(igb_t *igb) 5051 { 5052 igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap; 5053 igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap; 5054 igb->param_en_100t4_cap = igb->param_adv_100t4_cap; 5055 igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap; 5056 igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap; 5057 igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap; 5058 igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap; 5059 } 5060 5061 /* 5062 * igb_get_driver_control 5063 */ 5064 static void 5065 igb_get_driver_control(struct e1000_hw *hw) 5066 { 5067 uint32_t ctrl_ext; 5068 5069 /* Notify firmware that driver is in control of device */ 5070 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5071 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD; 5072 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5073 } 5074 5075 /* 5076 * igb_release_driver_control 5077 */ 5078 static void 5079 igb_release_driver_control(struct e1000_hw *hw) 5080 { 5081 uint32_t ctrl_ext; 5082 5083 /* Notify firmware that driver is no longer in control of device */ 5084 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5085 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD; 5086 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5087 } 5088 5089 /* 5090 * igb_atomic_reserve - Atomic decrease operation 5091 */ 5092 int 5093 igb_atomic_reserve(uint32_t *count_p, uint32_t n) 5094 { 5095 uint32_t oldval; 5096 uint32_t newval; 5097 5098 /* ATOMICALLY */ 5099 do { 5100 oldval = *count_p; 5101 if (oldval < n) 5102 return (-1); 5103 newval = oldval - n; 5104 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5105 5106 return (newval); 5107 } 5108 5109 /* 5110 * FMA support 5111 */ 5112 5113 int 5114 igb_check_acc_handle(ddi_acc_handle_t handle) 5115 { 5116 ddi_fm_error_t de; 5117 5118 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5119 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5120 return (de.fme_status); 5121 } 5122 5123 int 5124 igb_check_dma_handle(ddi_dma_handle_t handle) 5125 { 5126 ddi_fm_error_t de; 5127 5128 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5129 return (de.fme_status); 5130 } 5131 5132 /* 5133 * The IO fault service error handling callback function 5134 */ 5135 /*ARGSUSED*/ 5136 static int 5137 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5138 { 5139 /* 5140 * as the driver can always deal with an error in any dma or 5141 * access handle, we can just return the fme_status value. 5142 */ 5143 pci_ereport_post(dip, err, NULL); 5144 return (err->fme_status); 5145 } 5146 5147 static void 5148 igb_fm_init(igb_t *igb) 5149 { 5150 ddi_iblock_cookie_t iblk; 5151 int fma_dma_flag; 5152 5153 /* Only register with IO Fault Services if we have some capability */ 5154 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5155 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5156 } else { 5157 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5158 } 5159 5160 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5161 fma_dma_flag = 1; 5162 } else { 5163 fma_dma_flag = 0; 5164 } 5165 5166 (void) igb_set_fma_flags(fma_dma_flag); 5167 5168 if (igb->fm_capabilities) { 5169 5170 /* Register capabilities with IO Fault Services */ 5171 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk); 5172 5173 /* 5174 * Initialize pci ereport capabilities if ereport capable 5175 */ 5176 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5177 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5178 pci_ereport_setup(igb->dip); 5179 5180 /* 5181 * Register error callback if error callback capable 5182 */ 5183 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5184 ddi_fm_handler_register(igb->dip, 5185 igb_fm_error_cb, (void*) igb); 5186 } 5187 } 5188 5189 static void 5190 igb_fm_fini(igb_t *igb) 5191 { 5192 /* Only unregister FMA capabilities if we registered some */ 5193 if (igb->fm_capabilities) { 5194 5195 /* 5196 * Release any resources allocated by pci_ereport_setup() 5197 */ 5198 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5199 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5200 pci_ereport_teardown(igb->dip); 5201 5202 /* 5203 * Un-register error callback if error callback capable 5204 */ 5205 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5206 ddi_fm_handler_unregister(igb->dip); 5207 5208 /* Unregister from IO Fault Services */ 5209 ddi_fm_fini(igb->dip); 5210 } 5211 } 5212 5213 void 5214 igb_fm_ereport(igb_t *igb, char *detail) 5215 { 5216 uint64_t ena; 5217 char buf[FM_MAX_CLASS]; 5218 5219 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5220 ena = fm_ena_generate(0, FM_ENA_FMT1); 5221 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) { 5222 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP, 5223 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5224 } 5225 } 5226