1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 */ 29 30 #include "igb_sw.h" 31 32 static char ident[] = "Intel 1Gb Ethernet"; 33 static char igb_version[] = "igb 1.1.17"; 34 35 /* 36 * Local function protoypes 37 */ 38 static int igb_register_mac(igb_t *); 39 static int igb_identify_hardware(igb_t *); 40 static int igb_regs_map(igb_t *); 41 static void igb_init_properties(igb_t *); 42 static int igb_init_driver_settings(igb_t *); 43 static void igb_init_locks(igb_t *); 44 static void igb_destroy_locks(igb_t *); 45 static int igb_init_mac_address(igb_t *); 46 static int igb_init(igb_t *); 47 static int igb_init_adapter(igb_t *); 48 static void igb_stop_adapter(igb_t *); 49 static int igb_reset(igb_t *); 50 static void igb_tx_clean(igb_t *); 51 static boolean_t igb_tx_drain(igb_t *); 52 static boolean_t igb_rx_drain(igb_t *); 53 static int igb_alloc_rings(igb_t *); 54 static int igb_alloc_rx_data(igb_t *); 55 static void igb_free_rx_data(igb_t *); 56 static void igb_free_rings(igb_t *); 57 static void igb_setup_rings(igb_t *); 58 static void igb_setup_rx(igb_t *); 59 static void igb_setup_tx(igb_t *); 60 static void igb_setup_rx_ring(igb_rx_ring_t *); 61 static void igb_setup_tx_ring(igb_tx_ring_t *); 62 static void igb_setup_rss(igb_t *); 63 static void igb_setup_mac_rss_classify(igb_t *); 64 static void igb_setup_mac_classify(igb_t *); 65 static void igb_init_unicst(igb_t *); 66 static void igb_setup_multicst(igb_t *); 67 static void igb_get_phy_state(igb_t *); 68 static void igb_param_sync(igb_t *); 69 static void igb_get_conf(igb_t *); 70 static int igb_get_prop(igb_t *, char *, int, int, int); 71 static boolean_t igb_is_link_up(igb_t *); 72 static boolean_t igb_link_check(igb_t *); 73 static void igb_local_timer(void *); 74 static void igb_link_timer(void *); 75 static void igb_arm_watchdog_timer(igb_t *); 76 static void igb_start_watchdog_timer(igb_t *); 77 static void igb_restart_watchdog_timer(igb_t *); 78 static void igb_stop_watchdog_timer(igb_t *); 79 static void igb_start_link_timer(igb_t *); 80 static void igb_stop_link_timer(igb_t *); 81 static void igb_disable_adapter_interrupts(igb_t *); 82 static void igb_enable_adapter_interrupts_82575(igb_t *); 83 static void igb_enable_adapter_interrupts_82576(igb_t *); 84 static void igb_enable_adapter_interrupts_82580(igb_t *); 85 static boolean_t is_valid_mac_addr(uint8_t *); 86 static boolean_t igb_stall_check(igb_t *); 87 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t); 88 static void igb_set_external_loopback(igb_t *); 89 static void igb_set_internal_phy_loopback(igb_t *); 90 static void igb_set_internal_serdes_loopback(igb_t *); 91 static boolean_t igb_find_mac_address(igb_t *); 92 static int igb_alloc_intrs(igb_t *); 93 static int igb_alloc_intr_handles(igb_t *, int); 94 static int igb_add_intr_handlers(igb_t *); 95 static void igb_rem_intr_handlers(igb_t *); 96 static void igb_rem_intrs(igb_t *); 97 static int igb_enable_intrs(igb_t *); 98 static int igb_disable_intrs(igb_t *); 99 static void igb_setup_msix_82575(igb_t *); 100 static void igb_setup_msix_82576(igb_t *); 101 static void igb_setup_msix_82580(igb_t *); 102 static uint_t igb_intr_legacy(void *, void *); 103 static uint_t igb_intr_msi(void *, void *); 104 static uint_t igb_intr_rx(void *, void *); 105 static uint_t igb_intr_tx(void *, void *); 106 static uint_t igb_intr_tx_other(void *, void *); 107 static void igb_intr_rx_work(igb_rx_ring_t *); 108 static void igb_intr_tx_work(igb_tx_ring_t *); 109 static void igb_intr_link_work(igb_t *); 110 static void igb_get_driver_control(struct e1000_hw *); 111 static void igb_release_driver_control(struct e1000_hw *); 112 113 static int igb_attach(dev_info_t *, ddi_attach_cmd_t); 114 static int igb_detach(dev_info_t *, ddi_detach_cmd_t); 115 static int igb_resume(dev_info_t *); 116 static int igb_suspend(dev_info_t *); 117 static int igb_quiesce(dev_info_t *); 118 static void igb_unconfigure(dev_info_t *, igb_t *); 119 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *, 120 const void *); 121 static void igb_fm_init(igb_t *); 122 static void igb_fm_fini(igb_t *); 123 static void igb_release_multicast(igb_t *); 124 125 char *igb_priv_props[] = { 126 "_tx_copy_thresh", 127 "_tx_recycle_thresh", 128 "_tx_overload_thresh", 129 "_tx_resched_thresh", 130 "_rx_copy_thresh", 131 "_rx_limit_per_intr", 132 "_intr_throttling", 133 "_adv_pause_cap", 134 "_adv_asym_pause_cap", 135 NULL 136 }; 137 138 static struct cb_ops igb_cb_ops = { 139 nulldev, /* cb_open */ 140 nulldev, /* cb_close */ 141 nodev, /* cb_strategy */ 142 nodev, /* cb_print */ 143 nodev, /* cb_dump */ 144 nodev, /* cb_read */ 145 nodev, /* cb_write */ 146 nodev, /* cb_ioctl */ 147 nodev, /* cb_devmap */ 148 nodev, /* cb_mmap */ 149 nodev, /* cb_segmap */ 150 nochpoll, /* cb_chpoll */ 151 ddi_prop_op, /* cb_prop_op */ 152 NULL, /* cb_stream */ 153 D_MP | D_HOTPLUG, /* cb_flag */ 154 CB_REV, /* cb_rev */ 155 nodev, /* cb_aread */ 156 nodev /* cb_awrite */ 157 }; 158 159 static struct dev_ops igb_dev_ops = { 160 DEVO_REV, /* devo_rev */ 161 0, /* devo_refcnt */ 162 NULL, /* devo_getinfo */ 163 nulldev, /* devo_identify */ 164 nulldev, /* devo_probe */ 165 igb_attach, /* devo_attach */ 166 igb_detach, /* devo_detach */ 167 nodev, /* devo_reset */ 168 &igb_cb_ops, /* devo_cb_ops */ 169 NULL, /* devo_bus_ops */ 170 ddi_power, /* devo_power */ 171 igb_quiesce, /* devo_quiesce */ 172 }; 173 174 static struct modldrv igb_modldrv = { 175 &mod_driverops, /* Type of module. This one is a driver */ 176 ident, /* Discription string */ 177 &igb_dev_ops, /* driver ops */ 178 }; 179 180 static struct modlinkage igb_modlinkage = { 181 MODREV_1, &igb_modldrv, NULL 182 }; 183 184 /* Access attributes for register mapping */ 185 ddi_device_acc_attr_t igb_regs_acc_attr = { 186 DDI_DEVICE_ATTR_V1, 187 DDI_STRUCTURE_LE_ACC, 188 DDI_STRICTORDER_ACC, 189 DDI_FLAGERR_ACC 190 }; 191 192 #define IGB_M_CALLBACK_FLAGS \ 193 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 194 195 static mac_callbacks_t igb_m_callbacks = { 196 IGB_M_CALLBACK_FLAGS, 197 igb_m_stat, 198 igb_m_start, 199 igb_m_stop, 200 igb_m_promisc, 201 igb_m_multicst, 202 NULL, 203 NULL, 204 NULL, 205 igb_m_ioctl, 206 igb_m_getcapab, 207 NULL, 208 NULL, 209 igb_m_setprop, 210 igb_m_getprop, 211 igb_m_propinfo 212 }; 213 214 /* 215 * Initialize capabilities of each supported adapter type 216 */ 217 static adapter_info_t igb_82575_cap = { 218 /* limits */ 219 4, /* maximum number of rx queues */ 220 1, /* minimum number of rx queues */ 221 4, /* default number of rx queues */ 222 4, /* maximum number of tx queues */ 223 1, /* minimum number of tx queues */ 224 4, /* default number of tx queues */ 225 65535, /* maximum interrupt throttle rate */ 226 0, /* minimum interrupt throttle rate */ 227 200, /* default interrupt throttle rate */ 228 229 /* function pointers */ 230 igb_enable_adapter_interrupts_82575, 231 igb_setup_msix_82575, 232 233 /* capabilities */ 234 (IGB_FLAG_HAS_DCA | /* capability flags */ 235 IGB_FLAG_VMDQ_POOL), 236 237 0xffc00000 /* mask for RXDCTL register */ 238 }; 239 240 static adapter_info_t igb_82576_cap = { 241 /* limits */ 242 16, /* maximum number of rx queues */ 243 1, /* minimum number of rx queues */ 244 4, /* default number of rx queues */ 245 16, /* maximum number of tx queues */ 246 1, /* minimum number of tx queues */ 247 4, /* default number of tx queues */ 248 65535, /* maximum interrupt throttle rate */ 249 0, /* minimum interrupt throttle rate */ 250 200, /* default interrupt throttle rate */ 251 252 /* function pointers */ 253 igb_enable_adapter_interrupts_82576, 254 igb_setup_msix_82576, 255 256 /* capabilities */ 257 (IGB_FLAG_HAS_DCA | /* capability flags */ 258 IGB_FLAG_VMDQ_POOL | 259 IGB_FLAG_NEED_CTX_IDX), 260 261 0xffe00000 /* mask for RXDCTL register */ 262 }; 263 264 static adapter_info_t igb_82580_cap = { 265 /* limits */ 266 8, /* maximum number of rx queues */ 267 1, /* minimum number of rx queues */ 268 4, /* default number of rx queues */ 269 8, /* maximum number of tx queues */ 270 1, /* minimum number of tx queues */ 271 4, /* default number of tx queues */ 272 65535, /* maximum interrupt throttle rate */ 273 0, /* minimum interrupt throttle rate */ 274 200, /* default interrupt throttle rate */ 275 276 /* function pointers */ 277 igb_enable_adapter_interrupts_82580, 278 igb_setup_msix_82580, 279 280 /* capabilities */ 281 (IGB_FLAG_HAS_DCA | /* capability flags */ 282 IGB_FLAG_VMDQ_POOL | 283 IGB_FLAG_NEED_CTX_IDX), 284 285 0xffe00000 /* mask for RXDCTL register */ 286 }; 287 288 /* 289 * Module Initialization Functions 290 */ 291 292 int 293 _init(void) 294 { 295 int status; 296 297 mac_init_ops(&igb_dev_ops, MODULE_NAME); 298 299 status = mod_install(&igb_modlinkage); 300 301 if (status != DDI_SUCCESS) { 302 mac_fini_ops(&igb_dev_ops); 303 } 304 305 return (status); 306 } 307 308 int 309 _fini(void) 310 { 311 int status; 312 313 status = mod_remove(&igb_modlinkage); 314 315 if (status == DDI_SUCCESS) { 316 mac_fini_ops(&igb_dev_ops); 317 } 318 319 return (status); 320 321 } 322 323 int 324 _info(struct modinfo *modinfop) 325 { 326 int status; 327 328 status = mod_info(&igb_modlinkage, modinfop); 329 330 return (status); 331 } 332 333 /* 334 * igb_attach - driver attach 335 * 336 * This function is the device specific initialization entry 337 * point. This entry point is required and must be written. 338 * The DDI_ATTACH command must be provided in the attach entry 339 * point. When attach() is called with cmd set to DDI_ATTACH, 340 * all normal kernel services (such as kmem_alloc(9F)) are 341 * available for use by the driver. 342 * 343 * The attach() function will be called once for each instance 344 * of the device on the system with cmd set to DDI_ATTACH. 345 * Until attach() succeeds, the only driver entry points which 346 * may be called are open(9E) and getinfo(9E). 347 */ 348 static int 349 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 350 { 351 igb_t *igb; 352 struct igb_osdep *osdep; 353 struct e1000_hw *hw; 354 int instance; 355 356 /* 357 * Check the command and perform corresponding operations 358 */ 359 switch (cmd) { 360 default: 361 return (DDI_FAILURE); 362 363 case DDI_RESUME: 364 return (igb_resume(devinfo)); 365 366 case DDI_ATTACH: 367 break; 368 } 369 370 /* Get the device instance */ 371 instance = ddi_get_instance(devinfo); 372 373 /* Allocate memory for the instance data structure */ 374 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP); 375 376 igb->dip = devinfo; 377 igb->instance = instance; 378 379 hw = &igb->hw; 380 osdep = &igb->osdep; 381 hw->back = osdep; 382 osdep->igb = igb; 383 384 /* Attach the instance pointer to the dev_info data structure */ 385 ddi_set_driver_private(devinfo, igb); 386 387 388 /* Initialize for fma support */ 389 igb->fm_capabilities = igb_get_prop(igb, "fm-capable", 390 0, 0x0f, 391 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 392 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 393 igb_fm_init(igb); 394 igb->attach_progress |= ATTACH_PROGRESS_FMINIT; 395 396 /* 397 * Map PCI config space registers 398 */ 399 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 400 igb_error(igb, "Failed to map PCI configurations"); 401 goto attach_fail; 402 } 403 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 404 405 /* 406 * Identify the chipset family 407 */ 408 if (igb_identify_hardware(igb) != IGB_SUCCESS) { 409 igb_error(igb, "Failed to identify hardware"); 410 goto attach_fail; 411 } 412 413 /* 414 * Map device registers 415 */ 416 if (igb_regs_map(igb) != IGB_SUCCESS) { 417 igb_error(igb, "Failed to map device registers"); 418 goto attach_fail; 419 } 420 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 421 422 /* 423 * Initialize driver parameters 424 */ 425 igb_init_properties(igb); 426 igb->attach_progress |= ATTACH_PROGRESS_PROPS; 427 428 /* 429 * Allocate interrupts 430 */ 431 if (igb_alloc_intrs(igb) != IGB_SUCCESS) { 432 igb_error(igb, "Failed to allocate interrupts"); 433 goto attach_fail; 434 } 435 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 436 437 /* 438 * Allocate rx/tx rings based on the ring numbers. 439 * The actual numbers of rx/tx rings are decided by the number of 440 * allocated interrupt vectors, so we should allocate the rings after 441 * interrupts are allocated. 442 */ 443 if (igb_alloc_rings(igb) != IGB_SUCCESS) { 444 igb_error(igb, "Failed to allocate rx/tx rings or groups"); 445 goto attach_fail; 446 } 447 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 448 449 /* 450 * Add interrupt handlers 451 */ 452 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) { 453 igb_error(igb, "Failed to add interrupt handlers"); 454 goto attach_fail; 455 } 456 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 457 458 /* 459 * Initialize driver parameters 460 */ 461 if (igb_init_driver_settings(igb) != IGB_SUCCESS) { 462 igb_error(igb, "Failed to initialize driver settings"); 463 goto attach_fail; 464 } 465 466 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) { 467 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 468 goto attach_fail; 469 } 470 471 /* 472 * Initialize mutexes for this device. 473 * Do this before enabling the interrupt handler and 474 * register the softint to avoid the condition where 475 * interrupt handler can try using uninitialized mutex 476 */ 477 igb_init_locks(igb); 478 igb->attach_progress |= ATTACH_PROGRESS_LOCKS; 479 480 /* 481 * Initialize the adapter 482 */ 483 if (igb_init(igb) != IGB_SUCCESS) { 484 igb_error(igb, "Failed to initialize adapter"); 485 goto attach_fail; 486 } 487 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 488 489 /* 490 * Initialize statistics 491 */ 492 if (igb_init_stats(igb) != IGB_SUCCESS) { 493 igb_error(igb, "Failed to initialize statistics"); 494 goto attach_fail; 495 } 496 igb->attach_progress |= ATTACH_PROGRESS_STATS; 497 498 /* 499 * Register the driver to the MAC 500 */ 501 if (igb_register_mac(igb) != IGB_SUCCESS) { 502 igb_error(igb, "Failed to register MAC"); 503 goto attach_fail; 504 } 505 igb->attach_progress |= ATTACH_PROGRESS_MAC; 506 507 /* 508 * Now that mutex locks are initialized, and the chip is also 509 * initialized, enable interrupts. 510 */ 511 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 512 igb_error(igb, "Failed to enable DDI interrupts"); 513 goto attach_fail; 514 } 515 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 516 517 igb_log(igb, "%s", igb_version); 518 atomic_or_32(&igb->igb_state, IGB_INITIALIZED); 519 520 return (DDI_SUCCESS); 521 522 attach_fail: 523 igb_unconfigure(devinfo, igb); 524 return (DDI_FAILURE); 525 } 526 527 /* 528 * igb_detach - driver detach 529 * 530 * The detach() function is the complement of the attach routine. 531 * If cmd is set to DDI_DETACH, detach() is used to remove the 532 * state associated with a given instance of a device node 533 * prior to the removal of that instance from the system. 534 * 535 * The detach() function will be called once for each instance 536 * of the device for which there has been a successful attach() 537 * once there are no longer any opens on the device. 538 * 539 * Interrupts routine are disabled, All memory allocated by this 540 * driver are freed. 541 */ 542 static int 543 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 544 { 545 igb_t *igb; 546 547 /* 548 * Check detach command 549 */ 550 switch (cmd) { 551 default: 552 return (DDI_FAILURE); 553 554 case DDI_SUSPEND: 555 return (igb_suspend(devinfo)); 556 557 case DDI_DETACH: 558 break; 559 } 560 561 562 /* 563 * Get the pointer to the driver private data structure 564 */ 565 igb = (igb_t *)ddi_get_driver_private(devinfo); 566 if (igb == NULL) 567 return (DDI_FAILURE); 568 569 /* 570 * Unregister MAC. If failed, we have to fail the detach 571 */ 572 if (mac_unregister(igb->mac_hdl) != 0) { 573 igb_error(igb, "Failed to unregister MAC"); 574 return (DDI_FAILURE); 575 } 576 igb->attach_progress &= ~ATTACH_PROGRESS_MAC; 577 578 /* 579 * If the device is still running, it needs to be stopped first. 580 * This check is necessary because under some specific circumstances, 581 * the detach routine can be called without stopping the interface 582 * first. 583 */ 584 mutex_enter(&igb->gen_lock); 585 if (igb->igb_state & IGB_STARTED) { 586 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 587 igb_stop(igb, B_TRUE); 588 mutex_exit(&igb->gen_lock); 589 /* Disable and stop the watchdog timer */ 590 igb_disable_watchdog_timer(igb); 591 } else 592 mutex_exit(&igb->gen_lock); 593 594 /* 595 * Check if there are still rx buffers held by the upper layer. 596 * If so, fail the detach. 597 */ 598 if (!igb_rx_drain(igb)) 599 return (DDI_FAILURE); 600 601 /* 602 * Do the remaining unconfigure routines 603 */ 604 igb_unconfigure(devinfo, igb); 605 606 return (DDI_SUCCESS); 607 } 608 609 /* 610 * quiesce(9E) entry point. 611 * 612 * This function is called when the system is single-threaded at high 613 * PIL with preemption disabled. Therefore, this function must not be 614 * blocked. 615 * 616 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 617 * DDI_FAILURE indicates an error condition and should almost never happen. 618 */ 619 static int 620 igb_quiesce(dev_info_t *devinfo) 621 { 622 igb_t *igb; 623 struct e1000_hw *hw; 624 625 igb = (igb_t *)ddi_get_driver_private(devinfo); 626 627 if (igb == NULL) 628 return (DDI_FAILURE); 629 630 hw = &igb->hw; 631 632 /* 633 * Disable the adapter interrupts 634 */ 635 igb_disable_adapter_interrupts(igb); 636 637 /* Tell firmware driver is no longer in control */ 638 igb_release_driver_control(hw); 639 640 /* 641 * Reset the chipset 642 */ 643 (void) e1000_reset_hw(hw); 644 645 /* 646 * Reset PHY if possible 647 */ 648 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 649 (void) e1000_phy_hw_reset(hw); 650 651 return (DDI_SUCCESS); 652 } 653 654 /* 655 * igb_unconfigure - release all resources held by this instance 656 */ 657 static void 658 igb_unconfigure(dev_info_t *devinfo, igb_t *igb) 659 { 660 /* 661 * Disable interrupt 662 */ 663 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 664 (void) igb_disable_intrs(igb); 665 } 666 667 /* 668 * Unregister MAC 669 */ 670 if (igb->attach_progress & ATTACH_PROGRESS_MAC) { 671 (void) mac_unregister(igb->mac_hdl); 672 } 673 674 /* 675 * Free statistics 676 */ 677 if (igb->attach_progress & ATTACH_PROGRESS_STATS) { 678 kstat_delete((kstat_t *)igb->igb_ks); 679 } 680 681 /* 682 * Remove interrupt handlers 683 */ 684 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 685 igb_rem_intr_handlers(igb); 686 } 687 688 /* 689 * Remove interrupts 690 */ 691 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 692 igb_rem_intrs(igb); 693 } 694 695 /* 696 * Remove driver properties 697 */ 698 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) { 699 (void) ddi_prop_remove_all(devinfo); 700 } 701 702 /* 703 * Stop the adapter 704 */ 705 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) { 706 mutex_enter(&igb->gen_lock); 707 igb_stop_adapter(igb); 708 mutex_exit(&igb->gen_lock); 709 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 710 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED); 711 } 712 713 /* 714 * Free multicast table 715 */ 716 igb_release_multicast(igb); 717 718 /* 719 * Free register handle 720 */ 721 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 722 if (igb->osdep.reg_handle != NULL) 723 ddi_regs_map_free(&igb->osdep.reg_handle); 724 } 725 726 /* 727 * Free PCI config handle 728 */ 729 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 730 if (igb->osdep.cfg_handle != NULL) 731 pci_config_teardown(&igb->osdep.cfg_handle); 732 } 733 734 /* 735 * Free locks 736 */ 737 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) { 738 igb_destroy_locks(igb); 739 } 740 741 /* 742 * Free the rx/tx rings 743 */ 744 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 745 igb_free_rings(igb); 746 } 747 748 /* 749 * Remove FMA 750 */ 751 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) { 752 igb_fm_fini(igb); 753 } 754 755 /* 756 * Free the driver data structure 757 */ 758 kmem_free(igb, sizeof (igb_t)); 759 760 ddi_set_driver_private(devinfo, NULL); 761 } 762 763 /* 764 * igb_register_mac - Register the driver and its function pointers with 765 * the GLD interface 766 */ 767 static int 768 igb_register_mac(igb_t *igb) 769 { 770 struct e1000_hw *hw = &igb->hw; 771 mac_register_t *mac; 772 int status; 773 774 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 775 return (IGB_FAILURE); 776 777 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 778 mac->m_driver = igb; 779 mac->m_dip = igb->dip; 780 mac->m_src_addr = hw->mac.addr; 781 mac->m_callbacks = &igb_m_callbacks; 782 mac->m_min_sdu = 0; 783 mac->m_max_sdu = igb->max_frame_size - 784 sizeof (struct ether_vlan_header) - ETHERFCSL; 785 mac->m_margin = VLAN_TAGSZ; 786 mac->m_priv_props = igb_priv_props; 787 mac->m_v12n = MAC_VIRT_LEVEL1; 788 789 status = mac_register(mac, &igb->mac_hdl); 790 791 mac_free(mac); 792 793 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE); 794 } 795 796 /* 797 * igb_identify_hardware - Identify the type of the chipset 798 */ 799 static int 800 igb_identify_hardware(igb_t *igb) 801 { 802 struct e1000_hw *hw = &igb->hw; 803 struct igb_osdep *osdep = &igb->osdep; 804 805 /* 806 * Get the device id 807 */ 808 hw->vendor_id = 809 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 810 hw->device_id = 811 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 812 hw->revision_id = 813 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 814 hw->subsystem_device_id = 815 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 816 hw->subsystem_vendor_id = 817 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 818 819 /* 820 * Set the mac type of the adapter based on the device id 821 */ 822 if (e1000_set_mac_type(hw) != E1000_SUCCESS) { 823 return (IGB_FAILURE); 824 } 825 826 /* 827 * Install adapter capabilities based on mac type 828 */ 829 switch (hw->mac.type) { 830 case e1000_82575: 831 igb->capab = &igb_82575_cap; 832 break; 833 case e1000_82576: 834 igb->capab = &igb_82576_cap; 835 break; 836 case e1000_82580: 837 igb->capab = &igb_82580_cap; 838 break; 839 default: 840 return (IGB_FAILURE); 841 } 842 843 return (IGB_SUCCESS); 844 } 845 846 /* 847 * igb_regs_map - Map the device registers 848 */ 849 static int 850 igb_regs_map(igb_t *igb) 851 { 852 dev_info_t *devinfo = igb->dip; 853 struct e1000_hw *hw = &igb->hw; 854 struct igb_osdep *osdep = &igb->osdep; 855 off_t mem_size; 856 857 /* 858 * First get the size of device registers to be mapped. 859 */ 860 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) != 861 DDI_SUCCESS) { 862 return (IGB_FAILURE); 863 } 864 865 /* 866 * Call ddi_regs_map_setup() to map registers 867 */ 868 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET, 869 (caddr_t *)&hw->hw_addr, 0, 870 mem_size, &igb_regs_acc_attr, 871 &osdep->reg_handle)) != DDI_SUCCESS) { 872 return (IGB_FAILURE); 873 } 874 875 return (IGB_SUCCESS); 876 } 877 878 /* 879 * igb_init_properties - Initialize driver properties 880 */ 881 static void 882 igb_init_properties(igb_t *igb) 883 { 884 /* 885 * Get conf file properties, including link settings 886 * jumbo frames, ring number, descriptor number, etc. 887 */ 888 igb_get_conf(igb); 889 } 890 891 /* 892 * igb_init_driver_settings - Initialize driver settings 893 * 894 * The settings include hardware function pointers, bus information, 895 * rx/tx rings settings, link state, and any other parameters that 896 * need to be setup during driver initialization. 897 */ 898 static int 899 igb_init_driver_settings(igb_t *igb) 900 { 901 struct e1000_hw *hw = &igb->hw; 902 igb_rx_ring_t *rx_ring; 903 igb_tx_ring_t *tx_ring; 904 uint32_t rx_size; 905 uint32_t tx_size; 906 int i; 907 908 /* 909 * Initialize chipset specific hardware function pointers 910 */ 911 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) { 912 return (IGB_FAILURE); 913 } 914 915 /* 916 * Get bus information 917 */ 918 if (e1000_get_bus_info(hw) != E1000_SUCCESS) { 919 return (IGB_FAILURE); 920 } 921 922 /* 923 * Get the system page size 924 */ 925 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1); 926 927 /* 928 * Set rx buffer size 929 * The IP header alignment room is counted in the calculation. 930 * The rx buffer size is in unit of 1K that is required by the 931 * chipset hardware. 932 */ 933 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM; 934 igb->rx_buf_size = ((rx_size >> 10) + 935 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 936 937 /* 938 * Set tx buffer size 939 */ 940 tx_size = igb->max_frame_size; 941 igb->tx_buf_size = ((tx_size >> 10) + 942 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 943 944 /* 945 * Initialize rx/tx rings parameters 946 */ 947 for (i = 0; i < igb->num_rx_rings; i++) { 948 rx_ring = &igb->rx_rings[i]; 949 rx_ring->index = i; 950 rx_ring->igb = igb; 951 } 952 953 for (i = 0; i < igb->num_tx_rings; i++) { 954 tx_ring = &igb->tx_rings[i]; 955 tx_ring->index = i; 956 tx_ring->igb = igb; 957 if (igb->tx_head_wb_enable) 958 tx_ring->tx_recycle = igb_tx_recycle_head_wb; 959 else 960 tx_ring->tx_recycle = igb_tx_recycle_legacy; 961 962 tx_ring->ring_size = igb->tx_ring_size; 963 tx_ring->free_list_size = igb->tx_ring_size + 964 (igb->tx_ring_size >> 1); 965 } 966 967 /* 968 * Initialize values of interrupt throttling rates 969 */ 970 for (i = 1; i < MAX_NUM_EITR; i++) 971 igb->intr_throttling[i] = igb->intr_throttling[0]; 972 973 /* 974 * The initial link state should be "unknown" 975 */ 976 igb->link_state = LINK_STATE_UNKNOWN; 977 978 return (IGB_SUCCESS); 979 } 980 981 /* 982 * igb_init_locks - Initialize locks 983 */ 984 static void 985 igb_init_locks(igb_t *igb) 986 { 987 igb_rx_ring_t *rx_ring; 988 igb_tx_ring_t *tx_ring; 989 int i; 990 991 for (i = 0; i < igb->num_rx_rings; i++) { 992 rx_ring = &igb->rx_rings[i]; 993 mutex_init(&rx_ring->rx_lock, NULL, 994 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 995 } 996 997 for (i = 0; i < igb->num_tx_rings; i++) { 998 tx_ring = &igb->tx_rings[i]; 999 mutex_init(&tx_ring->tx_lock, NULL, 1000 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1001 mutex_init(&tx_ring->recycle_lock, NULL, 1002 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1003 mutex_init(&tx_ring->tcb_head_lock, NULL, 1004 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1005 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1006 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1007 } 1008 1009 mutex_init(&igb->gen_lock, NULL, 1010 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1011 1012 mutex_init(&igb->watchdog_lock, NULL, 1013 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1014 1015 mutex_init(&igb->link_lock, NULL, 1016 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1017 } 1018 1019 /* 1020 * igb_destroy_locks - Destroy locks 1021 */ 1022 static void 1023 igb_destroy_locks(igb_t *igb) 1024 { 1025 igb_rx_ring_t *rx_ring; 1026 igb_tx_ring_t *tx_ring; 1027 int i; 1028 1029 for (i = 0; i < igb->num_rx_rings; i++) { 1030 rx_ring = &igb->rx_rings[i]; 1031 mutex_destroy(&rx_ring->rx_lock); 1032 } 1033 1034 for (i = 0; i < igb->num_tx_rings; i++) { 1035 tx_ring = &igb->tx_rings[i]; 1036 mutex_destroy(&tx_ring->tx_lock); 1037 mutex_destroy(&tx_ring->recycle_lock); 1038 mutex_destroy(&tx_ring->tcb_head_lock); 1039 mutex_destroy(&tx_ring->tcb_tail_lock); 1040 } 1041 1042 mutex_destroy(&igb->gen_lock); 1043 mutex_destroy(&igb->watchdog_lock); 1044 mutex_destroy(&igb->link_lock); 1045 } 1046 1047 static int 1048 igb_resume(dev_info_t *devinfo) 1049 { 1050 igb_t *igb; 1051 1052 igb = (igb_t *)ddi_get_driver_private(devinfo); 1053 if (igb == NULL) 1054 return (DDI_FAILURE); 1055 1056 mutex_enter(&igb->gen_lock); 1057 1058 /* 1059 * Enable interrupts 1060 */ 1061 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1062 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 1063 igb_error(igb, "Failed to enable DDI interrupts"); 1064 mutex_exit(&igb->gen_lock); 1065 return (DDI_FAILURE); 1066 } 1067 } 1068 1069 if (igb->igb_state & IGB_STARTED) { 1070 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) { 1071 mutex_exit(&igb->gen_lock); 1072 return (DDI_FAILURE); 1073 } 1074 1075 /* 1076 * Enable and start the watchdog timer 1077 */ 1078 igb_enable_watchdog_timer(igb); 1079 } 1080 1081 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED); 1082 1083 mutex_exit(&igb->gen_lock); 1084 1085 return (DDI_SUCCESS); 1086 } 1087 1088 static int 1089 igb_suspend(dev_info_t *devinfo) 1090 { 1091 igb_t *igb; 1092 1093 igb = (igb_t *)ddi_get_driver_private(devinfo); 1094 if (igb == NULL) 1095 return (DDI_FAILURE); 1096 1097 mutex_enter(&igb->gen_lock); 1098 1099 atomic_or_32(&igb->igb_state, IGB_SUSPENDED); 1100 1101 /* 1102 * Disable interrupts 1103 */ 1104 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1105 (void) igb_disable_intrs(igb); 1106 } 1107 1108 if (!(igb->igb_state & IGB_STARTED)) { 1109 mutex_exit(&igb->gen_lock); 1110 return (DDI_SUCCESS); 1111 } 1112 1113 igb_stop(igb, B_FALSE); 1114 1115 mutex_exit(&igb->gen_lock); 1116 1117 /* 1118 * Disable and stop the watchdog timer 1119 */ 1120 igb_disable_watchdog_timer(igb); 1121 1122 return (DDI_SUCCESS); 1123 } 1124 1125 static int 1126 igb_init(igb_t *igb) 1127 { 1128 mutex_enter(&igb->gen_lock); 1129 1130 /* 1131 * Initilize the adapter 1132 */ 1133 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1134 mutex_exit(&igb->gen_lock); 1135 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1136 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1137 return (IGB_FAILURE); 1138 } 1139 1140 mutex_exit(&igb->gen_lock); 1141 1142 return (IGB_SUCCESS); 1143 } 1144 1145 /* 1146 * igb_init_mac_address - Initialize the default MAC address 1147 * 1148 * On success, the MAC address is entered in the igb->hw.mac.addr 1149 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive 1150 * address register. 1151 * 1152 * Important side effects: 1153 * 1. adapter is reset - this is required to put it in a known state. 1154 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where 1155 * MAC address and all default settings are stored, so a valid checksum 1156 * is required. 1157 */ 1158 static int 1159 igb_init_mac_address(igb_t *igb) 1160 { 1161 struct e1000_hw *hw = &igb->hw; 1162 1163 ASSERT(mutex_owned(&igb->gen_lock)); 1164 1165 /* 1166 * Reset chipset to put the hardware in a known state 1167 * before we try to get MAC address from NVM. 1168 */ 1169 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1170 igb_error(igb, "Adapter reset failed."); 1171 goto init_mac_fail; 1172 } 1173 1174 /* 1175 * NVM validation 1176 */ 1177 if (e1000_validate_nvm_checksum(hw) < 0) { 1178 /* 1179 * Some PCI-E parts fail the first check due to 1180 * the link being in sleep state. Call it again, 1181 * if it fails a second time its a real issue. 1182 */ 1183 if (e1000_validate_nvm_checksum(hw) < 0) { 1184 igb_error(igb, 1185 "Invalid NVM checksum. Please contact " 1186 "the vendor to update the NVM."); 1187 goto init_mac_fail; 1188 } 1189 } 1190 1191 /* 1192 * Get the mac address 1193 * This function should handle SPARC case correctly. 1194 */ 1195 if (!igb_find_mac_address(igb)) { 1196 igb_error(igb, "Failed to get the mac address"); 1197 goto init_mac_fail; 1198 } 1199 1200 /* Validate mac address */ 1201 if (!is_valid_mac_addr(hw->mac.addr)) { 1202 igb_error(igb, "Invalid mac address"); 1203 goto init_mac_fail; 1204 } 1205 1206 return (IGB_SUCCESS); 1207 1208 init_mac_fail: 1209 return (IGB_FAILURE); 1210 } 1211 1212 /* 1213 * igb_init_adapter - Initialize the adapter 1214 */ 1215 static int 1216 igb_init_adapter(igb_t *igb) 1217 { 1218 struct e1000_hw *hw = &igb->hw; 1219 uint32_t pba; 1220 uint32_t high_water; 1221 int i; 1222 1223 ASSERT(mutex_owned(&igb->gen_lock)); 1224 1225 /* 1226 * In order to obtain the default MAC address, this will reset the 1227 * adapter and validate the NVM that the address and many other 1228 * default settings come from. 1229 */ 1230 if (igb_init_mac_address(igb) != IGB_SUCCESS) { 1231 igb_error(igb, "Failed to initialize MAC address"); 1232 goto init_adapter_fail; 1233 } 1234 1235 /* 1236 * Setup flow control 1237 * 1238 * These parameters set thresholds for the adapter's generation(Tx) 1239 * and response(Rx) to Ethernet PAUSE frames. These are just threshold 1240 * settings. Flow control is enabled or disabled in the configuration 1241 * file. 1242 * High-water mark is set down from the top of the rx fifo (not 1243 * sensitive to max_frame_size) and low-water is set just below 1244 * high-water mark. 1245 * The high water mark must be low enough to fit one full frame above 1246 * it in the rx FIFO. Should be the lower of: 1247 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full 1248 * frame. 1249 */ 1250 /* 1251 * The default setting of PBA is correct for 82575 and other supported 1252 * adapters do not have the E1000_PBA register, so PBA value is only 1253 * used for calculation here and is never written to the adapter. 1254 */ 1255 if (hw->mac.type == e1000_82575) { 1256 pba = E1000_PBA_34K; 1257 } else { 1258 pba = E1000_PBA_64K; 1259 } 1260 1261 high_water = min(((pba << 10) * 9 / 10), 1262 ((pba << 10) - igb->max_frame_size)); 1263 1264 if (hw->mac.type == e1000_82575) { 1265 /* 8-byte granularity */ 1266 hw->fc.high_water = high_water & 0xFFF8; 1267 hw->fc.low_water = hw->fc.high_water - 8; 1268 } else { 1269 /* 16-byte granularity */ 1270 hw->fc.high_water = high_water & 0xFFF0; 1271 hw->fc.low_water = hw->fc.high_water - 16; 1272 } 1273 1274 hw->fc.pause_time = E1000_FC_PAUSE_TIME; 1275 hw->fc.send_xon = B_TRUE; 1276 1277 (void) e1000_validate_mdi_setting(hw); 1278 1279 /* 1280 * Reset the chipset hardware the second time to put PBA settings 1281 * into effect. 1282 */ 1283 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1284 igb_error(igb, "Second reset failed"); 1285 goto init_adapter_fail; 1286 } 1287 1288 /* 1289 * Don't wait for auto-negotiation to complete 1290 */ 1291 hw->phy.autoneg_wait_to_complete = B_FALSE; 1292 1293 /* 1294 * Copper options 1295 */ 1296 if (hw->phy.media_type == e1000_media_type_copper) { 1297 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 1298 hw->phy.disable_polarity_correction = B_FALSE; 1299 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */ 1300 } 1301 1302 /* 1303 * Initialize link settings 1304 */ 1305 (void) igb_setup_link(igb, B_FALSE); 1306 1307 /* 1308 * Configure/Initialize hardware 1309 */ 1310 if (e1000_init_hw(hw) != E1000_SUCCESS) { 1311 igb_error(igb, "Failed to initialize hardware"); 1312 goto init_adapter_fail; 1313 } 1314 1315 /* 1316 * Start the link setup timer 1317 */ 1318 igb_start_link_timer(igb); 1319 1320 /* 1321 * Disable wakeup control by default 1322 */ 1323 E1000_WRITE_REG(hw, E1000_WUC, 0); 1324 1325 /* 1326 * Record phy info in hw struct 1327 */ 1328 (void) e1000_get_phy_info(hw); 1329 1330 /* 1331 * Make sure driver has control 1332 */ 1333 igb_get_driver_control(hw); 1334 1335 /* 1336 * Restore LED settings to the default from EEPROM 1337 * to meet the standard for Sun platforms. 1338 */ 1339 (void) e1000_cleanup_led(hw); 1340 1341 /* 1342 * Setup MSI-X interrupts 1343 */ 1344 if (igb->intr_type == DDI_INTR_TYPE_MSIX) 1345 igb->capab->setup_msix(igb); 1346 1347 /* 1348 * Initialize unicast addresses. 1349 */ 1350 igb_init_unicst(igb); 1351 1352 /* 1353 * Setup and initialize the mctable structures. 1354 */ 1355 igb_setup_multicst(igb); 1356 1357 /* 1358 * Set interrupt throttling rate 1359 */ 1360 for (i = 0; i < igb->intr_cnt; i++) 1361 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]); 1362 1363 /* 1364 * Save the state of the phy 1365 */ 1366 igb_get_phy_state(igb); 1367 1368 igb_param_sync(igb); 1369 1370 return (IGB_SUCCESS); 1371 1372 init_adapter_fail: 1373 /* 1374 * Reset PHY if possible 1375 */ 1376 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 1377 (void) e1000_phy_hw_reset(hw); 1378 1379 return (IGB_FAILURE); 1380 } 1381 1382 /* 1383 * igb_stop_adapter - Stop the adapter 1384 */ 1385 static void 1386 igb_stop_adapter(igb_t *igb) 1387 { 1388 struct e1000_hw *hw = &igb->hw; 1389 1390 ASSERT(mutex_owned(&igb->gen_lock)); 1391 1392 /* Stop the link setup timer */ 1393 igb_stop_link_timer(igb); 1394 1395 /* Tell firmware driver is no longer in control */ 1396 igb_release_driver_control(hw); 1397 1398 /* 1399 * Reset the chipset 1400 */ 1401 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1402 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1403 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1404 } 1405 1406 /* 1407 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient 1408 */ 1409 } 1410 1411 /* 1412 * igb_reset - Reset the chipset and restart the driver. 1413 * 1414 * It involves stopping and re-starting the chipset, 1415 * and re-configuring the rx/tx rings. 1416 */ 1417 static int 1418 igb_reset(igb_t *igb) 1419 { 1420 int i; 1421 1422 mutex_enter(&igb->gen_lock); 1423 1424 ASSERT(igb->igb_state & IGB_STARTED); 1425 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 1426 1427 /* 1428 * Disable the adapter interrupts to stop any rx/tx activities 1429 * before draining pending data and resetting hardware. 1430 */ 1431 igb_disable_adapter_interrupts(igb); 1432 1433 /* 1434 * Drain the pending transmit packets 1435 */ 1436 (void) igb_tx_drain(igb); 1437 1438 for (i = 0; i < igb->num_rx_rings; i++) 1439 mutex_enter(&igb->rx_rings[i].rx_lock); 1440 for (i = 0; i < igb->num_tx_rings; i++) 1441 mutex_enter(&igb->tx_rings[i].tx_lock); 1442 1443 /* 1444 * Stop the adapter 1445 */ 1446 igb_stop_adapter(igb); 1447 1448 /* 1449 * Clean the pending tx data/resources 1450 */ 1451 igb_tx_clean(igb); 1452 1453 /* 1454 * Start the adapter 1455 */ 1456 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1457 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1458 goto reset_failure; 1459 } 1460 1461 /* 1462 * Setup the rx/tx rings 1463 */ 1464 igb->tx_ring_init = B_FALSE; 1465 igb_setup_rings(igb); 1466 1467 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL)); 1468 1469 /* 1470 * Enable adapter interrupts 1471 * The interrupts must be enabled after the driver state is START 1472 */ 1473 igb->capab->enable_intr(igb); 1474 1475 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1476 goto reset_failure; 1477 1478 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1479 goto reset_failure; 1480 1481 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1482 mutex_exit(&igb->tx_rings[i].tx_lock); 1483 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1484 mutex_exit(&igb->rx_rings[i].rx_lock); 1485 1486 atomic_or_32(&igb->igb_state, IGB_STARTED); 1487 1488 mutex_exit(&igb->gen_lock); 1489 1490 return (IGB_SUCCESS); 1491 1492 reset_failure: 1493 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1494 mutex_exit(&igb->tx_rings[i].tx_lock); 1495 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1496 mutex_exit(&igb->rx_rings[i].rx_lock); 1497 1498 mutex_exit(&igb->gen_lock); 1499 1500 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1501 1502 return (IGB_FAILURE); 1503 } 1504 1505 /* 1506 * igb_tx_clean - Clean the pending transmit packets and DMA resources 1507 */ 1508 static void 1509 igb_tx_clean(igb_t *igb) 1510 { 1511 igb_tx_ring_t *tx_ring; 1512 tx_control_block_t *tcb; 1513 link_list_t pending_list; 1514 uint32_t desc_num; 1515 int i, j; 1516 1517 LINK_LIST_INIT(&pending_list); 1518 1519 for (i = 0; i < igb->num_tx_rings; i++) { 1520 tx_ring = &igb->tx_rings[i]; 1521 1522 mutex_enter(&tx_ring->recycle_lock); 1523 1524 /* 1525 * Clean the pending tx data - the pending packets in the 1526 * work_list that have no chances to be transmitted again. 1527 * 1528 * We must ensure the chipset is stopped or the link is down 1529 * before cleaning the transmit packets. 1530 */ 1531 desc_num = 0; 1532 for (j = 0; j < tx_ring->ring_size; j++) { 1533 tcb = tx_ring->work_list[j]; 1534 if (tcb != NULL) { 1535 desc_num += tcb->desc_num; 1536 1537 tx_ring->work_list[j] = NULL; 1538 1539 igb_free_tcb(tcb); 1540 1541 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1542 } 1543 } 1544 1545 if (desc_num > 0) { 1546 atomic_add_32(&tx_ring->tbd_free, desc_num); 1547 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1548 1549 /* 1550 * Reset the head and tail pointers of the tbd ring; 1551 * Reset the head write-back if it is enabled. 1552 */ 1553 tx_ring->tbd_head = 0; 1554 tx_ring->tbd_tail = 0; 1555 if (igb->tx_head_wb_enable) 1556 *tx_ring->tbd_head_wb = 0; 1557 1558 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0); 1559 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0); 1560 } 1561 1562 mutex_exit(&tx_ring->recycle_lock); 1563 1564 /* 1565 * Add the tx control blocks in the pending list to 1566 * the free list. 1567 */ 1568 igb_put_free_list(tx_ring, &pending_list); 1569 } 1570 } 1571 1572 /* 1573 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted 1574 */ 1575 static boolean_t 1576 igb_tx_drain(igb_t *igb) 1577 { 1578 igb_tx_ring_t *tx_ring; 1579 boolean_t done; 1580 int i, j; 1581 1582 /* 1583 * Wait for a specific time to allow pending tx packets 1584 * to be transmitted. 1585 * 1586 * Check the counter tbd_free to see if transmission is done. 1587 * No lock protection is needed here. 1588 * 1589 * Return B_TRUE if all pending packets have been transmitted; 1590 * Otherwise return B_FALSE; 1591 */ 1592 for (i = 0; i < TX_DRAIN_TIME; i++) { 1593 1594 done = B_TRUE; 1595 for (j = 0; j < igb->num_tx_rings; j++) { 1596 tx_ring = &igb->tx_rings[j]; 1597 done = done && 1598 (tx_ring->tbd_free == tx_ring->ring_size); 1599 } 1600 1601 if (done) 1602 break; 1603 1604 msec_delay(1); 1605 } 1606 1607 return (done); 1608 } 1609 1610 /* 1611 * igb_rx_drain - Wait for all rx buffers to be released by upper layer 1612 */ 1613 static boolean_t 1614 igb_rx_drain(igb_t *igb) 1615 { 1616 boolean_t done; 1617 int i; 1618 1619 /* 1620 * Polling the rx free list to check if those rx buffers held by 1621 * the upper layer are released. 1622 * 1623 * Check the counter rcb_free to see if all pending buffers are 1624 * released. No lock protection is needed here. 1625 * 1626 * Return B_TRUE if all pending buffers have been released; 1627 * Otherwise return B_FALSE; 1628 */ 1629 for (i = 0; i < RX_DRAIN_TIME; i++) { 1630 done = (igb->rcb_pending == 0); 1631 1632 if (done) 1633 break; 1634 1635 msec_delay(1); 1636 } 1637 1638 return (done); 1639 } 1640 1641 /* 1642 * igb_start - Start the driver/chipset 1643 */ 1644 int 1645 igb_start(igb_t *igb, boolean_t alloc_buffer) 1646 { 1647 int i; 1648 1649 ASSERT(mutex_owned(&igb->gen_lock)); 1650 1651 if (alloc_buffer) { 1652 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) { 1653 igb_error(igb, 1654 "Failed to allocate software receive rings"); 1655 return (IGB_FAILURE); 1656 } 1657 1658 /* Allocate buffers for all the rx/tx rings */ 1659 if (igb_alloc_dma(igb) != IGB_SUCCESS) { 1660 igb_error(igb, "Failed to allocate DMA resource"); 1661 return (IGB_FAILURE); 1662 } 1663 1664 igb->tx_ring_init = B_TRUE; 1665 } else { 1666 igb->tx_ring_init = B_FALSE; 1667 } 1668 1669 for (i = 0; i < igb->num_rx_rings; i++) 1670 mutex_enter(&igb->rx_rings[i].rx_lock); 1671 for (i = 0; i < igb->num_tx_rings; i++) 1672 mutex_enter(&igb->tx_rings[i].tx_lock); 1673 1674 /* 1675 * Start the adapter 1676 */ 1677 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) { 1678 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1679 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1680 goto start_failure; 1681 } 1682 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 1683 } 1684 1685 /* 1686 * Setup the rx/tx rings 1687 */ 1688 igb_setup_rings(igb); 1689 1690 /* 1691 * Enable adapter interrupts 1692 * The interrupts must be enabled after the driver state is START 1693 */ 1694 igb->capab->enable_intr(igb); 1695 1696 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1697 goto start_failure; 1698 1699 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1700 goto start_failure; 1701 1702 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1703 mutex_exit(&igb->tx_rings[i].tx_lock); 1704 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1705 mutex_exit(&igb->rx_rings[i].rx_lock); 1706 1707 return (IGB_SUCCESS); 1708 1709 start_failure: 1710 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1711 mutex_exit(&igb->tx_rings[i].tx_lock); 1712 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1713 mutex_exit(&igb->rx_rings[i].rx_lock); 1714 1715 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1716 1717 return (IGB_FAILURE); 1718 } 1719 1720 /* 1721 * igb_stop - Stop the driver/chipset 1722 */ 1723 void 1724 igb_stop(igb_t *igb, boolean_t free_buffer) 1725 { 1726 int i; 1727 1728 ASSERT(mutex_owned(&igb->gen_lock)); 1729 1730 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER; 1731 1732 /* 1733 * Disable the adapter interrupts 1734 */ 1735 igb_disable_adapter_interrupts(igb); 1736 1737 /* 1738 * Drain the pending tx packets 1739 */ 1740 (void) igb_tx_drain(igb); 1741 1742 for (i = 0; i < igb->num_rx_rings; i++) 1743 mutex_enter(&igb->rx_rings[i].rx_lock); 1744 for (i = 0; i < igb->num_tx_rings; i++) 1745 mutex_enter(&igb->tx_rings[i].tx_lock); 1746 1747 /* 1748 * Stop the adapter 1749 */ 1750 igb_stop_adapter(igb); 1751 1752 /* 1753 * Clean the pending tx data/resources 1754 */ 1755 igb_tx_clean(igb); 1756 1757 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1758 mutex_exit(&igb->tx_rings[i].tx_lock); 1759 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1760 mutex_exit(&igb->rx_rings[i].rx_lock); 1761 1762 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1763 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1764 1765 if (igb->link_state == LINK_STATE_UP) { 1766 igb->link_state = LINK_STATE_UNKNOWN; 1767 mac_link_update(igb->mac_hdl, igb->link_state); 1768 } 1769 1770 if (free_buffer) { 1771 /* 1772 * Release the DMA/memory resources of rx/tx rings 1773 */ 1774 igb_free_dma(igb); 1775 igb_free_rx_data(igb); 1776 } 1777 } 1778 1779 /* 1780 * igb_alloc_rings - Allocate memory space for rx/tx rings 1781 */ 1782 static int 1783 igb_alloc_rings(igb_t *igb) 1784 { 1785 /* 1786 * Allocate memory space for rx rings 1787 */ 1788 igb->rx_rings = kmem_zalloc( 1789 sizeof (igb_rx_ring_t) * igb->num_rx_rings, 1790 KM_NOSLEEP); 1791 1792 if (igb->rx_rings == NULL) { 1793 return (IGB_FAILURE); 1794 } 1795 1796 /* 1797 * Allocate memory space for tx rings 1798 */ 1799 igb->tx_rings = kmem_zalloc( 1800 sizeof (igb_tx_ring_t) * igb->num_tx_rings, 1801 KM_NOSLEEP); 1802 1803 if (igb->tx_rings == NULL) { 1804 kmem_free(igb->rx_rings, 1805 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1806 igb->rx_rings = NULL; 1807 return (IGB_FAILURE); 1808 } 1809 1810 /* 1811 * Allocate memory space for rx ring groups 1812 */ 1813 igb->rx_groups = kmem_zalloc( 1814 sizeof (igb_rx_group_t) * igb->num_rx_groups, 1815 KM_NOSLEEP); 1816 1817 if (igb->rx_groups == NULL) { 1818 kmem_free(igb->rx_rings, 1819 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1820 kmem_free(igb->tx_rings, 1821 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1822 igb->rx_rings = NULL; 1823 igb->tx_rings = NULL; 1824 return (IGB_FAILURE); 1825 } 1826 1827 return (IGB_SUCCESS); 1828 } 1829 1830 /* 1831 * igb_free_rings - Free the memory space of rx/tx rings. 1832 */ 1833 static void 1834 igb_free_rings(igb_t *igb) 1835 { 1836 if (igb->rx_rings != NULL) { 1837 kmem_free(igb->rx_rings, 1838 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1839 igb->rx_rings = NULL; 1840 } 1841 1842 if (igb->tx_rings != NULL) { 1843 kmem_free(igb->tx_rings, 1844 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1845 igb->tx_rings = NULL; 1846 } 1847 1848 if (igb->rx_groups != NULL) { 1849 kmem_free(igb->rx_groups, 1850 sizeof (igb_rx_group_t) * igb->num_rx_groups); 1851 igb->rx_groups = NULL; 1852 } 1853 } 1854 1855 static int 1856 igb_alloc_rx_data(igb_t *igb) 1857 { 1858 igb_rx_ring_t *rx_ring; 1859 int i; 1860 1861 for (i = 0; i < igb->num_rx_rings; i++) { 1862 rx_ring = &igb->rx_rings[i]; 1863 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS) 1864 goto alloc_rx_rings_failure; 1865 } 1866 return (IGB_SUCCESS); 1867 1868 alloc_rx_rings_failure: 1869 igb_free_rx_data(igb); 1870 return (IGB_FAILURE); 1871 } 1872 1873 static void 1874 igb_free_rx_data(igb_t *igb) 1875 { 1876 igb_rx_ring_t *rx_ring; 1877 igb_rx_data_t *rx_data; 1878 int i; 1879 1880 for (i = 0; i < igb->num_rx_rings; i++) { 1881 rx_ring = &igb->rx_rings[i]; 1882 1883 mutex_enter(&igb->rx_pending_lock); 1884 rx_data = rx_ring->rx_data; 1885 1886 if (rx_data != NULL) { 1887 rx_data->flag |= IGB_RX_STOPPED; 1888 1889 if (rx_data->rcb_pending == 0) { 1890 igb_free_rx_ring_data(rx_data); 1891 rx_ring->rx_data = NULL; 1892 } 1893 } 1894 1895 mutex_exit(&igb->rx_pending_lock); 1896 } 1897 } 1898 1899 /* 1900 * igb_setup_rings - Setup rx/tx rings 1901 */ 1902 static void 1903 igb_setup_rings(igb_t *igb) 1904 { 1905 /* 1906 * Setup the rx/tx rings, including the following: 1907 * 1908 * 1. Setup the descriptor ring and the control block buffers; 1909 * 2. Initialize necessary registers for receive/transmit; 1910 * 3. Initialize software pointers/parameters for receive/transmit; 1911 */ 1912 igb_setup_rx(igb); 1913 1914 igb_setup_tx(igb); 1915 } 1916 1917 static void 1918 igb_setup_rx_ring(igb_rx_ring_t *rx_ring) 1919 { 1920 igb_t *igb = rx_ring->igb; 1921 igb_rx_data_t *rx_data = rx_ring->rx_data; 1922 struct e1000_hw *hw = &igb->hw; 1923 rx_control_block_t *rcb; 1924 union e1000_adv_rx_desc *rbd; 1925 uint32_t size; 1926 uint32_t buf_low; 1927 uint32_t buf_high; 1928 uint32_t rxdctl; 1929 int i; 1930 1931 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1932 ASSERT(mutex_owned(&igb->gen_lock)); 1933 1934 /* 1935 * Initialize descriptor ring with buffer addresses 1936 */ 1937 for (i = 0; i < igb->rx_ring_size; i++) { 1938 rcb = rx_data->work_list[i]; 1939 rbd = &rx_data->rbd_ring[i]; 1940 1941 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1942 rbd->read.hdr_addr = NULL; 1943 } 1944 1945 /* 1946 * Initialize the base address registers 1947 */ 1948 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 1949 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 1950 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high); 1951 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low); 1952 1953 /* 1954 * Initialize the length register 1955 */ 1956 size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc); 1957 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size); 1958 1959 /* 1960 * Initialize buffer size & descriptor type 1961 */ 1962 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index), 1963 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) | 1964 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF)); 1965 1966 /* 1967 * Setup the Receive Descriptor Control Register (RXDCTL) 1968 */ 1969 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index)); 1970 rxdctl &= igb->capab->rxdctl_mask; 1971 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 1972 rxdctl |= 16; /* pthresh */ 1973 rxdctl |= 8 << 8; /* hthresh */ 1974 rxdctl |= 1 << 16; /* wthresh */ 1975 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl); 1976 1977 rx_data->rbd_next = 0; 1978 } 1979 1980 static void 1981 igb_setup_rx(igb_t *igb) 1982 { 1983 igb_rx_ring_t *rx_ring; 1984 igb_rx_data_t *rx_data; 1985 igb_rx_group_t *rx_group; 1986 struct e1000_hw *hw = &igb->hw; 1987 uint32_t rctl, rxcsum; 1988 uint32_t ring_per_group; 1989 int i; 1990 1991 /* 1992 * Setup the Receive Control Register (RCTL), and enable the 1993 * receiver. The initial configuration is to: enable the receiver, 1994 * accept broadcasts, discard bad packets, accept long packets, 1995 * disable VLAN filter checking, and set receive buffer size to 1996 * 2k. For 82575, also set the receive descriptor minimum 1997 * threshold size to 1/2 the ring. 1998 */ 1999 rctl = E1000_READ_REG(hw, E1000_RCTL); 2000 2001 /* 2002 * Clear the field used for wakeup control. This driver doesn't do 2003 * wakeup but leave this here for completeness. 2004 */ 2005 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 2006 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 2007 2008 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */ 2009 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 2010 E1000_RCTL_LPE | /* Large Packet Enable */ 2011 /* Multicast filter offset */ 2012 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 2013 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */ 2014 E1000_RCTL_SECRC); /* Strip Ethernet CRC */ 2015 2016 for (i = 0; i < igb->num_rx_groups; i++) { 2017 rx_group = &igb->rx_groups[i]; 2018 rx_group->index = i; 2019 rx_group->igb = igb; 2020 } 2021 2022 /* 2023 * Set up all rx descriptor rings - must be called before receive unit 2024 * enabled. 2025 */ 2026 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2027 for (i = 0; i < igb->num_rx_rings; i++) { 2028 rx_ring = &igb->rx_rings[i]; 2029 igb_setup_rx_ring(rx_ring); 2030 2031 /* 2032 * Map a ring to a group by assigning a group index 2033 */ 2034 rx_ring->group_index = i / ring_per_group; 2035 } 2036 2037 /* 2038 * Setup the Rx Long Packet Max Length register 2039 */ 2040 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size); 2041 2042 /* 2043 * Hardware checksum settings 2044 */ 2045 if (igb->rx_hcksum_enable) { 2046 rxcsum = 2047 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */ 2048 E1000_RXCSUM_IPOFL; /* IP checksum */ 2049 2050 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2051 } 2052 2053 /* 2054 * Setup classify and RSS for multiple receive queues 2055 */ 2056 switch (igb->vmdq_mode) { 2057 case E1000_VMDQ_OFF: 2058 /* 2059 * One ring group, only RSS is needed when more than 2060 * one ring enabled. 2061 */ 2062 if (igb->num_rx_rings > 1) 2063 igb_setup_rss(igb); 2064 break; 2065 case E1000_VMDQ_MAC: 2066 /* 2067 * Multiple groups, each group has one ring, 2068 * only the MAC classification is needed. 2069 */ 2070 igb_setup_mac_classify(igb); 2071 break; 2072 case E1000_VMDQ_MAC_RSS: 2073 /* 2074 * Multiple groups and multiple rings, both 2075 * MAC classification and RSS are needed. 2076 */ 2077 igb_setup_mac_rss_classify(igb); 2078 break; 2079 } 2080 2081 /* 2082 * Enable the receive unit - must be done after all 2083 * the rx setup above. 2084 */ 2085 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2086 2087 /* 2088 * Initialize all adapter ring head & tail pointers - must 2089 * be done after receive unit is enabled 2090 */ 2091 for (i = 0; i < igb->num_rx_rings; i++) { 2092 rx_ring = &igb->rx_rings[i]; 2093 rx_data = rx_ring->rx_data; 2094 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 2095 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1); 2096 } 2097 2098 /* 2099 * 82575 with manageability enabled needs a special flush to make 2100 * sure the fifos start clean. 2101 */ 2102 if ((hw->mac.type == e1000_82575) && 2103 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) { 2104 e1000_rx_fifo_flush_82575(hw); 2105 } 2106 } 2107 2108 static void 2109 igb_setup_tx_ring(igb_tx_ring_t *tx_ring) 2110 { 2111 igb_t *igb = tx_ring->igb; 2112 struct e1000_hw *hw = &igb->hw; 2113 uint32_t size; 2114 uint32_t buf_low; 2115 uint32_t buf_high; 2116 uint32_t reg_val; 2117 2118 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2119 ASSERT(mutex_owned(&igb->gen_lock)); 2120 2121 2122 /* 2123 * Initialize the length register 2124 */ 2125 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc); 2126 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size); 2127 2128 /* 2129 * Initialize the base address registers 2130 */ 2131 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2132 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2133 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low); 2134 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high); 2135 2136 /* 2137 * Setup head & tail pointers 2138 */ 2139 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0); 2140 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0); 2141 2142 /* 2143 * Setup head write-back 2144 */ 2145 if (igb->tx_head_wb_enable) { 2146 /* 2147 * The memory of the head write-back is allocated using 2148 * the extra tbd beyond the tail of the tbd ring. 2149 */ 2150 tx_ring->tbd_head_wb = (uint32_t *) 2151 ((uintptr_t)tx_ring->tbd_area.address + size); 2152 *tx_ring->tbd_head_wb = 0; 2153 2154 buf_low = (uint32_t) 2155 (tx_ring->tbd_area.dma_address + size); 2156 buf_high = (uint32_t) 2157 ((tx_ring->tbd_area.dma_address + size) >> 32); 2158 2159 /* Set the head write-back enable bit */ 2160 buf_low |= E1000_TX_HEAD_WB_ENABLE; 2161 2162 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low); 2163 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high); 2164 2165 /* 2166 * Turn off relaxed ordering for head write back or it will 2167 * cause problems with the tx recycling 2168 */ 2169 reg_val = E1000_READ_REG(hw, 2170 E1000_DCA_TXCTRL(tx_ring->index)); 2171 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN; 2172 E1000_WRITE_REG(hw, 2173 E1000_DCA_TXCTRL(tx_ring->index), reg_val); 2174 } else { 2175 tx_ring->tbd_head_wb = NULL; 2176 } 2177 2178 tx_ring->tbd_head = 0; 2179 tx_ring->tbd_tail = 0; 2180 tx_ring->tbd_free = tx_ring->ring_size; 2181 2182 if (igb->tx_ring_init == B_TRUE) { 2183 tx_ring->tcb_head = 0; 2184 tx_ring->tcb_tail = 0; 2185 tx_ring->tcb_free = tx_ring->free_list_size; 2186 } 2187 2188 /* 2189 * Enable TXDCTL per queue 2190 */ 2191 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index)); 2192 reg_val |= E1000_TXDCTL_QUEUE_ENABLE; 2193 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val); 2194 2195 /* 2196 * Initialize hardware checksum offload settings 2197 */ 2198 bzero(&tx_ring->tx_context, sizeof (tx_context_t)); 2199 } 2200 2201 static void 2202 igb_setup_tx(igb_t *igb) 2203 { 2204 igb_tx_ring_t *tx_ring; 2205 struct e1000_hw *hw = &igb->hw; 2206 uint32_t reg_val; 2207 int i; 2208 2209 for (i = 0; i < igb->num_tx_rings; i++) { 2210 tx_ring = &igb->tx_rings[i]; 2211 igb_setup_tx_ring(tx_ring); 2212 } 2213 2214 /* 2215 * Setup the Transmit Control Register (TCTL) 2216 */ 2217 reg_val = E1000_READ_REG(hw, E1000_TCTL); 2218 reg_val &= ~E1000_TCTL_CT; 2219 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2220 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2221 2222 /* Enable transmits */ 2223 reg_val |= E1000_TCTL_EN; 2224 2225 E1000_WRITE_REG(hw, E1000_TCTL, reg_val); 2226 } 2227 2228 /* 2229 * igb_setup_rss - Setup receive-side scaling feature 2230 */ 2231 static void 2232 igb_setup_rss(igb_t *igb) 2233 { 2234 struct e1000_hw *hw = &igb->hw; 2235 uint32_t i, mrqc, rxcsum; 2236 int shift = 0; 2237 uint32_t random; 2238 union e1000_reta { 2239 uint32_t dword; 2240 uint8_t bytes[4]; 2241 } reta; 2242 2243 /* Setup the Redirection Table */ 2244 if (hw->mac.type == e1000_82576) { 2245 shift = 3; 2246 } else if (hw->mac.type == e1000_82575) { 2247 shift = 6; 2248 } 2249 for (i = 0; i < (32 * 4); i++) { 2250 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift; 2251 if ((i & 3) == 3) { 2252 E1000_WRITE_REG(hw, 2253 (E1000_RETA(0) + (i & ~3)), reta.dword); 2254 } 2255 } 2256 2257 /* Fill out hash function seeds */ 2258 for (i = 0; i < 10; i++) { 2259 (void) random_get_pseudo_bytes((uint8_t *)&random, 2260 sizeof (uint32_t)); 2261 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2262 } 2263 2264 /* Setup the Multiple Receive Queue Control register */ 2265 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2266 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2267 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2268 E1000_MRQC_RSS_FIELD_IPV6 | 2269 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2270 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2271 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2272 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2273 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2274 2275 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2276 2277 /* 2278 * Disable Packet Checksum to enable RSS for multiple receive queues. 2279 * 2280 * The Packet Checksum is not ethernet CRC. It is another kind of 2281 * checksum offloading provided by the 82575 chipset besides the IP 2282 * header checksum offloading and the TCP/UDP checksum offloading. 2283 * The Packet Checksum is by default computed over the entire packet 2284 * from the first byte of the DA through the last byte of the CRC, 2285 * including the Ethernet and IP headers. 2286 * 2287 * It is a hardware limitation that Packet Checksum is mutually 2288 * exclusive with RSS. 2289 */ 2290 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2291 rxcsum |= E1000_RXCSUM_PCSD; 2292 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2293 } 2294 2295 /* 2296 * igb_setup_mac_rss_classify - Setup MAC classification and rss 2297 */ 2298 static void 2299 igb_setup_mac_rss_classify(igb_t *igb) 2300 { 2301 struct e1000_hw *hw = &igb->hw; 2302 uint32_t i, mrqc, vmdctl, rxcsum; 2303 uint32_t ring_per_group; 2304 int shift_group0, shift_group1; 2305 uint32_t random; 2306 union e1000_reta { 2307 uint32_t dword; 2308 uint8_t bytes[4]; 2309 } reta; 2310 2311 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2312 2313 /* Setup the Redirection Table, it is shared between two groups */ 2314 shift_group0 = 2; 2315 shift_group1 = 6; 2316 for (i = 0; i < (32 * 4); i++) { 2317 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) | 2318 ((ring_per_group + (i % ring_per_group)) << shift_group1); 2319 if ((i & 3) == 3) { 2320 E1000_WRITE_REG(hw, 2321 (E1000_RETA(0) + (i & ~3)), reta.dword); 2322 } 2323 } 2324 2325 /* Fill out hash function seeds */ 2326 for (i = 0; i < 10; i++) { 2327 (void) random_get_pseudo_bytes((uint8_t *)&random, 2328 sizeof (uint32_t)); 2329 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2330 } 2331 2332 /* 2333 * Setup the Multiple Receive Queue Control register, 2334 * enable VMDq based on packet destination MAC address and RSS. 2335 */ 2336 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP; 2337 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2338 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2339 E1000_MRQC_RSS_FIELD_IPV6 | 2340 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2341 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2342 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2343 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2344 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2345 2346 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2347 2348 2349 /* Define the default group and default queues */ 2350 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE; 2351 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl); 2352 2353 /* 2354 * Disable Packet Checksum to enable RSS for multiple receive queues. 2355 * 2356 * The Packet Checksum is not ethernet CRC. It is another kind of 2357 * checksum offloading provided by the 82575 chipset besides the IP 2358 * header checksum offloading and the TCP/UDP checksum offloading. 2359 * The Packet Checksum is by default computed over the entire packet 2360 * from the first byte of the DA through the last byte of the CRC, 2361 * including the Ethernet and IP headers. 2362 * 2363 * It is a hardware limitation that Packet Checksum is mutually 2364 * exclusive with RSS. 2365 */ 2366 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2367 rxcsum |= E1000_RXCSUM_PCSD; 2368 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2369 } 2370 2371 /* 2372 * igb_setup_mac_classify - Setup MAC classification feature 2373 */ 2374 static void 2375 igb_setup_mac_classify(igb_t *igb) 2376 { 2377 struct e1000_hw *hw = &igb->hw; 2378 uint32_t mrqc, rxcsum; 2379 2380 /* 2381 * Setup the Multiple Receive Queue Control register, 2382 * enable VMDq based on packet destination MAC address. 2383 */ 2384 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP; 2385 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2386 2387 /* 2388 * Disable Packet Checksum to enable RSS for multiple receive queues. 2389 * 2390 * The Packet Checksum is not ethernet CRC. It is another kind of 2391 * checksum offloading provided by the 82575 chipset besides the IP 2392 * header checksum offloading and the TCP/UDP checksum offloading. 2393 * The Packet Checksum is by default computed over the entire packet 2394 * from the first byte of the DA through the last byte of the CRC, 2395 * including the Ethernet and IP headers. 2396 * 2397 * It is a hardware limitation that Packet Checksum is mutually 2398 * exclusive with RSS. 2399 */ 2400 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2401 rxcsum |= E1000_RXCSUM_PCSD; 2402 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2403 2404 } 2405 2406 /* 2407 * igb_init_unicst - Initialize the unicast addresses 2408 */ 2409 static void 2410 igb_init_unicst(igb_t *igb) 2411 { 2412 struct e1000_hw *hw = &igb->hw; 2413 int slot; 2414 2415 /* 2416 * Here we should consider two situations: 2417 * 2418 * 1. Chipset is initialized the first time 2419 * Initialize the multiple unicast addresses, and 2420 * save the default MAC address. 2421 * 2422 * 2. Chipset is reset 2423 * Recover the multiple unicast addresses from the 2424 * software data structure to the RAR registers. 2425 */ 2426 2427 /* 2428 * Clear the default MAC address in the RAR0 rgister, 2429 * which is loaded from EEPROM when system boot or chipreset, 2430 * this will cause the conficts with add_mac/rem_mac entry 2431 * points when VMDq is enabled. For this reason, the RAR0 2432 * must be cleared for both cases mentioned above. 2433 */ 2434 e1000_rar_clear(hw, 0); 2435 2436 if (!igb->unicst_init) { 2437 2438 /* Initialize the multiple unicast addresses */ 2439 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2440 igb->unicst_avail = igb->unicst_total; 2441 2442 for (slot = 0; slot < igb->unicst_total; slot++) 2443 igb->unicst_addr[slot].mac.set = 0; 2444 2445 igb->unicst_init = B_TRUE; 2446 } else { 2447 /* Re-configure the RAR registers */ 2448 for (slot = 0; slot < igb->unicst_total; slot++) { 2449 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr, 2450 slot, igb->vmdq_mode, 2451 igb->unicst_addr[slot].mac.group_index); 2452 } 2453 } 2454 } 2455 2456 /* 2457 * igb_unicst_find - Find the slot for the specified unicast address 2458 */ 2459 int 2460 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr) 2461 { 2462 int slot; 2463 2464 ASSERT(mutex_owned(&igb->gen_lock)); 2465 2466 for (slot = 0; slot < igb->unicst_total; slot++) { 2467 if (bcmp(igb->unicst_addr[slot].mac.addr, 2468 mac_addr, ETHERADDRL) == 0) 2469 return (slot); 2470 } 2471 2472 return (-1); 2473 } 2474 2475 /* 2476 * igb_unicst_set - Set the unicast address to the specified slot 2477 */ 2478 int 2479 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr, 2480 int slot) 2481 { 2482 struct e1000_hw *hw = &igb->hw; 2483 2484 ASSERT(mutex_owned(&igb->gen_lock)); 2485 2486 /* 2487 * Save the unicast address in the software data structure 2488 */ 2489 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL); 2490 2491 /* 2492 * Set the unicast address to the RAR register 2493 */ 2494 e1000_rar_set(hw, (uint8_t *)mac_addr, slot); 2495 2496 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2497 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2498 return (EIO); 2499 } 2500 2501 return (0); 2502 } 2503 2504 /* 2505 * igb_multicst_add - Add a multicst address 2506 */ 2507 int 2508 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr) 2509 { 2510 struct ether_addr *new_table; 2511 size_t new_len; 2512 size_t old_len; 2513 2514 ASSERT(mutex_owned(&igb->gen_lock)); 2515 2516 if ((multiaddr[0] & 01) == 0) { 2517 igb_error(igb, "Illegal multicast address"); 2518 return (EINVAL); 2519 } 2520 2521 if (igb->mcast_count >= igb->mcast_max_num) { 2522 igb_error(igb, "Adapter requested more than %d mcast addresses", 2523 igb->mcast_max_num); 2524 return (ENOENT); 2525 } 2526 2527 if (igb->mcast_count == igb->mcast_alloc_count) { 2528 old_len = igb->mcast_alloc_count * 2529 sizeof (struct ether_addr); 2530 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) * 2531 sizeof (struct ether_addr); 2532 2533 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2534 if (new_table == NULL) { 2535 igb_error(igb, 2536 "Not enough memory to alloc mcast table"); 2537 return (ENOMEM); 2538 } 2539 2540 if (igb->mcast_table != NULL) { 2541 bcopy(igb->mcast_table, new_table, old_len); 2542 kmem_free(igb->mcast_table, old_len); 2543 } 2544 igb->mcast_alloc_count += MCAST_ALLOC_COUNT; 2545 igb->mcast_table = new_table; 2546 } 2547 2548 bcopy(multiaddr, 2549 &igb->mcast_table[igb->mcast_count], ETHERADDRL); 2550 igb->mcast_count++; 2551 2552 /* 2553 * Update the multicast table in the hardware 2554 */ 2555 igb_setup_multicst(igb); 2556 2557 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2558 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2559 return (EIO); 2560 } 2561 2562 return (0); 2563 } 2564 2565 /* 2566 * igb_multicst_remove - Remove a multicst address 2567 */ 2568 int 2569 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr) 2570 { 2571 struct ether_addr *new_table; 2572 size_t new_len; 2573 size_t old_len; 2574 int i; 2575 2576 ASSERT(mutex_owned(&igb->gen_lock)); 2577 2578 for (i = 0; i < igb->mcast_count; i++) { 2579 if (bcmp(multiaddr, &igb->mcast_table[i], 2580 ETHERADDRL) == 0) { 2581 for (i++; i < igb->mcast_count; i++) { 2582 igb->mcast_table[i - 1] = 2583 igb->mcast_table[i]; 2584 } 2585 igb->mcast_count--; 2586 break; 2587 } 2588 } 2589 2590 if ((igb->mcast_alloc_count - igb->mcast_count) > 2591 MCAST_ALLOC_COUNT) { 2592 old_len = igb->mcast_alloc_count * 2593 sizeof (struct ether_addr); 2594 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) * 2595 sizeof (struct ether_addr); 2596 2597 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2598 if (new_table != NULL) { 2599 bcopy(igb->mcast_table, new_table, new_len); 2600 kmem_free(igb->mcast_table, old_len); 2601 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT; 2602 igb->mcast_table = new_table; 2603 } 2604 } 2605 2606 /* 2607 * Update the multicast table in the hardware 2608 */ 2609 igb_setup_multicst(igb); 2610 2611 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2612 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2613 return (EIO); 2614 } 2615 2616 return (0); 2617 } 2618 2619 static void 2620 igb_release_multicast(igb_t *igb) 2621 { 2622 if (igb->mcast_table != NULL) { 2623 kmem_free(igb->mcast_table, 2624 igb->mcast_alloc_count * sizeof (struct ether_addr)); 2625 igb->mcast_table = NULL; 2626 } 2627 } 2628 2629 /* 2630 * igb_setup_multicast - setup multicast data structures 2631 * 2632 * This routine initializes all of the multicast related structures 2633 * and save them in the hardware registers. 2634 */ 2635 static void 2636 igb_setup_multicst(igb_t *igb) 2637 { 2638 uint8_t *mc_addr_list; 2639 uint32_t mc_addr_count; 2640 struct e1000_hw *hw = &igb->hw; 2641 2642 ASSERT(mutex_owned(&igb->gen_lock)); 2643 ASSERT(igb->mcast_count <= igb->mcast_max_num); 2644 2645 mc_addr_list = (uint8_t *)igb->mcast_table; 2646 mc_addr_count = igb->mcast_count; 2647 2648 /* 2649 * Update the multicase addresses to the MTA registers 2650 */ 2651 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count); 2652 } 2653 2654 /* 2655 * igb_get_conf - Get driver configurations set in driver.conf 2656 * 2657 * This routine gets user-configured values out of the configuration 2658 * file igb.conf. 2659 * 2660 * For each configurable value, there is a minimum, a maximum, and a 2661 * default. 2662 * If user does not configure a value, use the default. 2663 * If user configures below the minimum, use the minumum. 2664 * If user configures above the maximum, use the maxumum. 2665 */ 2666 static void 2667 igb_get_conf(igb_t *igb) 2668 { 2669 struct e1000_hw *hw = &igb->hw; 2670 uint32_t default_mtu; 2671 uint32_t flow_control; 2672 uint32_t ring_per_group; 2673 int i; 2674 2675 /* 2676 * igb driver supports the following user configurations: 2677 * 2678 * Link configurations: 2679 * adv_autoneg_cap 2680 * adv_1000fdx_cap 2681 * adv_100fdx_cap 2682 * adv_100hdx_cap 2683 * adv_10fdx_cap 2684 * adv_10hdx_cap 2685 * Note: 1000hdx is not supported. 2686 * 2687 * Jumbo frame configuration: 2688 * default_mtu 2689 * 2690 * Ethernet flow control configuration: 2691 * flow_control 2692 * 2693 * Multiple rings configurations: 2694 * tx_queue_number 2695 * tx_ring_size 2696 * rx_queue_number 2697 * rx_ring_size 2698 * 2699 * Call igb_get_prop() to get the value for a specific 2700 * configuration parameter. 2701 */ 2702 2703 /* 2704 * Link configurations 2705 */ 2706 igb->param_adv_autoneg_cap = igb_get_prop(igb, 2707 PROP_ADV_AUTONEG_CAP, 0, 1, 1); 2708 igb->param_adv_1000fdx_cap = igb_get_prop(igb, 2709 PROP_ADV_1000FDX_CAP, 0, 1, 1); 2710 igb->param_adv_100fdx_cap = igb_get_prop(igb, 2711 PROP_ADV_100FDX_CAP, 0, 1, 1); 2712 igb->param_adv_100hdx_cap = igb_get_prop(igb, 2713 PROP_ADV_100HDX_CAP, 0, 1, 1); 2714 igb->param_adv_10fdx_cap = igb_get_prop(igb, 2715 PROP_ADV_10FDX_CAP, 0, 1, 1); 2716 igb->param_adv_10hdx_cap = igb_get_prop(igb, 2717 PROP_ADV_10HDX_CAP, 0, 1, 1); 2718 2719 /* 2720 * Jumbo frame configurations 2721 */ 2722 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU, 2723 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2724 2725 igb->max_frame_size = default_mtu + 2726 sizeof (struct ether_vlan_header) + ETHERFCSL; 2727 2728 /* 2729 * Ethernet flow control configuration 2730 */ 2731 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL, 2732 e1000_fc_none, 4, e1000_fc_full); 2733 if (flow_control == 4) 2734 flow_control = e1000_fc_default; 2735 2736 hw->fc.requested_mode = flow_control; 2737 2738 /* 2739 * Multiple rings configurations 2740 */ 2741 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE, 2742 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2743 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE, 2744 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2745 2746 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0); 2747 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM, 2748 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2749 /* 2750 * Currently we do not support VMDq for 82576 and 82580. 2751 * If it is e1000_82576, set num_rx_groups to 1. 2752 */ 2753 if (hw->mac.type >= e1000_82576) 2754 igb->num_rx_groups = 1; 2755 2756 if (igb->mr_enable) { 2757 igb->num_tx_rings = igb->capab->def_tx_que_num; 2758 igb->num_rx_rings = igb->capab->def_rx_que_num; 2759 } else { 2760 igb->num_tx_rings = 1; 2761 igb->num_rx_rings = 1; 2762 2763 if (igb->num_rx_groups > 1) { 2764 igb_error(igb, 2765 "Invalid rx groups number. Please enable multiple " 2766 "rings first"); 2767 igb->num_rx_groups = 1; 2768 } 2769 } 2770 2771 /* 2772 * Check the divisibility between rx rings and rx groups. 2773 */ 2774 for (i = igb->num_rx_groups; i > 0; i--) { 2775 if ((igb->num_rx_rings % i) == 0) 2776 break; 2777 } 2778 if (i != igb->num_rx_groups) { 2779 igb_error(igb, 2780 "Invalid rx groups number. Downgrade the rx group " 2781 "number to %d.", i); 2782 igb->num_rx_groups = i; 2783 } 2784 2785 /* 2786 * Get the ring number per group. 2787 */ 2788 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2789 2790 if (igb->num_rx_groups == 1) { 2791 /* 2792 * One rx ring group, the rx ring number is num_rx_rings. 2793 */ 2794 igb->vmdq_mode = E1000_VMDQ_OFF; 2795 } else if (ring_per_group == 1) { 2796 /* 2797 * Multiple rx groups, each group has one rx ring. 2798 */ 2799 igb->vmdq_mode = E1000_VMDQ_MAC; 2800 } else { 2801 /* 2802 * Multiple groups and multiple rings. 2803 */ 2804 igb->vmdq_mode = E1000_VMDQ_MAC_RSS; 2805 } 2806 2807 /* 2808 * Tunable used to force an interrupt type. The only use is 2809 * for testing of the lesser interrupt types. 2810 * 0 = don't force interrupt type 2811 * 1 = force interrupt type MSIX 2812 * 2 = force interrupt type MSI 2813 * 3 = force interrupt type Legacy 2814 */ 2815 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE, 2816 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE); 2817 2818 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE, 2819 0, 1, 1); 2820 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE, 2821 0, 1, 1); 2822 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE, 2823 0, 1, 1); 2824 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE, 2825 0, 1, 1); 2826 2827 /* 2828 * igb LSO needs the tx h/w checksum support. 2829 * Here LSO will be disabled if tx h/w checksum has been disabled. 2830 */ 2831 if (igb->tx_hcksum_enable == B_FALSE) 2832 igb->lso_enable = B_FALSE; 2833 2834 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD, 2835 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2836 DEFAULT_TX_COPY_THRESHOLD); 2837 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD, 2838 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, 2839 DEFAULT_TX_RECYCLE_THRESHOLD); 2840 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD, 2841 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, 2842 DEFAULT_TX_OVERLOAD_THRESHOLD); 2843 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD, 2844 MIN_TX_RESCHED_THRESHOLD, 2845 MIN(igb->tx_ring_size, MAX_TX_RESCHED_THRESHOLD), 2846 igb->tx_ring_size > DEFAULT_TX_RESCHED_THRESHOLD ? 2847 DEFAULT_TX_RESCHED_THRESHOLD : DEFAULT_TX_RESCHED_THRESHOLD_LOW); 2848 2849 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD, 2850 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2851 DEFAULT_RX_COPY_THRESHOLD); 2852 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR, 2853 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2854 DEFAULT_RX_LIMIT_PER_INTR); 2855 2856 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING, 2857 igb->capab->min_intr_throttle, 2858 igb->capab->max_intr_throttle, 2859 igb->capab->def_intr_throttle); 2860 2861 /* 2862 * Max number of multicast addresses 2863 */ 2864 igb->mcast_max_num = 2865 igb_get_prop(igb, PROP_MCAST_MAX_NUM, 2866 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM); 2867 } 2868 2869 /* 2870 * igb_get_prop - Get a property value out of the configuration file igb.conf 2871 * 2872 * Caller provides the name of the property, a default value, a minimum 2873 * value, and a maximum value. 2874 * 2875 * Return configured value of the property, with default, minimum and 2876 * maximum properly applied. 2877 */ 2878 static int 2879 igb_get_prop(igb_t *igb, 2880 char *propname, /* name of the property */ 2881 int minval, /* minimum acceptable value */ 2882 int maxval, /* maximim acceptable value */ 2883 int defval) /* default value */ 2884 { 2885 int value; 2886 2887 /* 2888 * Call ddi_prop_get_int() to read the conf settings 2889 */ 2890 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip, 2891 DDI_PROP_DONTPASS, propname, defval); 2892 2893 if (value > maxval) 2894 value = maxval; 2895 2896 if (value < minval) 2897 value = minval; 2898 2899 return (value); 2900 } 2901 2902 /* 2903 * igb_setup_link - Using the link properties to setup the link 2904 */ 2905 int 2906 igb_setup_link(igb_t *igb, boolean_t setup_hw) 2907 { 2908 struct e1000_mac_info *mac; 2909 struct e1000_phy_info *phy; 2910 boolean_t invalid; 2911 2912 mac = &igb->hw.mac; 2913 phy = &igb->hw.phy; 2914 invalid = B_FALSE; 2915 2916 if (igb->param_adv_autoneg_cap == 1) { 2917 mac->autoneg = B_TRUE; 2918 phy->autoneg_advertised = 0; 2919 2920 /* 2921 * 1000hdx is not supported for autonegotiation 2922 */ 2923 if (igb->param_adv_1000fdx_cap == 1) 2924 phy->autoneg_advertised |= ADVERTISE_1000_FULL; 2925 2926 if (igb->param_adv_100fdx_cap == 1) 2927 phy->autoneg_advertised |= ADVERTISE_100_FULL; 2928 2929 if (igb->param_adv_100hdx_cap == 1) 2930 phy->autoneg_advertised |= ADVERTISE_100_HALF; 2931 2932 if (igb->param_adv_10fdx_cap == 1) 2933 phy->autoneg_advertised |= ADVERTISE_10_FULL; 2934 2935 if (igb->param_adv_10hdx_cap == 1) 2936 phy->autoneg_advertised |= ADVERTISE_10_HALF; 2937 2938 if (phy->autoneg_advertised == 0) 2939 invalid = B_TRUE; 2940 } else { 2941 mac->autoneg = B_FALSE; 2942 2943 /* 2944 * 1000fdx and 1000hdx are not supported for forced link 2945 */ 2946 if (igb->param_adv_100fdx_cap == 1) 2947 mac->forced_speed_duplex = ADVERTISE_100_FULL; 2948 else if (igb->param_adv_100hdx_cap == 1) 2949 mac->forced_speed_duplex = ADVERTISE_100_HALF; 2950 else if (igb->param_adv_10fdx_cap == 1) 2951 mac->forced_speed_duplex = ADVERTISE_10_FULL; 2952 else if (igb->param_adv_10hdx_cap == 1) 2953 mac->forced_speed_duplex = ADVERTISE_10_HALF; 2954 else 2955 invalid = B_TRUE; 2956 } 2957 2958 if (invalid) { 2959 igb_notice(igb, "Invalid link settings. Setup link to " 2960 "autonegotiation with full link capabilities."); 2961 mac->autoneg = B_TRUE; 2962 phy->autoneg_advertised = ADVERTISE_1000_FULL | 2963 ADVERTISE_100_FULL | ADVERTISE_100_HALF | 2964 ADVERTISE_10_FULL | ADVERTISE_10_HALF; 2965 } 2966 2967 if (setup_hw) { 2968 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS) 2969 return (IGB_FAILURE); 2970 } 2971 2972 return (IGB_SUCCESS); 2973 } 2974 2975 2976 /* 2977 * igb_is_link_up - Check if the link is up 2978 */ 2979 static boolean_t 2980 igb_is_link_up(igb_t *igb) 2981 { 2982 struct e1000_hw *hw = &igb->hw; 2983 boolean_t link_up = B_FALSE; 2984 2985 ASSERT(mutex_owned(&igb->gen_lock)); 2986 2987 /* 2988 * get_link_status is set in the interrupt handler on link-status-change 2989 * or rx sequence error interrupt. get_link_status will stay 2990 * false until the e1000_check_for_link establishes link only 2991 * for copper adapters. 2992 */ 2993 switch (hw->phy.media_type) { 2994 case e1000_media_type_copper: 2995 if (hw->mac.get_link_status) { 2996 (void) e1000_check_for_link(hw); 2997 link_up = !hw->mac.get_link_status; 2998 } else { 2999 link_up = B_TRUE; 3000 } 3001 break; 3002 case e1000_media_type_fiber: 3003 (void) e1000_check_for_link(hw); 3004 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 3005 break; 3006 case e1000_media_type_internal_serdes: 3007 (void) e1000_check_for_link(hw); 3008 link_up = hw->mac.serdes_has_link; 3009 break; 3010 } 3011 3012 return (link_up); 3013 } 3014 3015 /* 3016 * igb_link_check - Link status processing 3017 */ 3018 static boolean_t 3019 igb_link_check(igb_t *igb) 3020 { 3021 struct e1000_hw *hw = &igb->hw; 3022 uint16_t speed = 0, duplex = 0; 3023 boolean_t link_changed = B_FALSE; 3024 3025 ASSERT(mutex_owned(&igb->gen_lock)); 3026 3027 if (igb_is_link_up(igb)) { 3028 /* 3029 * The Link is up, check whether it was marked as down earlier 3030 */ 3031 if (igb->link_state != LINK_STATE_UP) { 3032 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex); 3033 igb->link_speed = speed; 3034 igb->link_duplex = duplex; 3035 igb->link_state = LINK_STATE_UP; 3036 link_changed = B_TRUE; 3037 if (!igb->link_complete) 3038 igb_stop_link_timer(igb); 3039 } 3040 } else if (igb->link_complete) { 3041 if (igb->link_state != LINK_STATE_DOWN) { 3042 igb->link_speed = 0; 3043 igb->link_duplex = 0; 3044 igb->link_state = LINK_STATE_DOWN; 3045 link_changed = B_TRUE; 3046 } 3047 } 3048 3049 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3050 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3051 return (B_FALSE); 3052 } 3053 3054 return (link_changed); 3055 } 3056 3057 /* 3058 * igb_local_timer - driver watchdog function 3059 * 3060 * This function will handle the hardware stall check, link status 3061 * check and other routines. 3062 */ 3063 static void 3064 igb_local_timer(void *arg) 3065 { 3066 igb_t *igb = (igb_t *)arg; 3067 boolean_t link_changed = B_FALSE; 3068 3069 if (igb->igb_state & IGB_ERROR) { 3070 igb->reset_count++; 3071 if (igb_reset(igb) == IGB_SUCCESS) 3072 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3073 3074 igb_restart_watchdog_timer(igb); 3075 return; 3076 } 3077 3078 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) { 3079 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL); 3080 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 3081 igb->reset_count++; 3082 if (igb_reset(igb) == IGB_SUCCESS) 3083 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3084 3085 igb_restart_watchdog_timer(igb); 3086 return; 3087 } 3088 3089 mutex_enter(&igb->gen_lock); 3090 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED)) 3091 link_changed = igb_link_check(igb); 3092 mutex_exit(&igb->gen_lock); 3093 3094 if (link_changed) 3095 mac_link_update(igb->mac_hdl, igb->link_state); 3096 3097 igb_restart_watchdog_timer(igb); 3098 } 3099 3100 /* 3101 * igb_link_timer - link setup timer function 3102 * 3103 * It is called when the timer for link setup is expired, which indicates 3104 * the completion of the link setup. The link state will not be updated 3105 * until the link setup is completed. And the link state will not be sent 3106 * to the upper layer through mac_link_update() in this function. It will 3107 * be updated in the local timer routine or the interrupts service routine 3108 * after the interface is started (plumbed). 3109 */ 3110 static void 3111 igb_link_timer(void *arg) 3112 { 3113 igb_t *igb = (igb_t *)arg; 3114 3115 mutex_enter(&igb->link_lock); 3116 igb->link_complete = B_TRUE; 3117 igb->link_tid = 0; 3118 mutex_exit(&igb->link_lock); 3119 } 3120 /* 3121 * igb_stall_check - check for transmit stall 3122 * 3123 * This function checks if the adapter is stalled (in transmit). 3124 * 3125 * It is called each time the watchdog timeout is invoked. 3126 * If the transmit descriptor reclaim continuously fails, 3127 * the watchdog value will increment by 1. If the watchdog 3128 * value exceeds the threshold, the igb is assumed to 3129 * have stalled and need to be reset. 3130 */ 3131 static boolean_t 3132 igb_stall_check(igb_t *igb) 3133 { 3134 igb_tx_ring_t *tx_ring; 3135 struct e1000_hw *hw = &igb->hw; 3136 boolean_t result; 3137 int i; 3138 3139 if (igb->link_state != LINK_STATE_UP) 3140 return (B_FALSE); 3141 3142 /* 3143 * If any tx ring is stalled, we'll reset the chipset 3144 */ 3145 result = B_FALSE; 3146 for (i = 0; i < igb->num_tx_rings; i++) { 3147 tx_ring = &igb->tx_rings[i]; 3148 3149 if (tx_ring->recycle_fail > 0) 3150 tx_ring->stall_watchdog++; 3151 else 3152 tx_ring->stall_watchdog = 0; 3153 3154 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3155 result = B_TRUE; 3156 if (hw->mac.type == e1000_82580) { 3157 hw->dev_spec._82575.global_device_reset 3158 = B_TRUE; 3159 } 3160 break; 3161 } 3162 } 3163 3164 if (result) { 3165 tx_ring->stall_watchdog = 0; 3166 tx_ring->recycle_fail = 0; 3167 } 3168 3169 return (result); 3170 } 3171 3172 3173 /* 3174 * is_valid_mac_addr - Check if the mac address is valid 3175 */ 3176 static boolean_t 3177 is_valid_mac_addr(uint8_t *mac_addr) 3178 { 3179 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3180 const uint8_t addr_test2[6] = 3181 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3182 3183 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3184 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3185 return (B_FALSE); 3186 3187 return (B_TRUE); 3188 } 3189 3190 static boolean_t 3191 igb_find_mac_address(igb_t *igb) 3192 { 3193 struct e1000_hw *hw = &igb->hw; 3194 #ifdef __sparc 3195 uchar_t *bytes; 3196 struct ether_addr sysaddr; 3197 uint_t nelts; 3198 int err; 3199 boolean_t found = B_FALSE; 3200 3201 /* 3202 * The "vendor's factory-set address" may already have 3203 * been extracted from the chip, but if the property 3204 * "local-mac-address" is set we use that instead. 3205 * 3206 * We check whether it looks like an array of 6 3207 * bytes (which it should, if OBP set it). If we can't 3208 * make sense of it this way, we'll ignore it. 3209 */ 3210 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3211 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3212 if (err == DDI_PROP_SUCCESS) { 3213 if (nelts == ETHERADDRL) { 3214 while (nelts--) 3215 hw->mac.addr[nelts] = bytes[nelts]; 3216 found = B_TRUE; 3217 } 3218 ddi_prop_free(bytes); 3219 } 3220 3221 /* 3222 * Look up the OBP property "local-mac-address?". If the user has set 3223 * 'local-mac-address? = false', use "the system address" instead. 3224 */ 3225 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0, 3226 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3227 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3228 if (localetheraddr(NULL, &sysaddr) != 0) { 3229 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3230 found = B_TRUE; 3231 } 3232 } 3233 ddi_prop_free(bytes); 3234 } 3235 3236 /* 3237 * Finally(!), if there's a valid "mac-address" property (created 3238 * if we netbooted from this interface), we must use this instead 3239 * of any of the above to ensure that the NFS/install server doesn't 3240 * get confused by the address changing as Solaris takes over! 3241 */ 3242 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3243 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3244 if (err == DDI_PROP_SUCCESS) { 3245 if (nelts == ETHERADDRL) { 3246 while (nelts--) 3247 hw->mac.addr[nelts] = bytes[nelts]; 3248 found = B_TRUE; 3249 } 3250 ddi_prop_free(bytes); 3251 } 3252 3253 if (found) { 3254 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3255 return (B_TRUE); 3256 } 3257 #endif 3258 3259 /* 3260 * Read the device MAC address from the EEPROM 3261 */ 3262 if (e1000_read_mac_addr(hw) != E1000_SUCCESS) 3263 return (B_FALSE); 3264 3265 return (B_TRUE); 3266 } 3267 3268 #pragma inline(igb_arm_watchdog_timer) 3269 3270 static void 3271 igb_arm_watchdog_timer(igb_t *igb) 3272 { 3273 /* 3274 * Fire a watchdog timer 3275 */ 3276 igb->watchdog_tid = 3277 timeout(igb_local_timer, 3278 (void *)igb, 1 * drv_usectohz(1000000)); 3279 3280 } 3281 3282 /* 3283 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer 3284 */ 3285 void 3286 igb_enable_watchdog_timer(igb_t *igb) 3287 { 3288 mutex_enter(&igb->watchdog_lock); 3289 3290 if (!igb->watchdog_enable) { 3291 igb->watchdog_enable = B_TRUE; 3292 igb->watchdog_start = B_TRUE; 3293 igb_arm_watchdog_timer(igb); 3294 } 3295 3296 mutex_exit(&igb->watchdog_lock); 3297 3298 } 3299 3300 /* 3301 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer 3302 */ 3303 void 3304 igb_disable_watchdog_timer(igb_t *igb) 3305 { 3306 timeout_id_t tid; 3307 3308 mutex_enter(&igb->watchdog_lock); 3309 3310 igb->watchdog_enable = B_FALSE; 3311 igb->watchdog_start = B_FALSE; 3312 tid = igb->watchdog_tid; 3313 igb->watchdog_tid = 0; 3314 3315 mutex_exit(&igb->watchdog_lock); 3316 3317 if (tid != 0) 3318 (void) untimeout(tid); 3319 3320 } 3321 3322 /* 3323 * igb_start_watchdog_timer - Start the driver watchdog timer 3324 */ 3325 static void 3326 igb_start_watchdog_timer(igb_t *igb) 3327 { 3328 mutex_enter(&igb->watchdog_lock); 3329 3330 if (igb->watchdog_enable) { 3331 if (!igb->watchdog_start) { 3332 igb->watchdog_start = B_TRUE; 3333 igb_arm_watchdog_timer(igb); 3334 } 3335 } 3336 3337 mutex_exit(&igb->watchdog_lock); 3338 } 3339 3340 /* 3341 * igb_restart_watchdog_timer - Restart the driver watchdog timer 3342 */ 3343 static void 3344 igb_restart_watchdog_timer(igb_t *igb) 3345 { 3346 mutex_enter(&igb->watchdog_lock); 3347 3348 if (igb->watchdog_start) 3349 igb_arm_watchdog_timer(igb); 3350 3351 mutex_exit(&igb->watchdog_lock); 3352 } 3353 3354 /* 3355 * igb_stop_watchdog_timer - Stop the driver watchdog timer 3356 */ 3357 static void 3358 igb_stop_watchdog_timer(igb_t *igb) 3359 { 3360 timeout_id_t tid; 3361 3362 mutex_enter(&igb->watchdog_lock); 3363 3364 igb->watchdog_start = B_FALSE; 3365 tid = igb->watchdog_tid; 3366 igb->watchdog_tid = 0; 3367 3368 mutex_exit(&igb->watchdog_lock); 3369 3370 if (tid != 0) 3371 (void) untimeout(tid); 3372 } 3373 3374 /* 3375 * igb_start_link_timer - Start the link setup timer 3376 */ 3377 static void 3378 igb_start_link_timer(struct igb *igb) 3379 { 3380 struct e1000_hw *hw = &igb->hw; 3381 clock_t link_timeout; 3382 3383 if (hw->mac.autoneg) 3384 link_timeout = PHY_AUTO_NEG_LIMIT * 3385 drv_usectohz(100000); 3386 else 3387 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000); 3388 3389 mutex_enter(&igb->link_lock); 3390 if (hw->phy.autoneg_wait_to_complete) { 3391 igb->link_complete = B_TRUE; 3392 } else { 3393 igb->link_complete = B_FALSE; 3394 igb->link_tid = timeout(igb_link_timer, (void *)igb, 3395 link_timeout); 3396 } 3397 mutex_exit(&igb->link_lock); 3398 } 3399 3400 /* 3401 * igb_stop_link_timer - Stop the link setup timer 3402 */ 3403 static void 3404 igb_stop_link_timer(struct igb *igb) 3405 { 3406 timeout_id_t tid; 3407 3408 mutex_enter(&igb->link_lock); 3409 igb->link_complete = B_TRUE; 3410 tid = igb->link_tid; 3411 igb->link_tid = 0; 3412 mutex_exit(&igb->link_lock); 3413 3414 if (tid != 0) 3415 (void) untimeout(tid); 3416 } 3417 3418 /* 3419 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts 3420 */ 3421 static void 3422 igb_disable_adapter_interrupts(igb_t *igb) 3423 { 3424 struct e1000_hw *hw = &igb->hw; 3425 3426 /* 3427 * Set the IMC register to mask all the interrupts, 3428 * including the tx interrupts. 3429 */ 3430 E1000_WRITE_REG(hw, E1000_IMC, ~0); 3431 E1000_WRITE_REG(hw, E1000_IAM, 0); 3432 3433 /* 3434 * Additional disabling for MSI-X 3435 */ 3436 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3437 E1000_WRITE_REG(hw, E1000_EIMC, ~0); 3438 E1000_WRITE_REG(hw, E1000_EIAC, 0); 3439 E1000_WRITE_REG(hw, E1000_EIAM, 0); 3440 } 3441 3442 E1000_WRITE_FLUSH(hw); 3443 } 3444 3445 /* 3446 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580 3447 */ 3448 static void 3449 igb_enable_adapter_interrupts_82580(igb_t *igb) 3450 { 3451 struct e1000_hw *hw = &igb->hw; 3452 3453 /* Clear any pending interrupts */ 3454 (void) E1000_READ_REG(hw, E1000_ICR); 3455 igb->ims_mask |= E1000_IMS_DRSTA; 3456 3457 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3458 3459 /* Interrupt enabling for MSI-X */ 3460 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3461 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3462 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA); 3463 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3464 } else { /* Interrupt enabling for MSI and legacy */ 3465 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3466 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3467 igb->ims_mask |= E1000_IMS_DRSTA; 3468 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3469 } 3470 3471 /* Disable auto-mask for ICR interrupt bits */ 3472 E1000_WRITE_REG(hw, E1000_IAM, 0); 3473 3474 E1000_WRITE_FLUSH(hw); 3475 } 3476 3477 /* 3478 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576 3479 */ 3480 static void 3481 igb_enable_adapter_interrupts_82576(igb_t *igb) 3482 { 3483 struct e1000_hw *hw = &igb->hw; 3484 3485 /* Clear any pending interrupts */ 3486 (void) E1000_READ_REG(hw, E1000_ICR); 3487 3488 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3489 3490 /* Interrupt enabling for MSI-X */ 3491 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3492 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3493 igb->ims_mask = E1000_IMS_LSC; 3494 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3495 } else { 3496 /* Interrupt enabling for MSI and legacy */ 3497 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3498 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3499 E1000_WRITE_REG(hw, E1000_IMS, 3500 (IMS_ENABLE_MASK | E1000_IMS_TXQE)); 3501 } 3502 3503 /* Disable auto-mask for ICR interrupt bits */ 3504 E1000_WRITE_REG(hw, E1000_IAM, 0); 3505 3506 E1000_WRITE_FLUSH(hw); 3507 } 3508 3509 /* 3510 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575 3511 */ 3512 static void 3513 igb_enable_adapter_interrupts_82575(igb_t *igb) 3514 { 3515 struct e1000_hw *hw = &igb->hw; 3516 uint32_t reg; 3517 3518 /* Clear any pending interrupts */ 3519 (void) E1000_READ_REG(hw, E1000_ICR); 3520 3521 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3522 /* Interrupt enabling for MSI-X */ 3523 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3524 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3525 igb->ims_mask = E1000_IMS_LSC; 3526 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3527 3528 /* Enable MSI-X PBA support */ 3529 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 3530 reg |= E1000_CTRL_EXT_PBA_CLR; 3531 3532 /* Non-selective interrupt clear-on-read */ 3533 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */ 3534 3535 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 3536 } else { 3537 /* Interrupt enabling for MSI and legacy */ 3538 igb->ims_mask = IMS_ENABLE_MASK; 3539 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 3540 } 3541 3542 E1000_WRITE_FLUSH(hw); 3543 } 3544 3545 /* 3546 * Loopback Support 3547 */ 3548 static lb_property_t lb_normal = 3549 { normal, "normal", IGB_LB_NONE }; 3550 static lb_property_t lb_external = 3551 { external, "External", IGB_LB_EXTERNAL }; 3552 static lb_property_t lb_phy = 3553 { internal, "PHY", IGB_LB_INTERNAL_PHY }; 3554 static lb_property_t lb_serdes = 3555 { internal, "SerDes", IGB_LB_INTERNAL_SERDES }; 3556 3557 enum ioc_reply 3558 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp) 3559 { 3560 lb_info_sz_t *lbsp; 3561 lb_property_t *lbpp; 3562 struct e1000_hw *hw; 3563 uint32_t *lbmp; 3564 uint32_t size; 3565 uint32_t value; 3566 3567 hw = &igb->hw; 3568 3569 if (mp->b_cont == NULL) 3570 return (IOC_INVAL); 3571 3572 switch (iocp->ioc_cmd) { 3573 default: 3574 return (IOC_INVAL); 3575 3576 case LB_GET_INFO_SIZE: 3577 size = sizeof (lb_info_sz_t); 3578 if (iocp->ioc_count != size) 3579 return (IOC_INVAL); 3580 3581 value = sizeof (lb_normal); 3582 if (hw->phy.media_type == e1000_media_type_copper) 3583 value += sizeof (lb_phy); 3584 else 3585 value += sizeof (lb_serdes); 3586 value += sizeof (lb_external); 3587 3588 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3589 *lbsp = value; 3590 break; 3591 3592 case LB_GET_INFO: 3593 value = sizeof (lb_normal); 3594 if (hw->phy.media_type == e1000_media_type_copper) 3595 value += sizeof (lb_phy); 3596 else 3597 value += sizeof (lb_serdes); 3598 value += sizeof (lb_external); 3599 3600 size = value; 3601 if (iocp->ioc_count != size) 3602 return (IOC_INVAL); 3603 3604 value = 0; 3605 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3606 3607 lbpp[value++] = lb_normal; 3608 if (hw->phy.media_type == e1000_media_type_copper) 3609 lbpp[value++] = lb_phy; 3610 else 3611 lbpp[value++] = lb_serdes; 3612 lbpp[value++] = lb_external; 3613 break; 3614 3615 case LB_GET_MODE: 3616 size = sizeof (uint32_t); 3617 if (iocp->ioc_count != size) 3618 return (IOC_INVAL); 3619 3620 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3621 *lbmp = igb->loopback_mode; 3622 break; 3623 3624 case LB_SET_MODE: 3625 size = 0; 3626 if (iocp->ioc_count != sizeof (uint32_t)) 3627 return (IOC_INVAL); 3628 3629 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3630 if (!igb_set_loopback_mode(igb, *lbmp)) 3631 return (IOC_INVAL); 3632 break; 3633 } 3634 3635 iocp->ioc_count = size; 3636 iocp->ioc_error = 0; 3637 3638 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3639 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3640 return (IOC_INVAL); 3641 } 3642 3643 return (IOC_REPLY); 3644 } 3645 3646 /* 3647 * igb_set_loopback_mode - Setup loopback based on the loopback mode 3648 */ 3649 static boolean_t 3650 igb_set_loopback_mode(igb_t *igb, uint32_t mode) 3651 { 3652 struct e1000_hw *hw; 3653 int i; 3654 3655 if (mode == igb->loopback_mode) 3656 return (B_TRUE); 3657 3658 hw = &igb->hw; 3659 3660 igb->loopback_mode = mode; 3661 3662 if (mode == IGB_LB_NONE) { 3663 /* Reset the chip */ 3664 hw->phy.autoneg_wait_to_complete = B_TRUE; 3665 (void) igb_reset(igb); 3666 hw->phy.autoneg_wait_to_complete = B_FALSE; 3667 return (B_TRUE); 3668 } 3669 3670 mutex_enter(&igb->gen_lock); 3671 3672 switch (mode) { 3673 default: 3674 mutex_exit(&igb->gen_lock); 3675 return (B_FALSE); 3676 3677 case IGB_LB_EXTERNAL: 3678 igb_set_external_loopback(igb); 3679 break; 3680 3681 case IGB_LB_INTERNAL_PHY: 3682 igb_set_internal_phy_loopback(igb); 3683 break; 3684 3685 case IGB_LB_INTERNAL_SERDES: 3686 igb_set_internal_serdes_loopback(igb); 3687 break; 3688 } 3689 3690 mutex_exit(&igb->gen_lock); 3691 3692 /* 3693 * When external loopback is set, wait up to 1000ms to get the link up. 3694 * According to test, 1000ms can work and it's an experimental value. 3695 */ 3696 if (mode == IGB_LB_EXTERNAL) { 3697 for (i = 0; i <= 10; i++) { 3698 mutex_enter(&igb->gen_lock); 3699 (void) igb_link_check(igb); 3700 mutex_exit(&igb->gen_lock); 3701 3702 if (igb->link_state == LINK_STATE_UP) 3703 break; 3704 3705 msec_delay(100); 3706 } 3707 3708 if (igb->link_state != LINK_STATE_UP) { 3709 /* 3710 * Does not support external loopback. 3711 * Reset driver to loopback none. 3712 */ 3713 igb->loopback_mode = IGB_LB_NONE; 3714 3715 /* Reset the chip */ 3716 hw->phy.autoneg_wait_to_complete = B_TRUE; 3717 (void) igb_reset(igb); 3718 hw->phy.autoneg_wait_to_complete = B_FALSE; 3719 3720 IGB_DEBUGLOG_0(igb, "Set external loopback failed, " 3721 "reset to loopback none."); 3722 3723 return (B_FALSE); 3724 } 3725 } 3726 3727 return (B_TRUE); 3728 } 3729 3730 /* 3731 * igb_set_external_loopback - Set the external loopback mode 3732 */ 3733 static void 3734 igb_set_external_loopback(igb_t *igb) 3735 { 3736 struct e1000_hw *hw; 3737 uint32_t ctrl_ext; 3738 3739 hw = &igb->hw; 3740 3741 /* Set link mode to PHY (00b) in the Extended Control register */ 3742 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3743 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3744 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3745 3746 (void) e1000_write_phy_reg(hw, 0x0, 0x0140); 3747 (void) e1000_write_phy_reg(hw, 0x9, 0x1a00); 3748 (void) e1000_write_phy_reg(hw, 0x12, 0x1610); 3749 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c); 3750 } 3751 3752 /* 3753 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode 3754 */ 3755 static void 3756 igb_set_internal_phy_loopback(igb_t *igb) 3757 { 3758 struct e1000_hw *hw; 3759 uint32_t ctrl_ext; 3760 uint16_t phy_ctrl; 3761 uint16_t phy_pconf; 3762 3763 hw = &igb->hw; 3764 3765 /* Set link mode to PHY (00b) in the Extended Control register */ 3766 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3767 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3768 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3769 3770 /* 3771 * Set PHY control register (0x4140): 3772 * Set full duplex mode 3773 * Set loopback bit 3774 * Clear auto-neg enable bit 3775 * Set PHY speed 3776 */ 3777 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK; 3778 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3779 3780 /* Set the link disable bit in the Port Configuration register */ 3781 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf); 3782 phy_pconf |= (uint16_t)1 << 14; 3783 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf); 3784 } 3785 3786 /* 3787 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode 3788 */ 3789 static void 3790 igb_set_internal_serdes_loopback(igb_t *igb) 3791 { 3792 struct e1000_hw *hw; 3793 uint32_t ctrl_ext; 3794 uint32_t ctrl; 3795 uint32_t pcs_lctl; 3796 uint32_t connsw; 3797 3798 hw = &igb->hw; 3799 3800 /* Set link mode to SerDes (11b) in the Extended Control register */ 3801 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3802 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 3803 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3804 3805 /* Configure the SerDes to loopback */ 3806 E1000_WRITE_REG(hw, E1000_SCTL, 0x410); 3807 3808 /* Set Device Control register */ 3809 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3810 ctrl |= (E1000_CTRL_FD | /* Force full duplex */ 3811 E1000_CTRL_SLU); /* Force link up */ 3812 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */ 3813 E1000_CTRL_TFCE | /* Disable transmit flow control */ 3814 E1000_CTRL_LRST); /* Clear link reset */ 3815 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3816 3817 /* Set PCS Link Control register */ 3818 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL); 3819 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK | 3820 E1000_PCS_LCTL_FSD | 3821 E1000_PCS_LCTL_FDV_FULL | 3822 E1000_PCS_LCTL_FLV_LINK_UP); 3823 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE; 3824 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl); 3825 3826 /* Set the Copper/Fiber Switch Control - CONNSW register */ 3827 connsw = E1000_READ_REG(hw, E1000_CONNSW); 3828 connsw &= ~E1000_CONNSW_ENRGSRC; 3829 E1000_WRITE_REG(hw, E1000_CONNSW, connsw); 3830 } 3831 3832 #pragma inline(igb_intr_rx_work) 3833 /* 3834 * igb_intr_rx_work - rx processing of ISR 3835 */ 3836 static void 3837 igb_intr_rx_work(igb_rx_ring_t *rx_ring) 3838 { 3839 mblk_t *mp; 3840 3841 mutex_enter(&rx_ring->rx_lock); 3842 mp = igb_rx(rx_ring, IGB_NO_POLL); 3843 mutex_exit(&rx_ring->rx_lock); 3844 3845 if (mp != NULL) 3846 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp, 3847 rx_ring->ring_gen_num); 3848 } 3849 3850 #pragma inline(igb_intr_tx_work) 3851 /* 3852 * igb_intr_tx_work - tx processing of ISR 3853 */ 3854 static void 3855 igb_intr_tx_work(igb_tx_ring_t *tx_ring) 3856 { 3857 igb_t *igb = tx_ring->igb; 3858 3859 /* Recycle the tx descriptors */ 3860 tx_ring->tx_recycle(tx_ring); 3861 3862 /* Schedule the re-transmit */ 3863 if (tx_ring->reschedule && 3864 (tx_ring->tbd_free >= igb->tx_resched_thresh)) { 3865 tx_ring->reschedule = B_FALSE; 3866 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle); 3867 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3868 } 3869 } 3870 3871 #pragma inline(igb_intr_link_work) 3872 /* 3873 * igb_intr_link_work - link-status-change processing of ISR 3874 */ 3875 static void 3876 igb_intr_link_work(igb_t *igb) 3877 { 3878 boolean_t link_changed; 3879 3880 igb_stop_watchdog_timer(igb); 3881 3882 mutex_enter(&igb->gen_lock); 3883 3884 /* 3885 * Because we got a link-status-change interrupt, force 3886 * e1000_check_for_link() to look at phy 3887 */ 3888 igb->hw.mac.get_link_status = B_TRUE; 3889 3890 /* igb_link_check takes care of link status change */ 3891 link_changed = igb_link_check(igb); 3892 3893 /* Get new phy state */ 3894 igb_get_phy_state(igb); 3895 3896 mutex_exit(&igb->gen_lock); 3897 3898 if (link_changed) 3899 mac_link_update(igb->mac_hdl, igb->link_state); 3900 3901 igb_start_watchdog_timer(igb); 3902 } 3903 3904 /* 3905 * igb_intr_legacy - Interrupt handler for legacy interrupts 3906 */ 3907 static uint_t 3908 igb_intr_legacy(void *arg1, void *arg2) 3909 { 3910 igb_t *igb = (igb_t *)arg1; 3911 igb_tx_ring_t *tx_ring; 3912 uint32_t icr; 3913 mblk_t *mp; 3914 boolean_t tx_reschedule; 3915 boolean_t link_changed; 3916 uint_t result; 3917 3918 _NOTE(ARGUNUSED(arg2)); 3919 3920 mutex_enter(&igb->gen_lock); 3921 3922 if (igb->igb_state & IGB_SUSPENDED) { 3923 mutex_exit(&igb->gen_lock); 3924 return (DDI_INTR_UNCLAIMED); 3925 } 3926 3927 mp = NULL; 3928 tx_reschedule = B_FALSE; 3929 link_changed = B_FALSE; 3930 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3931 3932 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3933 mutex_exit(&igb->gen_lock); 3934 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3935 atomic_or_32(&igb->igb_state, IGB_ERROR); 3936 return (DDI_INTR_UNCLAIMED); 3937 } 3938 3939 if (icr & E1000_ICR_INT_ASSERTED) { 3940 /* 3941 * E1000_ICR_INT_ASSERTED bit was set: 3942 * Read(Clear) the ICR, claim this interrupt, 3943 * look for work to do. 3944 */ 3945 ASSERT(igb->num_rx_rings == 1); 3946 ASSERT(igb->num_tx_rings == 1); 3947 3948 /* Make sure all interrupt causes cleared */ 3949 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3950 3951 if (icr & E1000_ICR_RXT0) { 3952 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL); 3953 } 3954 3955 if (icr & E1000_ICR_TXDW) { 3956 tx_ring = &igb->tx_rings[0]; 3957 3958 /* Recycle the tx descriptors */ 3959 tx_ring->tx_recycle(tx_ring); 3960 3961 /* Schedule the re-transmit */ 3962 tx_reschedule = (tx_ring->reschedule && 3963 (tx_ring->tbd_free >= igb->tx_resched_thresh)); 3964 } 3965 3966 if (icr & E1000_ICR_LSC) { 3967 /* 3968 * Because we got a link-status-change interrupt, force 3969 * e1000_check_for_link() to look at phy 3970 */ 3971 igb->hw.mac.get_link_status = B_TRUE; 3972 3973 /* igb_link_check takes care of link status change */ 3974 link_changed = igb_link_check(igb); 3975 3976 /* Get new phy state */ 3977 igb_get_phy_state(igb); 3978 } 3979 3980 if (icr & E1000_ICR_DRSTA) { 3981 /* 82580 Full Device Reset needed */ 3982 atomic_or_32(&igb->igb_state, IGB_STALL); 3983 } 3984 3985 result = DDI_INTR_CLAIMED; 3986 } else { 3987 /* 3988 * E1000_ICR_INT_ASSERTED bit was not set: 3989 * Don't claim this interrupt. 3990 */ 3991 result = DDI_INTR_UNCLAIMED; 3992 } 3993 3994 mutex_exit(&igb->gen_lock); 3995 3996 /* 3997 * Do the following work outside of the gen_lock 3998 */ 3999 if (mp != NULL) 4000 mac_rx(igb->mac_hdl, NULL, mp); 4001 4002 if (tx_reschedule) { 4003 tx_ring->reschedule = B_FALSE; 4004 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle); 4005 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 4006 } 4007 4008 if (link_changed) 4009 mac_link_update(igb->mac_hdl, igb->link_state); 4010 4011 return (result); 4012 } 4013 4014 /* 4015 * igb_intr_msi - Interrupt handler for MSI 4016 */ 4017 static uint_t 4018 igb_intr_msi(void *arg1, void *arg2) 4019 { 4020 igb_t *igb = (igb_t *)arg1; 4021 uint32_t icr; 4022 4023 _NOTE(ARGUNUSED(arg2)); 4024 4025 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4026 4027 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4028 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4029 atomic_or_32(&igb->igb_state, IGB_ERROR); 4030 return (DDI_INTR_CLAIMED); 4031 } 4032 4033 /* Make sure all interrupt causes cleared */ 4034 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 4035 4036 /* 4037 * For MSI interrupt, we have only one vector, 4038 * so we have only one rx ring and one tx ring enabled. 4039 */ 4040 ASSERT(igb->num_rx_rings == 1); 4041 ASSERT(igb->num_tx_rings == 1); 4042 4043 if (icr & E1000_ICR_RXT0) { 4044 igb_intr_rx_work(&igb->rx_rings[0]); 4045 } 4046 4047 if (icr & E1000_ICR_TXDW) { 4048 igb_intr_tx_work(&igb->tx_rings[0]); 4049 } 4050 4051 if (icr & E1000_ICR_LSC) { 4052 igb_intr_link_work(igb); 4053 } 4054 4055 if (icr & E1000_ICR_DRSTA) { 4056 /* 82580 Full Device Reset needed */ 4057 atomic_or_32(&igb->igb_state, IGB_STALL); 4058 } 4059 4060 return (DDI_INTR_CLAIMED); 4061 } 4062 4063 /* 4064 * igb_intr_rx - Interrupt handler for rx 4065 */ 4066 static uint_t 4067 igb_intr_rx(void *arg1, void *arg2) 4068 { 4069 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1; 4070 4071 _NOTE(ARGUNUSED(arg2)); 4072 4073 /* 4074 * Only used via MSI-X vector so don't check cause bits 4075 * and only clean the given ring. 4076 */ 4077 igb_intr_rx_work(rx_ring); 4078 4079 return (DDI_INTR_CLAIMED); 4080 } 4081 4082 /* 4083 * igb_intr_tx - Interrupt handler for tx 4084 */ 4085 static uint_t 4086 igb_intr_tx(void *arg1, void *arg2) 4087 { 4088 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1; 4089 4090 _NOTE(ARGUNUSED(arg2)); 4091 4092 /* 4093 * Only used via MSI-X vector so don't check cause bits 4094 * and only clean the given ring. 4095 */ 4096 igb_intr_tx_work(tx_ring); 4097 4098 return (DDI_INTR_CLAIMED); 4099 } 4100 4101 /* 4102 * igb_intr_tx_other - Interrupt handler for both tx and other 4103 * 4104 */ 4105 static uint_t 4106 igb_intr_tx_other(void *arg1, void *arg2) 4107 { 4108 igb_t *igb = (igb_t *)arg1; 4109 uint32_t icr; 4110 4111 _NOTE(ARGUNUSED(arg2)); 4112 4113 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4114 4115 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4116 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4117 atomic_or_32(&igb->igb_state, IGB_ERROR); 4118 return (DDI_INTR_CLAIMED); 4119 } 4120 4121 /* 4122 * Look for tx reclaiming work first. Remember, in the 4123 * case of only interrupt sharing, only one tx ring is 4124 * used 4125 */ 4126 igb_intr_tx_work(&igb->tx_rings[0]); 4127 4128 /* 4129 * Check for "other" causes. 4130 */ 4131 if (icr & E1000_ICR_LSC) { 4132 igb_intr_link_work(igb); 4133 } 4134 4135 /* 4136 * The DOUTSYNC bit indicates a tx packet dropped because 4137 * DMA engine gets "out of sync". There isn't a real fix 4138 * for this. The Intel recommendation is to count the number 4139 * of occurrences so user can detect when it is happening. 4140 * The issue is non-fatal and there's no recovery action 4141 * available. 4142 */ 4143 if (icr & E1000_ICR_DOUTSYNC) { 4144 IGB_STAT(igb->dout_sync); 4145 } 4146 4147 if (icr & E1000_ICR_DRSTA) { 4148 /* 82580 Full Device Reset needed */ 4149 atomic_or_32(&igb->igb_state, IGB_STALL); 4150 } 4151 4152 return (DDI_INTR_CLAIMED); 4153 } 4154 4155 /* 4156 * igb_alloc_intrs - Allocate interrupts for the driver 4157 * 4158 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4159 * if not successful, try Legacy. 4160 * igb->intr_force can be used to force sequence to start with 4161 * any of the 3 types. 4162 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4163 */ 4164 static int 4165 igb_alloc_intrs(igb_t *igb) 4166 { 4167 dev_info_t *devinfo; 4168 int intr_types; 4169 int rc; 4170 4171 devinfo = igb->dip; 4172 4173 /* Get supported interrupt types */ 4174 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4175 4176 if (rc != DDI_SUCCESS) { 4177 igb_log(igb, 4178 "Get supported interrupt types failed: %d", rc); 4179 return (IGB_FAILURE); 4180 } 4181 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types); 4182 4183 igb->intr_type = 0; 4184 4185 /* Install MSI-X interrupts */ 4186 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4187 (igb->intr_force <= IGB_INTR_MSIX)) { 4188 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX); 4189 4190 if (rc == IGB_SUCCESS) 4191 return (IGB_SUCCESS); 4192 4193 igb_log(igb, 4194 "Allocate MSI-X failed, trying MSI interrupts..."); 4195 } 4196 4197 /* MSI-X not used, force rings to 1 */ 4198 igb->num_rx_rings = 1; 4199 igb->num_tx_rings = 1; 4200 igb_log(igb, 4201 "MSI-X not used, force rx and tx queue number to 1"); 4202 4203 /* Install MSI interrupts */ 4204 if ((intr_types & DDI_INTR_TYPE_MSI) && 4205 (igb->intr_force <= IGB_INTR_MSI)) { 4206 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI); 4207 4208 if (rc == IGB_SUCCESS) 4209 return (IGB_SUCCESS); 4210 4211 igb_log(igb, 4212 "Allocate MSI failed, trying Legacy interrupts..."); 4213 } 4214 4215 /* Install legacy interrupts */ 4216 if (intr_types & DDI_INTR_TYPE_FIXED) { 4217 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED); 4218 4219 if (rc == IGB_SUCCESS) 4220 return (IGB_SUCCESS); 4221 4222 igb_log(igb, 4223 "Allocate Legacy interrupts failed"); 4224 } 4225 4226 /* If none of the 3 types succeeded, return failure */ 4227 return (IGB_FAILURE); 4228 } 4229 4230 /* 4231 * igb_alloc_intr_handles - Allocate interrupt handles. 4232 * 4233 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4234 * if fewer than 2 handles are available, return failure. 4235 * Upon success, this sets the number of Rx rings to a number that 4236 * matches the handles available for Rx interrupts. 4237 */ 4238 static int 4239 igb_alloc_intr_handles(igb_t *igb, int intr_type) 4240 { 4241 dev_info_t *devinfo; 4242 int orig, request, count, avail, actual; 4243 int diff, minimum; 4244 int rc; 4245 4246 devinfo = igb->dip; 4247 4248 switch (intr_type) { 4249 case DDI_INTR_TYPE_FIXED: 4250 request = 1; /* Request 1 legacy interrupt handle */ 4251 minimum = 1; 4252 IGB_DEBUGLOG_0(igb, "interrupt type: legacy"); 4253 break; 4254 4255 case DDI_INTR_TYPE_MSI: 4256 request = 1; /* Request 1 MSI interrupt handle */ 4257 minimum = 1; 4258 IGB_DEBUGLOG_0(igb, "interrupt type: MSI"); 4259 break; 4260 4261 case DDI_INTR_TYPE_MSIX: 4262 /* 4263 * Number of vectors for the adapter is 4264 * # rx rings + # tx rings 4265 * One of tx vectors is for tx & other 4266 */ 4267 request = igb->num_rx_rings + igb->num_tx_rings; 4268 orig = request; 4269 minimum = 2; 4270 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X"); 4271 break; 4272 4273 default: 4274 igb_log(igb, 4275 "invalid call to igb_alloc_intr_handles(): %d\n", 4276 intr_type); 4277 return (IGB_FAILURE); 4278 } 4279 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d", 4280 request, minimum); 4281 4282 /* 4283 * Get number of supported interrupts 4284 */ 4285 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4286 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4287 igb_log(igb, 4288 "Get supported interrupt number failed. " 4289 "Return: %d, count: %d", rc, count); 4290 return (IGB_FAILURE); 4291 } 4292 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count); 4293 4294 /* 4295 * Get number of available interrupts 4296 */ 4297 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 4298 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 4299 igb_log(igb, 4300 "Get available interrupt number failed. " 4301 "Return: %d, available: %d", rc, avail); 4302 return (IGB_FAILURE); 4303 } 4304 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail); 4305 4306 if (avail < request) { 4307 igb_log(igb, "Request %d handles, %d available", 4308 request, avail); 4309 request = avail; 4310 } 4311 4312 actual = 0; 4313 igb->intr_cnt = 0; 4314 4315 /* 4316 * Allocate an array of interrupt handles 4317 */ 4318 igb->intr_size = request * sizeof (ddi_intr_handle_t); 4319 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP); 4320 4321 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0, 4322 request, &actual, DDI_INTR_ALLOC_NORMAL); 4323 if (rc != DDI_SUCCESS) { 4324 igb_log(igb, "Allocate interrupts failed. " 4325 "return: %d, request: %d, actual: %d", 4326 rc, request, actual); 4327 goto alloc_handle_fail; 4328 } 4329 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual); 4330 4331 igb->intr_cnt = actual; 4332 4333 if (actual < minimum) { 4334 igb_log(igb, "Insufficient interrupt handles allocated: %d", 4335 actual); 4336 goto alloc_handle_fail; 4337 } 4338 4339 /* 4340 * For MSI-X, actual might force us to reduce number of tx & rx rings 4341 */ 4342 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { 4343 diff = orig - actual; 4344 if (diff < igb->num_tx_rings) { 4345 igb_log(igb, 4346 "MSI-X vectors force Tx queue number to %d", 4347 igb->num_tx_rings - diff); 4348 igb->num_tx_rings -= diff; 4349 } else { 4350 igb_log(igb, 4351 "MSI-X vectors force Tx queue number to 1"); 4352 igb->num_tx_rings = 1; 4353 4354 igb_log(igb, 4355 "MSI-X vectors force Rx queue number to %d", 4356 actual - 1); 4357 igb->num_rx_rings = actual - 1; 4358 } 4359 } 4360 4361 /* 4362 * Get priority for first vector, assume remaining are all the same 4363 */ 4364 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri); 4365 if (rc != DDI_SUCCESS) { 4366 igb_log(igb, 4367 "Get interrupt priority failed: %d", rc); 4368 goto alloc_handle_fail; 4369 } 4370 4371 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap); 4372 if (rc != DDI_SUCCESS) { 4373 igb_log(igb, 4374 "Get interrupt cap failed: %d", rc); 4375 goto alloc_handle_fail; 4376 } 4377 4378 igb->intr_type = intr_type; 4379 4380 return (IGB_SUCCESS); 4381 4382 alloc_handle_fail: 4383 igb_rem_intrs(igb); 4384 4385 return (IGB_FAILURE); 4386 } 4387 4388 /* 4389 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type 4390 * 4391 * Before adding the interrupt handlers, the interrupt vectors have 4392 * been allocated, and the rx/tx rings have also been allocated. 4393 */ 4394 static int 4395 igb_add_intr_handlers(igb_t *igb) 4396 { 4397 igb_rx_ring_t *rx_ring; 4398 igb_tx_ring_t *tx_ring; 4399 int vector; 4400 int rc; 4401 int i; 4402 4403 vector = 0; 4404 4405 switch (igb->intr_type) { 4406 case DDI_INTR_TYPE_MSIX: 4407 /* Add interrupt handler for tx + other */ 4408 tx_ring = &igb->tx_rings[0]; 4409 rc = ddi_intr_add_handler(igb->htable[vector], 4410 (ddi_intr_handler_t *)igb_intr_tx_other, 4411 (void *)igb, NULL); 4412 4413 if (rc != DDI_SUCCESS) { 4414 igb_log(igb, 4415 "Add tx/other interrupt handler failed: %d", rc); 4416 return (IGB_FAILURE); 4417 } 4418 tx_ring->intr_vector = vector; 4419 vector++; 4420 4421 /* Add interrupt handler for each rx ring */ 4422 for (i = 0; i < igb->num_rx_rings; i++) { 4423 rx_ring = &igb->rx_rings[i]; 4424 4425 rc = ddi_intr_add_handler(igb->htable[vector], 4426 (ddi_intr_handler_t *)igb_intr_rx, 4427 (void *)rx_ring, NULL); 4428 4429 if (rc != DDI_SUCCESS) { 4430 igb_log(igb, 4431 "Add rx interrupt handler failed. " 4432 "return: %d, rx ring: %d", rc, i); 4433 for (vector--; vector >= 0; vector--) { 4434 (void) ddi_intr_remove_handler( 4435 igb->htable[vector]); 4436 } 4437 return (IGB_FAILURE); 4438 } 4439 4440 rx_ring->intr_vector = vector; 4441 4442 vector++; 4443 } 4444 4445 /* Add interrupt handler for each tx ring from 2nd ring */ 4446 for (i = 1; i < igb->num_tx_rings; i++) { 4447 tx_ring = &igb->tx_rings[i]; 4448 4449 rc = ddi_intr_add_handler(igb->htable[vector], 4450 (ddi_intr_handler_t *)igb_intr_tx, 4451 (void *)tx_ring, NULL); 4452 4453 if (rc != DDI_SUCCESS) { 4454 igb_log(igb, 4455 "Add tx interrupt handler failed. " 4456 "return: %d, tx ring: %d", rc, i); 4457 for (vector--; vector >= 0; vector--) { 4458 (void) ddi_intr_remove_handler( 4459 igb->htable[vector]); 4460 } 4461 return (IGB_FAILURE); 4462 } 4463 4464 tx_ring->intr_vector = vector; 4465 4466 vector++; 4467 } 4468 4469 break; 4470 4471 case DDI_INTR_TYPE_MSI: 4472 /* Add interrupt handlers for the only vector */ 4473 rc = ddi_intr_add_handler(igb->htable[vector], 4474 (ddi_intr_handler_t *)igb_intr_msi, 4475 (void *)igb, NULL); 4476 4477 if (rc != DDI_SUCCESS) { 4478 igb_log(igb, 4479 "Add MSI interrupt handler failed: %d", rc); 4480 return (IGB_FAILURE); 4481 } 4482 4483 rx_ring = &igb->rx_rings[0]; 4484 rx_ring->intr_vector = vector; 4485 4486 vector++; 4487 break; 4488 4489 case DDI_INTR_TYPE_FIXED: 4490 /* Add interrupt handlers for the only vector */ 4491 rc = ddi_intr_add_handler(igb->htable[vector], 4492 (ddi_intr_handler_t *)igb_intr_legacy, 4493 (void *)igb, NULL); 4494 4495 if (rc != DDI_SUCCESS) { 4496 igb_log(igb, 4497 "Add legacy interrupt handler failed: %d", rc); 4498 return (IGB_FAILURE); 4499 } 4500 4501 rx_ring = &igb->rx_rings[0]; 4502 rx_ring->intr_vector = vector; 4503 4504 vector++; 4505 break; 4506 4507 default: 4508 return (IGB_FAILURE); 4509 } 4510 4511 ASSERT(vector == igb->intr_cnt); 4512 4513 return (IGB_SUCCESS); 4514 } 4515 4516 /* 4517 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts 4518 * 4519 * For each vector enabled on the adapter, Set the MSIXBM register accordingly 4520 */ 4521 static void 4522 igb_setup_msix_82575(igb_t *igb) 4523 { 4524 uint32_t eims = 0; 4525 int i, vector; 4526 struct e1000_hw *hw = &igb->hw; 4527 4528 /* 4529 * Set vector for tx ring 0 and other causes. 4530 * NOTE assumption that it is vector 0. 4531 */ 4532 vector = 0; 4533 4534 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER; 4535 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask); 4536 vector++; 4537 4538 for (i = 0; i < igb->num_rx_rings; i++) { 4539 /* 4540 * Set vector for each rx ring 4541 */ 4542 eims = (E1000_EICR_RX_QUEUE0 << i); 4543 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4544 4545 /* 4546 * Accumulate bits to enable in 4547 * igb_enable_adapter_interrupts_82575() 4548 */ 4549 igb->eims_mask |= eims; 4550 4551 vector++; 4552 } 4553 4554 for (i = 1; i < igb->num_tx_rings; i++) { 4555 /* 4556 * Set vector for each tx ring from 2nd tx ring 4557 */ 4558 eims = (E1000_EICR_TX_QUEUE0 << i); 4559 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4560 4561 /* 4562 * Accumulate bits to enable in 4563 * igb_enable_adapter_interrupts_82575() 4564 */ 4565 igb->eims_mask |= eims; 4566 4567 vector++; 4568 } 4569 4570 ASSERT(vector == igb->intr_cnt); 4571 4572 /* 4573 * Disable IAM for ICR interrupt bits 4574 */ 4575 E1000_WRITE_REG(hw, E1000_IAM, 0); 4576 E1000_WRITE_FLUSH(hw); 4577 } 4578 4579 /* 4580 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts 4581 * 4582 * 82576 uses a table based method for assigning vectors. Each queue has a 4583 * single entry in the table to which we write a vector number along with a 4584 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors 4585 * take a different position in the 4-byte register depending on whether 4586 * they are numbered above or below 8. 4587 */ 4588 static void 4589 igb_setup_msix_82576(igb_t *igb) 4590 { 4591 struct e1000_hw *hw = &igb->hw; 4592 uint32_t ivar, index, vector; 4593 int i; 4594 4595 /* must enable msi-x capability before IVAR settings */ 4596 E1000_WRITE_REG(hw, E1000_GPIE, 4597 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR)); 4598 4599 /* 4600 * Set vector for tx ring 0 and other causes. 4601 * NOTE assumption that it is vector 0. 4602 * This is also interdependent with installation of interrupt service 4603 * routines in igb_add_intr_handlers(). 4604 */ 4605 4606 /* assign "other" causes to vector 0 */ 4607 vector = 0; 4608 ivar = ((vector | E1000_IVAR_VALID) << 8); 4609 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4610 4611 /* assign tx ring 0 to vector 0 */ 4612 ivar = ((vector | E1000_IVAR_VALID) << 8); 4613 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4614 4615 /* prepare to enable tx & other interrupt causes */ 4616 igb->eims_mask = (1 << vector); 4617 4618 vector ++; 4619 for (i = 0; i < igb->num_rx_rings; i++) { 4620 /* 4621 * Set vector for each rx ring 4622 */ 4623 index = (i & 0x7); 4624 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4625 4626 if (i < 8) { 4627 /* vector goes into low byte of register */ 4628 ivar = ivar & 0xFFFFFF00; 4629 ivar |= (vector | E1000_IVAR_VALID); 4630 } else { 4631 /* vector goes into third byte of register */ 4632 ivar = ivar & 0xFF00FFFF; 4633 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4634 } 4635 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4636 4637 /* Accumulate interrupt-cause bits to enable */ 4638 igb->eims_mask |= (1 << vector); 4639 4640 vector ++; 4641 } 4642 4643 for (i = 1; i < igb->num_tx_rings; i++) { 4644 /* 4645 * Set vector for each tx ring from 2nd tx ring. 4646 * Note assumption that tx vectors numericall follow rx vectors. 4647 */ 4648 index = (i & 0x7); 4649 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4650 4651 if (i < 8) { 4652 /* vector goes into second byte of register */ 4653 ivar = ivar & 0xFFFF00FF; 4654 ivar |= ((vector | E1000_IVAR_VALID) << 8); 4655 } else { 4656 /* vector goes into fourth byte of register */ 4657 ivar = ivar & 0x00FFFFFF; 4658 ivar |= (vector | E1000_IVAR_VALID) << 24; 4659 } 4660 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4661 4662 /* Accumulate interrupt-cause bits to enable */ 4663 igb->eims_mask |= (1 << vector); 4664 4665 vector ++; 4666 } 4667 4668 ASSERT(vector == igb->intr_cnt); 4669 } 4670 4671 /* 4672 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts 4673 * 4674 * 82580 uses same table approach at 82576 but has fewer entries. Each 4675 * queue has a single entry in the table to which we write a vector number 4676 * along with a "valid" bit. Vectors take a different position in the 4677 * register depending on * whether * they are numbered above or below 4. 4678 */ 4679 static void 4680 igb_setup_msix_82580(igb_t *igb) 4681 { 4682 struct e1000_hw *hw = &igb->hw; 4683 uint32_t ivar, index, vector; 4684 int i; 4685 4686 /* must enable msi-x capability before IVAR settings */ 4687 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE | 4688 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME)); 4689 /* 4690 * Set vector for tx ring 0 and other causes. 4691 * NOTE assumption that it is vector 0. 4692 * This is also interdependent with installation of interrupt service 4693 * routines in igb_add_intr_handlers(). 4694 */ 4695 4696 /* assign "other" causes to vector 0 */ 4697 vector = 0; 4698 ivar = ((vector | E1000_IVAR_VALID) << 8); 4699 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4700 4701 /* assign tx ring 0 to vector 0 */ 4702 ivar = ((vector | E1000_IVAR_VALID) << 8); 4703 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4704 4705 /* prepare to enable tx & other interrupt causes */ 4706 igb->eims_mask = (1 << vector); 4707 4708 vector ++; 4709 4710 for (i = 0; i < igb->num_rx_rings; i++) { 4711 /* 4712 * Set vector for each rx ring 4713 */ 4714 index = (i >> 1); 4715 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4716 4717 if (i & 1) { 4718 /* vector goes into third byte of register */ 4719 ivar = ivar & 0xFF00FFFF; 4720 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4721 } else { 4722 /* vector goes into low byte of register */ 4723 ivar = ivar & 0xFFFFFF00; 4724 ivar |= (vector | E1000_IVAR_VALID); 4725 } 4726 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4727 4728 /* Accumulate interrupt-cause bits to enable */ 4729 igb->eims_mask |= (1 << vector); 4730 4731 vector ++; 4732 } 4733 4734 for (i = 1; i < igb->num_tx_rings; i++) { 4735 /* 4736 * Set vector for each tx ring from 2nd tx ring. 4737 * Note assumption that tx vectors numericall follow rx vectors. 4738 */ 4739 index = (i >> 1); 4740 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4741 4742 if (i & 1) { 4743 /* vector goes into high byte of register */ 4744 ivar = ivar & 0x00FFFFFF; 4745 ivar |= ((vector | E1000_IVAR_VALID) << 24); 4746 } else { 4747 /* vector goes into second byte of register */ 4748 ivar = ivar & 0xFFFF00FF; 4749 ivar |= (vector | E1000_IVAR_VALID) << 8; 4750 } 4751 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4752 4753 /* Accumulate interrupt-cause bits to enable */ 4754 igb->eims_mask |= (1 << vector); 4755 4756 vector ++; 4757 } 4758 ASSERT(vector == igb->intr_cnt); 4759 } 4760 4761 /* 4762 * igb_rem_intr_handlers - remove the interrupt handlers 4763 */ 4764 static void 4765 igb_rem_intr_handlers(igb_t *igb) 4766 { 4767 int i; 4768 int rc; 4769 4770 for (i = 0; i < igb->intr_cnt; i++) { 4771 rc = ddi_intr_remove_handler(igb->htable[i]); 4772 if (rc != DDI_SUCCESS) { 4773 IGB_DEBUGLOG_1(igb, 4774 "Remove intr handler failed: %d", rc); 4775 } 4776 } 4777 } 4778 4779 /* 4780 * igb_rem_intrs - remove the allocated interrupts 4781 */ 4782 static void 4783 igb_rem_intrs(igb_t *igb) 4784 { 4785 int i; 4786 int rc; 4787 4788 for (i = 0; i < igb->intr_cnt; i++) { 4789 rc = ddi_intr_free(igb->htable[i]); 4790 if (rc != DDI_SUCCESS) { 4791 IGB_DEBUGLOG_1(igb, 4792 "Free intr failed: %d", rc); 4793 } 4794 } 4795 4796 kmem_free(igb->htable, igb->intr_size); 4797 igb->htable = NULL; 4798 } 4799 4800 /* 4801 * igb_enable_intrs - enable all the ddi interrupts 4802 */ 4803 static int 4804 igb_enable_intrs(igb_t *igb) 4805 { 4806 int i; 4807 int rc; 4808 4809 /* Enable interrupts */ 4810 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4811 /* Call ddi_intr_block_enable() for MSI */ 4812 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt); 4813 if (rc != DDI_SUCCESS) { 4814 igb_log(igb, 4815 "Enable block intr failed: %d", rc); 4816 return (IGB_FAILURE); 4817 } 4818 } else { 4819 /* Call ddi_intr_enable() for Legacy/MSI non block enable */ 4820 for (i = 0; i < igb->intr_cnt; i++) { 4821 rc = ddi_intr_enable(igb->htable[i]); 4822 if (rc != DDI_SUCCESS) { 4823 igb_log(igb, 4824 "Enable intr failed: %d", rc); 4825 return (IGB_FAILURE); 4826 } 4827 } 4828 } 4829 4830 return (IGB_SUCCESS); 4831 } 4832 4833 /* 4834 * igb_disable_intrs - disable all the ddi interrupts 4835 */ 4836 static int 4837 igb_disable_intrs(igb_t *igb) 4838 { 4839 int i; 4840 int rc; 4841 4842 /* Disable all interrupts */ 4843 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4844 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt); 4845 if (rc != DDI_SUCCESS) { 4846 igb_log(igb, 4847 "Disable block intr failed: %d", rc); 4848 return (IGB_FAILURE); 4849 } 4850 } else { 4851 for (i = 0; i < igb->intr_cnt; i++) { 4852 rc = ddi_intr_disable(igb->htable[i]); 4853 if (rc != DDI_SUCCESS) { 4854 igb_log(igb, 4855 "Disable intr failed: %d", rc); 4856 return (IGB_FAILURE); 4857 } 4858 } 4859 } 4860 4861 return (IGB_SUCCESS); 4862 } 4863 4864 /* 4865 * igb_get_phy_state - Get and save the parameters read from PHY registers 4866 */ 4867 static void 4868 igb_get_phy_state(igb_t *igb) 4869 { 4870 struct e1000_hw *hw = &igb->hw; 4871 uint16_t phy_ctrl; 4872 uint16_t phy_status; 4873 uint16_t phy_an_adv; 4874 uint16_t phy_an_exp; 4875 uint16_t phy_ext_status; 4876 uint16_t phy_1000t_ctrl; 4877 uint16_t phy_1000t_status; 4878 uint16_t phy_lp_able; 4879 4880 ASSERT(mutex_owned(&igb->gen_lock)); 4881 4882 if (hw->phy.media_type == e1000_media_type_copper) { 4883 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 4884 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 4885 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv); 4886 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp); 4887 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status); 4888 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl); 4889 (void) e1000_read_phy_reg(hw, 4890 PHY_1000T_STATUS, &phy_1000t_status); 4891 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able); 4892 4893 igb->param_autoneg_cap = 4894 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0; 4895 igb->param_pause_cap = 4896 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4897 igb->param_asym_pause_cap = 4898 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4899 igb->param_1000fdx_cap = 4900 ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) || 4901 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0; 4902 igb->param_1000hdx_cap = 4903 ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) || 4904 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0; 4905 igb->param_100t4_cap = 4906 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0; 4907 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) || 4908 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0; 4909 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) || 4910 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0; 4911 igb->param_10fdx_cap = 4912 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0; 4913 igb->param_10hdx_cap = 4914 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0; 4915 igb->param_rem_fault = 4916 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0; 4917 4918 igb->param_adv_autoneg_cap = hw->mac.autoneg; 4919 igb->param_adv_pause_cap = 4920 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4921 igb->param_adv_asym_pause_cap = 4922 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4923 igb->param_adv_1000hdx_cap = 4924 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0; 4925 igb->param_adv_100t4_cap = 4926 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0; 4927 igb->param_adv_rem_fault = 4928 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0; 4929 if (igb->param_adv_autoneg_cap == 1) { 4930 igb->param_adv_1000fdx_cap = 4931 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0; 4932 igb->param_adv_100fdx_cap = 4933 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0; 4934 igb->param_adv_100hdx_cap = 4935 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0; 4936 igb->param_adv_10fdx_cap = 4937 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0; 4938 igb->param_adv_10hdx_cap = 4939 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0; 4940 } 4941 4942 igb->param_lp_autoneg_cap = 4943 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0; 4944 igb->param_lp_pause_cap = 4945 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0; 4946 igb->param_lp_asym_pause_cap = 4947 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0; 4948 igb->param_lp_1000fdx_cap = 4949 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0; 4950 igb->param_lp_1000hdx_cap = 4951 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0; 4952 igb->param_lp_100t4_cap = 4953 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0; 4954 igb->param_lp_100fdx_cap = 4955 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0; 4956 igb->param_lp_100hdx_cap = 4957 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0; 4958 igb->param_lp_10fdx_cap = 4959 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0; 4960 igb->param_lp_10hdx_cap = 4961 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0; 4962 igb->param_lp_rem_fault = 4963 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0; 4964 } else { 4965 /* 4966 * 1Gig Fiber adapter only offers 1Gig Full Duplex. 4967 */ 4968 igb->param_autoneg_cap = 0; 4969 igb->param_pause_cap = 1; 4970 igb->param_asym_pause_cap = 1; 4971 igb->param_1000fdx_cap = 1; 4972 igb->param_1000hdx_cap = 0; 4973 igb->param_100t4_cap = 0; 4974 igb->param_100fdx_cap = 0; 4975 igb->param_100hdx_cap = 0; 4976 igb->param_10fdx_cap = 0; 4977 igb->param_10hdx_cap = 0; 4978 4979 igb->param_adv_autoneg_cap = 0; 4980 igb->param_adv_pause_cap = 1; 4981 igb->param_adv_asym_pause_cap = 1; 4982 igb->param_adv_1000fdx_cap = 1; 4983 igb->param_adv_1000hdx_cap = 0; 4984 igb->param_adv_100t4_cap = 0; 4985 igb->param_adv_100fdx_cap = 0; 4986 igb->param_adv_100hdx_cap = 0; 4987 igb->param_adv_10fdx_cap = 0; 4988 igb->param_adv_10hdx_cap = 0; 4989 4990 igb->param_lp_autoneg_cap = 0; 4991 igb->param_lp_pause_cap = 0; 4992 igb->param_lp_asym_pause_cap = 0; 4993 igb->param_lp_1000fdx_cap = 0; 4994 igb->param_lp_1000hdx_cap = 0; 4995 igb->param_lp_100t4_cap = 0; 4996 igb->param_lp_100fdx_cap = 0; 4997 igb->param_lp_100hdx_cap = 0; 4998 igb->param_lp_10fdx_cap = 0; 4999 igb->param_lp_10hdx_cap = 0; 5000 igb->param_lp_rem_fault = 0; 5001 } 5002 } 5003 5004 /* 5005 * synchronize the adv* and en* parameters. 5006 * 5007 * See comments in <sys/dld.h> for details of the *_en_* 5008 * parameters. The usage of ndd for setting adv parameters will 5009 * synchronize all the en parameters with the e1000g parameters, 5010 * implicitly disabling any settings made via dladm. 5011 */ 5012 static void 5013 igb_param_sync(igb_t *igb) 5014 { 5015 igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap; 5016 igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap; 5017 igb->param_en_100t4_cap = igb->param_adv_100t4_cap; 5018 igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap; 5019 igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap; 5020 igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap; 5021 igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap; 5022 } 5023 5024 /* 5025 * igb_get_driver_control 5026 */ 5027 static void 5028 igb_get_driver_control(struct e1000_hw *hw) 5029 { 5030 uint32_t ctrl_ext; 5031 5032 /* Notify firmware that driver is in control of device */ 5033 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5034 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD; 5035 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5036 } 5037 5038 /* 5039 * igb_release_driver_control 5040 */ 5041 static void 5042 igb_release_driver_control(struct e1000_hw *hw) 5043 { 5044 uint32_t ctrl_ext; 5045 5046 /* Notify firmware that driver is no longer in control of device */ 5047 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5048 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD; 5049 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5050 } 5051 5052 /* 5053 * igb_atomic_reserve - Atomic decrease operation 5054 */ 5055 int 5056 igb_atomic_reserve(uint32_t *count_p, uint32_t n) 5057 { 5058 uint32_t oldval; 5059 uint32_t newval; 5060 5061 /* ATOMICALLY */ 5062 do { 5063 oldval = *count_p; 5064 if (oldval < n) 5065 return (-1); 5066 newval = oldval - n; 5067 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5068 5069 return (newval); 5070 } 5071 5072 /* 5073 * FMA support 5074 */ 5075 5076 int 5077 igb_check_acc_handle(ddi_acc_handle_t handle) 5078 { 5079 ddi_fm_error_t de; 5080 5081 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5082 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5083 return (de.fme_status); 5084 } 5085 5086 int 5087 igb_check_dma_handle(ddi_dma_handle_t handle) 5088 { 5089 ddi_fm_error_t de; 5090 5091 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5092 return (de.fme_status); 5093 } 5094 5095 /* 5096 * The IO fault service error handling callback function 5097 */ 5098 /*ARGSUSED*/ 5099 static int 5100 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5101 { 5102 /* 5103 * as the driver can always deal with an error in any dma or 5104 * access handle, we can just return the fme_status value. 5105 */ 5106 pci_ereport_post(dip, err, NULL); 5107 return (err->fme_status); 5108 } 5109 5110 static void 5111 igb_fm_init(igb_t *igb) 5112 { 5113 ddi_iblock_cookie_t iblk; 5114 int fma_dma_flag; 5115 5116 /* Only register with IO Fault Services if we have some capability */ 5117 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5118 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5119 } else { 5120 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5121 } 5122 5123 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5124 fma_dma_flag = 1; 5125 } else { 5126 fma_dma_flag = 0; 5127 } 5128 5129 (void) igb_set_fma_flags(fma_dma_flag); 5130 5131 if (igb->fm_capabilities) { 5132 5133 /* Register capabilities with IO Fault Services */ 5134 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk); 5135 5136 /* 5137 * Initialize pci ereport capabilities if ereport capable 5138 */ 5139 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5140 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5141 pci_ereport_setup(igb->dip); 5142 5143 /* 5144 * Register error callback if error callback capable 5145 */ 5146 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5147 ddi_fm_handler_register(igb->dip, 5148 igb_fm_error_cb, (void*) igb); 5149 } 5150 } 5151 5152 static void 5153 igb_fm_fini(igb_t *igb) 5154 { 5155 /* Only unregister FMA capabilities if we registered some */ 5156 if (igb->fm_capabilities) { 5157 5158 /* 5159 * Release any resources allocated by pci_ereport_setup() 5160 */ 5161 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5162 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5163 pci_ereport_teardown(igb->dip); 5164 5165 /* 5166 * Un-register error callback if error callback capable 5167 */ 5168 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5169 ddi_fm_handler_unregister(igb->dip); 5170 5171 /* Unregister from IO Fault Services */ 5172 ddi_fm_fini(igb->dip); 5173 } 5174 } 5175 5176 void 5177 igb_fm_ereport(igb_t *igb, char *detail) 5178 { 5179 uint64_t ena; 5180 char buf[FM_MAX_CLASS]; 5181 5182 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5183 ena = fm_ena_generate(0, FM_ENA_FMT1); 5184 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) { 5185 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP, 5186 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5187 } 5188 } 5189