1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #include "igb_sw.h" 30 31 static char ident[] = "Intel 1Gb Ethernet"; 32 static char igb_version[] = "igb 1.1.10"; 33 34 /* 35 * Local function protoypes 36 */ 37 static int igb_register_mac(igb_t *); 38 static int igb_identify_hardware(igb_t *); 39 static int igb_regs_map(igb_t *); 40 static void igb_init_properties(igb_t *); 41 static int igb_init_driver_settings(igb_t *); 42 static void igb_init_locks(igb_t *); 43 static void igb_destroy_locks(igb_t *); 44 static int igb_init_mac_address(igb_t *); 45 static int igb_init(igb_t *); 46 static int igb_init_adapter(igb_t *); 47 static void igb_stop_adapter(igb_t *); 48 static int igb_reset(igb_t *); 49 static void igb_tx_clean(igb_t *); 50 static boolean_t igb_tx_drain(igb_t *); 51 static boolean_t igb_rx_drain(igb_t *); 52 static int igb_alloc_rings(igb_t *); 53 static void igb_free_rings(igb_t *); 54 static void igb_setup_rings(igb_t *); 55 static void igb_setup_rx(igb_t *); 56 static void igb_setup_tx(igb_t *); 57 static void igb_setup_rx_ring(igb_rx_ring_t *); 58 static void igb_setup_tx_ring(igb_tx_ring_t *); 59 static void igb_setup_rss(igb_t *); 60 static void igb_setup_mac_rss_classify(igb_t *); 61 static void igb_setup_mac_classify(igb_t *); 62 static void igb_init_unicst(igb_t *); 63 static void igb_setup_multicst(igb_t *); 64 static void igb_get_phy_state(igb_t *); 65 static void igb_get_conf(igb_t *); 66 static int igb_get_prop(igb_t *, char *, int, int, int); 67 static boolean_t igb_is_link_up(igb_t *); 68 static boolean_t igb_link_check(igb_t *); 69 static void igb_local_timer(void *); 70 static void igb_link_timer(void *); 71 static void igb_arm_watchdog_timer(igb_t *); 72 static void igb_start_watchdog_timer(igb_t *); 73 static void igb_restart_watchdog_timer(igb_t *); 74 static void igb_stop_watchdog_timer(igb_t *); 75 static void igb_start_link_timer(igb_t *); 76 static void igb_stop_link_timer(igb_t *); 77 static void igb_disable_adapter_interrupts(igb_t *); 78 static void igb_enable_adapter_interrupts_82575(igb_t *); 79 static void igb_enable_adapter_interrupts_82576(igb_t *); 80 static void igb_enable_adapter_interrupts_82580(igb_t *); 81 static boolean_t is_valid_mac_addr(uint8_t *); 82 static boolean_t igb_stall_check(igb_t *); 83 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t); 84 static void igb_set_external_loopback(igb_t *); 85 static void igb_set_internal_mac_loopback(igb_t *); 86 static void igb_set_internal_phy_loopback(igb_t *); 87 static void igb_set_internal_serdes_loopback(igb_t *); 88 static boolean_t igb_find_mac_address(igb_t *); 89 static int igb_alloc_intrs(igb_t *); 90 static int igb_alloc_intr_handles(igb_t *, int); 91 static int igb_add_intr_handlers(igb_t *); 92 static void igb_rem_intr_handlers(igb_t *); 93 static void igb_rem_intrs(igb_t *); 94 static int igb_enable_intrs(igb_t *); 95 static int igb_disable_intrs(igb_t *); 96 static void igb_setup_msix_82575(igb_t *); 97 static void igb_setup_msix_82576(igb_t *); 98 static void igb_setup_msix_82580(igb_t *); 99 static uint_t igb_intr_legacy(void *, void *); 100 static uint_t igb_intr_msi(void *, void *); 101 static uint_t igb_intr_rx(void *, void *); 102 static uint_t igb_intr_tx(void *, void *); 103 static uint_t igb_intr_tx_other(void *, void *); 104 static void igb_intr_rx_work(igb_rx_ring_t *); 105 static void igb_intr_tx_work(igb_tx_ring_t *); 106 static void igb_intr_link_work(igb_t *); 107 static void igb_get_driver_control(struct e1000_hw *); 108 static void igb_release_driver_control(struct e1000_hw *); 109 110 static int igb_attach(dev_info_t *, ddi_attach_cmd_t); 111 static int igb_detach(dev_info_t *, ddi_detach_cmd_t); 112 static int igb_resume(dev_info_t *); 113 static int igb_suspend(dev_info_t *); 114 static int igb_quiesce(dev_info_t *); 115 static void igb_unconfigure(dev_info_t *, igb_t *); 116 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *, 117 const void *); 118 static void igb_fm_init(igb_t *); 119 static void igb_fm_fini(igb_t *); 120 static void igb_release_multicast(igb_t *); 121 122 static struct cb_ops igb_cb_ops = { 123 nulldev, /* cb_open */ 124 nulldev, /* cb_close */ 125 nodev, /* cb_strategy */ 126 nodev, /* cb_print */ 127 nodev, /* cb_dump */ 128 nodev, /* cb_read */ 129 nodev, /* cb_write */ 130 nodev, /* cb_ioctl */ 131 nodev, /* cb_devmap */ 132 nodev, /* cb_mmap */ 133 nodev, /* cb_segmap */ 134 nochpoll, /* cb_chpoll */ 135 ddi_prop_op, /* cb_prop_op */ 136 NULL, /* cb_stream */ 137 D_MP | D_HOTPLUG, /* cb_flag */ 138 CB_REV, /* cb_rev */ 139 nodev, /* cb_aread */ 140 nodev /* cb_awrite */ 141 }; 142 143 static struct dev_ops igb_dev_ops = { 144 DEVO_REV, /* devo_rev */ 145 0, /* devo_refcnt */ 146 NULL, /* devo_getinfo */ 147 nulldev, /* devo_identify */ 148 nulldev, /* devo_probe */ 149 igb_attach, /* devo_attach */ 150 igb_detach, /* devo_detach */ 151 nodev, /* devo_reset */ 152 &igb_cb_ops, /* devo_cb_ops */ 153 NULL, /* devo_bus_ops */ 154 ddi_power, /* devo_power */ 155 igb_quiesce, /* devo_quiesce */ 156 }; 157 158 static struct modldrv igb_modldrv = { 159 &mod_driverops, /* Type of module. This one is a driver */ 160 ident, /* Discription string */ 161 &igb_dev_ops, /* driver ops */ 162 }; 163 164 static struct modlinkage igb_modlinkage = { 165 MODREV_1, &igb_modldrv, NULL 166 }; 167 168 /* Access attributes for register mapping */ 169 ddi_device_acc_attr_t igb_regs_acc_attr = { 170 DDI_DEVICE_ATTR_V1, 171 DDI_STRUCTURE_LE_ACC, 172 DDI_STRICTORDER_ACC, 173 DDI_FLAGERR_ACC 174 }; 175 176 #define IGB_M_CALLBACK_FLAGS (MC_IOCTL | MC_GETCAPAB) 177 178 static mac_callbacks_t igb_m_callbacks = { 179 IGB_M_CALLBACK_FLAGS, 180 igb_m_stat, 181 igb_m_start, 182 igb_m_stop, 183 igb_m_promisc, 184 igb_m_multicst, 185 NULL, 186 NULL, 187 igb_m_ioctl, 188 igb_m_getcapab 189 }; 190 191 /* 192 * Initialize capabilities of each supported adapter type 193 */ 194 static adapter_info_t igb_82575_cap = { 195 /* limits */ 196 4, /* maximum number of rx queues */ 197 1, /* minimum number of rx queues */ 198 4, /* default number of rx queues */ 199 4, /* maximum number of tx queues */ 200 1, /* minimum number of tx queues */ 201 4, /* default number of tx queues */ 202 65535, /* maximum interrupt throttle rate */ 203 0, /* minimum interrupt throttle rate */ 204 200, /* default interrupt throttle rate */ 205 206 /* function pointers */ 207 igb_enable_adapter_interrupts_82575, 208 igb_setup_msix_82575, 209 210 /* capabilities */ 211 (IGB_FLAG_HAS_DCA | /* capability flags */ 212 IGB_FLAG_VMDQ_POOL), 213 214 0xffc00000 /* mask for RXDCTL register */ 215 }; 216 217 static adapter_info_t igb_82576_cap = { 218 /* limits */ 219 16, /* maximum number of rx queues */ 220 1, /* minimum number of rx queues */ 221 4, /* default number of rx queues */ 222 16, /* maximum number of tx queues */ 223 1, /* minimum number of tx queues */ 224 4, /* default number of tx queues */ 225 65535, /* maximum interrupt throttle rate */ 226 0, /* minimum interrupt throttle rate */ 227 200, /* default interrupt throttle rate */ 228 229 /* function pointers */ 230 igb_enable_adapter_interrupts_82576, 231 igb_setup_msix_82576, 232 233 /* capabilities */ 234 (IGB_FLAG_HAS_DCA | /* capability flags */ 235 IGB_FLAG_VMDQ_POOL | 236 IGB_FLAG_NEED_CTX_IDX), 237 238 0xffe00000 /* mask for RXDCTL register */ 239 }; 240 241 static adapter_info_t igb_82580_cap = { 242 /* limits */ 243 8, /* maximum number of rx queues */ 244 1, /* minimum number of rx queues */ 245 4, /* default number of rx queues */ 246 8, /* maximum number of tx queues */ 247 1, /* minimum number of tx queues */ 248 4, /* default number of tx queues */ 249 65535, /* maximum interrupt throttle rate */ 250 0, /* minimum interrupt throttle rate */ 251 200, /* default interrupt throttle rate */ 252 253 /* function pointers */ 254 igb_enable_adapter_interrupts_82580, 255 igb_setup_msix_82580, 256 257 /* capabilities */ 258 (IGB_FLAG_HAS_DCA | /* capability flags */ 259 IGB_FLAG_VMDQ_POOL | 260 IGB_FLAG_NEED_CTX_IDX), 261 262 0xffe00000 /* mask for RXDCTL register */ 263 }; 264 265 /* 266 * Module Initialization Functions 267 */ 268 269 int 270 _init(void) 271 { 272 int status; 273 274 mac_init_ops(&igb_dev_ops, MODULE_NAME); 275 276 status = mod_install(&igb_modlinkage); 277 278 if (status != DDI_SUCCESS) { 279 mac_fini_ops(&igb_dev_ops); 280 } 281 282 return (status); 283 } 284 285 int 286 _fini(void) 287 { 288 int status; 289 290 status = mod_remove(&igb_modlinkage); 291 292 if (status == DDI_SUCCESS) { 293 mac_fini_ops(&igb_dev_ops); 294 } 295 296 return (status); 297 298 } 299 300 int 301 _info(struct modinfo *modinfop) 302 { 303 int status; 304 305 status = mod_info(&igb_modlinkage, modinfop); 306 307 return (status); 308 } 309 310 /* 311 * igb_attach - driver attach 312 * 313 * This function is the device specific initialization entry 314 * point. This entry point is required and must be written. 315 * The DDI_ATTACH command must be provided in the attach entry 316 * point. When attach() is called with cmd set to DDI_ATTACH, 317 * all normal kernel services (such as kmem_alloc(9F)) are 318 * available for use by the driver. 319 * 320 * The attach() function will be called once for each instance 321 * of the device on the system with cmd set to DDI_ATTACH. 322 * Until attach() succeeds, the only driver entry points which 323 * may be called are open(9E) and getinfo(9E). 324 */ 325 static int 326 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 327 { 328 igb_t *igb; 329 struct igb_osdep *osdep; 330 struct e1000_hw *hw; 331 int instance; 332 333 /* 334 * Check the command and perform corresponding operations 335 */ 336 switch (cmd) { 337 default: 338 return (DDI_FAILURE); 339 340 case DDI_RESUME: 341 return (igb_resume(devinfo)); 342 343 case DDI_ATTACH: 344 break; 345 } 346 347 /* Get the device instance */ 348 instance = ddi_get_instance(devinfo); 349 350 /* Allocate memory for the instance data structure */ 351 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP); 352 353 igb->dip = devinfo; 354 igb->instance = instance; 355 356 hw = &igb->hw; 357 osdep = &igb->osdep; 358 hw->back = osdep; 359 osdep->igb = igb; 360 361 /* Attach the instance pointer to the dev_info data structure */ 362 ddi_set_driver_private(devinfo, igb); 363 364 365 /* Initialize for fma support */ 366 igb->fm_capabilities = igb_get_prop(igb, "fm-capable", 367 0, 0x0f, 368 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 369 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 370 igb_fm_init(igb); 371 igb->attach_progress |= ATTACH_PROGRESS_FMINIT; 372 373 /* 374 * Map PCI config space registers 375 */ 376 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 377 igb_error(igb, "Failed to map PCI configurations"); 378 goto attach_fail; 379 } 380 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 381 382 /* 383 * Identify the chipset family 384 */ 385 if (igb_identify_hardware(igb) != IGB_SUCCESS) { 386 igb_error(igb, "Failed to identify hardware"); 387 goto attach_fail; 388 } 389 390 /* 391 * Map device registers 392 */ 393 if (igb_regs_map(igb) != IGB_SUCCESS) { 394 igb_error(igb, "Failed to map device registers"); 395 goto attach_fail; 396 } 397 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 398 399 /* 400 * Initialize driver parameters 401 */ 402 igb_init_properties(igb); 403 igb->attach_progress |= ATTACH_PROGRESS_PROPS; 404 405 /* 406 * Allocate interrupts 407 */ 408 if (igb_alloc_intrs(igb) != IGB_SUCCESS) { 409 igb_error(igb, "Failed to allocate interrupts"); 410 goto attach_fail; 411 } 412 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 413 414 /* 415 * Allocate rx/tx rings based on the ring numbers. 416 * The actual numbers of rx/tx rings are decided by the number of 417 * allocated interrupt vectors, so we should allocate the rings after 418 * interrupts are allocated. 419 */ 420 if (igb_alloc_rings(igb) != IGB_SUCCESS) { 421 igb_error(igb, "Failed to allocate rx/tx rings or groups"); 422 goto attach_fail; 423 } 424 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 425 426 /* 427 * Add interrupt handlers 428 */ 429 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) { 430 igb_error(igb, "Failed to add interrupt handlers"); 431 goto attach_fail; 432 } 433 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 434 435 /* 436 * Initialize driver parameters 437 */ 438 if (igb_init_driver_settings(igb) != IGB_SUCCESS) { 439 igb_error(igb, "Failed to initialize driver settings"); 440 goto attach_fail; 441 } 442 443 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) { 444 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 445 goto attach_fail; 446 } 447 448 /* 449 * Initialize mutexes for this device. 450 * Do this before enabling the interrupt handler and 451 * register the softint to avoid the condition where 452 * interrupt handler can try using uninitialized mutex 453 */ 454 igb_init_locks(igb); 455 igb->attach_progress |= ATTACH_PROGRESS_LOCKS; 456 457 /* 458 * Allocate DMA resources 459 */ 460 if (igb_alloc_dma(igb) != IGB_SUCCESS) { 461 igb_error(igb, "Failed to allocate DMA resources"); 462 goto attach_fail; 463 } 464 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_DMA; 465 466 /* 467 * Initialize the adapter and setup the rx/tx rings 468 */ 469 if (igb_init(igb) != IGB_SUCCESS) { 470 igb_error(igb, "Failed to initialize adapter"); 471 goto attach_fail; 472 } 473 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 474 475 /* 476 * Initialize statistics 477 */ 478 if (igb_init_stats(igb) != IGB_SUCCESS) { 479 igb_error(igb, "Failed to initialize statistics"); 480 goto attach_fail; 481 } 482 igb->attach_progress |= ATTACH_PROGRESS_STATS; 483 484 /* 485 * Initialize NDD parameters 486 */ 487 if (igb_nd_init(igb) != IGB_SUCCESS) { 488 igb_error(igb, "Failed to initialize ndd"); 489 goto attach_fail; 490 } 491 igb->attach_progress |= ATTACH_PROGRESS_NDD; 492 493 /* 494 * Register the driver to the MAC 495 */ 496 if (igb_register_mac(igb) != IGB_SUCCESS) { 497 igb_error(igb, "Failed to register MAC"); 498 goto attach_fail; 499 } 500 igb->attach_progress |= ATTACH_PROGRESS_MAC; 501 502 /* 503 * Now that mutex locks are initialized, and the chip is also 504 * initialized, enable interrupts. 505 */ 506 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 507 igb_error(igb, "Failed to enable DDI interrupts"); 508 goto attach_fail; 509 } 510 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 511 512 igb_log(igb, "%s", igb_version); 513 atomic_or_32(&igb->igb_state, IGB_INITIALIZED); 514 515 return (DDI_SUCCESS); 516 517 attach_fail: 518 igb_unconfigure(devinfo, igb); 519 return (DDI_FAILURE); 520 } 521 522 /* 523 * igb_detach - driver detach 524 * 525 * The detach() function is the complement of the attach routine. 526 * If cmd is set to DDI_DETACH, detach() is used to remove the 527 * state associated with a given instance of a device node 528 * prior to the removal of that instance from the system. 529 * 530 * The detach() function will be called once for each instance 531 * of the device for which there has been a successful attach() 532 * once there are no longer any opens on the device. 533 * 534 * Interrupts routine are disabled, All memory allocated by this 535 * driver are freed. 536 */ 537 static int 538 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 539 { 540 igb_t *igb; 541 542 /* 543 * Check detach command 544 */ 545 switch (cmd) { 546 default: 547 return (DDI_FAILURE); 548 549 case DDI_SUSPEND: 550 return (igb_suspend(devinfo)); 551 552 case DDI_DETACH: 553 break; 554 } 555 556 557 /* 558 * Get the pointer to the driver private data structure 559 */ 560 igb = (igb_t *)ddi_get_driver_private(devinfo); 561 if (igb == NULL) 562 return (DDI_FAILURE); 563 564 /* 565 * Unregister MAC. If failed, we have to fail the detach 566 */ 567 if (mac_unregister(igb->mac_hdl) != 0) { 568 igb_error(igb, "Failed to unregister MAC"); 569 return (DDI_FAILURE); 570 } 571 igb->attach_progress &= ~ATTACH_PROGRESS_MAC; 572 573 /* 574 * If the device is still running, it needs to be stopped first. 575 * This check is necessary because under some specific circumstances, 576 * the detach routine can be called without stopping the interface 577 * first. 578 */ 579 mutex_enter(&igb->gen_lock); 580 if (igb->igb_state & IGB_STARTED) { 581 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 582 igb_stop(igb); 583 mutex_exit(&igb->gen_lock); 584 /* Disable and stop the watchdog timer */ 585 igb_disable_watchdog_timer(igb); 586 } else 587 mutex_exit(&igb->gen_lock); 588 589 /* 590 * Check if there are still rx buffers held by the upper layer. 591 * If so, fail the detach. 592 */ 593 if (!igb_rx_drain(igb)) 594 return (DDI_FAILURE); 595 596 /* 597 * Do the remaining unconfigure routines 598 */ 599 igb_unconfigure(devinfo, igb); 600 601 return (DDI_SUCCESS); 602 } 603 604 /* 605 * quiesce(9E) entry point. 606 * 607 * This function is called when the system is single-threaded at high 608 * PIL with preemption disabled. Therefore, this function must not be 609 * blocked. 610 * 611 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 612 * DDI_FAILURE indicates an error condition and should almost never happen. 613 */ 614 static int 615 igb_quiesce(dev_info_t *devinfo) 616 { 617 igb_t *igb; 618 struct e1000_hw *hw; 619 620 igb = (igb_t *)ddi_get_driver_private(devinfo); 621 622 if (igb == NULL) 623 return (DDI_FAILURE); 624 625 hw = &igb->hw; 626 627 /* 628 * Disable the adapter interrupts 629 */ 630 igb_disable_adapter_interrupts(igb); 631 632 /* Tell firmware driver is no longer in control */ 633 igb_release_driver_control(hw); 634 635 /* 636 * Reset the chipset 637 */ 638 (void) e1000_reset_hw(hw); 639 640 /* 641 * Reset PHY if possible 642 */ 643 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 644 (void) e1000_phy_hw_reset(hw); 645 646 return (DDI_SUCCESS); 647 } 648 649 /* 650 * igb_unconfigure - release all resources held by this instance 651 */ 652 static void 653 igb_unconfigure(dev_info_t *devinfo, igb_t *igb) 654 { 655 /* 656 * Disable interrupt 657 */ 658 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 659 (void) igb_disable_intrs(igb); 660 } 661 662 /* 663 * Unregister MAC 664 */ 665 if (igb->attach_progress & ATTACH_PROGRESS_MAC) { 666 (void) mac_unregister(igb->mac_hdl); 667 } 668 669 /* 670 * Free ndd parameters 671 */ 672 if (igb->attach_progress & ATTACH_PROGRESS_NDD) { 673 igb_nd_cleanup(igb); 674 } 675 676 /* 677 * Free statistics 678 */ 679 if (igb->attach_progress & ATTACH_PROGRESS_STATS) { 680 kstat_delete((kstat_t *)igb->igb_ks); 681 } 682 683 /* 684 * Remove interrupt handlers 685 */ 686 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 687 igb_rem_intr_handlers(igb); 688 } 689 690 /* 691 * Remove interrupts 692 */ 693 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 694 igb_rem_intrs(igb); 695 } 696 697 /* 698 * Remove driver properties 699 */ 700 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) { 701 (void) ddi_prop_remove_all(devinfo); 702 } 703 704 /* 705 * Release the DMA resources of rx/tx rings 706 */ 707 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_DMA) { 708 igb_free_dma(igb); 709 } 710 711 /* 712 * Stop the adapter 713 */ 714 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) { 715 mutex_enter(&igb->gen_lock); 716 igb_stop_adapter(igb); 717 mutex_exit(&igb->gen_lock); 718 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 719 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED); 720 } 721 722 /* 723 * Free multicast table 724 */ 725 igb_release_multicast(igb); 726 727 /* 728 * Free register handle 729 */ 730 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 731 if (igb->osdep.reg_handle != NULL) 732 ddi_regs_map_free(&igb->osdep.reg_handle); 733 } 734 735 /* 736 * Free PCI config handle 737 */ 738 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 739 if (igb->osdep.cfg_handle != NULL) 740 pci_config_teardown(&igb->osdep.cfg_handle); 741 } 742 743 /* 744 * Free locks 745 */ 746 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) { 747 igb_destroy_locks(igb); 748 } 749 750 /* 751 * Free the rx/tx rings 752 */ 753 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 754 igb_free_rings(igb); 755 } 756 757 /* 758 * Remove FMA 759 */ 760 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) { 761 igb_fm_fini(igb); 762 } 763 764 /* 765 * Free the driver data structure 766 */ 767 kmem_free(igb, sizeof (igb_t)); 768 769 ddi_set_driver_private(devinfo, NULL); 770 } 771 772 /* 773 * igb_register_mac - Register the driver and its function pointers with 774 * the GLD interface 775 */ 776 static int 777 igb_register_mac(igb_t *igb) 778 { 779 struct e1000_hw *hw = &igb->hw; 780 mac_register_t *mac; 781 int status; 782 783 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 784 return (IGB_FAILURE); 785 786 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 787 mac->m_driver = igb; 788 mac->m_dip = igb->dip; 789 mac->m_src_addr = hw->mac.addr; 790 mac->m_callbacks = &igb_m_callbacks; 791 mac->m_min_sdu = 0; 792 mac->m_max_sdu = igb->max_frame_size - 793 sizeof (struct ether_vlan_header) - ETHERFCSL; 794 mac->m_margin = VLAN_TAGSZ; 795 mac->m_v12n = MAC_VIRT_LEVEL1; 796 797 status = mac_register(mac, &igb->mac_hdl); 798 799 mac_free(mac); 800 801 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE); 802 } 803 804 /* 805 * igb_identify_hardware - Identify the type of the chipset 806 */ 807 static int 808 igb_identify_hardware(igb_t *igb) 809 { 810 struct e1000_hw *hw = &igb->hw; 811 struct igb_osdep *osdep = &igb->osdep; 812 813 /* 814 * Get the device id 815 */ 816 hw->vendor_id = 817 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 818 hw->device_id = 819 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 820 hw->revision_id = 821 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 822 hw->subsystem_device_id = 823 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 824 hw->subsystem_vendor_id = 825 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 826 827 /* 828 * Set the mac type of the adapter based on the device id 829 */ 830 if (e1000_set_mac_type(hw) != E1000_SUCCESS) { 831 return (IGB_FAILURE); 832 } 833 834 /* 835 * Install adapter capabilities based on mac type 836 */ 837 switch (hw->mac.type) { 838 case e1000_82575: 839 igb->capab = &igb_82575_cap; 840 break; 841 case e1000_82576: 842 igb->capab = &igb_82576_cap; 843 break; 844 case e1000_82580: 845 igb->capab = &igb_82580_cap; 846 break; 847 default: 848 return (IGB_FAILURE); 849 } 850 851 return (IGB_SUCCESS); 852 } 853 854 /* 855 * igb_regs_map - Map the device registers 856 */ 857 static int 858 igb_regs_map(igb_t *igb) 859 { 860 dev_info_t *devinfo = igb->dip; 861 struct e1000_hw *hw = &igb->hw; 862 struct igb_osdep *osdep = &igb->osdep; 863 off_t mem_size; 864 865 /* 866 * First get the size of device registers to be mapped. 867 */ 868 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) != 869 DDI_SUCCESS) { 870 return (IGB_FAILURE); 871 } 872 873 /* 874 * Call ddi_regs_map_setup() to map registers 875 */ 876 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET, 877 (caddr_t *)&hw->hw_addr, 0, 878 mem_size, &igb_regs_acc_attr, 879 &osdep->reg_handle)) != DDI_SUCCESS) { 880 return (IGB_FAILURE); 881 } 882 883 return (IGB_SUCCESS); 884 } 885 886 /* 887 * igb_init_properties - Initialize driver properties 888 */ 889 static void 890 igb_init_properties(igb_t *igb) 891 { 892 /* 893 * Get conf file properties, including link settings 894 * jumbo frames, ring number, descriptor number, etc. 895 */ 896 igb_get_conf(igb); 897 } 898 899 /* 900 * igb_init_driver_settings - Initialize driver settings 901 * 902 * The settings include hardware function pointers, bus information, 903 * rx/tx rings settings, link state, and any other parameters that 904 * need to be setup during driver initialization. 905 */ 906 static int 907 igb_init_driver_settings(igb_t *igb) 908 { 909 struct e1000_hw *hw = &igb->hw; 910 igb_rx_ring_t *rx_ring; 911 igb_tx_ring_t *tx_ring; 912 uint32_t rx_size; 913 uint32_t tx_size; 914 int i; 915 916 /* 917 * Initialize chipset specific hardware function pointers 918 */ 919 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) { 920 return (IGB_FAILURE); 921 } 922 923 /* 924 * Get bus information 925 */ 926 if (e1000_get_bus_info(hw) != E1000_SUCCESS) { 927 return (IGB_FAILURE); 928 } 929 930 /* 931 * Get the system page size 932 */ 933 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1); 934 935 /* 936 * Set rx buffer size 937 * The IP header alignment room is counted in the calculation. 938 * The rx buffer size is in unit of 1K that is required by the 939 * chipset hardware. 940 */ 941 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM; 942 igb->rx_buf_size = ((rx_size >> 10) + 943 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 944 945 /* 946 * Set tx buffer size 947 */ 948 tx_size = igb->max_frame_size; 949 igb->tx_buf_size = ((tx_size >> 10) + 950 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 951 952 /* 953 * Initialize rx/tx rings parameters 954 */ 955 for (i = 0; i < igb->num_rx_rings; i++) { 956 rx_ring = &igb->rx_rings[i]; 957 rx_ring->index = i; 958 rx_ring->igb = igb; 959 960 rx_ring->ring_size = igb->rx_ring_size; 961 rx_ring->free_list_size = igb->rx_ring_size; 962 rx_ring->copy_thresh = igb->rx_copy_thresh; 963 rx_ring->limit_per_intr = igb->rx_limit_per_intr; 964 } 965 966 for (i = 0; i < igb->num_tx_rings; i++) { 967 tx_ring = &igb->tx_rings[i]; 968 tx_ring->index = i; 969 tx_ring->igb = igb; 970 if (igb->tx_head_wb_enable) 971 tx_ring->tx_recycle = igb_tx_recycle_head_wb; 972 else 973 tx_ring->tx_recycle = igb_tx_recycle_legacy; 974 975 tx_ring->ring_size = igb->tx_ring_size; 976 tx_ring->free_list_size = igb->tx_ring_size + 977 (igb->tx_ring_size >> 1); 978 tx_ring->copy_thresh = igb->tx_copy_thresh; 979 tx_ring->recycle_thresh = igb->tx_recycle_thresh; 980 tx_ring->overload_thresh = igb->tx_overload_thresh; 981 tx_ring->resched_thresh = igb->tx_resched_thresh; 982 } 983 984 /* 985 * Initialize values of interrupt throttling rates 986 */ 987 for (i = 1; i < MAX_NUM_EITR; i++) 988 igb->intr_throttling[i] = igb->intr_throttling[0]; 989 990 /* 991 * The initial link state should be "unknown" 992 */ 993 igb->link_state = LINK_STATE_UNKNOWN; 994 995 return (IGB_SUCCESS); 996 } 997 998 /* 999 * igb_init_locks - Initialize locks 1000 */ 1001 static void 1002 igb_init_locks(igb_t *igb) 1003 { 1004 igb_rx_ring_t *rx_ring; 1005 igb_tx_ring_t *tx_ring; 1006 int i; 1007 1008 for (i = 0; i < igb->num_rx_rings; i++) { 1009 rx_ring = &igb->rx_rings[i]; 1010 mutex_init(&rx_ring->rx_lock, NULL, 1011 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1012 mutex_init(&rx_ring->recycle_lock, NULL, 1013 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1014 } 1015 1016 for (i = 0; i < igb->num_tx_rings; i++) { 1017 tx_ring = &igb->tx_rings[i]; 1018 mutex_init(&tx_ring->tx_lock, NULL, 1019 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1020 mutex_init(&tx_ring->recycle_lock, NULL, 1021 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1022 mutex_init(&tx_ring->tcb_head_lock, NULL, 1023 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1024 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1025 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1026 } 1027 1028 mutex_init(&igb->gen_lock, NULL, 1029 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1030 1031 mutex_init(&igb->watchdog_lock, NULL, 1032 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1033 1034 mutex_init(&igb->link_lock, NULL, 1035 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1036 } 1037 1038 /* 1039 * igb_destroy_locks - Destroy locks 1040 */ 1041 static void 1042 igb_destroy_locks(igb_t *igb) 1043 { 1044 igb_rx_ring_t *rx_ring; 1045 igb_tx_ring_t *tx_ring; 1046 int i; 1047 1048 for (i = 0; i < igb->num_rx_rings; i++) { 1049 rx_ring = &igb->rx_rings[i]; 1050 mutex_destroy(&rx_ring->rx_lock); 1051 mutex_destroy(&rx_ring->recycle_lock); 1052 } 1053 1054 for (i = 0; i < igb->num_tx_rings; i++) { 1055 tx_ring = &igb->tx_rings[i]; 1056 mutex_destroy(&tx_ring->tx_lock); 1057 mutex_destroy(&tx_ring->recycle_lock); 1058 mutex_destroy(&tx_ring->tcb_head_lock); 1059 mutex_destroy(&tx_ring->tcb_tail_lock); 1060 } 1061 1062 mutex_destroy(&igb->gen_lock); 1063 mutex_destroy(&igb->watchdog_lock); 1064 mutex_destroy(&igb->link_lock); 1065 } 1066 1067 static int 1068 igb_resume(dev_info_t *devinfo) 1069 { 1070 igb_t *igb; 1071 1072 igb = (igb_t *)ddi_get_driver_private(devinfo); 1073 if (igb == NULL) 1074 return (DDI_FAILURE); 1075 1076 mutex_enter(&igb->gen_lock); 1077 1078 if (igb->igb_state & IGB_STARTED) { 1079 if (igb_start(igb) != IGB_SUCCESS) { 1080 mutex_exit(&igb->gen_lock); 1081 return (DDI_FAILURE); 1082 } 1083 1084 /* 1085 * Enable and start the watchdog timer 1086 */ 1087 igb_enable_watchdog_timer(igb); 1088 } 1089 1090 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED); 1091 1092 mutex_exit(&igb->gen_lock); 1093 1094 return (DDI_SUCCESS); 1095 } 1096 1097 static int 1098 igb_suspend(dev_info_t *devinfo) 1099 { 1100 igb_t *igb; 1101 1102 igb = (igb_t *)ddi_get_driver_private(devinfo); 1103 if (igb == NULL) 1104 return (DDI_FAILURE); 1105 1106 mutex_enter(&igb->gen_lock); 1107 1108 atomic_or_32(&igb->igb_state, IGB_SUSPENDED); 1109 1110 if (!(igb->igb_state & IGB_STARTED)) { 1111 mutex_exit(&igb->gen_lock); 1112 return (DDI_SUCCESS); 1113 } 1114 1115 igb_stop(igb); 1116 1117 mutex_exit(&igb->gen_lock); 1118 1119 /* 1120 * Disable and stop the watchdog timer 1121 */ 1122 igb_disable_watchdog_timer(igb); 1123 1124 return (DDI_SUCCESS); 1125 } 1126 1127 static int 1128 igb_init(igb_t *igb) 1129 { 1130 int i; 1131 1132 mutex_enter(&igb->gen_lock); 1133 1134 /* 1135 * Initilize the adapter 1136 */ 1137 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1138 mutex_exit(&igb->gen_lock); 1139 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1140 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1141 return (IGB_FAILURE); 1142 } 1143 1144 /* 1145 * Setup the rx/tx rings 1146 */ 1147 for (i = 0; i < igb->num_rx_rings; i++) 1148 mutex_enter(&igb->rx_rings[i].rx_lock); 1149 for (i = 0; i < igb->num_tx_rings; i++) 1150 mutex_enter(&igb->tx_rings[i].tx_lock); 1151 1152 igb_setup_rings(igb); 1153 1154 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1155 mutex_exit(&igb->tx_rings[i].tx_lock); 1156 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1157 mutex_exit(&igb->rx_rings[i].rx_lock); 1158 1159 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 1160 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1161 return (IGB_FAILURE); 1162 } 1163 1164 mutex_exit(&igb->gen_lock); 1165 1166 return (IGB_SUCCESS); 1167 } 1168 1169 /* 1170 * igb_init_mac_address - Initialize the default MAC address 1171 * 1172 * On success, the MAC address is entered in the igb->hw.mac.addr 1173 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive 1174 * address register. 1175 * 1176 * Important side effects: 1177 * 1. adapter is reset - this is required to put it in a known state. 1178 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where 1179 * MAC address and all default settings are stored, so a valid checksum 1180 * is required. 1181 */ 1182 static int 1183 igb_init_mac_address(igb_t *igb) 1184 { 1185 struct e1000_hw *hw = &igb->hw; 1186 1187 ASSERT(mutex_owned(&igb->gen_lock)); 1188 1189 /* 1190 * Reset chipset to put the hardware in a known state 1191 * before we try to get MAC address from NVM. 1192 */ 1193 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1194 igb_error(igb, "Adapter reset failed."); 1195 goto init_mac_fail; 1196 } 1197 1198 /* 1199 * NVM validation 1200 */ 1201 if (e1000_validate_nvm_checksum(hw) < 0) { 1202 /* 1203 * Some PCI-E parts fail the first check due to 1204 * the link being in sleep state. Call it again, 1205 * if it fails a second time its a real issue. 1206 */ 1207 if (e1000_validate_nvm_checksum(hw) < 0) { 1208 igb_error(igb, 1209 "Invalid NVM checksum. Please contact " 1210 "the vendor to update the NVM."); 1211 goto init_mac_fail; 1212 } 1213 } 1214 1215 /* 1216 * Get the mac address 1217 * This function should handle SPARC case correctly. 1218 */ 1219 if (!igb_find_mac_address(igb)) { 1220 igb_error(igb, "Failed to get the mac address"); 1221 goto init_mac_fail; 1222 } 1223 1224 /* Validate mac address */ 1225 if (!is_valid_mac_addr(hw->mac.addr)) { 1226 igb_error(igb, "Invalid mac address"); 1227 goto init_mac_fail; 1228 } 1229 1230 return (IGB_SUCCESS); 1231 1232 init_mac_fail: 1233 return (IGB_FAILURE); 1234 } 1235 1236 /* 1237 * igb_init_adapter - Initialize the adapter 1238 */ 1239 static int 1240 igb_init_adapter(igb_t *igb) 1241 { 1242 struct e1000_hw *hw = &igb->hw; 1243 uint32_t pba; 1244 uint32_t high_water; 1245 int i; 1246 1247 ASSERT(mutex_owned(&igb->gen_lock)); 1248 1249 /* 1250 * In order to obtain the default MAC address, this will reset the 1251 * adapter and validate the NVM that the address and many other 1252 * default settings come from. 1253 */ 1254 if (igb_init_mac_address(igb) != IGB_SUCCESS) { 1255 igb_error(igb, "Failed to initialize MAC address"); 1256 goto init_adapter_fail; 1257 } 1258 1259 /* 1260 * Setup flow control 1261 * 1262 * These parameters set thresholds for the adapter's generation(Tx) 1263 * and response(Rx) to Ethernet PAUSE frames. These are just threshold 1264 * settings. Flow control is enabled or disabled in the configuration 1265 * file. 1266 * High-water mark is set down from the top of the rx fifo (not 1267 * sensitive to max_frame_size) and low-water is set just below 1268 * high-water mark. 1269 * The high water mark must be low enough to fit one full frame above 1270 * it in the rx FIFO. Should be the lower of: 1271 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full 1272 * frame. 1273 */ 1274 /* 1275 * The default setting of PBA is correct for 82575 and other supported 1276 * adapters do not have the E1000_PBA register, so PBA value is only 1277 * used for calculation here and is never written to the adapter. 1278 */ 1279 if (hw->mac.type == e1000_82575) { 1280 pba = E1000_PBA_34K; 1281 } else { 1282 pba = E1000_PBA_64K; 1283 } 1284 1285 high_water = min(((pba << 10) * 9 / 10), 1286 ((pba << 10) - igb->max_frame_size)); 1287 1288 if (hw->mac.type == e1000_82575) { 1289 /* 8-byte granularity */ 1290 hw->fc.high_water = high_water & 0xFFF8; 1291 hw->fc.low_water = hw->fc.high_water - 8; 1292 } else { 1293 /* 16-byte granularity */ 1294 hw->fc.high_water = high_water & 0xFFF0; 1295 hw->fc.low_water = hw->fc.high_water - 16; 1296 } 1297 1298 hw->fc.pause_time = E1000_FC_PAUSE_TIME; 1299 hw->fc.send_xon = B_TRUE; 1300 1301 (void) e1000_validate_mdi_setting(hw); 1302 1303 /* 1304 * Reset the chipset hardware the second time to put PBA settings 1305 * into effect. 1306 */ 1307 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1308 igb_error(igb, "Second reset failed"); 1309 goto init_adapter_fail; 1310 } 1311 1312 /* 1313 * Don't wait for auto-negotiation to complete 1314 */ 1315 hw->phy.autoneg_wait_to_complete = B_FALSE; 1316 1317 /* 1318 * Copper options 1319 */ 1320 if (hw->phy.media_type == e1000_media_type_copper) { 1321 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 1322 hw->phy.disable_polarity_correction = B_FALSE; 1323 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */ 1324 } 1325 1326 /* 1327 * Initialize link settings 1328 */ 1329 (void) igb_setup_link(igb, B_FALSE); 1330 1331 /* 1332 * Configure/Initialize hardware 1333 */ 1334 if (e1000_init_hw(hw) != E1000_SUCCESS) { 1335 igb_error(igb, "Failed to initialize hardware"); 1336 goto init_adapter_fail; 1337 } 1338 1339 /* 1340 * Start the link setup timer 1341 */ 1342 igb_start_link_timer(igb); 1343 1344 /* 1345 * Disable wakeup control by default 1346 */ 1347 E1000_WRITE_REG(hw, E1000_WUC, 0); 1348 1349 /* 1350 * Record phy info in hw struct 1351 */ 1352 (void) e1000_get_phy_info(hw); 1353 1354 /* 1355 * Make sure driver has control 1356 */ 1357 igb_get_driver_control(hw); 1358 1359 /* 1360 * Restore LED settings to the default from EEPROM 1361 * to meet the standard for Sun platforms. 1362 */ 1363 (void) e1000_cleanup_led(hw); 1364 1365 /* 1366 * Setup MSI-X interrupts 1367 */ 1368 if (igb->intr_type == DDI_INTR_TYPE_MSIX) 1369 igb->capab->setup_msix(igb); 1370 1371 /* 1372 * Initialize unicast addresses. 1373 */ 1374 igb_init_unicst(igb); 1375 1376 /* 1377 * Setup and initialize the mctable structures. 1378 */ 1379 igb_setup_multicst(igb); 1380 1381 /* 1382 * Set interrupt throttling rate 1383 */ 1384 for (i = 0; i < igb->intr_cnt; i++) 1385 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]); 1386 1387 /* 1388 * Save the state of the phy 1389 */ 1390 igb_get_phy_state(igb); 1391 1392 return (IGB_SUCCESS); 1393 1394 init_adapter_fail: 1395 /* 1396 * Reset PHY if possible 1397 */ 1398 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 1399 (void) e1000_phy_hw_reset(hw); 1400 1401 return (IGB_FAILURE); 1402 } 1403 1404 /* 1405 * igb_stop_adapter - Stop the adapter 1406 */ 1407 static void 1408 igb_stop_adapter(igb_t *igb) 1409 { 1410 struct e1000_hw *hw = &igb->hw; 1411 1412 ASSERT(mutex_owned(&igb->gen_lock)); 1413 1414 /* Stop the link setup timer */ 1415 igb_stop_link_timer(igb); 1416 1417 /* Tell firmware driver is no longer in control */ 1418 igb_release_driver_control(hw); 1419 1420 /* 1421 * Reset the chipset 1422 */ 1423 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1424 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1425 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1426 } 1427 1428 /* 1429 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient 1430 */ 1431 } 1432 1433 /* 1434 * igb_reset - Reset the chipset and restart the driver. 1435 * 1436 * It involves stopping and re-starting the chipset, 1437 * and re-configuring the rx/tx rings. 1438 */ 1439 static int 1440 igb_reset(igb_t *igb) 1441 { 1442 int i; 1443 1444 mutex_enter(&igb->gen_lock); 1445 1446 ASSERT(igb->igb_state & IGB_STARTED); 1447 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 1448 1449 /* 1450 * Disable the adapter interrupts to stop any rx/tx activities 1451 * before draining pending data and resetting hardware. 1452 */ 1453 igb_disable_adapter_interrupts(igb); 1454 1455 /* 1456 * Drain the pending transmit packets 1457 */ 1458 (void) igb_tx_drain(igb); 1459 1460 for (i = 0; i < igb->num_rx_rings; i++) 1461 mutex_enter(&igb->rx_rings[i].rx_lock); 1462 for (i = 0; i < igb->num_tx_rings; i++) 1463 mutex_enter(&igb->tx_rings[i].tx_lock); 1464 1465 /* 1466 * Stop the adapter 1467 */ 1468 igb_stop_adapter(igb); 1469 1470 /* 1471 * Clean the pending tx data/resources 1472 */ 1473 igb_tx_clean(igb); 1474 1475 /* 1476 * Start the adapter 1477 */ 1478 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1479 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1480 goto reset_failure; 1481 } 1482 1483 /* 1484 * Setup the rx/tx rings 1485 */ 1486 igb_setup_rings(igb); 1487 1488 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL)); 1489 1490 /* 1491 * Enable adapter interrupts 1492 * The interrupts must be enabled after the driver state is START 1493 */ 1494 igb->capab->enable_intr(igb); 1495 1496 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1497 goto reset_failure; 1498 1499 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1500 goto reset_failure; 1501 1502 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1503 mutex_exit(&igb->tx_rings[i].tx_lock); 1504 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1505 mutex_exit(&igb->rx_rings[i].rx_lock); 1506 1507 atomic_or_32(&igb->igb_state, IGB_STARTED); 1508 1509 mutex_exit(&igb->gen_lock); 1510 1511 return (IGB_SUCCESS); 1512 1513 reset_failure: 1514 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1515 mutex_exit(&igb->tx_rings[i].tx_lock); 1516 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1517 mutex_exit(&igb->rx_rings[i].rx_lock); 1518 1519 mutex_exit(&igb->gen_lock); 1520 1521 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1522 1523 return (IGB_FAILURE); 1524 } 1525 1526 /* 1527 * igb_tx_clean - Clean the pending transmit packets and DMA resources 1528 */ 1529 static void 1530 igb_tx_clean(igb_t *igb) 1531 { 1532 igb_tx_ring_t *tx_ring; 1533 tx_control_block_t *tcb; 1534 link_list_t pending_list; 1535 uint32_t desc_num; 1536 int i, j; 1537 1538 LINK_LIST_INIT(&pending_list); 1539 1540 for (i = 0; i < igb->num_tx_rings; i++) { 1541 tx_ring = &igb->tx_rings[i]; 1542 1543 mutex_enter(&tx_ring->recycle_lock); 1544 1545 /* 1546 * Clean the pending tx data - the pending packets in the 1547 * work_list that have no chances to be transmitted again. 1548 * 1549 * We must ensure the chipset is stopped or the link is down 1550 * before cleaning the transmit packets. 1551 */ 1552 desc_num = 0; 1553 for (j = 0; j < tx_ring->ring_size; j++) { 1554 tcb = tx_ring->work_list[j]; 1555 if (tcb != NULL) { 1556 desc_num += tcb->desc_num; 1557 1558 tx_ring->work_list[j] = NULL; 1559 1560 igb_free_tcb(tcb); 1561 1562 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1563 } 1564 } 1565 1566 if (desc_num > 0) { 1567 atomic_add_32(&tx_ring->tbd_free, desc_num); 1568 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1569 1570 /* 1571 * Reset the head and tail pointers of the tbd ring; 1572 * Reset the head write-back if it is enabled. 1573 */ 1574 tx_ring->tbd_head = 0; 1575 tx_ring->tbd_tail = 0; 1576 if (igb->tx_head_wb_enable) 1577 *tx_ring->tbd_head_wb = 0; 1578 1579 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0); 1580 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0); 1581 } 1582 1583 mutex_exit(&tx_ring->recycle_lock); 1584 1585 /* 1586 * Add the tx control blocks in the pending list to 1587 * the free list. 1588 */ 1589 igb_put_free_list(tx_ring, &pending_list); 1590 } 1591 } 1592 1593 /* 1594 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted 1595 */ 1596 static boolean_t 1597 igb_tx_drain(igb_t *igb) 1598 { 1599 igb_tx_ring_t *tx_ring; 1600 boolean_t done; 1601 int i, j; 1602 1603 /* 1604 * Wait for a specific time to allow pending tx packets 1605 * to be transmitted. 1606 * 1607 * Check the counter tbd_free to see if transmission is done. 1608 * No lock protection is needed here. 1609 * 1610 * Return B_TRUE if all pending packets have been transmitted; 1611 * Otherwise return B_FALSE; 1612 */ 1613 for (i = 0; i < TX_DRAIN_TIME; i++) { 1614 1615 done = B_TRUE; 1616 for (j = 0; j < igb->num_tx_rings; j++) { 1617 tx_ring = &igb->tx_rings[j]; 1618 done = done && 1619 (tx_ring->tbd_free == tx_ring->ring_size); 1620 } 1621 1622 if (done) 1623 break; 1624 1625 msec_delay(1); 1626 } 1627 1628 return (done); 1629 } 1630 1631 /* 1632 * igb_rx_drain - Wait for all rx buffers to be released by upper layer 1633 */ 1634 static boolean_t 1635 igb_rx_drain(igb_t *igb) 1636 { 1637 igb_rx_ring_t *rx_ring; 1638 boolean_t done; 1639 int i, j; 1640 1641 /* 1642 * Polling the rx free list to check if those rx buffers held by 1643 * the upper layer are released. 1644 * 1645 * Check the counter rcb_free to see if all pending buffers are 1646 * released. No lock protection is needed here. 1647 * 1648 * Return B_TRUE if all pending buffers have been released; 1649 * Otherwise return B_FALSE; 1650 */ 1651 for (i = 0; i < RX_DRAIN_TIME; i++) { 1652 1653 done = B_TRUE; 1654 for (j = 0; j < igb->num_rx_rings; j++) { 1655 rx_ring = &igb->rx_rings[j]; 1656 done = done && 1657 (rx_ring->rcb_free == rx_ring->free_list_size); 1658 } 1659 1660 if (done) 1661 break; 1662 1663 msec_delay(1); 1664 } 1665 1666 return (done); 1667 } 1668 1669 /* 1670 * igb_start - Start the driver/chipset 1671 */ 1672 int 1673 igb_start(igb_t *igb) 1674 { 1675 int i; 1676 1677 ASSERT(mutex_owned(&igb->gen_lock)); 1678 1679 for (i = 0; i < igb->num_rx_rings; i++) 1680 mutex_enter(&igb->rx_rings[i].rx_lock); 1681 for (i = 0; i < igb->num_tx_rings; i++) 1682 mutex_enter(&igb->tx_rings[i].tx_lock); 1683 1684 /* 1685 * Start the adapter 1686 */ 1687 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) { 1688 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1689 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1690 goto start_failure; 1691 } 1692 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 1693 1694 /* 1695 * Setup the rx/tx rings 1696 */ 1697 igb_setup_rings(igb); 1698 } 1699 1700 /* 1701 * Enable adapter interrupts 1702 * The interrupts must be enabled after the driver state is START 1703 */ 1704 igb->capab->enable_intr(igb); 1705 1706 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1707 goto start_failure; 1708 1709 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1710 goto start_failure; 1711 1712 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1713 mutex_exit(&igb->tx_rings[i].tx_lock); 1714 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1715 mutex_exit(&igb->rx_rings[i].rx_lock); 1716 1717 return (IGB_SUCCESS); 1718 1719 start_failure: 1720 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1721 mutex_exit(&igb->tx_rings[i].tx_lock); 1722 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1723 mutex_exit(&igb->rx_rings[i].rx_lock); 1724 1725 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1726 1727 return (IGB_FAILURE); 1728 } 1729 1730 /* 1731 * igb_stop - Stop the driver/chipset 1732 */ 1733 void 1734 igb_stop(igb_t *igb) 1735 { 1736 int i; 1737 1738 ASSERT(mutex_owned(&igb->gen_lock)); 1739 1740 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER; 1741 1742 /* 1743 * Disable the adapter interrupts 1744 */ 1745 igb_disable_adapter_interrupts(igb); 1746 1747 /* 1748 * Drain the pending tx packets 1749 */ 1750 (void) igb_tx_drain(igb); 1751 1752 for (i = 0; i < igb->num_rx_rings; i++) 1753 mutex_enter(&igb->rx_rings[i].rx_lock); 1754 for (i = 0; i < igb->num_tx_rings; i++) 1755 mutex_enter(&igb->tx_rings[i].tx_lock); 1756 1757 /* 1758 * Stop the adapter 1759 */ 1760 igb_stop_adapter(igb); 1761 1762 /* 1763 * Clean the pending tx data/resources 1764 */ 1765 igb_tx_clean(igb); 1766 1767 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1768 mutex_exit(&igb->tx_rings[i].tx_lock); 1769 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1770 mutex_exit(&igb->rx_rings[i].rx_lock); 1771 1772 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1773 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1774 } 1775 1776 /* 1777 * igb_alloc_rings - Allocate memory space for rx/tx rings 1778 */ 1779 static int 1780 igb_alloc_rings(igb_t *igb) 1781 { 1782 /* 1783 * Allocate memory space for rx rings 1784 */ 1785 igb->rx_rings = kmem_zalloc( 1786 sizeof (igb_rx_ring_t) * igb->num_rx_rings, 1787 KM_NOSLEEP); 1788 1789 if (igb->rx_rings == NULL) { 1790 return (IGB_FAILURE); 1791 } 1792 1793 /* 1794 * Allocate memory space for tx rings 1795 */ 1796 igb->tx_rings = kmem_zalloc( 1797 sizeof (igb_tx_ring_t) * igb->num_tx_rings, 1798 KM_NOSLEEP); 1799 1800 if (igb->tx_rings == NULL) { 1801 kmem_free(igb->rx_rings, 1802 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1803 igb->rx_rings = NULL; 1804 return (IGB_FAILURE); 1805 } 1806 1807 /* 1808 * Allocate memory space for rx ring groups 1809 */ 1810 igb->rx_groups = kmem_zalloc( 1811 sizeof (igb_rx_group_t) * igb->num_rx_groups, 1812 KM_NOSLEEP); 1813 1814 if (igb->rx_groups == NULL) { 1815 kmem_free(igb->rx_rings, 1816 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1817 kmem_free(igb->tx_rings, 1818 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1819 igb->rx_rings = NULL; 1820 igb->tx_rings = NULL; 1821 return (IGB_FAILURE); 1822 } 1823 1824 return (IGB_SUCCESS); 1825 } 1826 1827 /* 1828 * igb_free_rings - Free the memory space of rx/tx rings. 1829 */ 1830 static void 1831 igb_free_rings(igb_t *igb) 1832 { 1833 if (igb->rx_rings != NULL) { 1834 kmem_free(igb->rx_rings, 1835 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1836 igb->rx_rings = NULL; 1837 } 1838 1839 if (igb->tx_rings != NULL) { 1840 kmem_free(igb->tx_rings, 1841 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1842 igb->tx_rings = NULL; 1843 } 1844 1845 if (igb->rx_groups != NULL) { 1846 kmem_free(igb->rx_groups, 1847 sizeof (igb_rx_group_t) * igb->num_rx_groups); 1848 igb->rx_groups = NULL; 1849 } 1850 } 1851 1852 /* 1853 * igb_setup_rings - Setup rx/tx rings 1854 */ 1855 static void 1856 igb_setup_rings(igb_t *igb) 1857 { 1858 /* 1859 * Setup the rx/tx rings, including the following: 1860 * 1861 * 1. Setup the descriptor ring and the control block buffers; 1862 * 2. Initialize necessary registers for receive/transmit; 1863 * 3. Initialize software pointers/parameters for receive/transmit; 1864 */ 1865 igb_setup_rx(igb); 1866 1867 igb_setup_tx(igb); 1868 } 1869 1870 static void 1871 igb_setup_rx_ring(igb_rx_ring_t *rx_ring) 1872 { 1873 igb_t *igb = rx_ring->igb; 1874 struct e1000_hw *hw = &igb->hw; 1875 rx_control_block_t *rcb; 1876 union e1000_adv_rx_desc *rbd; 1877 uint32_t size; 1878 uint32_t buf_low; 1879 uint32_t buf_high; 1880 uint32_t rxdctl; 1881 int i; 1882 1883 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1884 ASSERT(mutex_owned(&igb->gen_lock)); 1885 1886 /* 1887 * Initialize descriptor ring with buffer addresses 1888 */ 1889 for (i = 0; i < igb->rx_ring_size; i++) { 1890 rcb = rx_ring->work_list[i]; 1891 rbd = &rx_ring->rbd_ring[i]; 1892 1893 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1894 rbd->read.hdr_addr = NULL; 1895 } 1896 1897 /* 1898 * Initialize the base address registers 1899 */ 1900 buf_low = (uint32_t)rx_ring->rbd_area.dma_address; 1901 buf_high = (uint32_t)(rx_ring->rbd_area.dma_address >> 32); 1902 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high); 1903 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low); 1904 1905 /* 1906 * Initialize the length register 1907 */ 1908 size = rx_ring->ring_size * sizeof (union e1000_adv_rx_desc); 1909 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size); 1910 1911 /* 1912 * Initialize buffer size & descriptor type 1913 */ 1914 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index), 1915 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) | 1916 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF)); 1917 1918 /* 1919 * Setup the Receive Descriptor Control Register (RXDCTL) 1920 */ 1921 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index)); 1922 rxdctl &= igb->capab->rxdctl_mask; 1923 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 1924 rxdctl |= 16; /* pthresh */ 1925 rxdctl |= 8 << 8; /* hthresh */ 1926 rxdctl |= 1 << 16; /* wthresh */ 1927 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl); 1928 1929 rx_ring->rbd_next = 0; 1930 1931 /* 1932 * Note: Considering the case that the chipset is being reset 1933 * and there are still some buffers held by the upper layer, 1934 * we should not reset the values of rcb_head, rcb_tail and 1935 * rcb_free; 1936 */ 1937 if (igb->igb_state == IGB_UNKNOWN) { 1938 rx_ring->rcb_head = 0; 1939 rx_ring->rcb_tail = 0; 1940 rx_ring->rcb_free = rx_ring->free_list_size; 1941 } 1942 } 1943 1944 static void 1945 igb_setup_rx(igb_t *igb) 1946 { 1947 igb_rx_ring_t *rx_ring; 1948 igb_rx_group_t *rx_group; 1949 struct e1000_hw *hw = &igb->hw; 1950 uint32_t rctl, rxcsum; 1951 uint32_t ring_per_group; 1952 int i; 1953 1954 /* 1955 * Setup the Receive Control Register (RCTL), and enable the 1956 * receiver. The initial configuration is to: enable the receiver, 1957 * accept broadcasts, discard bad packets, accept long packets, 1958 * disable VLAN filter checking, and set receive buffer size to 1959 * 2k. For 82575, also set the receive descriptor minimum 1960 * threshold size to 1/2 the ring. 1961 */ 1962 rctl = E1000_READ_REG(hw, E1000_RCTL); 1963 1964 /* 1965 * Clear the field used for wakeup control. This driver doesn't do 1966 * wakeup but leave this here for completeness. 1967 */ 1968 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 1969 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 1970 1971 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */ 1972 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 1973 E1000_RCTL_LPE | /* Large Packet Enable */ 1974 /* Multicast filter offset */ 1975 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 1976 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */ 1977 E1000_RCTL_SECRC); /* Strip Ethernet CRC */ 1978 1979 for (i = 0; i < igb->num_rx_groups; i++) { 1980 rx_group = &igb->rx_groups[i]; 1981 rx_group->index = i; 1982 rx_group->igb = igb; 1983 } 1984 1985 /* 1986 * Set up all rx descriptor rings - must be called before receive unit 1987 * enabled. 1988 */ 1989 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 1990 for (i = 0; i < igb->num_rx_rings; i++) { 1991 rx_ring = &igb->rx_rings[i]; 1992 igb_setup_rx_ring(rx_ring); 1993 1994 /* 1995 * Map a ring to a group by assigning a group index 1996 */ 1997 rx_ring->group_index = i / ring_per_group; 1998 } 1999 2000 /* 2001 * Setup the Rx Long Packet Max Length register 2002 */ 2003 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size); 2004 2005 /* 2006 * Hardware checksum settings 2007 */ 2008 if (igb->rx_hcksum_enable) { 2009 rxcsum = 2010 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */ 2011 E1000_RXCSUM_IPOFL; /* IP checksum */ 2012 2013 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2014 } 2015 2016 /* 2017 * Setup classify and RSS for multiple receive queues 2018 */ 2019 switch (igb->vmdq_mode) { 2020 case E1000_VMDQ_OFF: 2021 /* 2022 * One ring group, only RSS is needed when more than 2023 * one ring enabled. 2024 */ 2025 if (igb->num_rx_rings > 1) 2026 igb_setup_rss(igb); 2027 break; 2028 case E1000_VMDQ_MAC: 2029 /* 2030 * Multiple groups, each group has one ring, 2031 * only the MAC classification is needed. 2032 */ 2033 igb_setup_mac_classify(igb); 2034 break; 2035 case E1000_VMDQ_MAC_RSS: 2036 /* 2037 * Multiple groups and multiple rings, both 2038 * MAC classification and RSS are needed. 2039 */ 2040 igb_setup_mac_rss_classify(igb); 2041 break; 2042 } 2043 2044 /* 2045 * Enable the receive unit - must be done after all 2046 * the rx setup above. 2047 */ 2048 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2049 2050 /* 2051 * Initialize all adapter ring head & tail pointers - must 2052 * be done after receive unit is enabled 2053 */ 2054 for (i = 0; i < igb->num_rx_rings; i++) { 2055 rx_ring = &igb->rx_rings[i]; 2056 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 2057 E1000_WRITE_REG(hw, E1000_RDT(i), rx_ring->ring_size - 1); 2058 } 2059 2060 /* 2061 * 82575 with manageability enabled needs a special flush to make 2062 * sure the fifos start clean. 2063 */ 2064 if ((hw->mac.type == e1000_82575) && 2065 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) { 2066 e1000_rx_fifo_flush_82575(hw); 2067 } 2068 } 2069 2070 static void 2071 igb_setup_tx_ring(igb_tx_ring_t *tx_ring) 2072 { 2073 igb_t *igb = tx_ring->igb; 2074 struct e1000_hw *hw = &igb->hw; 2075 uint32_t size; 2076 uint32_t buf_low; 2077 uint32_t buf_high; 2078 uint32_t reg_val; 2079 2080 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2081 ASSERT(mutex_owned(&igb->gen_lock)); 2082 2083 2084 /* 2085 * Initialize the length register 2086 */ 2087 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc); 2088 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size); 2089 2090 /* 2091 * Initialize the base address registers 2092 */ 2093 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2094 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2095 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low); 2096 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high); 2097 2098 /* 2099 * Setup head & tail pointers 2100 */ 2101 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0); 2102 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0); 2103 2104 /* 2105 * Setup head write-back 2106 */ 2107 if (igb->tx_head_wb_enable) { 2108 /* 2109 * The memory of the head write-back is allocated using 2110 * the extra tbd beyond the tail of the tbd ring. 2111 */ 2112 tx_ring->tbd_head_wb = (uint32_t *) 2113 ((uintptr_t)tx_ring->tbd_area.address + size); 2114 *tx_ring->tbd_head_wb = 0; 2115 2116 buf_low = (uint32_t) 2117 (tx_ring->tbd_area.dma_address + size); 2118 buf_high = (uint32_t) 2119 ((tx_ring->tbd_area.dma_address + size) >> 32); 2120 2121 /* Set the head write-back enable bit */ 2122 buf_low |= E1000_TX_HEAD_WB_ENABLE; 2123 2124 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low); 2125 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high); 2126 2127 /* 2128 * Turn off relaxed ordering for head write back or it will 2129 * cause problems with the tx recycling 2130 */ 2131 reg_val = E1000_READ_REG(hw, 2132 E1000_DCA_TXCTRL(tx_ring->index)); 2133 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN; 2134 E1000_WRITE_REG(hw, 2135 E1000_DCA_TXCTRL(tx_ring->index), reg_val); 2136 } else { 2137 tx_ring->tbd_head_wb = NULL; 2138 } 2139 2140 tx_ring->tbd_head = 0; 2141 tx_ring->tbd_tail = 0; 2142 tx_ring->tbd_free = tx_ring->ring_size; 2143 2144 /* 2145 * Note: for the case that the chipset is being reset, we should not 2146 * reset the values of tcb_head, tcb_tail. And considering there might 2147 * still be some packets kept in the pending_list, we should not assert 2148 * (tcb_free == free_list_size) here. 2149 */ 2150 if (igb->igb_state == IGB_UNKNOWN) { 2151 tx_ring->tcb_head = 0; 2152 tx_ring->tcb_tail = 0; 2153 tx_ring->tcb_free = tx_ring->free_list_size; 2154 } 2155 2156 /* 2157 * Enable TXDCTL per queue 2158 */ 2159 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index)); 2160 reg_val |= E1000_TXDCTL_QUEUE_ENABLE; 2161 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val); 2162 2163 /* 2164 * Initialize hardware checksum offload settings 2165 */ 2166 bzero(&tx_ring->tx_context, sizeof (tx_context_t)); 2167 } 2168 2169 static void 2170 igb_setup_tx(igb_t *igb) 2171 { 2172 igb_tx_ring_t *tx_ring; 2173 struct e1000_hw *hw = &igb->hw; 2174 uint32_t reg_val; 2175 int i; 2176 2177 for (i = 0; i < igb->num_tx_rings; i++) { 2178 tx_ring = &igb->tx_rings[i]; 2179 igb_setup_tx_ring(tx_ring); 2180 } 2181 2182 /* 2183 * Setup the Transmit Control Register (TCTL) 2184 */ 2185 reg_val = E1000_READ_REG(hw, E1000_TCTL); 2186 reg_val &= ~E1000_TCTL_CT; 2187 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2188 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2189 2190 /* Enable transmits */ 2191 reg_val |= E1000_TCTL_EN; 2192 2193 E1000_WRITE_REG(hw, E1000_TCTL, reg_val); 2194 } 2195 2196 /* 2197 * igb_setup_rss - Setup receive-side scaling feature 2198 */ 2199 static void 2200 igb_setup_rss(igb_t *igb) 2201 { 2202 struct e1000_hw *hw = &igb->hw; 2203 uint32_t i, mrqc, rxcsum; 2204 int shift = 0; 2205 uint32_t random; 2206 union e1000_reta { 2207 uint32_t dword; 2208 uint8_t bytes[4]; 2209 } reta; 2210 2211 /* Setup the Redirection Table */ 2212 if (hw->mac.type == e1000_82576) { 2213 shift = 3; 2214 } else if (hw->mac.type == e1000_82575) { 2215 shift = 6; 2216 } 2217 for (i = 0; i < (32 * 4); i++) { 2218 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift; 2219 if ((i & 3) == 3) { 2220 E1000_WRITE_REG(hw, 2221 (E1000_RETA(0) + (i & ~3)), reta.dword); 2222 } 2223 } 2224 2225 /* Fill out hash function seeds */ 2226 for (i = 0; i < 10; i++) { 2227 (void) random_get_pseudo_bytes((uint8_t *)&random, 2228 sizeof (uint32_t)); 2229 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2230 } 2231 2232 /* Setup the Multiple Receive Queue Control register */ 2233 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2234 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2235 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2236 E1000_MRQC_RSS_FIELD_IPV6 | 2237 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2238 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2239 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2240 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2241 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2242 2243 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2244 2245 /* 2246 * Disable Packet Checksum to enable RSS for multiple receive queues. 2247 * 2248 * The Packet Checksum is not ethernet CRC. It is another kind of 2249 * checksum offloading provided by the 82575 chipset besides the IP 2250 * header checksum offloading and the TCP/UDP checksum offloading. 2251 * The Packet Checksum is by default computed over the entire packet 2252 * from the first byte of the DA through the last byte of the CRC, 2253 * including the Ethernet and IP headers. 2254 * 2255 * It is a hardware limitation that Packet Checksum is mutually 2256 * exclusive with RSS. 2257 */ 2258 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2259 rxcsum |= E1000_RXCSUM_PCSD; 2260 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2261 } 2262 2263 /* 2264 * igb_setup_mac_rss_classify - Setup MAC classification and rss 2265 */ 2266 static void 2267 igb_setup_mac_rss_classify(igb_t *igb) 2268 { 2269 struct e1000_hw *hw = &igb->hw; 2270 uint32_t i, mrqc, vmdctl, rxcsum; 2271 uint32_t ring_per_group; 2272 int shift_group0, shift_group1; 2273 uint32_t random; 2274 union e1000_reta { 2275 uint32_t dword; 2276 uint8_t bytes[4]; 2277 } reta; 2278 2279 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2280 2281 /* Setup the Redirection Table, it is shared between two groups */ 2282 shift_group0 = 2; 2283 shift_group1 = 6; 2284 for (i = 0; i < (32 * 4); i++) { 2285 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) | 2286 ((ring_per_group + (i % ring_per_group)) << shift_group1); 2287 if ((i & 3) == 3) { 2288 E1000_WRITE_REG(hw, 2289 (E1000_RETA(0) + (i & ~3)), reta.dword); 2290 } 2291 } 2292 2293 /* Fill out hash function seeds */ 2294 for (i = 0; i < 10; i++) { 2295 (void) random_get_pseudo_bytes((uint8_t *)&random, 2296 sizeof (uint32_t)); 2297 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2298 } 2299 2300 /* 2301 * Setup the Multiple Receive Queue Control register, 2302 * enable VMDq based on packet destination MAC address and RSS. 2303 */ 2304 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP; 2305 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2306 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2307 E1000_MRQC_RSS_FIELD_IPV6 | 2308 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2309 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2310 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2311 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2312 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2313 2314 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2315 2316 2317 /* Define the default group and default queues */ 2318 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE; 2319 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl); 2320 2321 /* 2322 * Disable Packet Checksum to enable RSS for multiple receive queues. 2323 * 2324 * The Packet Checksum is not ethernet CRC. It is another kind of 2325 * checksum offloading provided by the 82575 chipset besides the IP 2326 * header checksum offloading and the TCP/UDP checksum offloading. 2327 * The Packet Checksum is by default computed over the entire packet 2328 * from the first byte of the DA through the last byte of the CRC, 2329 * including the Ethernet and IP headers. 2330 * 2331 * It is a hardware limitation that Packet Checksum is mutually 2332 * exclusive with RSS. 2333 */ 2334 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2335 rxcsum |= E1000_RXCSUM_PCSD; 2336 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2337 } 2338 2339 /* 2340 * igb_setup_mac_classify - Setup MAC classification feature 2341 */ 2342 static void 2343 igb_setup_mac_classify(igb_t *igb) 2344 { 2345 struct e1000_hw *hw = &igb->hw; 2346 uint32_t mrqc, rxcsum; 2347 2348 /* 2349 * Setup the Multiple Receive Queue Control register, 2350 * enable VMDq based on packet destination MAC address. 2351 */ 2352 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP; 2353 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2354 2355 /* 2356 * Disable Packet Checksum to enable RSS for multiple receive queues. 2357 * 2358 * The Packet Checksum is not ethernet CRC. It is another kind of 2359 * checksum offloading provided by the 82575 chipset besides the IP 2360 * header checksum offloading and the TCP/UDP checksum offloading. 2361 * The Packet Checksum is by default computed over the entire packet 2362 * from the first byte of the DA through the last byte of the CRC, 2363 * including the Ethernet and IP headers. 2364 * 2365 * It is a hardware limitation that Packet Checksum is mutually 2366 * exclusive with RSS. 2367 */ 2368 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2369 rxcsum |= E1000_RXCSUM_PCSD; 2370 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2371 2372 } 2373 2374 /* 2375 * igb_init_unicst - Initialize the unicast addresses 2376 */ 2377 static void 2378 igb_init_unicst(igb_t *igb) 2379 { 2380 struct e1000_hw *hw = &igb->hw; 2381 int slot; 2382 2383 /* 2384 * Here we should consider two situations: 2385 * 2386 * 1. Chipset is initialized the first time 2387 * Initialize the multiple unicast addresses, and 2388 * save the default MAC address. 2389 * 2390 * 2. Chipset is reset 2391 * Recover the multiple unicast addresses from the 2392 * software data structure to the RAR registers. 2393 */ 2394 2395 /* 2396 * Clear the default MAC address in the RAR0 rgister, 2397 * which is loaded from EEPROM when system boot or chipreset, 2398 * this will cause the conficts with add_mac/rem_mac entry 2399 * points when VMDq is enabled. For this reason, the RAR0 2400 * must be cleared for both cases mentioned above. 2401 */ 2402 e1000_rar_clear(hw, 0); 2403 2404 if (!igb->unicst_init) { 2405 2406 /* Initialize the multiple unicast addresses */ 2407 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2408 igb->unicst_avail = igb->unicst_total; 2409 2410 for (slot = 0; slot < igb->unicst_total; slot++) 2411 igb->unicst_addr[slot].mac.set = 0; 2412 2413 igb->unicst_init = B_TRUE; 2414 } else { 2415 /* Re-configure the RAR registers */ 2416 for (slot = 0; slot < igb->unicst_total; slot++) { 2417 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr, 2418 slot, igb->vmdq_mode, 2419 igb->unicst_addr[slot].mac.group_index); 2420 } 2421 } 2422 } 2423 2424 /* 2425 * igb_unicst_find - Find the slot for the specified unicast address 2426 */ 2427 int 2428 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr) 2429 { 2430 int slot; 2431 2432 ASSERT(mutex_owned(&igb->gen_lock)); 2433 2434 for (slot = 0; slot < igb->unicst_total; slot++) { 2435 if (bcmp(igb->unicst_addr[slot].mac.addr, 2436 mac_addr, ETHERADDRL) == 0) 2437 return (slot); 2438 } 2439 2440 return (-1); 2441 } 2442 2443 /* 2444 * igb_unicst_set - Set the unicast address to the specified slot 2445 */ 2446 int 2447 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr, 2448 int slot) 2449 { 2450 struct e1000_hw *hw = &igb->hw; 2451 2452 ASSERT(mutex_owned(&igb->gen_lock)); 2453 2454 /* 2455 * Save the unicast address in the software data structure 2456 */ 2457 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL); 2458 2459 /* 2460 * Set the unicast address to the RAR register 2461 */ 2462 e1000_rar_set(hw, (uint8_t *)mac_addr, slot); 2463 2464 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2465 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2466 return (EIO); 2467 } 2468 2469 return (0); 2470 } 2471 2472 /* 2473 * igb_multicst_add - Add a multicst address 2474 */ 2475 int 2476 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr) 2477 { 2478 struct ether_addr *new_table; 2479 size_t new_len; 2480 size_t old_len; 2481 2482 ASSERT(mutex_owned(&igb->gen_lock)); 2483 2484 if ((multiaddr[0] & 01) == 0) { 2485 igb_error(igb, "Illegal multicast address"); 2486 return (EINVAL); 2487 } 2488 2489 if (igb->mcast_count >= igb->mcast_max_num) { 2490 igb_error(igb, "Adapter requested more than %d mcast addresses", 2491 igb->mcast_max_num); 2492 return (ENOENT); 2493 } 2494 2495 if (igb->mcast_count == igb->mcast_alloc_count) { 2496 old_len = igb->mcast_alloc_count * 2497 sizeof (struct ether_addr); 2498 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) * 2499 sizeof (struct ether_addr); 2500 2501 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2502 if (new_table == NULL) { 2503 igb_error(igb, 2504 "Not enough memory to alloc mcast table"); 2505 return (ENOMEM); 2506 } 2507 2508 if (igb->mcast_table != NULL) { 2509 bcopy(igb->mcast_table, new_table, old_len); 2510 kmem_free(igb->mcast_table, old_len); 2511 } 2512 igb->mcast_alloc_count += MCAST_ALLOC_COUNT; 2513 igb->mcast_table = new_table; 2514 } 2515 2516 bcopy(multiaddr, 2517 &igb->mcast_table[igb->mcast_count], ETHERADDRL); 2518 igb->mcast_count++; 2519 2520 /* 2521 * Update the multicast table in the hardware 2522 */ 2523 igb_setup_multicst(igb); 2524 2525 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2526 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2527 return (EIO); 2528 } 2529 2530 return (0); 2531 } 2532 2533 /* 2534 * igb_multicst_remove - Remove a multicst address 2535 */ 2536 int 2537 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr) 2538 { 2539 struct ether_addr *new_table; 2540 size_t new_len; 2541 size_t old_len; 2542 int i; 2543 2544 ASSERT(mutex_owned(&igb->gen_lock)); 2545 2546 for (i = 0; i < igb->mcast_count; i++) { 2547 if (bcmp(multiaddr, &igb->mcast_table[i], 2548 ETHERADDRL) == 0) { 2549 for (i++; i < igb->mcast_count; i++) { 2550 igb->mcast_table[i - 1] = 2551 igb->mcast_table[i]; 2552 } 2553 igb->mcast_count--; 2554 break; 2555 } 2556 } 2557 2558 if ((igb->mcast_alloc_count - igb->mcast_count) > 2559 MCAST_ALLOC_COUNT) { 2560 old_len = igb->mcast_alloc_count * 2561 sizeof (struct ether_addr); 2562 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) * 2563 sizeof (struct ether_addr); 2564 2565 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2566 if (new_table != NULL) { 2567 bcopy(igb->mcast_table, new_table, new_len); 2568 kmem_free(igb->mcast_table, old_len); 2569 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT; 2570 igb->mcast_table = new_table; 2571 } 2572 } 2573 2574 /* 2575 * Update the multicast table in the hardware 2576 */ 2577 igb_setup_multicst(igb); 2578 2579 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2580 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2581 return (EIO); 2582 } 2583 2584 return (0); 2585 } 2586 2587 static void 2588 igb_release_multicast(igb_t *igb) 2589 { 2590 if (igb->mcast_table != NULL) { 2591 kmem_free(igb->mcast_table, 2592 igb->mcast_alloc_count * sizeof (struct ether_addr)); 2593 igb->mcast_table = NULL; 2594 } 2595 } 2596 2597 /* 2598 * igb_setup_multicast - setup multicast data structures 2599 * 2600 * This routine initializes all of the multicast related structures 2601 * and save them in the hardware registers. 2602 */ 2603 static void 2604 igb_setup_multicst(igb_t *igb) 2605 { 2606 uint8_t *mc_addr_list; 2607 uint32_t mc_addr_count; 2608 struct e1000_hw *hw = &igb->hw; 2609 2610 ASSERT(mutex_owned(&igb->gen_lock)); 2611 ASSERT(igb->mcast_count <= igb->mcast_max_num); 2612 2613 mc_addr_list = (uint8_t *)igb->mcast_table; 2614 mc_addr_count = igb->mcast_count; 2615 2616 /* 2617 * Update the multicase addresses to the MTA registers 2618 */ 2619 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count); 2620 } 2621 2622 /* 2623 * igb_get_conf - Get driver configurations set in driver.conf 2624 * 2625 * This routine gets user-configured values out of the configuration 2626 * file igb.conf. 2627 * 2628 * For each configurable value, there is a minimum, a maximum, and a 2629 * default. 2630 * If user does not configure a value, use the default. 2631 * If user configures below the minimum, use the minumum. 2632 * If user configures above the maximum, use the maxumum. 2633 */ 2634 static void 2635 igb_get_conf(igb_t *igb) 2636 { 2637 struct e1000_hw *hw = &igb->hw; 2638 uint32_t default_mtu; 2639 uint32_t flow_control; 2640 uint32_t ring_per_group; 2641 int i; 2642 2643 /* 2644 * igb driver supports the following user configurations: 2645 * 2646 * Link configurations: 2647 * adv_autoneg_cap 2648 * adv_1000fdx_cap 2649 * adv_100fdx_cap 2650 * adv_100hdx_cap 2651 * adv_10fdx_cap 2652 * adv_10hdx_cap 2653 * Note: 1000hdx is not supported. 2654 * 2655 * Jumbo frame configuration: 2656 * default_mtu 2657 * 2658 * Ethernet flow control configuration: 2659 * flow_control 2660 * 2661 * Multiple rings configurations: 2662 * tx_queue_number 2663 * tx_ring_size 2664 * rx_queue_number 2665 * rx_ring_size 2666 * 2667 * Call igb_get_prop() to get the value for a specific 2668 * configuration parameter. 2669 */ 2670 2671 /* 2672 * Link configurations 2673 */ 2674 igb->param_adv_autoneg_cap = igb_get_prop(igb, 2675 PROP_ADV_AUTONEG_CAP, 0, 1, 1); 2676 igb->param_adv_1000fdx_cap = igb_get_prop(igb, 2677 PROP_ADV_1000FDX_CAP, 0, 1, 1); 2678 igb->param_adv_100fdx_cap = igb_get_prop(igb, 2679 PROP_ADV_100FDX_CAP, 0, 1, 1); 2680 igb->param_adv_100hdx_cap = igb_get_prop(igb, 2681 PROP_ADV_100HDX_CAP, 0, 1, 1); 2682 igb->param_adv_10fdx_cap = igb_get_prop(igb, 2683 PROP_ADV_10FDX_CAP, 0, 1, 1); 2684 igb->param_adv_10hdx_cap = igb_get_prop(igb, 2685 PROP_ADV_10HDX_CAP, 0, 1, 1); 2686 2687 /* 2688 * Jumbo frame configurations 2689 */ 2690 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU, 2691 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2692 2693 igb->max_frame_size = default_mtu + 2694 sizeof (struct ether_vlan_header) + ETHERFCSL; 2695 2696 /* 2697 * Ethernet flow control configuration 2698 */ 2699 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL, 2700 e1000_fc_none, 4, e1000_fc_full); 2701 if (flow_control == 4) 2702 flow_control = e1000_fc_default; 2703 2704 hw->fc.requested_mode = flow_control; 2705 2706 /* 2707 * Multiple rings configurations 2708 */ 2709 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE, 2710 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2711 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE, 2712 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2713 2714 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0); 2715 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM, 2716 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2717 /* 2718 * Currently we do not support VMDq for 82576 and 82580. 2719 * If it is e1000_82576, set num_rx_groups to 1. 2720 */ 2721 if (hw->mac.type >= e1000_82576) 2722 igb->num_rx_groups = 1; 2723 2724 if (igb->mr_enable) { 2725 igb->num_tx_rings = igb->capab->def_tx_que_num; 2726 igb->num_rx_rings = igb->capab->def_rx_que_num; 2727 } else { 2728 igb->num_tx_rings = 1; 2729 igb->num_rx_rings = 1; 2730 2731 if (igb->num_rx_groups > 1) { 2732 igb_error(igb, 2733 "Invalid rx groups number. Please enable multiple " 2734 "rings first"); 2735 igb->num_rx_groups = 1; 2736 } 2737 } 2738 2739 /* 2740 * Check the divisibility between rx rings and rx groups. 2741 */ 2742 for (i = igb->num_rx_groups; i > 0; i--) { 2743 if ((igb->num_rx_rings % i) == 0) 2744 break; 2745 } 2746 if (i != igb->num_rx_groups) { 2747 igb_error(igb, 2748 "Invalid rx groups number. Downgrade the rx group " 2749 "number to %d.", i); 2750 igb->num_rx_groups = i; 2751 } 2752 2753 /* 2754 * Get the ring number per group. 2755 */ 2756 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2757 2758 if (igb->num_rx_groups == 1) { 2759 /* 2760 * One rx ring group, the rx ring number is num_rx_rings. 2761 */ 2762 igb->vmdq_mode = E1000_VMDQ_OFF; 2763 } else if (ring_per_group == 1) { 2764 /* 2765 * Multiple rx groups, each group has one rx ring. 2766 */ 2767 igb->vmdq_mode = E1000_VMDQ_MAC; 2768 } else { 2769 /* 2770 * Multiple groups and multiple rings. 2771 */ 2772 igb->vmdq_mode = E1000_VMDQ_MAC_RSS; 2773 } 2774 2775 /* 2776 * Tunable used to force an interrupt type. The only use is 2777 * for testing of the lesser interrupt types. 2778 * 0 = don't force interrupt type 2779 * 1 = force interrupt type MSIX 2780 * 2 = force interrupt type MSI 2781 * 3 = force interrupt type Legacy 2782 */ 2783 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE, 2784 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE); 2785 2786 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE, 2787 0, 1, 1); 2788 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE, 2789 0, 1, 1); 2790 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE, 2791 0, 1, 1); 2792 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE, 2793 0, 1, 1); 2794 2795 /* 2796 * igb LSO needs the tx h/w checksum support. 2797 * Here LSO will be disabled if tx h/w checksum has been disabled. 2798 */ 2799 if (igb->tx_hcksum_enable == B_FALSE) 2800 igb->lso_enable = B_FALSE; 2801 2802 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD, 2803 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2804 DEFAULT_TX_COPY_THRESHOLD); 2805 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD, 2806 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, 2807 DEFAULT_TX_RECYCLE_THRESHOLD); 2808 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD, 2809 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, 2810 DEFAULT_TX_OVERLOAD_THRESHOLD); 2811 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD, 2812 MIN_TX_RESCHED_THRESHOLD, MAX_TX_RESCHED_THRESHOLD, 2813 DEFAULT_TX_RESCHED_THRESHOLD); 2814 2815 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD, 2816 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2817 DEFAULT_RX_COPY_THRESHOLD); 2818 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR, 2819 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2820 DEFAULT_RX_LIMIT_PER_INTR); 2821 2822 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING, 2823 igb->capab->min_intr_throttle, 2824 igb->capab->max_intr_throttle, 2825 igb->capab->def_intr_throttle); 2826 2827 /* 2828 * Max number of multicast addresses 2829 */ 2830 igb->mcast_max_num = 2831 igb_get_prop(igb, PROP_MCAST_MAX_NUM, 2832 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM); 2833 } 2834 2835 /* 2836 * igb_get_prop - Get a property value out of the configuration file igb.conf 2837 * 2838 * Caller provides the name of the property, a default value, a minimum 2839 * value, and a maximum value. 2840 * 2841 * Return configured value of the property, with default, minimum and 2842 * maximum properly applied. 2843 */ 2844 static int 2845 igb_get_prop(igb_t *igb, 2846 char *propname, /* name of the property */ 2847 int minval, /* minimum acceptable value */ 2848 int maxval, /* maximim acceptable value */ 2849 int defval) /* default value */ 2850 { 2851 int value; 2852 2853 /* 2854 * Call ddi_prop_get_int() to read the conf settings 2855 */ 2856 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip, 2857 DDI_PROP_DONTPASS, propname, defval); 2858 2859 if (value > maxval) 2860 value = maxval; 2861 2862 if (value < minval) 2863 value = minval; 2864 2865 return (value); 2866 } 2867 2868 /* 2869 * igb_setup_link - Using the link properties to setup the link 2870 */ 2871 int 2872 igb_setup_link(igb_t *igb, boolean_t setup_hw) 2873 { 2874 struct e1000_mac_info *mac; 2875 struct e1000_phy_info *phy; 2876 boolean_t invalid; 2877 2878 mac = &igb->hw.mac; 2879 phy = &igb->hw.phy; 2880 invalid = B_FALSE; 2881 2882 if (igb->param_adv_autoneg_cap == 1) { 2883 mac->autoneg = B_TRUE; 2884 phy->autoneg_advertised = 0; 2885 2886 /* 2887 * 1000hdx is not supported for autonegotiation 2888 */ 2889 if (igb->param_adv_1000fdx_cap == 1) 2890 phy->autoneg_advertised |= ADVERTISE_1000_FULL; 2891 2892 if (igb->param_adv_100fdx_cap == 1) 2893 phy->autoneg_advertised |= ADVERTISE_100_FULL; 2894 2895 if (igb->param_adv_100hdx_cap == 1) 2896 phy->autoneg_advertised |= ADVERTISE_100_HALF; 2897 2898 if (igb->param_adv_10fdx_cap == 1) 2899 phy->autoneg_advertised |= ADVERTISE_10_FULL; 2900 2901 if (igb->param_adv_10hdx_cap == 1) 2902 phy->autoneg_advertised |= ADVERTISE_10_HALF; 2903 2904 if (phy->autoneg_advertised == 0) 2905 invalid = B_TRUE; 2906 } else { 2907 mac->autoneg = B_FALSE; 2908 2909 /* 2910 * 1000fdx and 1000hdx are not supported for forced link 2911 */ 2912 if (igb->param_adv_100fdx_cap == 1) 2913 mac->forced_speed_duplex = ADVERTISE_100_FULL; 2914 else if (igb->param_adv_100hdx_cap == 1) 2915 mac->forced_speed_duplex = ADVERTISE_100_HALF; 2916 else if (igb->param_adv_10fdx_cap == 1) 2917 mac->forced_speed_duplex = ADVERTISE_10_FULL; 2918 else if (igb->param_adv_10hdx_cap == 1) 2919 mac->forced_speed_duplex = ADVERTISE_10_HALF; 2920 else 2921 invalid = B_TRUE; 2922 } 2923 2924 if (invalid) { 2925 igb_notice(igb, "Invalid link settings. Setup link to " 2926 "autonegotiation with full link capabilities."); 2927 mac->autoneg = B_TRUE; 2928 phy->autoneg_advertised = ADVERTISE_1000_FULL | 2929 ADVERTISE_100_FULL | ADVERTISE_100_HALF | 2930 ADVERTISE_10_FULL | ADVERTISE_10_HALF; 2931 } 2932 2933 if (setup_hw) { 2934 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS) 2935 return (IGB_FAILURE); 2936 } 2937 2938 return (IGB_SUCCESS); 2939 } 2940 2941 2942 /* 2943 * igb_is_link_up - Check if the link is up 2944 */ 2945 static boolean_t 2946 igb_is_link_up(igb_t *igb) 2947 { 2948 struct e1000_hw *hw = &igb->hw; 2949 boolean_t link_up = B_FALSE; 2950 2951 ASSERT(mutex_owned(&igb->gen_lock)); 2952 2953 /* 2954 * get_link_status is set in the interrupt handler on link-status-change 2955 * or rx sequence error interrupt. get_link_status will stay 2956 * false until the e1000_check_for_link establishes link only 2957 * for copper adapters. 2958 */ 2959 switch (hw->phy.media_type) { 2960 case e1000_media_type_copper: 2961 if (hw->mac.get_link_status) { 2962 (void) e1000_check_for_link(hw); 2963 link_up = !hw->mac.get_link_status; 2964 } else { 2965 link_up = B_TRUE; 2966 } 2967 break; 2968 case e1000_media_type_fiber: 2969 (void) e1000_check_for_link(hw); 2970 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 2971 break; 2972 case e1000_media_type_internal_serdes: 2973 (void) e1000_check_for_link(hw); 2974 link_up = hw->mac.serdes_has_link; 2975 break; 2976 } 2977 2978 return (link_up); 2979 } 2980 2981 /* 2982 * igb_link_check - Link status processing 2983 */ 2984 static boolean_t 2985 igb_link_check(igb_t *igb) 2986 { 2987 struct e1000_hw *hw = &igb->hw; 2988 uint16_t speed = 0, duplex = 0; 2989 boolean_t link_changed = B_FALSE; 2990 2991 ASSERT(mutex_owned(&igb->gen_lock)); 2992 2993 if (igb_is_link_up(igb)) { 2994 /* 2995 * The Link is up, check whether it was marked as down earlier 2996 */ 2997 if (igb->link_state != LINK_STATE_UP) { 2998 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex); 2999 igb->link_speed = speed; 3000 igb->link_duplex = duplex; 3001 igb->link_state = LINK_STATE_UP; 3002 igb->link_down_timeout = 0; 3003 link_changed = B_TRUE; 3004 if (!igb->link_complete) 3005 igb_stop_link_timer(igb); 3006 } 3007 } else if (igb->link_complete) { 3008 if (igb->link_state != LINK_STATE_DOWN) { 3009 igb->link_speed = 0; 3010 igb->link_duplex = 0; 3011 igb->link_state = LINK_STATE_DOWN; 3012 link_changed = B_TRUE; 3013 } 3014 3015 if (igb->igb_state & IGB_STARTED) { 3016 if (igb->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) { 3017 igb->link_down_timeout++; 3018 } else if (igb->link_down_timeout == 3019 MAX_LINK_DOWN_TIMEOUT) { 3020 igb_tx_clean(igb); 3021 igb->link_down_timeout++; 3022 } 3023 } 3024 } 3025 3026 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3027 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3028 return (B_FALSE); 3029 } 3030 3031 return (link_changed); 3032 } 3033 3034 /* 3035 * igb_local_timer - driver watchdog function 3036 * 3037 * This function will handle the hardware stall check, link status 3038 * check and other routines. 3039 */ 3040 static void 3041 igb_local_timer(void *arg) 3042 { 3043 igb_t *igb = (igb_t *)arg; 3044 boolean_t link_changed = B_FALSE; 3045 3046 if (igb->igb_state & IGB_ERROR) { 3047 igb->reset_count++; 3048 if (igb_reset(igb) == IGB_SUCCESS) 3049 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3050 3051 igb_restart_watchdog_timer(igb); 3052 return; 3053 } 3054 3055 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) { 3056 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL); 3057 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 3058 igb->reset_count++; 3059 if (igb_reset(igb) == IGB_SUCCESS) 3060 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3061 3062 igb_restart_watchdog_timer(igb); 3063 return; 3064 } 3065 3066 mutex_enter(&igb->gen_lock); 3067 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED)) 3068 link_changed = igb_link_check(igb); 3069 mutex_exit(&igb->gen_lock); 3070 3071 if (link_changed) 3072 mac_link_update(igb->mac_hdl, igb->link_state); 3073 3074 igb_restart_watchdog_timer(igb); 3075 } 3076 3077 /* 3078 * igb_link_timer - link setup timer function 3079 * 3080 * It is called when the timer for link setup is expired, which indicates 3081 * the completion of the link setup. The link state will not be updated 3082 * until the link setup is completed. And the link state will not be sent 3083 * to the upper layer through mac_link_update() in this function. It will 3084 * be updated in the local timer routine or the interrupts service routine 3085 * after the interface is started (plumbed). 3086 */ 3087 static void 3088 igb_link_timer(void *arg) 3089 { 3090 igb_t *igb = (igb_t *)arg; 3091 3092 mutex_enter(&igb->link_lock); 3093 igb->link_complete = B_TRUE; 3094 igb->link_tid = 0; 3095 mutex_exit(&igb->link_lock); 3096 } 3097 /* 3098 * igb_stall_check - check for transmit stall 3099 * 3100 * This function checks if the adapter is stalled (in transmit). 3101 * 3102 * It is called each time the watchdog timeout is invoked. 3103 * If the transmit descriptor reclaim continuously fails, 3104 * the watchdog value will increment by 1. If the watchdog 3105 * value exceeds the threshold, the igb is assumed to 3106 * have stalled and need to be reset. 3107 */ 3108 static boolean_t 3109 igb_stall_check(igb_t *igb) 3110 { 3111 igb_tx_ring_t *tx_ring; 3112 struct e1000_hw *hw = &igb->hw; 3113 boolean_t result; 3114 int i; 3115 3116 if (igb->link_state != LINK_STATE_UP) 3117 return (B_FALSE); 3118 3119 /* 3120 * If any tx ring is stalled, we'll reset the chipset 3121 */ 3122 result = B_FALSE; 3123 for (i = 0; i < igb->num_tx_rings; i++) { 3124 tx_ring = &igb->tx_rings[i]; 3125 3126 if (tx_ring->recycle_fail > 0) 3127 tx_ring->stall_watchdog++; 3128 else 3129 tx_ring->stall_watchdog = 0; 3130 3131 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3132 result = B_TRUE; 3133 if (hw->mac.type == e1000_82580) { 3134 hw->dev_spec._82575.global_device_reset 3135 = B_TRUE; 3136 } 3137 break; 3138 } 3139 } 3140 3141 if (result) { 3142 tx_ring->stall_watchdog = 0; 3143 tx_ring->recycle_fail = 0; 3144 } 3145 3146 return (result); 3147 } 3148 3149 3150 /* 3151 * is_valid_mac_addr - Check if the mac address is valid 3152 */ 3153 static boolean_t 3154 is_valid_mac_addr(uint8_t *mac_addr) 3155 { 3156 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3157 const uint8_t addr_test2[6] = 3158 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3159 3160 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3161 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3162 return (B_FALSE); 3163 3164 return (B_TRUE); 3165 } 3166 3167 static boolean_t 3168 igb_find_mac_address(igb_t *igb) 3169 { 3170 struct e1000_hw *hw = &igb->hw; 3171 #ifdef __sparc 3172 uchar_t *bytes; 3173 struct ether_addr sysaddr; 3174 uint_t nelts; 3175 int err; 3176 boolean_t found = B_FALSE; 3177 3178 /* 3179 * The "vendor's factory-set address" may already have 3180 * been extracted from the chip, but if the property 3181 * "local-mac-address" is set we use that instead. 3182 * 3183 * We check whether it looks like an array of 6 3184 * bytes (which it should, if OBP set it). If we can't 3185 * make sense of it this way, we'll ignore it. 3186 */ 3187 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3188 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3189 if (err == DDI_PROP_SUCCESS) { 3190 if (nelts == ETHERADDRL) { 3191 while (nelts--) 3192 hw->mac.addr[nelts] = bytes[nelts]; 3193 found = B_TRUE; 3194 } 3195 ddi_prop_free(bytes); 3196 } 3197 3198 /* 3199 * Look up the OBP property "local-mac-address?". If the user has set 3200 * 'local-mac-address? = false', use "the system address" instead. 3201 */ 3202 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0, 3203 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3204 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3205 if (localetheraddr(NULL, &sysaddr) != 0) { 3206 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3207 found = B_TRUE; 3208 } 3209 } 3210 ddi_prop_free(bytes); 3211 } 3212 3213 /* 3214 * Finally(!), if there's a valid "mac-address" property (created 3215 * if we netbooted from this interface), we must use this instead 3216 * of any of the above to ensure that the NFS/install server doesn't 3217 * get confused by the address changing as Solaris takes over! 3218 */ 3219 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3220 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3221 if (err == DDI_PROP_SUCCESS) { 3222 if (nelts == ETHERADDRL) { 3223 while (nelts--) 3224 hw->mac.addr[nelts] = bytes[nelts]; 3225 found = B_TRUE; 3226 } 3227 ddi_prop_free(bytes); 3228 } 3229 3230 if (found) { 3231 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3232 return (B_TRUE); 3233 } 3234 #endif 3235 3236 /* 3237 * Read the device MAC address from the EEPROM 3238 */ 3239 if (e1000_read_mac_addr(hw) != E1000_SUCCESS) 3240 return (B_FALSE); 3241 3242 return (B_TRUE); 3243 } 3244 3245 #pragma inline(igb_arm_watchdog_timer) 3246 3247 static void 3248 igb_arm_watchdog_timer(igb_t *igb) 3249 { 3250 /* 3251 * Fire a watchdog timer 3252 */ 3253 igb->watchdog_tid = 3254 timeout(igb_local_timer, 3255 (void *)igb, 1 * drv_usectohz(1000000)); 3256 3257 } 3258 3259 /* 3260 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer 3261 */ 3262 void 3263 igb_enable_watchdog_timer(igb_t *igb) 3264 { 3265 mutex_enter(&igb->watchdog_lock); 3266 3267 if (!igb->watchdog_enable) { 3268 igb->watchdog_enable = B_TRUE; 3269 igb->watchdog_start = B_TRUE; 3270 igb_arm_watchdog_timer(igb); 3271 } 3272 3273 mutex_exit(&igb->watchdog_lock); 3274 3275 } 3276 3277 /* 3278 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer 3279 */ 3280 void 3281 igb_disable_watchdog_timer(igb_t *igb) 3282 { 3283 timeout_id_t tid; 3284 3285 mutex_enter(&igb->watchdog_lock); 3286 3287 igb->watchdog_enable = B_FALSE; 3288 igb->watchdog_start = B_FALSE; 3289 tid = igb->watchdog_tid; 3290 igb->watchdog_tid = 0; 3291 3292 mutex_exit(&igb->watchdog_lock); 3293 3294 if (tid != 0) 3295 (void) untimeout(tid); 3296 3297 } 3298 3299 /* 3300 * igb_start_watchdog_timer - Start the driver watchdog timer 3301 */ 3302 static void 3303 igb_start_watchdog_timer(igb_t *igb) 3304 { 3305 mutex_enter(&igb->watchdog_lock); 3306 3307 if (igb->watchdog_enable) { 3308 if (!igb->watchdog_start) { 3309 igb->watchdog_start = B_TRUE; 3310 igb_arm_watchdog_timer(igb); 3311 } 3312 } 3313 3314 mutex_exit(&igb->watchdog_lock); 3315 } 3316 3317 /* 3318 * igb_restart_watchdog_timer - Restart the driver watchdog timer 3319 */ 3320 static void 3321 igb_restart_watchdog_timer(igb_t *igb) 3322 { 3323 mutex_enter(&igb->watchdog_lock); 3324 3325 if (igb->watchdog_start) 3326 igb_arm_watchdog_timer(igb); 3327 3328 mutex_exit(&igb->watchdog_lock); 3329 } 3330 3331 /* 3332 * igb_stop_watchdog_timer - Stop the driver watchdog timer 3333 */ 3334 static void 3335 igb_stop_watchdog_timer(igb_t *igb) 3336 { 3337 timeout_id_t tid; 3338 3339 mutex_enter(&igb->watchdog_lock); 3340 3341 igb->watchdog_start = B_FALSE; 3342 tid = igb->watchdog_tid; 3343 igb->watchdog_tid = 0; 3344 3345 mutex_exit(&igb->watchdog_lock); 3346 3347 if (tid != 0) 3348 (void) untimeout(tid); 3349 } 3350 3351 /* 3352 * igb_start_link_timer - Start the link setup timer 3353 */ 3354 static void 3355 igb_start_link_timer(struct igb *igb) 3356 { 3357 struct e1000_hw *hw = &igb->hw; 3358 clock_t link_timeout; 3359 3360 if (hw->mac.autoneg) 3361 link_timeout = PHY_AUTO_NEG_LIMIT * 3362 drv_usectohz(100000); 3363 else 3364 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000); 3365 3366 mutex_enter(&igb->link_lock); 3367 if (hw->phy.autoneg_wait_to_complete) { 3368 igb->link_complete = B_TRUE; 3369 } else { 3370 igb->link_complete = B_FALSE; 3371 igb->link_tid = timeout(igb_link_timer, (void *)igb, 3372 link_timeout); 3373 } 3374 mutex_exit(&igb->link_lock); 3375 } 3376 3377 /* 3378 * igb_stop_link_timer - Stop the link setup timer 3379 */ 3380 static void 3381 igb_stop_link_timer(struct igb *igb) 3382 { 3383 timeout_id_t tid; 3384 3385 mutex_enter(&igb->link_lock); 3386 igb->link_complete = B_TRUE; 3387 tid = igb->link_tid; 3388 igb->link_tid = 0; 3389 mutex_exit(&igb->link_lock); 3390 3391 if (tid != 0) 3392 (void) untimeout(tid); 3393 } 3394 3395 /* 3396 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts 3397 */ 3398 static void 3399 igb_disable_adapter_interrupts(igb_t *igb) 3400 { 3401 struct e1000_hw *hw = &igb->hw; 3402 3403 /* 3404 * Set the IMC register to mask all the interrupts, 3405 * including the tx interrupts. 3406 */ 3407 E1000_WRITE_REG(hw, E1000_IMC, ~0); 3408 E1000_WRITE_REG(hw, E1000_IAM, 0); 3409 3410 /* 3411 * Additional disabling for MSI-X 3412 */ 3413 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3414 E1000_WRITE_REG(hw, E1000_EIMC, ~0); 3415 E1000_WRITE_REG(hw, E1000_EIAC, 0); 3416 E1000_WRITE_REG(hw, E1000_EIAM, 0); 3417 } 3418 3419 E1000_WRITE_FLUSH(hw); 3420 } 3421 3422 /* 3423 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580 3424 */ 3425 static void 3426 igb_enable_adapter_interrupts_82580(igb_t *igb) 3427 { 3428 struct e1000_hw *hw = &igb->hw; 3429 3430 /* Clear any pending interrupts */ 3431 (void) E1000_READ_REG(hw, E1000_ICR); 3432 igb->ims_mask |= E1000_IMS_DRSTA; 3433 3434 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3435 3436 /* Interrupt enabling for MSI-X */ 3437 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3438 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3439 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA); 3440 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3441 } else { /* Interrupt enabling for MSI and legacy */ 3442 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3443 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3444 igb->ims_mask |= E1000_IMS_DRSTA; 3445 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3446 } 3447 3448 /* Disable auto-mask for ICR interrupt bits */ 3449 E1000_WRITE_REG(hw, E1000_IAM, 0); 3450 3451 E1000_WRITE_FLUSH(hw); 3452 } 3453 3454 /* 3455 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576 3456 */ 3457 static void 3458 igb_enable_adapter_interrupts_82576(igb_t *igb) 3459 { 3460 struct e1000_hw *hw = &igb->hw; 3461 3462 /* Clear any pending interrupts */ 3463 (void) E1000_READ_REG(hw, E1000_ICR); 3464 3465 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3466 3467 /* Interrupt enabling for MSI-X */ 3468 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3469 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3470 igb->ims_mask = E1000_IMS_LSC; 3471 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3472 } else { 3473 /* Interrupt enabling for MSI and legacy */ 3474 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3475 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3476 E1000_WRITE_REG(hw, E1000_IMS, 3477 (IMS_ENABLE_MASK | E1000_IMS_TXQE)); 3478 } 3479 3480 /* Disable auto-mask for ICR interrupt bits */ 3481 E1000_WRITE_REG(hw, E1000_IAM, 0); 3482 3483 E1000_WRITE_FLUSH(hw); 3484 } 3485 3486 /* 3487 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575 3488 */ 3489 static void 3490 igb_enable_adapter_interrupts_82575(igb_t *igb) 3491 { 3492 struct e1000_hw *hw = &igb->hw; 3493 uint32_t reg; 3494 3495 /* Clear any pending interrupts */ 3496 (void) E1000_READ_REG(hw, E1000_ICR); 3497 3498 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3499 /* Interrupt enabling for MSI-X */ 3500 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3501 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3502 igb->ims_mask = E1000_IMS_LSC; 3503 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3504 3505 /* Enable MSI-X PBA support */ 3506 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 3507 reg |= E1000_CTRL_EXT_PBA_CLR; 3508 3509 /* Non-selective interrupt clear-on-read */ 3510 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */ 3511 3512 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 3513 } else { 3514 /* Interrupt enabling for MSI and legacy */ 3515 igb->ims_mask = IMS_ENABLE_MASK; 3516 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 3517 } 3518 3519 E1000_WRITE_FLUSH(hw); 3520 } 3521 3522 /* 3523 * Loopback Support 3524 */ 3525 static lb_property_t lb_normal = 3526 { normal, "normal", IGB_LB_NONE }; 3527 static lb_property_t lb_external = 3528 { external, "External", IGB_LB_EXTERNAL }; 3529 static lb_property_t lb_mac = 3530 { internal, "MAC", IGB_LB_INTERNAL_MAC }; 3531 static lb_property_t lb_phy = 3532 { internal, "PHY", IGB_LB_INTERNAL_PHY }; 3533 static lb_property_t lb_serdes = 3534 { internal, "SerDes", IGB_LB_INTERNAL_SERDES }; 3535 3536 enum ioc_reply 3537 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp) 3538 { 3539 lb_info_sz_t *lbsp; 3540 lb_property_t *lbpp; 3541 struct e1000_hw *hw; 3542 uint32_t *lbmp; 3543 uint32_t size; 3544 uint32_t value; 3545 3546 hw = &igb->hw; 3547 3548 if (mp->b_cont == NULL) 3549 return (IOC_INVAL); 3550 3551 switch (iocp->ioc_cmd) { 3552 default: 3553 return (IOC_INVAL); 3554 3555 case LB_GET_INFO_SIZE: 3556 size = sizeof (lb_info_sz_t); 3557 if (iocp->ioc_count != size) 3558 return (IOC_INVAL); 3559 3560 value = sizeof (lb_normal); 3561 value += sizeof (lb_mac); 3562 if (hw->phy.media_type == e1000_media_type_copper) 3563 value += sizeof (lb_phy); 3564 else 3565 value += sizeof (lb_serdes); 3566 value += sizeof (lb_external); 3567 3568 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3569 *lbsp = value; 3570 break; 3571 3572 case LB_GET_INFO: 3573 value = sizeof (lb_normal); 3574 value += sizeof (lb_mac); 3575 if (hw->phy.media_type == e1000_media_type_copper) 3576 value += sizeof (lb_phy); 3577 else 3578 value += sizeof (lb_serdes); 3579 value += sizeof (lb_external); 3580 3581 size = value; 3582 if (iocp->ioc_count != size) 3583 return (IOC_INVAL); 3584 3585 value = 0; 3586 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3587 3588 lbpp[value++] = lb_normal; 3589 lbpp[value++] = lb_mac; 3590 if (hw->phy.media_type == e1000_media_type_copper) 3591 lbpp[value++] = lb_phy; 3592 else 3593 lbpp[value++] = lb_serdes; 3594 lbpp[value++] = lb_external; 3595 break; 3596 3597 case LB_GET_MODE: 3598 size = sizeof (uint32_t); 3599 if (iocp->ioc_count != size) 3600 return (IOC_INVAL); 3601 3602 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3603 *lbmp = igb->loopback_mode; 3604 break; 3605 3606 case LB_SET_MODE: 3607 size = 0; 3608 if (iocp->ioc_count != sizeof (uint32_t)) 3609 return (IOC_INVAL); 3610 3611 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3612 if (!igb_set_loopback_mode(igb, *lbmp)) 3613 return (IOC_INVAL); 3614 break; 3615 } 3616 3617 iocp->ioc_count = size; 3618 iocp->ioc_error = 0; 3619 3620 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3621 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3622 return (IOC_INVAL); 3623 } 3624 3625 return (IOC_REPLY); 3626 } 3627 3628 /* 3629 * igb_set_loopback_mode - Setup loopback based on the loopback mode 3630 */ 3631 static boolean_t 3632 igb_set_loopback_mode(igb_t *igb, uint32_t mode) 3633 { 3634 struct e1000_hw *hw; 3635 3636 if (mode == igb->loopback_mode) 3637 return (B_TRUE); 3638 3639 hw = &igb->hw; 3640 3641 igb->loopback_mode = mode; 3642 3643 if (mode == IGB_LB_NONE) { 3644 /* Reset the chip */ 3645 hw->phy.autoneg_wait_to_complete = B_TRUE; 3646 (void) igb_reset(igb); 3647 hw->phy.autoneg_wait_to_complete = B_FALSE; 3648 return (B_TRUE); 3649 } 3650 3651 mutex_enter(&igb->gen_lock); 3652 3653 switch (mode) { 3654 default: 3655 mutex_exit(&igb->gen_lock); 3656 return (B_FALSE); 3657 3658 case IGB_LB_EXTERNAL: 3659 igb_set_external_loopback(igb); 3660 break; 3661 3662 case IGB_LB_INTERNAL_MAC: 3663 igb_set_internal_mac_loopback(igb); 3664 break; 3665 3666 case IGB_LB_INTERNAL_PHY: 3667 igb_set_internal_phy_loopback(igb); 3668 break; 3669 3670 case IGB_LB_INTERNAL_SERDES: 3671 igb_set_internal_serdes_loopback(igb); 3672 break; 3673 } 3674 3675 mutex_exit(&igb->gen_lock); 3676 3677 return (B_TRUE); 3678 } 3679 3680 /* 3681 * igb_set_external_loopback - Set the external loopback mode 3682 */ 3683 static void 3684 igb_set_external_loopback(igb_t *igb) 3685 { 3686 struct e1000_hw *hw; 3687 3688 hw = &igb->hw; 3689 3690 /* Set phy to known state */ 3691 (void) e1000_phy_hw_reset(hw); 3692 3693 (void) e1000_write_phy_reg(hw, 0x0, 0x0140); 3694 (void) e1000_write_phy_reg(hw, 0x9, 0x1b00); 3695 (void) e1000_write_phy_reg(hw, 0x12, 0x1610); 3696 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c); 3697 } 3698 3699 /* 3700 * igb_set_internal_mac_loopback - Set the internal MAC loopback mode 3701 */ 3702 static void 3703 igb_set_internal_mac_loopback(igb_t *igb) 3704 { 3705 struct e1000_hw *hw; 3706 uint32_t ctrl; 3707 uint32_t rctl; 3708 uint32_t ctrl_ext; 3709 uint16_t phy_ctrl; 3710 uint16_t phy_status; 3711 3712 hw = &igb->hw; 3713 3714 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 3715 phy_ctrl &= ~MII_CR_AUTO_NEG_EN; 3716 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3717 3718 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 3719 3720 /* Set link mode to PHY (00b) in the Extended Control register */ 3721 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3722 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3723 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3724 3725 /* Set the Device Control register */ 3726 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3727 if (!(phy_status & MII_SR_LINK_STATUS)) 3728 ctrl |= E1000_CTRL_ILOS; /* Set ILOS when the link is down */ 3729 ctrl &= ~E1000_CTRL_SPD_SEL; /* Clear the speed sel bits */ 3730 ctrl |= (E1000_CTRL_SLU | /* Force link up */ 3731 E1000_CTRL_FRCSPD | /* Force speed */ 3732 E1000_CTRL_FRCDPX | /* Force duplex */ 3733 E1000_CTRL_SPD_1000 | /* Force speed to 1000 */ 3734 E1000_CTRL_FD); /* Force full duplex */ 3735 3736 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3737 3738 /* Set the Receive Control register */ 3739 rctl = E1000_READ_REG(hw, E1000_RCTL); 3740 rctl &= ~E1000_RCTL_LBM_TCVR; 3741 rctl |= E1000_RCTL_LBM_MAC; 3742 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3743 } 3744 3745 /* 3746 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode 3747 */ 3748 static void 3749 igb_set_internal_phy_loopback(igb_t *igb) 3750 { 3751 struct e1000_hw *hw; 3752 uint32_t ctrl_ext; 3753 uint16_t phy_ctrl; 3754 uint16_t phy_pconf; 3755 3756 hw = &igb->hw; 3757 3758 /* Set link mode to PHY (00b) in the Extended Control register */ 3759 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3760 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3761 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3762 3763 /* 3764 * Set PHY control register (0x4140): 3765 * Set full duplex mode 3766 * Set loopback bit 3767 * Clear auto-neg enable bit 3768 * Set PHY speed 3769 */ 3770 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK; 3771 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3772 3773 /* Set the link disable bit in the Port Configuration register */ 3774 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf); 3775 phy_pconf |= (uint16_t)1 << 14; 3776 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf); 3777 } 3778 3779 /* 3780 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode 3781 */ 3782 static void 3783 igb_set_internal_serdes_loopback(igb_t *igb) 3784 { 3785 struct e1000_hw *hw; 3786 uint32_t ctrl_ext; 3787 uint32_t ctrl; 3788 uint32_t pcs_lctl; 3789 uint32_t connsw; 3790 3791 hw = &igb->hw; 3792 3793 /* Set link mode to SerDes (11b) in the Extended Control register */ 3794 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3795 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 3796 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3797 3798 /* Configure the SerDes to loopback */ 3799 E1000_WRITE_REG(hw, E1000_SCTL, 0x410); 3800 3801 /* Set Device Control register */ 3802 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3803 ctrl |= (E1000_CTRL_FD | /* Force full duplex */ 3804 E1000_CTRL_SLU); /* Force link up */ 3805 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */ 3806 E1000_CTRL_TFCE | /* Disable transmit flow control */ 3807 E1000_CTRL_LRST); /* Clear link reset */ 3808 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3809 3810 /* Set PCS Link Control register */ 3811 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL); 3812 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK | 3813 E1000_PCS_LCTL_FSD | 3814 E1000_PCS_LCTL_FDV_FULL | 3815 E1000_PCS_LCTL_FLV_LINK_UP); 3816 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE; 3817 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl); 3818 3819 /* Set the Copper/Fiber Switch Control - CONNSW register */ 3820 connsw = E1000_READ_REG(hw, E1000_CONNSW); 3821 connsw &= ~E1000_CONNSW_ENRGSRC; 3822 E1000_WRITE_REG(hw, E1000_CONNSW, connsw); 3823 } 3824 3825 #pragma inline(igb_intr_rx_work) 3826 /* 3827 * igb_intr_rx_work - rx processing of ISR 3828 */ 3829 static void 3830 igb_intr_rx_work(igb_rx_ring_t *rx_ring) 3831 { 3832 mblk_t *mp; 3833 3834 mutex_enter(&rx_ring->rx_lock); 3835 mp = igb_rx(rx_ring, IGB_NO_POLL); 3836 mutex_exit(&rx_ring->rx_lock); 3837 3838 if (mp != NULL) 3839 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp, 3840 rx_ring->ring_gen_num); 3841 } 3842 3843 #pragma inline(igb_intr_tx_work) 3844 /* 3845 * igb_intr_tx_work - tx processing of ISR 3846 */ 3847 static void 3848 igb_intr_tx_work(igb_tx_ring_t *tx_ring) 3849 { 3850 /* Recycle the tx descriptors */ 3851 tx_ring->tx_recycle(tx_ring); 3852 3853 /* Schedule the re-transmit */ 3854 if (tx_ring->reschedule && 3855 (tx_ring->tbd_free >= tx_ring->resched_thresh)) { 3856 tx_ring->reschedule = B_FALSE; 3857 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle); 3858 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3859 } 3860 } 3861 3862 #pragma inline(igb_intr_link_work) 3863 /* 3864 * igb_intr_link_work - link-status-change processing of ISR 3865 */ 3866 static void 3867 igb_intr_link_work(igb_t *igb) 3868 { 3869 boolean_t link_changed; 3870 3871 igb_stop_watchdog_timer(igb); 3872 3873 mutex_enter(&igb->gen_lock); 3874 3875 /* 3876 * Because we got a link-status-change interrupt, force 3877 * e1000_check_for_link() to look at phy 3878 */ 3879 igb->hw.mac.get_link_status = B_TRUE; 3880 3881 /* igb_link_check takes care of link status change */ 3882 link_changed = igb_link_check(igb); 3883 3884 /* Get new phy state */ 3885 igb_get_phy_state(igb); 3886 3887 mutex_exit(&igb->gen_lock); 3888 3889 if (link_changed) 3890 mac_link_update(igb->mac_hdl, igb->link_state); 3891 3892 igb_start_watchdog_timer(igb); 3893 } 3894 3895 /* 3896 * igb_intr_legacy - Interrupt handler for legacy interrupts 3897 */ 3898 static uint_t 3899 igb_intr_legacy(void *arg1, void *arg2) 3900 { 3901 igb_t *igb = (igb_t *)arg1; 3902 igb_tx_ring_t *tx_ring; 3903 uint32_t icr; 3904 mblk_t *mp; 3905 boolean_t tx_reschedule; 3906 boolean_t link_changed; 3907 uint_t result; 3908 3909 _NOTE(ARGUNUSED(arg2)); 3910 3911 mutex_enter(&igb->gen_lock); 3912 3913 if (igb->igb_state & IGB_SUSPENDED) { 3914 mutex_exit(&igb->gen_lock); 3915 return (DDI_INTR_UNCLAIMED); 3916 } 3917 3918 mp = NULL; 3919 tx_reschedule = B_FALSE; 3920 link_changed = B_FALSE; 3921 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3922 3923 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3924 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3925 atomic_or_32(&igb->igb_state, IGB_ERROR); 3926 return (DDI_INTR_UNCLAIMED); 3927 } 3928 3929 if (icr & E1000_ICR_INT_ASSERTED) { 3930 /* 3931 * E1000_ICR_INT_ASSERTED bit was set: 3932 * Read(Clear) the ICR, claim this interrupt, 3933 * look for work to do. 3934 */ 3935 ASSERT(igb->num_rx_rings == 1); 3936 ASSERT(igb->num_tx_rings == 1); 3937 3938 /* Make sure all interrupt causes cleared */ 3939 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3940 3941 if (icr & E1000_ICR_RXT0) { 3942 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL); 3943 } 3944 3945 if (icr & E1000_ICR_TXDW) { 3946 tx_ring = &igb->tx_rings[0]; 3947 3948 /* Recycle the tx descriptors */ 3949 tx_ring->tx_recycle(tx_ring); 3950 3951 /* Schedule the re-transmit */ 3952 tx_reschedule = (tx_ring->reschedule && 3953 (tx_ring->tbd_free >= tx_ring->resched_thresh)); 3954 } 3955 3956 if (icr & E1000_ICR_LSC) { 3957 /* 3958 * Because we got a link-status-change interrupt, force 3959 * e1000_check_for_link() to look at phy 3960 */ 3961 igb->hw.mac.get_link_status = B_TRUE; 3962 3963 /* igb_link_check takes care of link status change */ 3964 link_changed = igb_link_check(igb); 3965 3966 /* Get new phy state */ 3967 igb_get_phy_state(igb); 3968 } 3969 3970 if (icr & E1000_ICR_DRSTA) { 3971 /* 82580 Full Device Reset needed */ 3972 atomic_or_32(&igb->igb_state, IGB_STALL); 3973 } 3974 3975 result = DDI_INTR_CLAIMED; 3976 } else { 3977 /* 3978 * E1000_ICR_INT_ASSERTED bit was not set: 3979 * Don't claim this interrupt. 3980 */ 3981 result = DDI_INTR_UNCLAIMED; 3982 } 3983 3984 mutex_exit(&igb->gen_lock); 3985 3986 /* 3987 * Do the following work outside of the gen_lock 3988 */ 3989 if (mp != NULL) 3990 mac_rx(igb->mac_hdl, NULL, mp); 3991 3992 if (tx_reschedule) { 3993 tx_ring->reschedule = B_FALSE; 3994 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle); 3995 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3996 } 3997 3998 if (link_changed) 3999 mac_link_update(igb->mac_hdl, igb->link_state); 4000 4001 return (result); 4002 } 4003 4004 /* 4005 * igb_intr_msi - Interrupt handler for MSI 4006 */ 4007 static uint_t 4008 igb_intr_msi(void *arg1, void *arg2) 4009 { 4010 igb_t *igb = (igb_t *)arg1; 4011 uint32_t icr; 4012 4013 _NOTE(ARGUNUSED(arg2)); 4014 4015 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4016 4017 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4018 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4019 atomic_or_32(&igb->igb_state, IGB_ERROR); 4020 return (DDI_INTR_CLAIMED); 4021 } 4022 4023 /* Make sure all interrupt causes cleared */ 4024 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 4025 4026 /* 4027 * For MSI interrupt, we have only one vector, 4028 * so we have only one rx ring and one tx ring enabled. 4029 */ 4030 ASSERT(igb->num_rx_rings == 1); 4031 ASSERT(igb->num_tx_rings == 1); 4032 4033 if (icr & E1000_ICR_RXT0) { 4034 igb_intr_rx_work(&igb->rx_rings[0]); 4035 } 4036 4037 if (icr & E1000_ICR_TXDW) { 4038 igb_intr_tx_work(&igb->tx_rings[0]); 4039 } 4040 4041 if (icr & E1000_ICR_LSC) { 4042 igb_intr_link_work(igb); 4043 } 4044 4045 if (icr & E1000_ICR_DRSTA) { 4046 /* 82580 Full Device Reset needed */ 4047 atomic_or_32(&igb->igb_state, IGB_STALL); 4048 } 4049 4050 return (DDI_INTR_CLAIMED); 4051 } 4052 4053 /* 4054 * igb_intr_rx - Interrupt handler for rx 4055 */ 4056 static uint_t 4057 igb_intr_rx(void *arg1, void *arg2) 4058 { 4059 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1; 4060 4061 _NOTE(ARGUNUSED(arg2)); 4062 4063 /* 4064 * Only used via MSI-X vector so don't check cause bits 4065 * and only clean the given ring. 4066 */ 4067 igb_intr_rx_work(rx_ring); 4068 4069 return (DDI_INTR_CLAIMED); 4070 } 4071 4072 /* 4073 * igb_intr_tx - Interrupt handler for tx 4074 */ 4075 static uint_t 4076 igb_intr_tx(void *arg1, void *arg2) 4077 { 4078 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1; 4079 4080 _NOTE(ARGUNUSED(arg2)); 4081 4082 /* 4083 * Only used via MSI-X vector so don't check cause bits 4084 * and only clean the given ring. 4085 */ 4086 igb_intr_tx_work(tx_ring); 4087 4088 return (DDI_INTR_CLAIMED); 4089 } 4090 4091 /* 4092 * igb_intr_tx_other - Interrupt handler for both tx and other 4093 * 4094 */ 4095 static uint_t 4096 igb_intr_tx_other(void *arg1, void *arg2) 4097 { 4098 igb_t *igb = (igb_t *)arg1; 4099 uint32_t icr; 4100 4101 _NOTE(ARGUNUSED(arg2)); 4102 4103 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4104 4105 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4106 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4107 atomic_or_32(&igb->igb_state, IGB_ERROR); 4108 return (DDI_INTR_CLAIMED); 4109 } 4110 4111 /* 4112 * Look for tx reclaiming work first. Remember, in the 4113 * case of only interrupt sharing, only one tx ring is 4114 * used 4115 */ 4116 igb_intr_tx_work(&igb->tx_rings[0]); 4117 4118 /* 4119 * Check for "other" causes. 4120 */ 4121 if (icr & E1000_ICR_LSC) { 4122 igb_intr_link_work(igb); 4123 } 4124 4125 /* 4126 * The DOUTSYNC bit indicates a tx packet dropped because 4127 * DMA engine gets "out of sync". There isn't a real fix 4128 * for this. The Intel recommendation is to count the number 4129 * of occurrences so user can detect when it is happening. 4130 * The issue is non-fatal and there's no recovery action 4131 * available. 4132 */ 4133 if (icr & E1000_ICR_DOUTSYNC) { 4134 IGB_STAT(igb->dout_sync); 4135 } 4136 4137 if (icr & E1000_ICR_DRSTA) { 4138 /* 82580 Full Device Reset needed */ 4139 atomic_or_32(&igb->igb_state, IGB_STALL); 4140 } 4141 4142 return (DDI_INTR_CLAIMED); 4143 } 4144 4145 /* 4146 * igb_alloc_intrs - Allocate interrupts for the driver 4147 * 4148 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4149 * if not successful, try Legacy. 4150 * igb->intr_force can be used to force sequence to start with 4151 * any of the 3 types. 4152 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4153 */ 4154 static int 4155 igb_alloc_intrs(igb_t *igb) 4156 { 4157 dev_info_t *devinfo; 4158 int intr_types; 4159 int rc; 4160 4161 devinfo = igb->dip; 4162 4163 /* Get supported interrupt types */ 4164 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4165 4166 if (rc != DDI_SUCCESS) { 4167 igb_log(igb, 4168 "Get supported interrupt types failed: %d", rc); 4169 return (IGB_FAILURE); 4170 } 4171 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types); 4172 4173 igb->intr_type = 0; 4174 4175 /* Install MSI-X interrupts */ 4176 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4177 (igb->intr_force <= IGB_INTR_MSIX)) { 4178 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX); 4179 4180 if (rc == IGB_SUCCESS) 4181 return (IGB_SUCCESS); 4182 4183 igb_log(igb, 4184 "Allocate MSI-X failed, trying MSI interrupts..."); 4185 } 4186 4187 /* MSI-X not used, force rings to 1 */ 4188 igb->num_rx_rings = 1; 4189 igb->num_tx_rings = 1; 4190 igb_log(igb, 4191 "MSI-X not used, force rx and tx queue number to 1"); 4192 4193 /* Install MSI interrupts */ 4194 if ((intr_types & DDI_INTR_TYPE_MSI) && 4195 (igb->intr_force <= IGB_INTR_MSI)) { 4196 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI); 4197 4198 if (rc == IGB_SUCCESS) 4199 return (IGB_SUCCESS); 4200 4201 igb_log(igb, 4202 "Allocate MSI failed, trying Legacy interrupts..."); 4203 } 4204 4205 /* Install legacy interrupts */ 4206 if (intr_types & DDI_INTR_TYPE_FIXED) { 4207 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED); 4208 4209 if (rc == IGB_SUCCESS) 4210 return (IGB_SUCCESS); 4211 4212 igb_log(igb, 4213 "Allocate Legacy interrupts failed"); 4214 } 4215 4216 /* If none of the 3 types succeeded, return failure */ 4217 return (IGB_FAILURE); 4218 } 4219 4220 /* 4221 * igb_alloc_intr_handles - Allocate interrupt handles. 4222 * 4223 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4224 * if fewer than 2 handles are available, return failure. 4225 * Upon success, this sets the number of Rx rings to a number that 4226 * matches the handles available for Rx interrupts. 4227 */ 4228 static int 4229 igb_alloc_intr_handles(igb_t *igb, int intr_type) 4230 { 4231 dev_info_t *devinfo; 4232 int orig, request, count, avail, actual; 4233 int diff, minimum; 4234 int rc; 4235 4236 devinfo = igb->dip; 4237 4238 switch (intr_type) { 4239 case DDI_INTR_TYPE_FIXED: 4240 request = 1; /* Request 1 legacy interrupt handle */ 4241 minimum = 1; 4242 IGB_DEBUGLOG_0(igb, "interrupt type: legacy"); 4243 break; 4244 4245 case DDI_INTR_TYPE_MSI: 4246 request = 1; /* Request 1 MSI interrupt handle */ 4247 minimum = 1; 4248 IGB_DEBUGLOG_0(igb, "interrupt type: MSI"); 4249 break; 4250 4251 case DDI_INTR_TYPE_MSIX: 4252 /* 4253 * Number of vectors for the adapter is 4254 * # rx rings + # tx rings 4255 * One of tx vectors is for tx & other 4256 */ 4257 request = igb->num_rx_rings + igb->num_tx_rings; 4258 orig = request; 4259 minimum = 2; 4260 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X"); 4261 break; 4262 4263 default: 4264 igb_log(igb, 4265 "invalid call to igb_alloc_intr_handles(): %d\n", 4266 intr_type); 4267 return (IGB_FAILURE); 4268 } 4269 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d", 4270 request, minimum); 4271 4272 /* 4273 * Get number of supported interrupts 4274 */ 4275 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4276 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4277 igb_log(igb, 4278 "Get supported interrupt number failed. " 4279 "Return: %d, count: %d", rc, count); 4280 return (IGB_FAILURE); 4281 } 4282 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count); 4283 4284 /* 4285 * Get number of available interrupts 4286 */ 4287 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 4288 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 4289 igb_log(igb, 4290 "Get available interrupt number failed. " 4291 "Return: %d, available: %d", rc, avail); 4292 return (IGB_FAILURE); 4293 } 4294 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail); 4295 4296 if (avail < request) { 4297 igb_log(igb, "Request %d handles, %d available", 4298 request, avail); 4299 request = avail; 4300 } 4301 4302 actual = 0; 4303 igb->intr_cnt = 0; 4304 4305 /* 4306 * Allocate an array of interrupt handles 4307 */ 4308 igb->intr_size = request * sizeof (ddi_intr_handle_t); 4309 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP); 4310 4311 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0, 4312 request, &actual, DDI_INTR_ALLOC_NORMAL); 4313 if (rc != DDI_SUCCESS) { 4314 igb_log(igb, "Allocate interrupts failed. " 4315 "return: %d, request: %d, actual: %d", 4316 rc, request, actual); 4317 goto alloc_handle_fail; 4318 } 4319 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual); 4320 4321 igb->intr_cnt = actual; 4322 4323 if (actual < minimum) { 4324 igb_log(igb, "Insufficient interrupt handles allocated: %d", 4325 actual); 4326 goto alloc_handle_fail; 4327 } 4328 4329 /* 4330 * For MSI-X, actual might force us to reduce number of tx & rx rings 4331 */ 4332 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { 4333 diff = orig - actual; 4334 if (diff < igb->num_tx_rings) { 4335 igb_log(igb, 4336 "MSI-X vectors force Tx queue number to %d", 4337 igb->num_tx_rings - diff); 4338 igb->num_tx_rings -= diff; 4339 } else { 4340 igb_log(igb, 4341 "MSI-X vectors force Tx queue number to 1"); 4342 igb->num_tx_rings = 1; 4343 4344 igb_log(igb, 4345 "MSI-X vectors force Rx queue number to %d", 4346 actual - 1); 4347 igb->num_rx_rings = actual - 1; 4348 } 4349 } 4350 4351 /* 4352 * Get priority for first vector, assume remaining are all the same 4353 */ 4354 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri); 4355 if (rc != DDI_SUCCESS) { 4356 igb_log(igb, 4357 "Get interrupt priority failed: %d", rc); 4358 goto alloc_handle_fail; 4359 } 4360 4361 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap); 4362 if (rc != DDI_SUCCESS) { 4363 igb_log(igb, 4364 "Get interrupt cap failed: %d", rc); 4365 goto alloc_handle_fail; 4366 } 4367 4368 igb->intr_type = intr_type; 4369 4370 return (IGB_SUCCESS); 4371 4372 alloc_handle_fail: 4373 igb_rem_intrs(igb); 4374 4375 return (IGB_FAILURE); 4376 } 4377 4378 /* 4379 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type 4380 * 4381 * Before adding the interrupt handlers, the interrupt vectors have 4382 * been allocated, and the rx/tx rings have also been allocated. 4383 */ 4384 static int 4385 igb_add_intr_handlers(igb_t *igb) 4386 { 4387 igb_rx_ring_t *rx_ring; 4388 igb_tx_ring_t *tx_ring; 4389 int vector; 4390 int rc; 4391 int i; 4392 4393 vector = 0; 4394 4395 switch (igb->intr_type) { 4396 case DDI_INTR_TYPE_MSIX: 4397 /* Add interrupt handler for tx + other */ 4398 tx_ring = &igb->tx_rings[0]; 4399 rc = ddi_intr_add_handler(igb->htable[vector], 4400 (ddi_intr_handler_t *)igb_intr_tx_other, 4401 (void *)igb, NULL); 4402 4403 if (rc != DDI_SUCCESS) { 4404 igb_log(igb, 4405 "Add tx/other interrupt handler failed: %d", rc); 4406 return (IGB_FAILURE); 4407 } 4408 tx_ring->intr_vector = vector; 4409 vector++; 4410 4411 /* Add interrupt handler for each rx ring */ 4412 for (i = 0; i < igb->num_rx_rings; i++) { 4413 rx_ring = &igb->rx_rings[i]; 4414 4415 rc = ddi_intr_add_handler(igb->htable[vector], 4416 (ddi_intr_handler_t *)igb_intr_rx, 4417 (void *)rx_ring, NULL); 4418 4419 if (rc != DDI_SUCCESS) { 4420 igb_log(igb, 4421 "Add rx interrupt handler failed. " 4422 "return: %d, rx ring: %d", rc, i); 4423 for (vector--; vector >= 0; vector--) { 4424 (void) ddi_intr_remove_handler( 4425 igb->htable[vector]); 4426 } 4427 return (IGB_FAILURE); 4428 } 4429 4430 rx_ring->intr_vector = vector; 4431 4432 vector++; 4433 } 4434 4435 /* Add interrupt handler for each tx ring from 2nd ring */ 4436 for (i = 1; i < igb->num_tx_rings; i++) { 4437 tx_ring = &igb->tx_rings[i]; 4438 4439 rc = ddi_intr_add_handler(igb->htable[vector], 4440 (ddi_intr_handler_t *)igb_intr_tx, 4441 (void *)tx_ring, NULL); 4442 4443 if (rc != DDI_SUCCESS) { 4444 igb_log(igb, 4445 "Add tx interrupt handler failed. " 4446 "return: %d, tx ring: %d", rc, i); 4447 for (vector--; vector >= 0; vector--) { 4448 (void) ddi_intr_remove_handler( 4449 igb->htable[vector]); 4450 } 4451 return (IGB_FAILURE); 4452 } 4453 4454 tx_ring->intr_vector = vector; 4455 4456 vector++; 4457 } 4458 4459 break; 4460 4461 case DDI_INTR_TYPE_MSI: 4462 /* Add interrupt handlers for the only vector */ 4463 rc = ddi_intr_add_handler(igb->htable[vector], 4464 (ddi_intr_handler_t *)igb_intr_msi, 4465 (void *)igb, NULL); 4466 4467 if (rc != DDI_SUCCESS) { 4468 igb_log(igb, 4469 "Add MSI interrupt handler failed: %d", rc); 4470 return (IGB_FAILURE); 4471 } 4472 4473 rx_ring = &igb->rx_rings[0]; 4474 rx_ring->intr_vector = vector; 4475 4476 vector++; 4477 break; 4478 4479 case DDI_INTR_TYPE_FIXED: 4480 /* Add interrupt handlers for the only vector */ 4481 rc = ddi_intr_add_handler(igb->htable[vector], 4482 (ddi_intr_handler_t *)igb_intr_legacy, 4483 (void *)igb, NULL); 4484 4485 if (rc != DDI_SUCCESS) { 4486 igb_log(igb, 4487 "Add legacy interrupt handler failed: %d", rc); 4488 return (IGB_FAILURE); 4489 } 4490 4491 rx_ring = &igb->rx_rings[0]; 4492 rx_ring->intr_vector = vector; 4493 4494 vector++; 4495 break; 4496 4497 default: 4498 return (IGB_FAILURE); 4499 } 4500 4501 ASSERT(vector == igb->intr_cnt); 4502 4503 return (IGB_SUCCESS); 4504 } 4505 4506 /* 4507 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts 4508 * 4509 * For each vector enabled on the adapter, Set the MSIXBM register accordingly 4510 */ 4511 static void 4512 igb_setup_msix_82575(igb_t *igb) 4513 { 4514 uint32_t eims = 0; 4515 int i, vector; 4516 struct e1000_hw *hw = &igb->hw; 4517 4518 /* 4519 * Set vector for tx ring 0 and other causes. 4520 * NOTE assumption that it is vector 0. 4521 */ 4522 vector = 0; 4523 4524 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER; 4525 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask); 4526 vector++; 4527 4528 for (i = 0; i < igb->num_rx_rings; i++) { 4529 /* 4530 * Set vector for each rx ring 4531 */ 4532 eims = (E1000_EICR_RX_QUEUE0 << i); 4533 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4534 4535 /* 4536 * Accumulate bits to enable in 4537 * igb_enable_adapter_interrupts_82575() 4538 */ 4539 igb->eims_mask |= eims; 4540 4541 vector++; 4542 } 4543 4544 for (i = 1; i < igb->num_tx_rings; i++) { 4545 /* 4546 * Set vector for each tx ring from 2nd tx ring 4547 */ 4548 eims = (E1000_EICR_TX_QUEUE0 << i); 4549 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4550 4551 /* 4552 * Accumulate bits to enable in 4553 * igb_enable_adapter_interrupts_82575() 4554 */ 4555 igb->eims_mask |= eims; 4556 4557 vector++; 4558 } 4559 4560 ASSERT(vector == igb->intr_cnt); 4561 4562 /* 4563 * Disable IAM for ICR interrupt bits 4564 */ 4565 E1000_WRITE_REG(hw, E1000_IAM, 0); 4566 E1000_WRITE_FLUSH(hw); 4567 } 4568 4569 /* 4570 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts 4571 * 4572 * 82576 uses a table based method for assigning vectors. Each queue has a 4573 * single entry in the table to which we write a vector number along with a 4574 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors 4575 * take a different position in the 4-byte register depending on whether 4576 * they are numbered above or below 8. 4577 */ 4578 static void 4579 igb_setup_msix_82576(igb_t *igb) 4580 { 4581 struct e1000_hw *hw = &igb->hw; 4582 uint32_t ivar, index, vector; 4583 int i; 4584 4585 /* must enable msi-x capability before IVAR settings */ 4586 E1000_WRITE_REG(hw, E1000_GPIE, 4587 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR)); 4588 4589 /* 4590 * Set vector for tx ring 0 and other causes. 4591 * NOTE assumption that it is vector 0. 4592 * This is also interdependent with installation of interrupt service 4593 * routines in igb_add_intr_handlers(). 4594 */ 4595 4596 /* assign "other" causes to vector 0 */ 4597 vector = 0; 4598 ivar = ((vector | E1000_IVAR_VALID) << 8); 4599 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4600 4601 /* assign tx ring 0 to vector 0 */ 4602 ivar = ((vector | E1000_IVAR_VALID) << 8); 4603 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4604 4605 /* prepare to enable tx & other interrupt causes */ 4606 igb->eims_mask = (1 << vector); 4607 4608 vector ++; 4609 for (i = 0; i < igb->num_rx_rings; i++) { 4610 /* 4611 * Set vector for each rx ring 4612 */ 4613 index = (i & 0x7); 4614 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4615 4616 if (i < 8) { 4617 /* vector goes into low byte of register */ 4618 ivar = ivar & 0xFFFFFF00; 4619 ivar |= (vector | E1000_IVAR_VALID); 4620 } else { 4621 /* vector goes into third byte of register */ 4622 ivar = ivar & 0xFF00FFFF; 4623 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4624 } 4625 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4626 4627 /* Accumulate interrupt-cause bits to enable */ 4628 igb->eims_mask |= (1 << vector); 4629 4630 vector ++; 4631 } 4632 4633 for (i = 1; i < igb->num_tx_rings; i++) { 4634 /* 4635 * Set vector for each tx ring from 2nd tx ring. 4636 * Note assumption that tx vectors numericall follow rx vectors. 4637 */ 4638 index = (i & 0x7); 4639 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4640 4641 if (i < 8) { 4642 /* vector goes into second byte of register */ 4643 ivar = ivar & 0xFFFF00FF; 4644 ivar |= ((vector | E1000_IVAR_VALID) << 8); 4645 } else { 4646 /* vector goes into fourth byte of register */ 4647 ivar = ivar & 0x00FFFFFF; 4648 ivar |= (vector | E1000_IVAR_VALID) << 24; 4649 } 4650 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4651 4652 /* Accumulate interrupt-cause bits to enable */ 4653 igb->eims_mask |= (1 << vector); 4654 4655 vector ++; 4656 } 4657 4658 ASSERT(vector == igb->intr_cnt); 4659 } 4660 4661 /* 4662 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts 4663 * 4664 * 82580 uses same table approach at 82576 but has fewer entries. Each 4665 * queue has a single entry in the table to which we write a vector number 4666 * along with a "valid" bit. Vectors take a different position in the 4667 * register depending on * whether * they are numbered above or below 4. 4668 */ 4669 static void 4670 igb_setup_msix_82580(igb_t *igb) 4671 { 4672 struct e1000_hw *hw = &igb->hw; 4673 uint32_t ivar, index, vector; 4674 int i; 4675 4676 /* must enable msi-x capability before IVAR settings */ 4677 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE | 4678 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME)); 4679 /* 4680 * Set vector for tx ring 0 and other causes. 4681 * NOTE assumption that it is vector 0. 4682 * This is also interdependent with installation of interrupt service 4683 * routines in igb_add_intr_handlers(). 4684 */ 4685 4686 /* assign "other" causes to vector 0 */ 4687 vector = 0; 4688 ivar = ((vector | E1000_IVAR_VALID) << 8); 4689 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4690 4691 /* assign tx ring 0 to vector 0 */ 4692 ivar = ((vector | E1000_IVAR_VALID) << 8); 4693 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4694 4695 /* prepare to enable tx & other interrupt causes */ 4696 igb->eims_mask = (1 << vector); 4697 4698 vector ++; 4699 4700 for (i = 0; i < igb->num_rx_rings; i++) { 4701 /* 4702 * Set vector for each rx ring 4703 */ 4704 index = (i >> 1); 4705 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4706 4707 if (i & 1) { 4708 /* vector goes into third byte of register */ 4709 ivar = ivar & 0xFF00FFFF; 4710 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4711 } else { 4712 /* vector goes into low byte of register */ 4713 ivar = ivar & 0xFFFFFF00; 4714 ivar |= (vector | E1000_IVAR_VALID); 4715 } 4716 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4717 4718 /* Accumulate interrupt-cause bits to enable */ 4719 igb->eims_mask |= (1 << vector); 4720 4721 vector ++; 4722 } 4723 4724 for (i = 1; i < igb->num_tx_rings; i++) { 4725 /* 4726 * Set vector for each tx ring from 2nd tx ring. 4727 * Note assumption that tx vectors numericall follow rx vectors. 4728 */ 4729 index = (i >> 1); 4730 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4731 4732 if (i & 1) { 4733 /* vector goes into high byte of register */ 4734 ivar = ivar & 0x00FFFFFF; 4735 ivar |= ((vector | E1000_IVAR_VALID) << 24); 4736 } else { 4737 /* vector goes into second byte of register */ 4738 ivar = ivar & 0xFFFF00FF; 4739 ivar |= (vector | E1000_IVAR_VALID) << 8; 4740 } 4741 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4742 4743 /* Accumulate interrupt-cause bits to enable */ 4744 igb->eims_mask |= (1 << vector); 4745 4746 vector ++; 4747 } 4748 ASSERT(vector == igb->intr_cnt); 4749 } 4750 4751 /* 4752 * igb_rem_intr_handlers - remove the interrupt handlers 4753 */ 4754 static void 4755 igb_rem_intr_handlers(igb_t *igb) 4756 { 4757 int i; 4758 int rc; 4759 4760 for (i = 0; i < igb->intr_cnt; i++) { 4761 rc = ddi_intr_remove_handler(igb->htable[i]); 4762 if (rc != DDI_SUCCESS) { 4763 IGB_DEBUGLOG_1(igb, 4764 "Remove intr handler failed: %d", rc); 4765 } 4766 } 4767 } 4768 4769 /* 4770 * igb_rem_intrs - remove the allocated interrupts 4771 */ 4772 static void 4773 igb_rem_intrs(igb_t *igb) 4774 { 4775 int i; 4776 int rc; 4777 4778 for (i = 0; i < igb->intr_cnt; i++) { 4779 rc = ddi_intr_free(igb->htable[i]); 4780 if (rc != DDI_SUCCESS) { 4781 IGB_DEBUGLOG_1(igb, 4782 "Free intr failed: %d", rc); 4783 } 4784 } 4785 4786 kmem_free(igb->htable, igb->intr_size); 4787 igb->htable = NULL; 4788 } 4789 4790 /* 4791 * igb_enable_intrs - enable all the ddi interrupts 4792 */ 4793 static int 4794 igb_enable_intrs(igb_t *igb) 4795 { 4796 int i; 4797 int rc; 4798 4799 /* Enable interrupts */ 4800 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4801 /* Call ddi_intr_block_enable() for MSI */ 4802 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt); 4803 if (rc != DDI_SUCCESS) { 4804 igb_log(igb, 4805 "Enable block intr failed: %d", rc); 4806 return (IGB_FAILURE); 4807 } 4808 } else { 4809 /* Call ddi_intr_enable() for Legacy/MSI non block enable */ 4810 for (i = 0; i < igb->intr_cnt; i++) { 4811 rc = ddi_intr_enable(igb->htable[i]); 4812 if (rc != DDI_SUCCESS) { 4813 igb_log(igb, 4814 "Enable intr failed: %d", rc); 4815 return (IGB_FAILURE); 4816 } 4817 } 4818 } 4819 4820 return (IGB_SUCCESS); 4821 } 4822 4823 /* 4824 * igb_disable_intrs - disable all the ddi interrupts 4825 */ 4826 static int 4827 igb_disable_intrs(igb_t *igb) 4828 { 4829 int i; 4830 int rc; 4831 4832 /* Disable all interrupts */ 4833 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4834 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt); 4835 if (rc != DDI_SUCCESS) { 4836 igb_log(igb, 4837 "Disable block intr failed: %d", rc); 4838 return (IGB_FAILURE); 4839 } 4840 } else { 4841 for (i = 0; i < igb->intr_cnt; i++) { 4842 rc = ddi_intr_disable(igb->htable[i]); 4843 if (rc != DDI_SUCCESS) { 4844 igb_log(igb, 4845 "Disable intr failed: %d", rc); 4846 return (IGB_FAILURE); 4847 } 4848 } 4849 } 4850 4851 return (IGB_SUCCESS); 4852 } 4853 4854 /* 4855 * igb_get_phy_state - Get and save the parameters read from PHY registers 4856 */ 4857 static void 4858 igb_get_phy_state(igb_t *igb) 4859 { 4860 struct e1000_hw *hw = &igb->hw; 4861 uint16_t phy_ctrl; 4862 uint16_t phy_status; 4863 uint16_t phy_an_adv; 4864 uint16_t phy_an_exp; 4865 uint16_t phy_ext_status; 4866 uint16_t phy_1000t_ctrl; 4867 uint16_t phy_1000t_status; 4868 uint16_t phy_lp_able; 4869 4870 ASSERT(mutex_owned(&igb->gen_lock)); 4871 4872 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 4873 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 4874 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv); 4875 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp); 4876 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status); 4877 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl); 4878 (void) e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_1000t_status); 4879 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able); 4880 4881 igb->param_autoneg_cap = 4882 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0; 4883 igb->param_pause_cap = 4884 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4885 igb->param_asym_pause_cap = 4886 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4887 igb->param_1000fdx_cap = ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) || 4888 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0; 4889 igb->param_1000hdx_cap = ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) || 4890 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0; 4891 igb->param_100t4_cap = 4892 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0; 4893 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) || 4894 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0; 4895 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) || 4896 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0; 4897 igb->param_10fdx_cap = 4898 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0; 4899 igb->param_10hdx_cap = 4900 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0; 4901 igb->param_rem_fault = 4902 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0; 4903 4904 igb->param_adv_autoneg_cap = hw->mac.autoneg; 4905 igb->param_adv_pause_cap = 4906 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4907 igb->param_adv_asym_pause_cap = 4908 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4909 igb->param_adv_1000hdx_cap = 4910 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0; 4911 igb->param_adv_100t4_cap = 4912 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0; 4913 igb->param_adv_rem_fault = 4914 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0; 4915 if (igb->param_adv_autoneg_cap == 1) { 4916 igb->param_adv_1000fdx_cap = 4917 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0; 4918 igb->param_adv_100fdx_cap = 4919 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0; 4920 igb->param_adv_100hdx_cap = 4921 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0; 4922 igb->param_adv_10fdx_cap = 4923 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0; 4924 igb->param_adv_10hdx_cap = 4925 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0; 4926 } 4927 4928 igb->param_lp_autoneg_cap = 4929 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0; 4930 igb->param_lp_pause_cap = 4931 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0; 4932 igb->param_lp_asym_pause_cap = 4933 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0; 4934 igb->param_lp_1000fdx_cap = 4935 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0; 4936 igb->param_lp_1000hdx_cap = 4937 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0; 4938 igb->param_lp_100t4_cap = 4939 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0; 4940 igb->param_lp_100fdx_cap = 4941 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0; 4942 igb->param_lp_100hdx_cap = 4943 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0; 4944 igb->param_lp_10fdx_cap = 4945 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0; 4946 igb->param_lp_10hdx_cap = 4947 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0; 4948 igb->param_lp_rem_fault = 4949 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0; 4950 } 4951 4952 /* 4953 * igb_get_driver_control 4954 */ 4955 static void 4956 igb_get_driver_control(struct e1000_hw *hw) 4957 { 4958 uint32_t ctrl_ext; 4959 4960 /* Notify firmware that driver is in control of device */ 4961 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 4962 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD; 4963 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 4964 } 4965 4966 /* 4967 * igb_release_driver_control 4968 */ 4969 static void 4970 igb_release_driver_control(struct e1000_hw *hw) 4971 { 4972 uint32_t ctrl_ext; 4973 4974 /* Notify firmware that driver is no longer in control of device */ 4975 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 4976 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD; 4977 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 4978 } 4979 4980 /* 4981 * igb_atomic_reserve - Atomic decrease operation 4982 */ 4983 int 4984 igb_atomic_reserve(uint32_t *count_p, uint32_t n) 4985 { 4986 uint32_t oldval; 4987 uint32_t newval; 4988 4989 /* ATOMICALLY */ 4990 do { 4991 oldval = *count_p; 4992 if (oldval < n) 4993 return (-1); 4994 newval = oldval - n; 4995 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 4996 4997 return (newval); 4998 } 4999 5000 /* 5001 * FMA support 5002 */ 5003 5004 int 5005 igb_check_acc_handle(ddi_acc_handle_t handle) 5006 { 5007 ddi_fm_error_t de; 5008 5009 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5010 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5011 return (de.fme_status); 5012 } 5013 5014 int 5015 igb_check_dma_handle(ddi_dma_handle_t handle) 5016 { 5017 ddi_fm_error_t de; 5018 5019 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5020 return (de.fme_status); 5021 } 5022 5023 /* 5024 * The IO fault service error handling callback function 5025 */ 5026 /*ARGSUSED*/ 5027 static int 5028 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5029 { 5030 /* 5031 * as the driver can always deal with an error in any dma or 5032 * access handle, we can just return the fme_status value. 5033 */ 5034 pci_ereport_post(dip, err, NULL); 5035 return (err->fme_status); 5036 } 5037 5038 static void 5039 igb_fm_init(igb_t *igb) 5040 { 5041 ddi_iblock_cookie_t iblk; 5042 int fma_dma_flag; 5043 5044 /* Only register with IO Fault Services if we have some capability */ 5045 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5046 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5047 } else { 5048 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5049 } 5050 5051 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5052 fma_dma_flag = 1; 5053 } else { 5054 fma_dma_flag = 0; 5055 } 5056 5057 (void) igb_set_fma_flags(fma_dma_flag); 5058 5059 if (igb->fm_capabilities) { 5060 5061 /* Register capabilities with IO Fault Services */ 5062 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk); 5063 5064 /* 5065 * Initialize pci ereport capabilities if ereport capable 5066 */ 5067 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5068 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5069 pci_ereport_setup(igb->dip); 5070 5071 /* 5072 * Register error callback if error callback capable 5073 */ 5074 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5075 ddi_fm_handler_register(igb->dip, 5076 igb_fm_error_cb, (void*) igb); 5077 } 5078 } 5079 5080 static void 5081 igb_fm_fini(igb_t *igb) 5082 { 5083 /* Only unregister FMA capabilities if we registered some */ 5084 if (igb->fm_capabilities) { 5085 5086 /* 5087 * Release any resources allocated by pci_ereport_setup() 5088 */ 5089 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5090 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5091 pci_ereport_teardown(igb->dip); 5092 5093 /* 5094 * Un-register error callback if error callback capable 5095 */ 5096 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5097 ddi_fm_handler_unregister(igb->dip); 5098 5099 /* Unregister from IO Fault Services */ 5100 ddi_fm_fini(igb->dip); 5101 } 5102 } 5103 5104 void 5105 igb_fm_ereport(igb_t *igb, char *detail) 5106 { 5107 uint64_t ena; 5108 char buf[FM_MAX_CLASS]; 5109 5110 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5111 ena = fm_ena_generate(0, FM_ENA_FMT1); 5112 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) { 5113 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP, 5114 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5115 } 5116 } 5117