1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #include "igb_sw.h" 30 31 static char ident[] = "Intel 1Gb Ethernet"; 32 static char igb_version[] = "igb 1.1.9"; 33 34 /* 35 * Local function protoypes 36 */ 37 static int igb_register_mac(igb_t *); 38 static int igb_identify_hardware(igb_t *); 39 static int igb_regs_map(igb_t *); 40 static void igb_init_properties(igb_t *); 41 static int igb_init_driver_settings(igb_t *); 42 static void igb_init_locks(igb_t *); 43 static void igb_destroy_locks(igb_t *); 44 static int igb_init_mac_address(igb_t *); 45 static int igb_init(igb_t *); 46 static int igb_init_adapter(igb_t *); 47 static void igb_stop_adapter(igb_t *); 48 static int igb_reset(igb_t *); 49 static void igb_tx_clean(igb_t *); 50 static boolean_t igb_tx_drain(igb_t *); 51 static boolean_t igb_rx_drain(igb_t *); 52 static int igb_alloc_rings(igb_t *); 53 static void igb_free_rings(igb_t *); 54 static void igb_setup_rings(igb_t *); 55 static void igb_setup_rx(igb_t *); 56 static void igb_setup_tx(igb_t *); 57 static void igb_setup_rx_ring(igb_rx_ring_t *); 58 static void igb_setup_tx_ring(igb_tx_ring_t *); 59 static void igb_setup_rss(igb_t *); 60 static void igb_setup_mac_rss_classify(igb_t *); 61 static void igb_setup_mac_classify(igb_t *); 62 static void igb_init_unicst(igb_t *); 63 static void igb_setup_multicst(igb_t *); 64 static void igb_get_phy_state(igb_t *); 65 static void igb_get_conf(igb_t *); 66 static int igb_get_prop(igb_t *, char *, int, int, int); 67 static boolean_t igb_is_link_up(igb_t *); 68 static boolean_t igb_link_check(igb_t *); 69 static void igb_local_timer(void *); 70 static void igb_arm_watchdog_timer(igb_t *); 71 static void igb_start_watchdog_timer(igb_t *); 72 static void igb_restart_watchdog_timer(igb_t *); 73 static void igb_stop_watchdog_timer(igb_t *); 74 static void igb_disable_adapter_interrupts(igb_t *); 75 static void igb_enable_adapter_interrupts_82575(igb_t *); 76 static void igb_enable_adapter_interrupts_82576(igb_t *); 77 static void igb_enable_adapter_interrupts_82580(igb_t *); 78 static boolean_t is_valid_mac_addr(uint8_t *); 79 static boolean_t igb_stall_check(igb_t *); 80 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t); 81 static void igb_set_external_loopback(igb_t *); 82 static void igb_set_internal_mac_loopback(igb_t *); 83 static void igb_set_internal_phy_loopback(igb_t *); 84 static void igb_set_internal_serdes_loopback(igb_t *); 85 static boolean_t igb_find_mac_address(igb_t *); 86 static int igb_alloc_intrs(igb_t *); 87 static int igb_alloc_intr_handles(igb_t *, int); 88 static int igb_add_intr_handlers(igb_t *); 89 static void igb_rem_intr_handlers(igb_t *); 90 static void igb_rem_intrs(igb_t *); 91 static int igb_enable_intrs(igb_t *); 92 static int igb_disable_intrs(igb_t *); 93 static void igb_setup_msix_82575(igb_t *); 94 static void igb_setup_msix_82576(igb_t *); 95 static void igb_setup_msix_82580(igb_t *); 96 static uint_t igb_intr_legacy(void *, void *); 97 static uint_t igb_intr_msi(void *, void *); 98 static uint_t igb_intr_rx(void *, void *); 99 static uint_t igb_intr_tx(void *, void *); 100 static uint_t igb_intr_tx_other(void *, void *); 101 static void igb_intr_rx_work(igb_rx_ring_t *); 102 static void igb_intr_tx_work(igb_tx_ring_t *); 103 static void igb_intr_link_work(igb_t *); 104 static void igb_get_driver_control(struct e1000_hw *); 105 static void igb_release_driver_control(struct e1000_hw *); 106 107 static int igb_attach(dev_info_t *, ddi_attach_cmd_t); 108 static int igb_detach(dev_info_t *, ddi_detach_cmd_t); 109 static int igb_resume(dev_info_t *); 110 static int igb_suspend(dev_info_t *); 111 static int igb_quiesce(dev_info_t *); 112 static void igb_unconfigure(dev_info_t *, igb_t *); 113 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *, 114 const void *); 115 static void igb_fm_init(igb_t *); 116 static void igb_fm_fini(igb_t *); 117 static void igb_release_multicast(igb_t *); 118 119 static struct cb_ops igb_cb_ops = { 120 nulldev, /* cb_open */ 121 nulldev, /* cb_close */ 122 nodev, /* cb_strategy */ 123 nodev, /* cb_print */ 124 nodev, /* cb_dump */ 125 nodev, /* cb_read */ 126 nodev, /* cb_write */ 127 nodev, /* cb_ioctl */ 128 nodev, /* cb_devmap */ 129 nodev, /* cb_mmap */ 130 nodev, /* cb_segmap */ 131 nochpoll, /* cb_chpoll */ 132 ddi_prop_op, /* cb_prop_op */ 133 NULL, /* cb_stream */ 134 D_MP | D_HOTPLUG, /* cb_flag */ 135 CB_REV, /* cb_rev */ 136 nodev, /* cb_aread */ 137 nodev /* cb_awrite */ 138 }; 139 140 static struct dev_ops igb_dev_ops = { 141 DEVO_REV, /* devo_rev */ 142 0, /* devo_refcnt */ 143 NULL, /* devo_getinfo */ 144 nulldev, /* devo_identify */ 145 nulldev, /* devo_probe */ 146 igb_attach, /* devo_attach */ 147 igb_detach, /* devo_detach */ 148 nodev, /* devo_reset */ 149 &igb_cb_ops, /* devo_cb_ops */ 150 NULL, /* devo_bus_ops */ 151 ddi_power, /* devo_power */ 152 igb_quiesce, /* devo_quiesce */ 153 }; 154 155 static struct modldrv igb_modldrv = { 156 &mod_driverops, /* Type of module. This one is a driver */ 157 ident, /* Discription string */ 158 &igb_dev_ops, /* driver ops */ 159 }; 160 161 static struct modlinkage igb_modlinkage = { 162 MODREV_1, &igb_modldrv, NULL 163 }; 164 165 /* Access attributes for register mapping */ 166 ddi_device_acc_attr_t igb_regs_acc_attr = { 167 DDI_DEVICE_ATTR_V0, 168 DDI_STRUCTURE_LE_ACC, 169 DDI_STRICTORDER_ACC, 170 DDI_FLAGERR_ACC 171 }; 172 173 #define IGB_M_CALLBACK_FLAGS (MC_IOCTL | MC_GETCAPAB) 174 175 static mac_callbacks_t igb_m_callbacks = { 176 IGB_M_CALLBACK_FLAGS, 177 igb_m_stat, 178 igb_m_start, 179 igb_m_stop, 180 igb_m_promisc, 181 igb_m_multicst, 182 NULL, 183 NULL, 184 igb_m_ioctl, 185 igb_m_getcapab 186 }; 187 188 /* 189 * Initialize capabilities of each supported adapter type 190 */ 191 static adapter_info_t igb_82575_cap = { 192 /* limits */ 193 4, /* maximum number of rx queues */ 194 1, /* minimum number of rx queues */ 195 4, /* default number of rx queues */ 196 4, /* maximum number of tx queues */ 197 1, /* minimum number of tx queues */ 198 4, /* default number of tx queues */ 199 65535, /* maximum interrupt throttle rate */ 200 0, /* minimum interrupt throttle rate */ 201 200, /* default interrupt throttle rate */ 202 203 /* function pointers */ 204 igb_enable_adapter_interrupts_82575, 205 igb_setup_msix_82575, 206 207 /* capabilities */ 208 (IGB_FLAG_HAS_DCA | /* capability flags */ 209 IGB_FLAG_VMDQ_POOL), 210 211 0xffc00000 /* mask for RXDCTL register */ 212 }; 213 214 static adapter_info_t igb_82576_cap = { 215 /* limits */ 216 16, /* maximum number of rx queues */ 217 1, /* minimum number of rx queues */ 218 4, /* default number of rx queues */ 219 16, /* maximum number of tx queues */ 220 1, /* minimum number of tx queues */ 221 4, /* default number of tx queues */ 222 65535, /* maximum interrupt throttle rate */ 223 0, /* minimum interrupt throttle rate */ 224 200, /* default interrupt throttle rate */ 225 226 /* function pointers */ 227 igb_enable_adapter_interrupts_82576, 228 igb_setup_msix_82576, 229 230 /* capabilities */ 231 (IGB_FLAG_HAS_DCA | /* capability flags */ 232 IGB_FLAG_VMDQ_POOL | 233 IGB_FLAG_NEED_CTX_IDX), 234 235 0xffe00000 /* mask for RXDCTL register */ 236 }; 237 238 static adapter_info_t igb_82580_cap = { 239 /* limits */ 240 8, /* maximum number of rx queues */ 241 1, /* minimum number of rx queues */ 242 4, /* default number of rx queues */ 243 8, /* maximum number of tx queues */ 244 1, /* minimum number of tx queues */ 245 4, /* default number of tx queues */ 246 65535, /* maximum interrupt throttle rate */ 247 0, /* minimum interrupt throttle rate */ 248 200, /* default interrupt throttle rate */ 249 250 /* function pointers */ 251 igb_enable_adapter_interrupts_82580, 252 igb_setup_msix_82580, 253 254 /* capabilities */ 255 (IGB_FLAG_HAS_DCA | /* capability flags */ 256 IGB_FLAG_VMDQ_POOL | 257 IGB_FLAG_NEED_CTX_IDX), 258 259 0xffe00000 /* mask for RXDCTL register */ 260 }; 261 262 /* 263 * Module Initialization Functions 264 */ 265 266 int 267 _init(void) 268 { 269 int status; 270 271 mac_init_ops(&igb_dev_ops, MODULE_NAME); 272 273 status = mod_install(&igb_modlinkage); 274 275 if (status != DDI_SUCCESS) { 276 mac_fini_ops(&igb_dev_ops); 277 } 278 279 return (status); 280 } 281 282 int 283 _fini(void) 284 { 285 int status; 286 287 status = mod_remove(&igb_modlinkage); 288 289 if (status == DDI_SUCCESS) { 290 mac_fini_ops(&igb_dev_ops); 291 } 292 293 return (status); 294 295 } 296 297 int 298 _info(struct modinfo *modinfop) 299 { 300 int status; 301 302 status = mod_info(&igb_modlinkage, modinfop); 303 304 return (status); 305 } 306 307 /* 308 * igb_attach - driver attach 309 * 310 * This function is the device specific initialization entry 311 * point. This entry point is required and must be written. 312 * The DDI_ATTACH command must be provided in the attach entry 313 * point. When attach() is called with cmd set to DDI_ATTACH, 314 * all normal kernel services (such as kmem_alloc(9F)) are 315 * available for use by the driver. 316 * 317 * The attach() function will be called once for each instance 318 * of the device on the system with cmd set to DDI_ATTACH. 319 * Until attach() succeeds, the only driver entry points which 320 * may be called are open(9E) and getinfo(9E). 321 */ 322 static int 323 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 324 { 325 igb_t *igb; 326 struct igb_osdep *osdep; 327 struct e1000_hw *hw; 328 int instance; 329 330 /* 331 * Check the command and perform corresponding operations 332 */ 333 switch (cmd) { 334 default: 335 return (DDI_FAILURE); 336 337 case DDI_RESUME: 338 return (igb_resume(devinfo)); 339 340 case DDI_ATTACH: 341 break; 342 } 343 344 /* Get the device instance */ 345 instance = ddi_get_instance(devinfo); 346 347 /* Allocate memory for the instance data structure */ 348 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP); 349 350 igb->dip = devinfo; 351 igb->instance = instance; 352 353 hw = &igb->hw; 354 osdep = &igb->osdep; 355 hw->back = osdep; 356 osdep->igb = igb; 357 358 /* Attach the instance pointer to the dev_info data structure */ 359 ddi_set_driver_private(devinfo, igb); 360 361 362 /* Initialize for fma support */ 363 igb->fm_capabilities = igb_get_prop(igb, "fm-capable", 364 0, 0x0f, 365 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 366 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 367 igb_fm_init(igb); 368 igb->attach_progress |= ATTACH_PROGRESS_FMINIT; 369 370 /* 371 * Map PCI config space registers 372 */ 373 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 374 igb_error(igb, "Failed to map PCI configurations"); 375 goto attach_fail; 376 } 377 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 378 379 /* 380 * Identify the chipset family 381 */ 382 if (igb_identify_hardware(igb) != IGB_SUCCESS) { 383 igb_error(igb, "Failed to identify hardware"); 384 goto attach_fail; 385 } 386 387 /* 388 * Map device registers 389 */ 390 if (igb_regs_map(igb) != IGB_SUCCESS) { 391 igb_error(igb, "Failed to map device registers"); 392 goto attach_fail; 393 } 394 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 395 396 /* 397 * Initialize driver parameters 398 */ 399 igb_init_properties(igb); 400 igb->attach_progress |= ATTACH_PROGRESS_PROPS; 401 402 /* 403 * Allocate interrupts 404 */ 405 if (igb_alloc_intrs(igb) != IGB_SUCCESS) { 406 igb_error(igb, "Failed to allocate interrupts"); 407 goto attach_fail; 408 } 409 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 410 411 /* 412 * Allocate rx/tx rings based on the ring numbers. 413 * The actual numbers of rx/tx rings are decided by the number of 414 * allocated interrupt vectors, so we should allocate the rings after 415 * interrupts are allocated. 416 */ 417 if (igb_alloc_rings(igb) != IGB_SUCCESS) { 418 igb_error(igb, "Failed to allocate rx/tx rings or groups"); 419 goto attach_fail; 420 } 421 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 422 423 /* 424 * Add interrupt handlers 425 */ 426 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) { 427 igb_error(igb, "Failed to add interrupt handlers"); 428 goto attach_fail; 429 } 430 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 431 432 /* 433 * Initialize driver parameters 434 */ 435 if (igb_init_driver_settings(igb) != IGB_SUCCESS) { 436 igb_error(igb, "Failed to initialize driver settings"); 437 goto attach_fail; 438 } 439 440 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) { 441 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 442 goto attach_fail; 443 } 444 445 /* 446 * Initialize mutexes for this device. 447 * Do this before enabling the interrupt handler and 448 * register the softint to avoid the condition where 449 * interrupt handler can try using uninitialized mutex 450 */ 451 igb_init_locks(igb); 452 igb->attach_progress |= ATTACH_PROGRESS_LOCKS; 453 454 /* 455 * Allocate DMA resources 456 */ 457 if (igb_alloc_dma(igb) != IGB_SUCCESS) { 458 igb_error(igb, "Failed to allocate DMA resources"); 459 goto attach_fail; 460 } 461 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_DMA; 462 463 /* 464 * Initialize the adapter and setup the rx/tx rings 465 */ 466 if (igb_init(igb) != IGB_SUCCESS) { 467 igb_error(igb, "Failed to initialize adapter"); 468 goto attach_fail; 469 } 470 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 471 472 /* 473 * Initialize statistics 474 */ 475 if (igb_init_stats(igb) != IGB_SUCCESS) { 476 igb_error(igb, "Failed to initialize statistics"); 477 goto attach_fail; 478 } 479 igb->attach_progress |= ATTACH_PROGRESS_STATS; 480 481 /* 482 * Initialize NDD parameters 483 */ 484 if (igb_nd_init(igb) != IGB_SUCCESS) { 485 igb_error(igb, "Failed to initialize ndd"); 486 goto attach_fail; 487 } 488 igb->attach_progress |= ATTACH_PROGRESS_NDD; 489 490 /* 491 * Register the driver to the MAC 492 */ 493 if (igb_register_mac(igb) != IGB_SUCCESS) { 494 igb_error(igb, "Failed to register MAC"); 495 goto attach_fail; 496 } 497 igb->attach_progress |= ATTACH_PROGRESS_MAC; 498 499 /* 500 * Now that mutex locks are initialized, and the chip is also 501 * initialized, enable interrupts. 502 */ 503 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 504 igb_error(igb, "Failed to enable DDI interrupts"); 505 goto attach_fail; 506 } 507 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 508 509 igb_log(igb, "%s", igb_version); 510 igb->igb_state |= IGB_INITIALIZED; 511 512 return (DDI_SUCCESS); 513 514 attach_fail: 515 igb_unconfigure(devinfo, igb); 516 return (DDI_FAILURE); 517 } 518 519 /* 520 * igb_detach - driver detach 521 * 522 * The detach() function is the complement of the attach routine. 523 * If cmd is set to DDI_DETACH, detach() is used to remove the 524 * state associated with a given instance of a device node 525 * prior to the removal of that instance from the system. 526 * 527 * The detach() function will be called once for each instance 528 * of the device for which there has been a successful attach() 529 * once there are no longer any opens on the device. 530 * 531 * Interrupts routine are disabled, All memory allocated by this 532 * driver are freed. 533 */ 534 static int 535 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 536 { 537 igb_t *igb; 538 539 /* 540 * Check detach command 541 */ 542 switch (cmd) { 543 default: 544 return (DDI_FAILURE); 545 546 case DDI_SUSPEND: 547 return (igb_suspend(devinfo)); 548 549 case DDI_DETACH: 550 break; 551 } 552 553 554 /* 555 * Get the pointer to the driver private data structure 556 */ 557 igb = (igb_t *)ddi_get_driver_private(devinfo); 558 if (igb == NULL) 559 return (DDI_FAILURE); 560 561 /* 562 * Unregister MAC. If failed, we have to fail the detach 563 */ 564 if (mac_unregister(igb->mac_hdl) != 0) { 565 igb_error(igb, "Failed to unregister MAC"); 566 return (DDI_FAILURE); 567 } 568 igb->attach_progress &= ~ATTACH_PROGRESS_MAC; 569 570 /* 571 * If the device is still running, it needs to be stopped first. 572 * This check is necessary because under some specific circumstances, 573 * the detach routine can be called without stopping the interface 574 * first. 575 */ 576 mutex_enter(&igb->gen_lock); 577 if (igb->igb_state & IGB_STARTED) { 578 igb->igb_state &= ~IGB_STARTED; 579 igb_stop(igb); 580 mutex_exit(&igb->gen_lock); 581 /* Disable and stop the watchdog timer */ 582 igb_disable_watchdog_timer(igb); 583 } else 584 mutex_exit(&igb->gen_lock); 585 586 /* 587 * Check if there are still rx buffers held by the upper layer. 588 * If so, fail the detach. 589 */ 590 if (!igb_rx_drain(igb)) 591 return (DDI_FAILURE); 592 593 /* 594 * Do the remaining unconfigure routines 595 */ 596 igb_unconfigure(devinfo, igb); 597 598 return (DDI_SUCCESS); 599 } 600 601 /* 602 * quiesce(9E) entry point. 603 * 604 * This function is called when the system is single-threaded at high 605 * PIL with preemption disabled. Therefore, this function must not be 606 * blocked. 607 * 608 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 609 * DDI_FAILURE indicates an error condition and should almost never happen. 610 */ 611 static int 612 igb_quiesce(dev_info_t *devinfo) 613 { 614 igb_t *igb; 615 struct e1000_hw *hw; 616 617 igb = (igb_t *)ddi_get_driver_private(devinfo); 618 619 if (igb == NULL) 620 return (DDI_FAILURE); 621 622 hw = &igb->hw; 623 624 /* 625 * Disable the adapter interrupts 626 */ 627 igb_disable_adapter_interrupts(igb); 628 629 /* Tell firmware driver is no longer in control */ 630 igb_release_driver_control(hw); 631 632 /* 633 * Reset the chipset 634 */ 635 (void) e1000_reset_hw(hw); 636 637 /* 638 * Reset PHY if possible 639 */ 640 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 641 (void) e1000_phy_hw_reset(hw); 642 643 return (DDI_SUCCESS); 644 } 645 646 /* 647 * igb_unconfigure - release all resources held by this instance 648 */ 649 static void 650 igb_unconfigure(dev_info_t *devinfo, igb_t *igb) 651 { 652 /* 653 * Disable interrupt 654 */ 655 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 656 (void) igb_disable_intrs(igb); 657 } 658 659 /* 660 * Unregister MAC 661 */ 662 if (igb->attach_progress & ATTACH_PROGRESS_MAC) { 663 (void) mac_unregister(igb->mac_hdl); 664 } 665 666 /* 667 * Free ndd parameters 668 */ 669 if (igb->attach_progress & ATTACH_PROGRESS_NDD) { 670 igb_nd_cleanup(igb); 671 } 672 673 /* 674 * Free statistics 675 */ 676 if (igb->attach_progress & ATTACH_PROGRESS_STATS) { 677 kstat_delete((kstat_t *)igb->igb_ks); 678 } 679 680 /* 681 * Remove interrupt handlers 682 */ 683 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 684 igb_rem_intr_handlers(igb); 685 } 686 687 /* 688 * Remove interrupts 689 */ 690 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 691 igb_rem_intrs(igb); 692 } 693 694 /* 695 * Remove driver properties 696 */ 697 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) { 698 (void) ddi_prop_remove_all(devinfo); 699 } 700 701 /* 702 * Release the DMA resources of rx/tx rings 703 */ 704 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_DMA) { 705 igb_free_dma(igb); 706 } 707 708 /* 709 * Stop the adapter 710 */ 711 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) { 712 mutex_enter(&igb->gen_lock); 713 igb_stop_adapter(igb); 714 mutex_exit(&igb->gen_lock); 715 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 716 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED); 717 } 718 719 /* 720 * Free multicast table 721 */ 722 igb_release_multicast(igb); 723 724 /* 725 * Free register handle 726 */ 727 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 728 if (igb->osdep.reg_handle != NULL) 729 ddi_regs_map_free(&igb->osdep.reg_handle); 730 } 731 732 /* 733 * Free PCI config handle 734 */ 735 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 736 if (igb->osdep.cfg_handle != NULL) 737 pci_config_teardown(&igb->osdep.cfg_handle); 738 } 739 740 /* 741 * Free locks 742 */ 743 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) { 744 igb_destroy_locks(igb); 745 } 746 747 /* 748 * Free the rx/tx rings 749 */ 750 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 751 igb_free_rings(igb); 752 } 753 754 /* 755 * Remove FMA 756 */ 757 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) { 758 igb_fm_fini(igb); 759 } 760 761 /* 762 * Free the driver data structure 763 */ 764 kmem_free(igb, sizeof (igb_t)); 765 766 ddi_set_driver_private(devinfo, NULL); 767 } 768 769 /* 770 * igb_register_mac - Register the driver and its function pointers with 771 * the GLD interface 772 */ 773 static int 774 igb_register_mac(igb_t *igb) 775 { 776 struct e1000_hw *hw = &igb->hw; 777 mac_register_t *mac; 778 int status; 779 780 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 781 return (IGB_FAILURE); 782 783 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 784 mac->m_driver = igb; 785 mac->m_dip = igb->dip; 786 mac->m_src_addr = hw->mac.addr; 787 mac->m_callbacks = &igb_m_callbacks; 788 mac->m_min_sdu = 0; 789 mac->m_max_sdu = igb->max_frame_size - 790 sizeof (struct ether_vlan_header) - ETHERFCSL; 791 mac->m_margin = VLAN_TAGSZ; 792 mac->m_v12n = MAC_VIRT_LEVEL1; 793 794 status = mac_register(mac, &igb->mac_hdl); 795 796 mac_free(mac); 797 798 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE); 799 } 800 801 /* 802 * igb_identify_hardware - Identify the type of the chipset 803 */ 804 static int 805 igb_identify_hardware(igb_t *igb) 806 { 807 struct e1000_hw *hw = &igb->hw; 808 struct igb_osdep *osdep = &igb->osdep; 809 810 /* 811 * Get the device id 812 */ 813 hw->vendor_id = 814 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 815 hw->device_id = 816 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 817 hw->revision_id = 818 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 819 hw->subsystem_device_id = 820 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 821 hw->subsystem_vendor_id = 822 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 823 824 /* 825 * Set the mac type of the adapter based on the device id 826 */ 827 if (e1000_set_mac_type(hw) != E1000_SUCCESS) { 828 return (IGB_FAILURE); 829 } 830 831 /* 832 * Install adapter capabilities based on mac type 833 */ 834 switch (hw->mac.type) { 835 case e1000_82575: 836 igb->capab = &igb_82575_cap; 837 break; 838 case e1000_82576: 839 igb->capab = &igb_82576_cap; 840 break; 841 case e1000_82580: 842 igb->capab = &igb_82580_cap; 843 break; 844 default: 845 return (IGB_FAILURE); 846 } 847 848 return (IGB_SUCCESS); 849 } 850 851 /* 852 * igb_regs_map - Map the device registers 853 */ 854 static int 855 igb_regs_map(igb_t *igb) 856 { 857 dev_info_t *devinfo = igb->dip; 858 struct e1000_hw *hw = &igb->hw; 859 struct igb_osdep *osdep = &igb->osdep; 860 off_t mem_size; 861 862 /* 863 * First get the size of device registers to be mapped. 864 */ 865 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) != 866 DDI_SUCCESS) { 867 return (IGB_FAILURE); 868 } 869 870 /* 871 * Call ddi_regs_map_setup() to map registers 872 */ 873 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET, 874 (caddr_t *)&hw->hw_addr, 0, 875 mem_size, &igb_regs_acc_attr, 876 &osdep->reg_handle)) != DDI_SUCCESS) { 877 return (IGB_FAILURE); 878 } 879 880 return (IGB_SUCCESS); 881 } 882 883 /* 884 * igb_init_properties - Initialize driver properties 885 */ 886 static void 887 igb_init_properties(igb_t *igb) 888 { 889 /* 890 * Get conf file properties, including link settings 891 * jumbo frames, ring number, descriptor number, etc. 892 */ 893 igb_get_conf(igb); 894 } 895 896 /* 897 * igb_init_driver_settings - Initialize driver settings 898 * 899 * The settings include hardware function pointers, bus information, 900 * rx/tx rings settings, link state, and any other parameters that 901 * need to be setup during driver initialization. 902 */ 903 static int 904 igb_init_driver_settings(igb_t *igb) 905 { 906 struct e1000_hw *hw = &igb->hw; 907 igb_rx_ring_t *rx_ring; 908 igb_tx_ring_t *tx_ring; 909 uint32_t rx_size; 910 uint32_t tx_size; 911 int i; 912 913 /* 914 * Initialize chipset specific hardware function pointers 915 */ 916 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) { 917 return (IGB_FAILURE); 918 } 919 920 /* 921 * Get bus information 922 */ 923 if (e1000_get_bus_info(hw) != E1000_SUCCESS) { 924 return (IGB_FAILURE); 925 } 926 927 /* 928 * Get the system page size 929 */ 930 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1); 931 932 /* 933 * Set rx buffer size 934 * The IP header alignment room is counted in the calculation. 935 * The rx buffer size is in unit of 1K that is required by the 936 * chipset hardware. 937 */ 938 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM; 939 igb->rx_buf_size = ((rx_size >> 10) + 940 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 941 942 /* 943 * Set tx buffer size 944 */ 945 tx_size = igb->max_frame_size; 946 igb->tx_buf_size = ((tx_size >> 10) + 947 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 948 949 /* 950 * Initialize rx/tx rings parameters 951 */ 952 for (i = 0; i < igb->num_rx_rings; i++) { 953 rx_ring = &igb->rx_rings[i]; 954 rx_ring->index = i; 955 rx_ring->igb = igb; 956 957 rx_ring->ring_size = igb->rx_ring_size; 958 rx_ring->free_list_size = igb->rx_ring_size; 959 rx_ring->copy_thresh = igb->rx_copy_thresh; 960 rx_ring->limit_per_intr = igb->rx_limit_per_intr; 961 } 962 963 for (i = 0; i < igb->num_tx_rings; i++) { 964 tx_ring = &igb->tx_rings[i]; 965 tx_ring->index = i; 966 tx_ring->igb = igb; 967 if (igb->tx_head_wb_enable) 968 tx_ring->tx_recycle = igb_tx_recycle_head_wb; 969 else 970 tx_ring->tx_recycle = igb_tx_recycle_legacy; 971 972 tx_ring->ring_size = igb->tx_ring_size; 973 tx_ring->free_list_size = igb->tx_ring_size + 974 (igb->tx_ring_size >> 1); 975 tx_ring->copy_thresh = igb->tx_copy_thresh; 976 tx_ring->recycle_thresh = igb->tx_recycle_thresh; 977 tx_ring->overload_thresh = igb->tx_overload_thresh; 978 tx_ring->resched_thresh = igb->tx_resched_thresh; 979 } 980 981 /* 982 * Initialize values of interrupt throttling rates 983 */ 984 for (i = 1; i < MAX_NUM_EITR; i++) 985 igb->intr_throttling[i] = igb->intr_throttling[0]; 986 987 /* 988 * The initial link state should be "unknown" 989 */ 990 igb->link_state = LINK_STATE_UNKNOWN; 991 992 return (IGB_SUCCESS); 993 } 994 995 /* 996 * igb_init_locks - Initialize locks 997 */ 998 static void 999 igb_init_locks(igb_t *igb) 1000 { 1001 igb_rx_ring_t *rx_ring; 1002 igb_tx_ring_t *tx_ring; 1003 int i; 1004 1005 for (i = 0; i < igb->num_rx_rings; i++) { 1006 rx_ring = &igb->rx_rings[i]; 1007 mutex_init(&rx_ring->rx_lock, NULL, 1008 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1009 mutex_init(&rx_ring->recycle_lock, NULL, 1010 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1011 } 1012 1013 for (i = 0; i < igb->num_tx_rings; i++) { 1014 tx_ring = &igb->tx_rings[i]; 1015 mutex_init(&tx_ring->tx_lock, NULL, 1016 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1017 mutex_init(&tx_ring->recycle_lock, NULL, 1018 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1019 mutex_init(&tx_ring->tcb_head_lock, NULL, 1020 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1021 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1022 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1023 } 1024 1025 mutex_init(&igb->gen_lock, NULL, 1026 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1027 1028 mutex_init(&igb->watchdog_lock, NULL, 1029 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1030 } 1031 1032 /* 1033 * igb_destroy_locks - Destroy locks 1034 */ 1035 static void 1036 igb_destroy_locks(igb_t *igb) 1037 { 1038 igb_rx_ring_t *rx_ring; 1039 igb_tx_ring_t *tx_ring; 1040 int i; 1041 1042 for (i = 0; i < igb->num_rx_rings; i++) { 1043 rx_ring = &igb->rx_rings[i]; 1044 mutex_destroy(&rx_ring->rx_lock); 1045 mutex_destroy(&rx_ring->recycle_lock); 1046 } 1047 1048 for (i = 0; i < igb->num_tx_rings; i++) { 1049 tx_ring = &igb->tx_rings[i]; 1050 mutex_destroy(&tx_ring->tx_lock); 1051 mutex_destroy(&tx_ring->recycle_lock); 1052 mutex_destroy(&tx_ring->tcb_head_lock); 1053 mutex_destroy(&tx_ring->tcb_tail_lock); 1054 } 1055 1056 mutex_destroy(&igb->gen_lock); 1057 mutex_destroy(&igb->watchdog_lock); 1058 } 1059 1060 static int 1061 igb_resume(dev_info_t *devinfo) 1062 { 1063 igb_t *igb; 1064 1065 igb = (igb_t *)ddi_get_driver_private(devinfo); 1066 if (igb == NULL) 1067 return (DDI_FAILURE); 1068 1069 mutex_enter(&igb->gen_lock); 1070 1071 if (igb->igb_state & IGB_STARTED) { 1072 if (igb_start(igb) != IGB_SUCCESS) { 1073 mutex_exit(&igb->gen_lock); 1074 return (DDI_FAILURE); 1075 } 1076 1077 /* 1078 * Enable and start the watchdog timer 1079 */ 1080 igb_enable_watchdog_timer(igb); 1081 } 1082 1083 igb->igb_state &= ~IGB_SUSPENDED; 1084 1085 mutex_exit(&igb->gen_lock); 1086 1087 return (DDI_SUCCESS); 1088 } 1089 1090 static int 1091 igb_suspend(dev_info_t *devinfo) 1092 { 1093 igb_t *igb; 1094 1095 igb = (igb_t *)ddi_get_driver_private(devinfo); 1096 if (igb == NULL) 1097 return (DDI_FAILURE); 1098 1099 mutex_enter(&igb->gen_lock); 1100 1101 igb->igb_state |= IGB_SUSPENDED; 1102 1103 if (!(igb->igb_state & IGB_STARTED)) { 1104 mutex_exit(&igb->gen_lock); 1105 return (DDI_SUCCESS); 1106 } 1107 1108 igb_stop(igb); 1109 1110 mutex_exit(&igb->gen_lock); 1111 1112 /* 1113 * Disable and stop the watchdog timer 1114 */ 1115 igb_disable_watchdog_timer(igb); 1116 1117 return (DDI_SUCCESS); 1118 } 1119 1120 static int 1121 igb_init(igb_t *igb) 1122 { 1123 int i; 1124 1125 mutex_enter(&igb->gen_lock); 1126 1127 /* 1128 * Initilize the adapter 1129 */ 1130 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1131 mutex_exit(&igb->gen_lock); 1132 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1133 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1134 return (IGB_FAILURE); 1135 } 1136 1137 /* 1138 * Setup the rx/tx rings 1139 */ 1140 for (i = 0; i < igb->num_rx_rings; i++) 1141 mutex_enter(&igb->rx_rings[i].rx_lock); 1142 for (i = 0; i < igb->num_tx_rings; i++) 1143 mutex_enter(&igb->tx_rings[i].tx_lock); 1144 1145 igb_setup_rings(igb); 1146 1147 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1148 mutex_exit(&igb->tx_rings[i].tx_lock); 1149 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1150 mutex_exit(&igb->rx_rings[i].rx_lock); 1151 1152 mutex_exit(&igb->gen_lock); 1153 1154 return (IGB_SUCCESS); 1155 } 1156 1157 /* 1158 * igb_init_mac_address - Initialize the default MAC address 1159 * 1160 * On success, the MAC address is entered in the igb->hw.mac.addr 1161 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive 1162 * address register. 1163 * 1164 * Important side effects: 1165 * 1. adapter is reset - this is required to put it in a known state. 1166 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where 1167 * MAC address and all default settings are stored, so a valid checksum 1168 * is required. 1169 */ 1170 static int 1171 igb_init_mac_address(igb_t *igb) 1172 { 1173 struct e1000_hw *hw = &igb->hw; 1174 1175 ASSERT(mutex_owned(&igb->gen_lock)); 1176 1177 /* 1178 * Reset chipset to put the hardware in a known state 1179 * before we try to get MAC address from NVM. 1180 */ 1181 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1182 igb_error(igb, "Adapter reset failed."); 1183 goto init_mac_fail; 1184 } 1185 1186 /* 1187 * NVM validation 1188 */ 1189 if (e1000_validate_nvm_checksum(hw) < 0) { 1190 /* 1191 * Some PCI-E parts fail the first check due to 1192 * the link being in sleep state. Call it again, 1193 * if it fails a second time its a real issue. 1194 */ 1195 if (e1000_validate_nvm_checksum(hw) < 0) { 1196 igb_error(igb, 1197 "Invalid NVM checksum. Please contact " 1198 "the vendor to update the NVM."); 1199 goto init_mac_fail; 1200 } 1201 } 1202 1203 /* 1204 * Get the mac address 1205 * This function should handle SPARC case correctly. 1206 */ 1207 if (!igb_find_mac_address(igb)) { 1208 igb_error(igb, "Failed to get the mac address"); 1209 goto init_mac_fail; 1210 } 1211 1212 /* Validate mac address */ 1213 if (!is_valid_mac_addr(hw->mac.addr)) { 1214 igb_error(igb, "Invalid mac address"); 1215 goto init_mac_fail; 1216 } 1217 1218 return (IGB_SUCCESS); 1219 1220 init_mac_fail: 1221 return (IGB_FAILURE); 1222 } 1223 1224 /* 1225 * igb_init_adapter - Initialize the adapter 1226 */ 1227 static int 1228 igb_init_adapter(igb_t *igb) 1229 { 1230 struct e1000_hw *hw = &igb->hw; 1231 uint32_t pba; 1232 uint32_t high_water; 1233 int i; 1234 1235 ASSERT(mutex_owned(&igb->gen_lock)); 1236 1237 /* 1238 * In order to obtain the default MAC address, this will reset the 1239 * adapter and validate the NVM that the address and many other 1240 * default settings come from. 1241 */ 1242 if (igb_init_mac_address(igb) != IGB_SUCCESS) { 1243 igb_error(igb, "Failed to initialize MAC address"); 1244 goto init_adapter_fail; 1245 } 1246 1247 /* 1248 * Setup flow control 1249 * 1250 * These parameters set thresholds for the adapter's generation(Tx) 1251 * and response(Rx) to Ethernet PAUSE frames. These are just threshold 1252 * settings. Flow control is enabled or disabled in the configuration 1253 * file. 1254 * High-water mark is set down from the top of the rx fifo (not 1255 * sensitive to max_frame_size) and low-water is set just below 1256 * high-water mark. 1257 * The high water mark must be low enough to fit one full frame above 1258 * it in the rx FIFO. Should be the lower of: 1259 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full 1260 * frame. 1261 */ 1262 /* 1263 * The default setting of PBA is correct for 82575 and other supported 1264 * adapters do not have the E1000_PBA register, so PBA value is only 1265 * used for calculation here and is never written to the adapter. 1266 */ 1267 if (hw->mac.type == e1000_82575) { 1268 pba = E1000_PBA_34K; 1269 } else { 1270 pba = E1000_PBA_64K; 1271 } 1272 1273 high_water = min(((pba << 10) * 9 / 10), 1274 ((pba << 10) - igb->max_frame_size)); 1275 1276 if (hw->mac.type == e1000_82575) { 1277 /* 8-byte granularity */ 1278 hw->fc.high_water = high_water & 0xFFF8; 1279 hw->fc.low_water = hw->fc.high_water - 8; 1280 } else { 1281 /* 16-byte granularity */ 1282 hw->fc.high_water = high_water & 0xFFF0; 1283 hw->fc.low_water = hw->fc.high_water - 16; 1284 } 1285 1286 hw->fc.pause_time = E1000_FC_PAUSE_TIME; 1287 hw->fc.send_xon = B_TRUE; 1288 1289 (void) e1000_validate_mdi_setting(hw); 1290 1291 /* 1292 * Reset the chipset hardware the second time to put PBA settings 1293 * into effect. 1294 */ 1295 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1296 igb_error(igb, "Second reset failed"); 1297 goto init_adapter_fail; 1298 } 1299 1300 /* 1301 * Don't wait for auto-negotiation to complete 1302 */ 1303 hw->phy.autoneg_wait_to_complete = B_FALSE; 1304 1305 /* 1306 * Copper options 1307 */ 1308 if (hw->phy.media_type == e1000_media_type_copper) { 1309 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 1310 hw->phy.disable_polarity_correction = B_FALSE; 1311 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */ 1312 } 1313 1314 /* 1315 * Initialize link settings 1316 */ 1317 (void) igb_setup_link(igb, B_FALSE); 1318 1319 /* 1320 * Configure/Initialize hardware 1321 */ 1322 if (e1000_init_hw(hw) != E1000_SUCCESS) { 1323 igb_error(igb, "Failed to initialize hardware"); 1324 goto init_adapter_fail; 1325 } 1326 1327 /* 1328 * Disable wakeup control by default 1329 */ 1330 E1000_WRITE_REG(hw, E1000_WUC, 0); 1331 1332 /* 1333 * Record phy info in hw struct 1334 */ 1335 (void) e1000_get_phy_info(hw); 1336 1337 /* 1338 * Make sure driver has control 1339 */ 1340 igb_get_driver_control(hw); 1341 1342 /* 1343 * Restore LED settings to the default from EEPROM 1344 * to meet the standard for Sun platforms. 1345 */ 1346 (void) e1000_cleanup_led(hw); 1347 1348 /* 1349 * Setup MSI-X interrupts 1350 */ 1351 if (igb->intr_type == DDI_INTR_TYPE_MSIX) 1352 igb->capab->setup_msix(igb); 1353 1354 /* 1355 * Initialize unicast addresses. 1356 */ 1357 igb_init_unicst(igb); 1358 1359 /* 1360 * Setup and initialize the mctable structures. 1361 */ 1362 igb_setup_multicst(igb); 1363 1364 /* 1365 * Set interrupt throttling rate 1366 */ 1367 for (i = 0; i < igb->intr_cnt; i++) 1368 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]); 1369 1370 /* 1371 * Save the state of the phy 1372 */ 1373 igb_get_phy_state(igb); 1374 1375 return (IGB_SUCCESS); 1376 1377 init_adapter_fail: 1378 /* 1379 * Reset PHY if possible 1380 */ 1381 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 1382 (void) e1000_phy_hw_reset(hw); 1383 1384 return (IGB_FAILURE); 1385 } 1386 1387 /* 1388 * igb_stop_adapter - Stop the adapter 1389 */ 1390 static void 1391 igb_stop_adapter(igb_t *igb) 1392 { 1393 struct e1000_hw *hw = &igb->hw; 1394 1395 ASSERT(mutex_owned(&igb->gen_lock)); 1396 1397 /* Tell firmware driver is no longer in control */ 1398 igb_release_driver_control(hw); 1399 1400 /* 1401 * Reset the chipset 1402 */ 1403 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1404 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1405 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1406 } 1407 1408 /* 1409 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient 1410 */ 1411 } 1412 1413 /* 1414 * igb_reset - Reset the chipset and restart the driver. 1415 * 1416 * It involves stopping and re-starting the chipset, 1417 * and re-configuring the rx/tx rings. 1418 */ 1419 static int 1420 igb_reset(igb_t *igb) 1421 { 1422 int i; 1423 1424 mutex_enter(&igb->gen_lock); 1425 1426 ASSERT(igb->igb_state & IGB_STARTED); 1427 1428 /* 1429 * Disable the adapter interrupts to stop any rx/tx activities 1430 * before draining pending data and resetting hardware. 1431 */ 1432 igb_disable_adapter_interrupts(igb); 1433 1434 /* 1435 * Drain the pending transmit packets 1436 */ 1437 (void) igb_tx_drain(igb); 1438 1439 for (i = 0; i < igb->num_rx_rings; i++) 1440 mutex_enter(&igb->rx_rings[i].rx_lock); 1441 for (i = 0; i < igb->num_tx_rings; i++) 1442 mutex_enter(&igb->tx_rings[i].tx_lock); 1443 1444 /* 1445 * Stop the adapter 1446 */ 1447 igb_stop_adapter(igb); 1448 1449 /* 1450 * Clean the pending tx data/resources 1451 */ 1452 igb_tx_clean(igb); 1453 1454 /* 1455 * Start the adapter 1456 */ 1457 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1458 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1459 goto reset_failure; 1460 } 1461 1462 /* 1463 * Setup the rx/tx rings 1464 */ 1465 igb_setup_rings(igb); 1466 1467 /* 1468 * Enable adapter interrupts 1469 * The interrupts must be enabled after the driver state is START 1470 */ 1471 igb->capab->enable_intr(igb); 1472 1473 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1474 goto reset_failure; 1475 1476 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1477 goto reset_failure; 1478 1479 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1480 mutex_exit(&igb->tx_rings[i].tx_lock); 1481 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1482 mutex_exit(&igb->rx_rings[i].rx_lock); 1483 1484 mutex_exit(&igb->gen_lock); 1485 1486 return (IGB_SUCCESS); 1487 1488 reset_failure: 1489 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1490 mutex_exit(&igb->tx_rings[i].tx_lock); 1491 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1492 mutex_exit(&igb->rx_rings[i].rx_lock); 1493 1494 mutex_exit(&igb->gen_lock); 1495 1496 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1497 1498 return (IGB_FAILURE); 1499 } 1500 1501 /* 1502 * igb_tx_clean - Clean the pending transmit packets and DMA resources 1503 */ 1504 static void 1505 igb_tx_clean(igb_t *igb) 1506 { 1507 igb_tx_ring_t *tx_ring; 1508 tx_control_block_t *tcb; 1509 link_list_t pending_list; 1510 uint32_t desc_num; 1511 int i, j; 1512 1513 LINK_LIST_INIT(&pending_list); 1514 1515 for (i = 0; i < igb->num_tx_rings; i++) { 1516 tx_ring = &igb->tx_rings[i]; 1517 1518 mutex_enter(&tx_ring->recycle_lock); 1519 1520 /* 1521 * Clean the pending tx data - the pending packets in the 1522 * work_list that have no chances to be transmitted again. 1523 * 1524 * We must ensure the chipset is stopped or the link is down 1525 * before cleaning the transmit packets. 1526 */ 1527 desc_num = 0; 1528 for (j = 0; j < tx_ring->ring_size; j++) { 1529 tcb = tx_ring->work_list[j]; 1530 if (tcb != NULL) { 1531 desc_num += tcb->desc_num; 1532 1533 tx_ring->work_list[j] = NULL; 1534 1535 igb_free_tcb(tcb); 1536 1537 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1538 } 1539 } 1540 1541 if (desc_num > 0) { 1542 atomic_add_32(&tx_ring->tbd_free, desc_num); 1543 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1544 1545 /* 1546 * Reset the head and tail pointers of the tbd ring; 1547 * Reset the head write-back if it is enabled. 1548 */ 1549 tx_ring->tbd_head = 0; 1550 tx_ring->tbd_tail = 0; 1551 if (igb->tx_head_wb_enable) 1552 *tx_ring->tbd_head_wb = 0; 1553 1554 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0); 1555 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0); 1556 } 1557 1558 mutex_exit(&tx_ring->recycle_lock); 1559 1560 /* 1561 * Add the tx control blocks in the pending list to 1562 * the free list. 1563 */ 1564 igb_put_free_list(tx_ring, &pending_list); 1565 } 1566 } 1567 1568 /* 1569 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted 1570 */ 1571 static boolean_t 1572 igb_tx_drain(igb_t *igb) 1573 { 1574 igb_tx_ring_t *tx_ring; 1575 boolean_t done; 1576 int i, j; 1577 1578 /* 1579 * Wait for a specific time to allow pending tx packets 1580 * to be transmitted. 1581 * 1582 * Check the counter tbd_free to see if transmission is done. 1583 * No lock protection is needed here. 1584 * 1585 * Return B_TRUE if all pending packets have been transmitted; 1586 * Otherwise return B_FALSE; 1587 */ 1588 for (i = 0; i < TX_DRAIN_TIME; i++) { 1589 1590 done = B_TRUE; 1591 for (j = 0; j < igb->num_tx_rings; j++) { 1592 tx_ring = &igb->tx_rings[j]; 1593 done = done && 1594 (tx_ring->tbd_free == tx_ring->ring_size); 1595 } 1596 1597 if (done) 1598 break; 1599 1600 msec_delay(1); 1601 } 1602 1603 return (done); 1604 } 1605 1606 /* 1607 * igb_rx_drain - Wait for all rx buffers to be released by upper layer 1608 */ 1609 static boolean_t 1610 igb_rx_drain(igb_t *igb) 1611 { 1612 igb_rx_ring_t *rx_ring; 1613 boolean_t done; 1614 int i, j; 1615 1616 /* 1617 * Polling the rx free list to check if those rx buffers held by 1618 * the upper layer are released. 1619 * 1620 * Check the counter rcb_free to see if all pending buffers are 1621 * released. No lock protection is needed here. 1622 * 1623 * Return B_TRUE if all pending buffers have been released; 1624 * Otherwise return B_FALSE; 1625 */ 1626 for (i = 0; i < RX_DRAIN_TIME; i++) { 1627 1628 done = B_TRUE; 1629 for (j = 0; j < igb->num_rx_rings; j++) { 1630 rx_ring = &igb->rx_rings[j]; 1631 done = done && 1632 (rx_ring->rcb_free == rx_ring->free_list_size); 1633 } 1634 1635 if (done) 1636 break; 1637 1638 msec_delay(1); 1639 } 1640 1641 return (done); 1642 } 1643 1644 /* 1645 * igb_start - Start the driver/chipset 1646 */ 1647 int 1648 igb_start(igb_t *igb) 1649 { 1650 int i; 1651 1652 ASSERT(mutex_owned(&igb->gen_lock)); 1653 1654 for (i = 0; i < igb->num_rx_rings; i++) 1655 mutex_enter(&igb->rx_rings[i].rx_lock); 1656 for (i = 0; i < igb->num_tx_rings; i++) 1657 mutex_enter(&igb->tx_rings[i].tx_lock); 1658 1659 /* 1660 * Start the adapter 1661 */ 1662 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) { 1663 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1664 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1665 goto start_failure; 1666 } 1667 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 1668 1669 /* 1670 * Setup the rx/tx rings 1671 */ 1672 igb_setup_rings(igb); 1673 } 1674 1675 /* 1676 * Enable adapter interrupts 1677 * The interrupts must be enabled after the driver state is START 1678 */ 1679 igb->capab->enable_intr(igb); 1680 1681 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1682 goto start_failure; 1683 1684 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1685 goto start_failure; 1686 1687 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1688 mutex_exit(&igb->tx_rings[i].tx_lock); 1689 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1690 mutex_exit(&igb->rx_rings[i].rx_lock); 1691 1692 return (IGB_SUCCESS); 1693 1694 start_failure: 1695 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1696 mutex_exit(&igb->tx_rings[i].tx_lock); 1697 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1698 mutex_exit(&igb->rx_rings[i].rx_lock); 1699 1700 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1701 1702 return (IGB_FAILURE); 1703 } 1704 1705 /* 1706 * igb_stop - Stop the driver/chipset 1707 */ 1708 void 1709 igb_stop(igb_t *igb) 1710 { 1711 int i; 1712 1713 ASSERT(mutex_owned(&igb->gen_lock)); 1714 1715 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER; 1716 1717 /* 1718 * Disable the adapter interrupts 1719 */ 1720 igb_disable_adapter_interrupts(igb); 1721 1722 /* 1723 * Drain the pending tx packets 1724 */ 1725 (void) igb_tx_drain(igb); 1726 1727 for (i = 0; i < igb->num_rx_rings; i++) 1728 mutex_enter(&igb->rx_rings[i].rx_lock); 1729 for (i = 0; i < igb->num_tx_rings; i++) 1730 mutex_enter(&igb->tx_rings[i].tx_lock); 1731 1732 /* 1733 * Stop the adapter 1734 */ 1735 igb_stop_adapter(igb); 1736 1737 /* 1738 * Clean the pending tx data/resources 1739 */ 1740 igb_tx_clean(igb); 1741 1742 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1743 mutex_exit(&igb->tx_rings[i].tx_lock); 1744 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1745 mutex_exit(&igb->rx_rings[i].rx_lock); 1746 1747 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1748 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1749 } 1750 1751 /* 1752 * igb_alloc_rings - Allocate memory space for rx/tx rings 1753 */ 1754 static int 1755 igb_alloc_rings(igb_t *igb) 1756 { 1757 /* 1758 * Allocate memory space for rx rings 1759 */ 1760 igb->rx_rings = kmem_zalloc( 1761 sizeof (igb_rx_ring_t) * igb->num_rx_rings, 1762 KM_NOSLEEP); 1763 1764 if (igb->rx_rings == NULL) { 1765 return (IGB_FAILURE); 1766 } 1767 1768 /* 1769 * Allocate memory space for tx rings 1770 */ 1771 igb->tx_rings = kmem_zalloc( 1772 sizeof (igb_tx_ring_t) * igb->num_tx_rings, 1773 KM_NOSLEEP); 1774 1775 if (igb->tx_rings == NULL) { 1776 kmem_free(igb->rx_rings, 1777 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1778 igb->rx_rings = NULL; 1779 return (IGB_FAILURE); 1780 } 1781 1782 /* 1783 * Allocate memory space for rx ring groups 1784 */ 1785 igb->rx_groups = kmem_zalloc( 1786 sizeof (igb_rx_group_t) * igb->num_rx_groups, 1787 KM_NOSLEEP); 1788 1789 if (igb->rx_groups == NULL) { 1790 kmem_free(igb->rx_rings, 1791 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1792 kmem_free(igb->tx_rings, 1793 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1794 igb->rx_rings = NULL; 1795 igb->tx_rings = NULL; 1796 return (IGB_FAILURE); 1797 } 1798 1799 return (IGB_SUCCESS); 1800 } 1801 1802 /* 1803 * igb_free_rings - Free the memory space of rx/tx rings. 1804 */ 1805 static void 1806 igb_free_rings(igb_t *igb) 1807 { 1808 if (igb->rx_rings != NULL) { 1809 kmem_free(igb->rx_rings, 1810 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1811 igb->rx_rings = NULL; 1812 } 1813 1814 if (igb->tx_rings != NULL) { 1815 kmem_free(igb->tx_rings, 1816 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1817 igb->tx_rings = NULL; 1818 } 1819 1820 if (igb->rx_groups != NULL) { 1821 kmem_free(igb->rx_groups, 1822 sizeof (igb_rx_group_t) * igb->num_rx_groups); 1823 igb->rx_groups = NULL; 1824 } 1825 } 1826 1827 /* 1828 * igb_setup_rings - Setup rx/tx rings 1829 */ 1830 static void 1831 igb_setup_rings(igb_t *igb) 1832 { 1833 /* 1834 * Setup the rx/tx rings, including the following: 1835 * 1836 * 1. Setup the descriptor ring and the control block buffers; 1837 * 2. Initialize necessary registers for receive/transmit; 1838 * 3. Initialize software pointers/parameters for receive/transmit; 1839 */ 1840 igb_setup_rx(igb); 1841 1842 igb_setup_tx(igb); 1843 1844 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1845 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1846 } 1847 1848 static void 1849 igb_setup_rx_ring(igb_rx_ring_t *rx_ring) 1850 { 1851 igb_t *igb = rx_ring->igb; 1852 struct e1000_hw *hw = &igb->hw; 1853 rx_control_block_t *rcb; 1854 union e1000_adv_rx_desc *rbd; 1855 uint32_t size; 1856 uint32_t buf_low; 1857 uint32_t buf_high; 1858 uint32_t rxdctl; 1859 int i; 1860 1861 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1862 ASSERT(mutex_owned(&igb->gen_lock)); 1863 1864 /* 1865 * Initialize descriptor ring with buffer addresses 1866 */ 1867 for (i = 0; i < igb->rx_ring_size; i++) { 1868 rcb = rx_ring->work_list[i]; 1869 rbd = &rx_ring->rbd_ring[i]; 1870 1871 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1872 rbd->read.hdr_addr = NULL; 1873 } 1874 1875 /* 1876 * Initialize the base address registers 1877 */ 1878 buf_low = (uint32_t)rx_ring->rbd_area.dma_address; 1879 buf_high = (uint32_t)(rx_ring->rbd_area.dma_address >> 32); 1880 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high); 1881 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low); 1882 1883 /* 1884 * Initialize the length register 1885 */ 1886 size = rx_ring->ring_size * sizeof (union e1000_adv_rx_desc); 1887 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size); 1888 1889 /* 1890 * Initialize buffer size & descriptor type 1891 */ 1892 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index), 1893 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) | 1894 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF)); 1895 1896 /* 1897 * Setup the Receive Descriptor Control Register (RXDCTL) 1898 */ 1899 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index)); 1900 rxdctl &= igb->capab->rxdctl_mask; 1901 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 1902 rxdctl |= 16; /* pthresh */ 1903 rxdctl |= 8 << 8; /* hthresh */ 1904 rxdctl |= 1 << 16; /* wthresh */ 1905 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl); 1906 1907 rx_ring->rbd_next = 0; 1908 1909 /* 1910 * Note: Considering the case that the chipset is being reset 1911 * and there are still some buffers held by the upper layer, 1912 * we should not reset the values of rcb_head, rcb_tail and 1913 * rcb_free; 1914 */ 1915 if (igb->igb_state == IGB_UNKNOWN) { 1916 rx_ring->rcb_head = 0; 1917 rx_ring->rcb_tail = 0; 1918 rx_ring->rcb_free = rx_ring->free_list_size; 1919 } 1920 } 1921 1922 static void 1923 igb_setup_rx(igb_t *igb) 1924 { 1925 igb_rx_ring_t *rx_ring; 1926 igb_rx_group_t *rx_group; 1927 struct e1000_hw *hw = &igb->hw; 1928 uint32_t rctl, rxcsum; 1929 uint32_t ring_per_group; 1930 int i; 1931 1932 /* 1933 * Setup the Receive Control Register (RCTL), and enable the 1934 * receiver. The initial configuration is to: enable the receiver, 1935 * accept broadcasts, discard bad packets, accept long packets, 1936 * disable VLAN filter checking, and set receive buffer size to 1937 * 2k. For 82575, also set the receive descriptor minimum 1938 * threshold size to 1/2 the ring. 1939 */ 1940 rctl = E1000_READ_REG(hw, E1000_RCTL); 1941 1942 /* 1943 * Clear the field used for wakeup control. This driver doesn't do 1944 * wakeup but leave this here for completeness. 1945 */ 1946 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 1947 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 1948 1949 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */ 1950 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 1951 E1000_RCTL_LPE | /* Large Packet Enable */ 1952 /* Multicast filter offset */ 1953 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 1954 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */ 1955 E1000_RCTL_SECRC); /* Strip Ethernet CRC */ 1956 1957 for (i = 0; i < igb->num_rx_groups; i++) { 1958 rx_group = &igb->rx_groups[i]; 1959 rx_group->index = i; 1960 rx_group->igb = igb; 1961 } 1962 1963 /* 1964 * Set up all rx descriptor rings - must be called before receive unit 1965 * enabled. 1966 */ 1967 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 1968 for (i = 0; i < igb->num_rx_rings; i++) { 1969 rx_ring = &igb->rx_rings[i]; 1970 igb_setup_rx_ring(rx_ring); 1971 1972 /* 1973 * Map a ring to a group by assigning a group index 1974 */ 1975 rx_ring->group_index = i / ring_per_group; 1976 } 1977 1978 /* 1979 * Setup the Rx Long Packet Max Length register 1980 */ 1981 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size); 1982 1983 /* 1984 * Hardware checksum settings 1985 */ 1986 if (igb->rx_hcksum_enable) { 1987 rxcsum = 1988 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */ 1989 E1000_RXCSUM_IPOFL; /* IP checksum */ 1990 1991 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 1992 } 1993 1994 /* 1995 * Setup classify and RSS for multiple receive queues 1996 */ 1997 switch (igb->vmdq_mode) { 1998 case E1000_VMDQ_OFF: 1999 /* 2000 * One ring group, only RSS is needed when more than 2001 * one ring enabled. 2002 */ 2003 if (igb->num_rx_rings > 1) 2004 igb_setup_rss(igb); 2005 break; 2006 case E1000_VMDQ_MAC: 2007 /* 2008 * Multiple groups, each group has one ring, 2009 * only the MAC classification is needed. 2010 */ 2011 igb_setup_mac_classify(igb); 2012 break; 2013 case E1000_VMDQ_MAC_RSS: 2014 /* 2015 * Multiple groups and multiple rings, both 2016 * MAC classification and RSS are needed. 2017 */ 2018 igb_setup_mac_rss_classify(igb); 2019 break; 2020 } 2021 2022 /* 2023 * Enable the receive unit - must be done after all 2024 * the rx setup above. 2025 */ 2026 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2027 2028 /* 2029 * Initialize all adapter ring head & tail pointers - must 2030 * be done after receive unit is enabled 2031 */ 2032 for (i = 0; i < igb->num_rx_rings; i++) { 2033 rx_ring = &igb->rx_rings[i]; 2034 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 2035 E1000_WRITE_REG(hw, E1000_RDT(i), rx_ring->ring_size - 1); 2036 } 2037 2038 /* 2039 * 82575 with manageability enabled needs a special flush to make 2040 * sure the fifos start clean. 2041 */ 2042 if ((hw->mac.type == e1000_82575) && 2043 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) { 2044 e1000_rx_fifo_flush_82575(hw); 2045 } 2046 } 2047 2048 static void 2049 igb_setup_tx_ring(igb_tx_ring_t *tx_ring) 2050 { 2051 igb_t *igb = tx_ring->igb; 2052 struct e1000_hw *hw = &igb->hw; 2053 uint32_t size; 2054 uint32_t buf_low; 2055 uint32_t buf_high; 2056 uint32_t reg_val; 2057 2058 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2059 ASSERT(mutex_owned(&igb->gen_lock)); 2060 2061 2062 /* 2063 * Initialize the length register 2064 */ 2065 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc); 2066 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size); 2067 2068 /* 2069 * Initialize the base address registers 2070 */ 2071 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2072 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2073 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low); 2074 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high); 2075 2076 /* 2077 * Setup head & tail pointers 2078 */ 2079 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0); 2080 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0); 2081 2082 /* 2083 * Setup head write-back 2084 */ 2085 if (igb->tx_head_wb_enable) { 2086 /* 2087 * The memory of the head write-back is allocated using 2088 * the extra tbd beyond the tail of the tbd ring. 2089 */ 2090 tx_ring->tbd_head_wb = (uint32_t *) 2091 ((uintptr_t)tx_ring->tbd_area.address + size); 2092 *tx_ring->tbd_head_wb = 0; 2093 2094 buf_low = (uint32_t) 2095 (tx_ring->tbd_area.dma_address + size); 2096 buf_high = (uint32_t) 2097 ((tx_ring->tbd_area.dma_address + size) >> 32); 2098 2099 /* Set the head write-back enable bit */ 2100 buf_low |= E1000_TX_HEAD_WB_ENABLE; 2101 2102 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low); 2103 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high); 2104 2105 /* 2106 * Turn off relaxed ordering for head write back or it will 2107 * cause problems with the tx recycling 2108 */ 2109 reg_val = E1000_READ_REG(hw, 2110 E1000_DCA_TXCTRL(tx_ring->index)); 2111 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN; 2112 E1000_WRITE_REG(hw, 2113 E1000_DCA_TXCTRL(tx_ring->index), reg_val); 2114 } else { 2115 tx_ring->tbd_head_wb = NULL; 2116 } 2117 2118 tx_ring->tbd_head = 0; 2119 tx_ring->tbd_tail = 0; 2120 tx_ring->tbd_free = tx_ring->ring_size; 2121 2122 /* 2123 * Note: for the case that the chipset is being reset, we should not 2124 * reset the values of tcb_head, tcb_tail. And considering there might 2125 * still be some packets kept in the pending_list, we should not assert 2126 * (tcb_free == free_list_size) here. 2127 */ 2128 if (igb->igb_state == IGB_UNKNOWN) { 2129 tx_ring->tcb_head = 0; 2130 tx_ring->tcb_tail = 0; 2131 tx_ring->tcb_free = tx_ring->free_list_size; 2132 } 2133 2134 /* 2135 * Enable TXDCTL per queue 2136 */ 2137 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index)); 2138 reg_val |= E1000_TXDCTL_QUEUE_ENABLE; 2139 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val); 2140 2141 /* 2142 * Initialize hardware checksum offload settings 2143 */ 2144 bzero(&tx_ring->tx_context, sizeof (tx_context_t)); 2145 } 2146 2147 static void 2148 igb_setup_tx(igb_t *igb) 2149 { 2150 igb_tx_ring_t *tx_ring; 2151 struct e1000_hw *hw = &igb->hw; 2152 uint32_t reg_val; 2153 int i; 2154 2155 for (i = 0; i < igb->num_tx_rings; i++) { 2156 tx_ring = &igb->tx_rings[i]; 2157 igb_setup_tx_ring(tx_ring); 2158 } 2159 2160 /* 2161 * Setup the Transmit Control Register (TCTL) 2162 */ 2163 reg_val = E1000_READ_REG(hw, E1000_TCTL); 2164 reg_val &= ~E1000_TCTL_CT; 2165 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2166 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2167 2168 /* Enable transmits */ 2169 reg_val |= E1000_TCTL_EN; 2170 2171 E1000_WRITE_REG(hw, E1000_TCTL, reg_val); 2172 } 2173 2174 /* 2175 * igb_setup_rss - Setup receive-side scaling feature 2176 */ 2177 static void 2178 igb_setup_rss(igb_t *igb) 2179 { 2180 struct e1000_hw *hw = &igb->hw; 2181 uint32_t i, mrqc, rxcsum; 2182 int shift = 0; 2183 uint32_t random; 2184 union e1000_reta { 2185 uint32_t dword; 2186 uint8_t bytes[4]; 2187 } reta; 2188 2189 /* Setup the Redirection Table */ 2190 if (hw->mac.type == e1000_82576) { 2191 shift = 3; 2192 } else if (hw->mac.type == e1000_82575) { 2193 shift = 6; 2194 } 2195 for (i = 0; i < (32 * 4); i++) { 2196 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift; 2197 if ((i & 3) == 3) { 2198 E1000_WRITE_REG(hw, 2199 (E1000_RETA(0) + (i & ~3)), reta.dword); 2200 } 2201 } 2202 2203 /* Fill out hash function seeds */ 2204 for (i = 0; i < 10; i++) { 2205 (void) random_get_pseudo_bytes((uint8_t *)&random, 2206 sizeof (uint32_t)); 2207 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2208 } 2209 2210 /* Setup the Multiple Receive Queue Control register */ 2211 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2212 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2213 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2214 E1000_MRQC_RSS_FIELD_IPV6 | 2215 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2216 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2217 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2218 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2219 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2220 2221 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2222 2223 /* 2224 * Disable Packet Checksum to enable RSS for multiple receive queues. 2225 * 2226 * The Packet Checksum is not ethernet CRC. It is another kind of 2227 * checksum offloading provided by the 82575 chipset besides the IP 2228 * header checksum offloading and the TCP/UDP checksum offloading. 2229 * The Packet Checksum is by default computed over the entire packet 2230 * from the first byte of the DA through the last byte of the CRC, 2231 * including the Ethernet and IP headers. 2232 * 2233 * It is a hardware limitation that Packet Checksum is mutually 2234 * exclusive with RSS. 2235 */ 2236 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2237 rxcsum |= E1000_RXCSUM_PCSD; 2238 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2239 } 2240 2241 /* 2242 * igb_setup_mac_rss_classify - Setup MAC classification and rss 2243 */ 2244 static void 2245 igb_setup_mac_rss_classify(igb_t *igb) 2246 { 2247 struct e1000_hw *hw = &igb->hw; 2248 uint32_t i, mrqc, vmdctl, rxcsum; 2249 uint32_t ring_per_group; 2250 int shift_group0, shift_group1; 2251 uint32_t random; 2252 union e1000_reta { 2253 uint32_t dword; 2254 uint8_t bytes[4]; 2255 } reta; 2256 2257 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2258 2259 /* Setup the Redirection Table, it is shared between two groups */ 2260 shift_group0 = 2; 2261 shift_group1 = 6; 2262 for (i = 0; i < (32 * 4); i++) { 2263 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) | 2264 ((ring_per_group + (i % ring_per_group)) << shift_group1); 2265 if ((i & 3) == 3) { 2266 E1000_WRITE_REG(hw, 2267 (E1000_RETA(0) + (i & ~3)), reta.dword); 2268 } 2269 } 2270 2271 /* Fill out hash function seeds */ 2272 for (i = 0; i < 10; i++) { 2273 (void) random_get_pseudo_bytes((uint8_t *)&random, 2274 sizeof (uint32_t)); 2275 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2276 } 2277 2278 /* 2279 * Setup the Multiple Receive Queue Control register, 2280 * enable VMDq based on packet destination MAC address and RSS. 2281 */ 2282 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP; 2283 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2284 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2285 E1000_MRQC_RSS_FIELD_IPV6 | 2286 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2287 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2288 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2289 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2290 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2291 2292 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2293 2294 2295 /* Define the default group and default queues */ 2296 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE; 2297 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl); 2298 2299 /* 2300 * Disable Packet Checksum to enable RSS for multiple receive queues. 2301 * 2302 * The Packet Checksum is not ethernet CRC. It is another kind of 2303 * checksum offloading provided by the 82575 chipset besides the IP 2304 * header checksum offloading and the TCP/UDP checksum offloading. 2305 * The Packet Checksum is by default computed over the entire packet 2306 * from the first byte of the DA through the last byte of the CRC, 2307 * including the Ethernet and IP headers. 2308 * 2309 * It is a hardware limitation that Packet Checksum is mutually 2310 * exclusive with RSS. 2311 */ 2312 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2313 rxcsum |= E1000_RXCSUM_PCSD; 2314 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2315 } 2316 2317 /* 2318 * igb_setup_mac_classify - Setup MAC classification feature 2319 */ 2320 static void 2321 igb_setup_mac_classify(igb_t *igb) 2322 { 2323 struct e1000_hw *hw = &igb->hw; 2324 uint32_t mrqc, rxcsum; 2325 2326 /* 2327 * Setup the Multiple Receive Queue Control register, 2328 * enable VMDq based on packet destination MAC address. 2329 */ 2330 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP; 2331 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2332 2333 /* 2334 * Disable Packet Checksum to enable RSS for multiple receive queues. 2335 * 2336 * The Packet Checksum is not ethernet CRC. It is another kind of 2337 * checksum offloading provided by the 82575 chipset besides the IP 2338 * header checksum offloading and the TCP/UDP checksum offloading. 2339 * The Packet Checksum is by default computed over the entire packet 2340 * from the first byte of the DA through the last byte of the CRC, 2341 * including the Ethernet and IP headers. 2342 * 2343 * It is a hardware limitation that Packet Checksum is mutually 2344 * exclusive with RSS. 2345 */ 2346 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2347 rxcsum |= E1000_RXCSUM_PCSD; 2348 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2349 2350 } 2351 2352 /* 2353 * igb_init_unicst - Initialize the unicast addresses 2354 */ 2355 static void 2356 igb_init_unicst(igb_t *igb) 2357 { 2358 struct e1000_hw *hw = &igb->hw; 2359 int slot; 2360 2361 /* 2362 * Here we should consider two situations: 2363 * 2364 * 1. Chipset is initialized the first time 2365 * Initialize the multiple unicast addresses, and 2366 * save the default MAC address. 2367 * 2368 * 2. Chipset is reset 2369 * Recover the multiple unicast addresses from the 2370 * software data structure to the RAR registers. 2371 */ 2372 2373 /* 2374 * Clear the default MAC address in the RAR0 rgister, 2375 * which is loaded from EEPROM when system boot or chipreset, 2376 * this will cause the conficts with add_mac/rem_mac entry 2377 * points when VMDq is enabled. For this reason, the RAR0 2378 * must be cleared for both cases mentioned above. 2379 */ 2380 e1000_rar_clear(hw, 0); 2381 2382 if (!igb->unicst_init) { 2383 2384 /* Initialize the multiple unicast addresses */ 2385 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2386 igb->unicst_avail = igb->unicst_total; 2387 2388 for (slot = 0; slot < igb->unicst_total; slot++) 2389 igb->unicst_addr[slot].mac.set = 0; 2390 2391 igb->unicst_init = B_TRUE; 2392 } else { 2393 /* Re-configure the RAR registers */ 2394 for (slot = 0; slot < igb->unicst_total; slot++) { 2395 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr, 2396 slot, igb->vmdq_mode, 2397 igb->unicst_addr[slot].mac.group_index); 2398 } 2399 } 2400 2401 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 2402 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2403 } 2404 2405 /* 2406 * igb_unicst_find - Find the slot for the specified unicast address 2407 */ 2408 int 2409 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr) 2410 { 2411 int slot; 2412 2413 ASSERT(mutex_owned(&igb->gen_lock)); 2414 2415 for (slot = 0; slot < igb->unicst_total; slot++) { 2416 if (bcmp(igb->unicst_addr[slot].mac.addr, 2417 mac_addr, ETHERADDRL) == 0) 2418 return (slot); 2419 } 2420 2421 return (-1); 2422 } 2423 2424 /* 2425 * igb_unicst_set - Set the unicast address to the specified slot 2426 */ 2427 int 2428 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr, 2429 int slot) 2430 { 2431 struct e1000_hw *hw = &igb->hw; 2432 2433 ASSERT(mutex_owned(&igb->gen_lock)); 2434 2435 /* 2436 * Save the unicast address in the software data structure 2437 */ 2438 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL); 2439 2440 /* 2441 * Set the unicast address to the RAR register 2442 */ 2443 e1000_rar_set(hw, (uint8_t *)mac_addr, slot); 2444 2445 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2446 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2447 return (EIO); 2448 } 2449 2450 return (0); 2451 } 2452 2453 /* 2454 * igb_multicst_add - Add a multicst address 2455 */ 2456 int 2457 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr) 2458 { 2459 struct ether_addr *new_table; 2460 size_t new_len; 2461 size_t old_len; 2462 2463 ASSERT(mutex_owned(&igb->gen_lock)); 2464 2465 if ((multiaddr[0] & 01) == 0) { 2466 igb_error(igb, "Illegal multicast address"); 2467 return (EINVAL); 2468 } 2469 2470 if (igb->mcast_count >= igb->mcast_max_num) { 2471 igb_error(igb, "Adapter requested more than %d mcast addresses", 2472 igb->mcast_max_num); 2473 return (ENOENT); 2474 } 2475 2476 if (igb->mcast_count == igb->mcast_alloc_count) { 2477 old_len = igb->mcast_alloc_count * 2478 sizeof (struct ether_addr); 2479 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) * 2480 sizeof (struct ether_addr); 2481 2482 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2483 if (new_table == NULL) { 2484 igb_error(igb, 2485 "Not enough memory to alloc mcast table"); 2486 return (ENOMEM); 2487 } 2488 2489 if (igb->mcast_table != NULL) { 2490 bcopy(igb->mcast_table, new_table, old_len); 2491 kmem_free(igb->mcast_table, old_len); 2492 } 2493 igb->mcast_alloc_count += MCAST_ALLOC_COUNT; 2494 igb->mcast_table = new_table; 2495 } 2496 2497 bcopy(multiaddr, 2498 &igb->mcast_table[igb->mcast_count], ETHERADDRL); 2499 igb->mcast_count++; 2500 2501 /* 2502 * Update the multicast table in the hardware 2503 */ 2504 igb_setup_multicst(igb); 2505 2506 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2507 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2508 return (EIO); 2509 } 2510 2511 return (0); 2512 } 2513 2514 /* 2515 * igb_multicst_remove - Remove a multicst address 2516 */ 2517 int 2518 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr) 2519 { 2520 struct ether_addr *new_table; 2521 size_t new_len; 2522 size_t old_len; 2523 int i; 2524 2525 ASSERT(mutex_owned(&igb->gen_lock)); 2526 2527 for (i = 0; i < igb->mcast_count; i++) { 2528 if (bcmp(multiaddr, &igb->mcast_table[i], 2529 ETHERADDRL) == 0) { 2530 for (i++; i < igb->mcast_count; i++) { 2531 igb->mcast_table[i - 1] = 2532 igb->mcast_table[i]; 2533 } 2534 igb->mcast_count--; 2535 break; 2536 } 2537 } 2538 2539 if ((igb->mcast_alloc_count - igb->mcast_count) > 2540 MCAST_ALLOC_COUNT) { 2541 old_len = igb->mcast_alloc_count * 2542 sizeof (struct ether_addr); 2543 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) * 2544 sizeof (struct ether_addr); 2545 2546 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2547 if (new_table != NULL) { 2548 bcopy(igb->mcast_table, new_table, new_len); 2549 kmem_free(igb->mcast_table, old_len); 2550 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT; 2551 igb->mcast_table = new_table; 2552 } 2553 } 2554 2555 /* 2556 * Update the multicast table in the hardware 2557 */ 2558 igb_setup_multicst(igb); 2559 2560 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2561 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2562 return (EIO); 2563 } 2564 2565 return (0); 2566 } 2567 2568 static void 2569 igb_release_multicast(igb_t *igb) 2570 { 2571 if (igb->mcast_table != NULL) { 2572 kmem_free(igb->mcast_table, 2573 igb->mcast_alloc_count * sizeof (struct ether_addr)); 2574 igb->mcast_table = NULL; 2575 } 2576 } 2577 2578 /* 2579 * igb_setup_multicast - setup multicast data structures 2580 * 2581 * This routine initializes all of the multicast related structures 2582 * and save them in the hardware registers. 2583 */ 2584 static void 2585 igb_setup_multicst(igb_t *igb) 2586 { 2587 uint8_t *mc_addr_list; 2588 uint32_t mc_addr_count; 2589 struct e1000_hw *hw = &igb->hw; 2590 2591 ASSERT(mutex_owned(&igb->gen_lock)); 2592 ASSERT(igb->mcast_count <= igb->mcast_max_num); 2593 2594 mc_addr_list = (uint8_t *)igb->mcast_table; 2595 mc_addr_count = igb->mcast_count; 2596 2597 /* 2598 * Update the multicase addresses to the MTA registers 2599 */ 2600 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count); 2601 } 2602 2603 /* 2604 * igb_get_conf - Get driver configurations set in driver.conf 2605 * 2606 * This routine gets user-configured values out of the configuration 2607 * file igb.conf. 2608 * 2609 * For each configurable value, there is a minimum, a maximum, and a 2610 * default. 2611 * If user does not configure a value, use the default. 2612 * If user configures below the minimum, use the minumum. 2613 * If user configures above the maximum, use the maxumum. 2614 */ 2615 static void 2616 igb_get_conf(igb_t *igb) 2617 { 2618 struct e1000_hw *hw = &igb->hw; 2619 uint32_t default_mtu; 2620 uint32_t flow_control; 2621 uint32_t ring_per_group; 2622 int i; 2623 2624 /* 2625 * igb driver supports the following user configurations: 2626 * 2627 * Link configurations: 2628 * adv_autoneg_cap 2629 * adv_1000fdx_cap 2630 * adv_100fdx_cap 2631 * adv_100hdx_cap 2632 * adv_10fdx_cap 2633 * adv_10hdx_cap 2634 * Note: 1000hdx is not supported. 2635 * 2636 * Jumbo frame configuration: 2637 * default_mtu 2638 * 2639 * Ethernet flow control configuration: 2640 * flow_control 2641 * 2642 * Multiple rings configurations: 2643 * tx_queue_number 2644 * tx_ring_size 2645 * rx_queue_number 2646 * rx_ring_size 2647 * 2648 * Call igb_get_prop() to get the value for a specific 2649 * configuration parameter. 2650 */ 2651 2652 /* 2653 * Link configurations 2654 */ 2655 igb->param_adv_autoneg_cap = igb_get_prop(igb, 2656 PROP_ADV_AUTONEG_CAP, 0, 1, 1); 2657 igb->param_adv_1000fdx_cap = igb_get_prop(igb, 2658 PROP_ADV_1000FDX_CAP, 0, 1, 1); 2659 igb->param_adv_100fdx_cap = igb_get_prop(igb, 2660 PROP_ADV_100FDX_CAP, 0, 1, 1); 2661 igb->param_adv_100hdx_cap = igb_get_prop(igb, 2662 PROP_ADV_100HDX_CAP, 0, 1, 1); 2663 igb->param_adv_10fdx_cap = igb_get_prop(igb, 2664 PROP_ADV_10FDX_CAP, 0, 1, 1); 2665 igb->param_adv_10hdx_cap = igb_get_prop(igb, 2666 PROP_ADV_10HDX_CAP, 0, 1, 1); 2667 2668 /* 2669 * Jumbo frame configurations 2670 */ 2671 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU, 2672 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2673 2674 igb->max_frame_size = default_mtu + 2675 sizeof (struct ether_vlan_header) + ETHERFCSL; 2676 2677 /* 2678 * Ethernet flow control configuration 2679 */ 2680 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL, 2681 e1000_fc_none, 4, e1000_fc_full); 2682 if (flow_control == 4) 2683 flow_control = e1000_fc_default; 2684 2685 hw->fc.requested_mode = flow_control; 2686 2687 /* 2688 * Multiple rings configurations 2689 */ 2690 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE, 2691 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2692 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE, 2693 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2694 2695 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0); 2696 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM, 2697 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2698 /* 2699 * Currently we do not support VMDq for 82576 and 82580. 2700 * If it is e1000_82576, set num_rx_groups to 1. 2701 */ 2702 if (hw->mac.type >= e1000_82576) 2703 igb->num_rx_groups = 1; 2704 2705 if (igb->mr_enable) { 2706 igb->num_tx_rings = igb->capab->def_tx_que_num; 2707 igb->num_rx_rings = igb->capab->def_rx_que_num; 2708 } else { 2709 igb->num_tx_rings = 1; 2710 igb->num_rx_rings = 1; 2711 2712 if (igb->num_rx_groups > 1) { 2713 igb_error(igb, 2714 "Invalid rx groups number. Please enable multiple " 2715 "rings first"); 2716 igb->num_rx_groups = 1; 2717 } 2718 } 2719 2720 /* 2721 * Check the divisibility between rx rings and rx groups. 2722 */ 2723 for (i = igb->num_rx_groups; i > 0; i--) { 2724 if ((igb->num_rx_rings % i) == 0) 2725 break; 2726 } 2727 if (i != igb->num_rx_groups) { 2728 igb_error(igb, 2729 "Invalid rx groups number. Downgrade the rx group " 2730 "number to %d.", i); 2731 igb->num_rx_groups = i; 2732 } 2733 2734 /* 2735 * Get the ring number per group. 2736 */ 2737 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2738 2739 if (igb->num_rx_groups == 1) { 2740 /* 2741 * One rx ring group, the rx ring number is num_rx_rings. 2742 */ 2743 igb->vmdq_mode = E1000_VMDQ_OFF; 2744 } else if (ring_per_group == 1) { 2745 /* 2746 * Multiple rx groups, each group has one rx ring. 2747 */ 2748 igb->vmdq_mode = E1000_VMDQ_MAC; 2749 } else { 2750 /* 2751 * Multiple groups and multiple rings. 2752 */ 2753 igb->vmdq_mode = E1000_VMDQ_MAC_RSS; 2754 } 2755 2756 /* 2757 * Tunable used to force an interrupt type. The only use is 2758 * for testing of the lesser interrupt types. 2759 * 0 = don't force interrupt type 2760 * 1 = force interrupt type MSIX 2761 * 2 = force interrupt type MSI 2762 * 3 = force interrupt type Legacy 2763 */ 2764 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE, 2765 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE); 2766 2767 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE, 2768 0, 1, 1); 2769 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE, 2770 0, 1, 1); 2771 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE, 2772 0, 1, 1); 2773 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE, 2774 0, 1, 1); 2775 2776 /* 2777 * igb LSO needs the tx h/w checksum support. 2778 * Here LSO will be disabled if tx h/w checksum has been disabled. 2779 */ 2780 if (igb->tx_hcksum_enable == B_FALSE) 2781 igb->lso_enable = B_FALSE; 2782 2783 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD, 2784 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2785 DEFAULT_TX_COPY_THRESHOLD); 2786 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD, 2787 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, 2788 DEFAULT_TX_RECYCLE_THRESHOLD); 2789 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD, 2790 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, 2791 DEFAULT_TX_OVERLOAD_THRESHOLD); 2792 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD, 2793 MIN_TX_RESCHED_THRESHOLD, MAX_TX_RESCHED_THRESHOLD, 2794 DEFAULT_TX_RESCHED_THRESHOLD); 2795 2796 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD, 2797 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2798 DEFAULT_RX_COPY_THRESHOLD); 2799 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR, 2800 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2801 DEFAULT_RX_LIMIT_PER_INTR); 2802 2803 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING, 2804 igb->capab->min_intr_throttle, 2805 igb->capab->max_intr_throttle, 2806 igb->capab->def_intr_throttle); 2807 2808 /* 2809 * Max number of multicast addresses 2810 */ 2811 igb->mcast_max_num = 2812 igb_get_prop(igb, PROP_MCAST_MAX_NUM, 2813 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM); 2814 } 2815 2816 /* 2817 * igb_get_prop - Get a property value out of the configuration file igb.conf 2818 * 2819 * Caller provides the name of the property, a default value, a minimum 2820 * value, and a maximum value. 2821 * 2822 * Return configured value of the property, with default, minimum and 2823 * maximum properly applied. 2824 */ 2825 static int 2826 igb_get_prop(igb_t *igb, 2827 char *propname, /* name of the property */ 2828 int minval, /* minimum acceptable value */ 2829 int maxval, /* maximim acceptable value */ 2830 int defval) /* default value */ 2831 { 2832 int value; 2833 2834 /* 2835 * Call ddi_prop_get_int() to read the conf settings 2836 */ 2837 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip, 2838 DDI_PROP_DONTPASS, propname, defval); 2839 2840 if (value > maxval) 2841 value = maxval; 2842 2843 if (value < minval) 2844 value = minval; 2845 2846 return (value); 2847 } 2848 2849 /* 2850 * igb_setup_link - Using the link properties to setup the link 2851 */ 2852 int 2853 igb_setup_link(igb_t *igb, boolean_t setup_hw) 2854 { 2855 struct e1000_mac_info *mac; 2856 struct e1000_phy_info *phy; 2857 boolean_t invalid; 2858 2859 mac = &igb->hw.mac; 2860 phy = &igb->hw.phy; 2861 invalid = B_FALSE; 2862 2863 if (igb->param_adv_autoneg_cap == 1) { 2864 mac->autoneg = B_TRUE; 2865 phy->autoneg_advertised = 0; 2866 2867 /* 2868 * 1000hdx is not supported for autonegotiation 2869 */ 2870 if (igb->param_adv_1000fdx_cap == 1) 2871 phy->autoneg_advertised |= ADVERTISE_1000_FULL; 2872 2873 if (igb->param_adv_100fdx_cap == 1) 2874 phy->autoneg_advertised |= ADVERTISE_100_FULL; 2875 2876 if (igb->param_adv_100hdx_cap == 1) 2877 phy->autoneg_advertised |= ADVERTISE_100_HALF; 2878 2879 if (igb->param_adv_10fdx_cap == 1) 2880 phy->autoneg_advertised |= ADVERTISE_10_FULL; 2881 2882 if (igb->param_adv_10hdx_cap == 1) 2883 phy->autoneg_advertised |= ADVERTISE_10_HALF; 2884 2885 if (phy->autoneg_advertised == 0) 2886 invalid = B_TRUE; 2887 } else { 2888 mac->autoneg = B_FALSE; 2889 2890 /* 2891 * 1000fdx and 1000hdx are not supported for forced link 2892 */ 2893 if (igb->param_adv_100fdx_cap == 1) 2894 mac->forced_speed_duplex = ADVERTISE_100_FULL; 2895 else if (igb->param_adv_100hdx_cap == 1) 2896 mac->forced_speed_duplex = ADVERTISE_100_HALF; 2897 else if (igb->param_adv_10fdx_cap == 1) 2898 mac->forced_speed_duplex = ADVERTISE_10_FULL; 2899 else if (igb->param_adv_10hdx_cap == 1) 2900 mac->forced_speed_duplex = ADVERTISE_10_HALF; 2901 else 2902 invalid = B_TRUE; 2903 } 2904 2905 if (invalid) { 2906 igb_notice(igb, "Invalid link settings. Setup link to " 2907 "autonegotiation with full link capabilities."); 2908 mac->autoneg = B_TRUE; 2909 phy->autoneg_advertised = ADVERTISE_1000_FULL | 2910 ADVERTISE_100_FULL | ADVERTISE_100_HALF | 2911 ADVERTISE_10_FULL | ADVERTISE_10_HALF; 2912 } 2913 2914 if (setup_hw) { 2915 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS) 2916 return (IGB_FAILURE); 2917 } 2918 2919 return (IGB_SUCCESS); 2920 } 2921 2922 2923 /* 2924 * igb_is_link_up - Check if the link is up 2925 */ 2926 static boolean_t 2927 igb_is_link_up(igb_t *igb) 2928 { 2929 struct e1000_hw *hw = &igb->hw; 2930 boolean_t link_up = B_FALSE; 2931 2932 ASSERT(mutex_owned(&igb->gen_lock)); 2933 2934 /* 2935 * get_link_status is set in the interrupt handler on link-status-change 2936 * or rx sequence error interrupt. get_link_status will stay 2937 * false until the e1000_check_for_link establishes link only 2938 * for copper adapters. 2939 */ 2940 switch (hw->phy.media_type) { 2941 case e1000_media_type_copper: 2942 if (hw->mac.get_link_status) { 2943 (void) e1000_check_for_link(hw); 2944 link_up = !hw->mac.get_link_status; 2945 } else { 2946 link_up = B_TRUE; 2947 } 2948 break; 2949 case e1000_media_type_fiber: 2950 (void) e1000_check_for_link(hw); 2951 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 2952 break; 2953 case e1000_media_type_internal_serdes: 2954 (void) e1000_check_for_link(hw); 2955 link_up = hw->mac.serdes_has_link; 2956 break; 2957 } 2958 2959 return (link_up); 2960 } 2961 2962 /* 2963 * igb_link_check - Link status processing 2964 */ 2965 static boolean_t 2966 igb_link_check(igb_t *igb) 2967 { 2968 struct e1000_hw *hw = &igb->hw; 2969 uint16_t speed = 0, duplex = 0; 2970 boolean_t link_changed = B_FALSE; 2971 2972 ASSERT(mutex_owned(&igb->gen_lock)); 2973 2974 if (igb_is_link_up(igb)) { 2975 /* 2976 * The Link is up, check whether it was marked as down earlier 2977 */ 2978 if (igb->link_state != LINK_STATE_UP) { 2979 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex); 2980 igb->link_speed = speed; 2981 igb->link_duplex = duplex; 2982 igb->link_state = LINK_STATE_UP; 2983 igb->link_down_timeout = 0; 2984 link_changed = B_TRUE; 2985 } 2986 } else { 2987 if (igb->link_state != LINK_STATE_DOWN) { 2988 igb->link_speed = 0; 2989 igb->link_duplex = 0; 2990 igb->link_state = LINK_STATE_DOWN; 2991 link_changed = B_TRUE; 2992 } 2993 2994 if (igb->igb_state & IGB_STARTED) { 2995 if (igb->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) { 2996 igb->link_down_timeout++; 2997 } else if (igb->link_down_timeout == 2998 MAX_LINK_DOWN_TIMEOUT) { 2999 igb_tx_clean(igb); 3000 igb->link_down_timeout++; 3001 } 3002 } 3003 } 3004 3005 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 3006 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3007 3008 return (link_changed); 3009 } 3010 3011 /* 3012 * igb_local_timer - driver watchdog function 3013 * 3014 * This function will handle the hardware stall check, link status 3015 * check and other routines. 3016 */ 3017 static void 3018 igb_local_timer(void *arg) 3019 { 3020 igb_t *igb = (igb_t *)arg; 3021 boolean_t link_changed = B_FALSE; 3022 3023 if (igb_stall_check(igb)) 3024 igb->igb_state |= IGB_STALL; 3025 3026 if (igb->igb_state & IGB_STALL) { 3027 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL); 3028 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 3029 igb->reset_count++; 3030 igb->igb_state &= ~IGB_STALL; 3031 if (igb_reset(igb) == IGB_SUCCESS) 3032 ddi_fm_service_impact(igb->dip, 3033 DDI_SERVICE_RESTORED); 3034 } 3035 3036 mutex_enter(&igb->gen_lock); 3037 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED)) 3038 link_changed = igb_link_check(igb); 3039 mutex_exit(&igb->gen_lock); 3040 3041 if (link_changed) 3042 mac_link_update(igb->mac_hdl, igb->link_state); 3043 3044 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 3045 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3046 3047 igb_restart_watchdog_timer(igb); 3048 } 3049 3050 /* 3051 * igb_stall_check - check for transmit stall 3052 * 3053 * This function checks if the adapter is stalled (in transmit). 3054 * 3055 * It is called each time the watchdog timeout is invoked. 3056 * If the transmit descriptor reclaim continuously fails, 3057 * the watchdog value will increment by 1. If the watchdog 3058 * value exceeds the threshold, the igb is assumed to 3059 * have stalled and need to be reset. 3060 */ 3061 static boolean_t 3062 igb_stall_check(igb_t *igb) 3063 { 3064 igb_tx_ring_t *tx_ring; 3065 struct e1000_hw *hw = &igb->hw; 3066 boolean_t result; 3067 int i; 3068 3069 if (igb->link_state != LINK_STATE_UP) 3070 return (B_FALSE); 3071 3072 /* 3073 * If any tx ring is stalled, we'll reset the chipset 3074 */ 3075 result = B_FALSE; 3076 for (i = 0; i < igb->num_tx_rings; i++) { 3077 tx_ring = &igb->tx_rings[i]; 3078 3079 if (tx_ring->recycle_fail > 0) 3080 tx_ring->stall_watchdog++; 3081 else 3082 tx_ring->stall_watchdog = 0; 3083 3084 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3085 result = B_TRUE; 3086 if (hw->mac.type == e1000_82580) { 3087 hw->dev_spec._82575.global_device_reset 3088 = B_TRUE; 3089 } 3090 break; 3091 } 3092 } 3093 3094 if (result) { 3095 tx_ring->stall_watchdog = 0; 3096 tx_ring->recycle_fail = 0; 3097 } 3098 3099 return (result); 3100 } 3101 3102 3103 /* 3104 * is_valid_mac_addr - Check if the mac address is valid 3105 */ 3106 static boolean_t 3107 is_valid_mac_addr(uint8_t *mac_addr) 3108 { 3109 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3110 const uint8_t addr_test2[6] = 3111 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3112 3113 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3114 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3115 return (B_FALSE); 3116 3117 return (B_TRUE); 3118 } 3119 3120 static boolean_t 3121 igb_find_mac_address(igb_t *igb) 3122 { 3123 struct e1000_hw *hw = &igb->hw; 3124 #ifdef __sparc 3125 uchar_t *bytes; 3126 struct ether_addr sysaddr; 3127 uint_t nelts; 3128 int err; 3129 boolean_t found = B_FALSE; 3130 3131 /* 3132 * The "vendor's factory-set address" may already have 3133 * been extracted from the chip, but if the property 3134 * "local-mac-address" is set we use that instead. 3135 * 3136 * We check whether it looks like an array of 6 3137 * bytes (which it should, if OBP set it). If we can't 3138 * make sense of it this way, we'll ignore it. 3139 */ 3140 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3141 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3142 if (err == DDI_PROP_SUCCESS) { 3143 if (nelts == ETHERADDRL) { 3144 while (nelts--) 3145 hw->mac.addr[nelts] = bytes[nelts]; 3146 found = B_TRUE; 3147 } 3148 ddi_prop_free(bytes); 3149 } 3150 3151 /* 3152 * Look up the OBP property "local-mac-address?". If the user has set 3153 * 'local-mac-address? = false', use "the system address" instead. 3154 */ 3155 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0, 3156 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3157 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3158 if (localetheraddr(NULL, &sysaddr) != 0) { 3159 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3160 found = B_TRUE; 3161 } 3162 } 3163 ddi_prop_free(bytes); 3164 } 3165 3166 /* 3167 * Finally(!), if there's a valid "mac-address" property (created 3168 * if we netbooted from this interface), we must use this instead 3169 * of any of the above to ensure that the NFS/install server doesn't 3170 * get confused by the address changing as Solaris takes over! 3171 */ 3172 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3173 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3174 if (err == DDI_PROP_SUCCESS) { 3175 if (nelts == ETHERADDRL) { 3176 while (nelts--) 3177 hw->mac.addr[nelts] = bytes[nelts]; 3178 found = B_TRUE; 3179 } 3180 ddi_prop_free(bytes); 3181 } 3182 3183 if (found) { 3184 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3185 return (B_TRUE); 3186 } 3187 #endif 3188 3189 /* 3190 * Read the device MAC address from the EEPROM 3191 */ 3192 if (e1000_read_mac_addr(hw) != E1000_SUCCESS) 3193 return (B_FALSE); 3194 3195 return (B_TRUE); 3196 } 3197 3198 #pragma inline(igb_arm_watchdog_timer) 3199 3200 static void 3201 igb_arm_watchdog_timer(igb_t *igb) 3202 { 3203 /* 3204 * Fire a watchdog timer 3205 */ 3206 igb->watchdog_tid = 3207 timeout(igb_local_timer, 3208 (void *)igb, 1 * drv_usectohz(1000000)); 3209 3210 } 3211 3212 /* 3213 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer 3214 */ 3215 void 3216 igb_enable_watchdog_timer(igb_t *igb) 3217 { 3218 mutex_enter(&igb->watchdog_lock); 3219 3220 if (!igb->watchdog_enable) { 3221 igb->watchdog_enable = B_TRUE; 3222 igb->watchdog_start = B_TRUE; 3223 igb_arm_watchdog_timer(igb); 3224 } 3225 3226 mutex_exit(&igb->watchdog_lock); 3227 3228 } 3229 3230 /* 3231 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer 3232 */ 3233 void 3234 igb_disable_watchdog_timer(igb_t *igb) 3235 { 3236 timeout_id_t tid; 3237 3238 mutex_enter(&igb->watchdog_lock); 3239 3240 igb->watchdog_enable = B_FALSE; 3241 igb->watchdog_start = B_FALSE; 3242 tid = igb->watchdog_tid; 3243 igb->watchdog_tid = 0; 3244 3245 mutex_exit(&igb->watchdog_lock); 3246 3247 if (tid != 0) 3248 (void) untimeout(tid); 3249 3250 } 3251 3252 /* 3253 * igb_start_watchdog_timer - Start the driver watchdog timer 3254 */ 3255 static void 3256 igb_start_watchdog_timer(igb_t *igb) 3257 { 3258 mutex_enter(&igb->watchdog_lock); 3259 3260 if (igb->watchdog_enable) { 3261 if (!igb->watchdog_start) { 3262 igb->watchdog_start = B_TRUE; 3263 igb_arm_watchdog_timer(igb); 3264 } 3265 } 3266 3267 mutex_exit(&igb->watchdog_lock); 3268 } 3269 3270 /* 3271 * igb_restart_watchdog_timer - Restart the driver watchdog timer 3272 */ 3273 static void 3274 igb_restart_watchdog_timer(igb_t *igb) 3275 { 3276 mutex_enter(&igb->watchdog_lock); 3277 3278 if (igb->watchdog_start) 3279 igb_arm_watchdog_timer(igb); 3280 3281 mutex_exit(&igb->watchdog_lock); 3282 } 3283 3284 /* 3285 * igb_stop_watchdog_timer - Stop the driver watchdog timer 3286 */ 3287 static void 3288 igb_stop_watchdog_timer(igb_t *igb) 3289 { 3290 timeout_id_t tid; 3291 3292 mutex_enter(&igb->watchdog_lock); 3293 3294 igb->watchdog_start = B_FALSE; 3295 tid = igb->watchdog_tid; 3296 igb->watchdog_tid = 0; 3297 3298 mutex_exit(&igb->watchdog_lock); 3299 3300 if (tid != 0) 3301 (void) untimeout(tid); 3302 } 3303 3304 /* 3305 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts 3306 */ 3307 static void 3308 igb_disable_adapter_interrupts(igb_t *igb) 3309 { 3310 struct e1000_hw *hw = &igb->hw; 3311 3312 /* 3313 * Set the IMC register to mask all the interrupts, 3314 * including the tx interrupts. 3315 */ 3316 E1000_WRITE_REG(hw, E1000_IMC, ~0); 3317 E1000_WRITE_REG(hw, E1000_IAM, 0); 3318 3319 /* 3320 * Additional disabling for MSI-X 3321 */ 3322 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3323 E1000_WRITE_REG(hw, E1000_EIMC, ~0); 3324 E1000_WRITE_REG(hw, E1000_EIAC, 0); 3325 E1000_WRITE_REG(hw, E1000_EIAM, 0); 3326 } 3327 3328 E1000_WRITE_FLUSH(hw); 3329 } 3330 3331 /* 3332 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580 3333 */ 3334 static void 3335 igb_enable_adapter_interrupts_82580(igb_t *igb) 3336 { 3337 struct e1000_hw *hw = &igb->hw; 3338 3339 /* Clear any pending interrupts */ 3340 (void) E1000_READ_REG(hw, E1000_ICR); 3341 igb->ims_mask |= E1000_IMS_DRSTA; 3342 3343 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3344 3345 /* Interrupt enabling for MSI-X */ 3346 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3347 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3348 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA); 3349 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3350 } else { /* Interrupt enabling for MSI and legacy */ 3351 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3352 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3353 igb->ims_mask |= E1000_IMS_DRSTA; 3354 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3355 } 3356 3357 /* Disable auto-mask for ICR interrupt bits */ 3358 E1000_WRITE_REG(hw, E1000_IAM, 0); 3359 3360 E1000_WRITE_FLUSH(hw); 3361 } 3362 3363 /* 3364 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576 3365 */ 3366 static void 3367 igb_enable_adapter_interrupts_82576(igb_t *igb) 3368 { 3369 struct e1000_hw *hw = &igb->hw; 3370 3371 /* Clear any pending interrupts */ 3372 (void) E1000_READ_REG(hw, E1000_ICR); 3373 3374 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3375 3376 /* Interrupt enabling for MSI-X */ 3377 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3378 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3379 igb->ims_mask = E1000_IMS_LSC; 3380 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3381 } else { 3382 /* Interrupt enabling for MSI and legacy */ 3383 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3384 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3385 E1000_WRITE_REG(hw, E1000_IMS, 3386 (IMS_ENABLE_MASK | E1000_IMS_TXQE)); 3387 } 3388 3389 /* Disable auto-mask for ICR interrupt bits */ 3390 E1000_WRITE_REG(hw, E1000_IAM, 0); 3391 3392 E1000_WRITE_FLUSH(hw); 3393 } 3394 3395 /* 3396 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575 3397 */ 3398 static void 3399 igb_enable_adapter_interrupts_82575(igb_t *igb) 3400 { 3401 struct e1000_hw *hw = &igb->hw; 3402 uint32_t reg; 3403 3404 /* Clear any pending interrupts */ 3405 (void) E1000_READ_REG(hw, E1000_ICR); 3406 3407 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3408 /* Interrupt enabling for MSI-X */ 3409 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3410 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3411 igb->ims_mask = E1000_IMS_LSC; 3412 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3413 3414 /* Enable MSI-X PBA support */ 3415 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 3416 reg |= E1000_CTRL_EXT_PBA_CLR; 3417 3418 /* Non-selective interrupt clear-on-read */ 3419 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */ 3420 3421 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 3422 } else { 3423 /* Interrupt enabling for MSI and legacy */ 3424 igb->ims_mask = IMS_ENABLE_MASK; 3425 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 3426 } 3427 3428 E1000_WRITE_FLUSH(hw); 3429 } 3430 3431 /* 3432 * Loopback Support 3433 */ 3434 static lb_property_t lb_normal = 3435 { normal, "normal", IGB_LB_NONE }; 3436 static lb_property_t lb_external = 3437 { external, "External", IGB_LB_EXTERNAL }; 3438 static lb_property_t lb_mac = 3439 { internal, "MAC", IGB_LB_INTERNAL_MAC }; 3440 static lb_property_t lb_phy = 3441 { internal, "PHY", IGB_LB_INTERNAL_PHY }; 3442 static lb_property_t lb_serdes = 3443 { internal, "SerDes", IGB_LB_INTERNAL_SERDES }; 3444 3445 enum ioc_reply 3446 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp) 3447 { 3448 lb_info_sz_t *lbsp; 3449 lb_property_t *lbpp; 3450 struct e1000_hw *hw; 3451 uint32_t *lbmp; 3452 uint32_t size; 3453 uint32_t value; 3454 3455 hw = &igb->hw; 3456 3457 if (mp->b_cont == NULL) 3458 return (IOC_INVAL); 3459 3460 switch (iocp->ioc_cmd) { 3461 default: 3462 return (IOC_INVAL); 3463 3464 case LB_GET_INFO_SIZE: 3465 size = sizeof (lb_info_sz_t); 3466 if (iocp->ioc_count != size) 3467 return (IOC_INVAL); 3468 3469 value = sizeof (lb_normal); 3470 value += sizeof (lb_mac); 3471 if (hw->phy.media_type == e1000_media_type_copper) 3472 value += sizeof (lb_phy); 3473 else 3474 value += sizeof (lb_serdes); 3475 value += sizeof (lb_external); 3476 3477 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3478 *lbsp = value; 3479 break; 3480 3481 case LB_GET_INFO: 3482 value = sizeof (lb_normal); 3483 value += sizeof (lb_mac); 3484 if (hw->phy.media_type == e1000_media_type_copper) 3485 value += sizeof (lb_phy); 3486 else 3487 value += sizeof (lb_serdes); 3488 value += sizeof (lb_external); 3489 3490 size = value; 3491 if (iocp->ioc_count != size) 3492 return (IOC_INVAL); 3493 3494 value = 0; 3495 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3496 3497 lbpp[value++] = lb_normal; 3498 lbpp[value++] = lb_mac; 3499 if (hw->phy.media_type == e1000_media_type_copper) 3500 lbpp[value++] = lb_phy; 3501 else 3502 lbpp[value++] = lb_serdes; 3503 lbpp[value++] = lb_external; 3504 break; 3505 3506 case LB_GET_MODE: 3507 size = sizeof (uint32_t); 3508 if (iocp->ioc_count != size) 3509 return (IOC_INVAL); 3510 3511 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3512 *lbmp = igb->loopback_mode; 3513 break; 3514 3515 case LB_SET_MODE: 3516 size = 0; 3517 if (iocp->ioc_count != sizeof (uint32_t)) 3518 return (IOC_INVAL); 3519 3520 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3521 if (!igb_set_loopback_mode(igb, *lbmp)) 3522 return (IOC_INVAL); 3523 break; 3524 } 3525 3526 iocp->ioc_count = size; 3527 iocp->ioc_error = 0; 3528 3529 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3530 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3531 return (IOC_INVAL); 3532 } 3533 3534 return (IOC_REPLY); 3535 } 3536 3537 /* 3538 * igb_set_loopback_mode - Setup loopback based on the loopback mode 3539 */ 3540 static boolean_t 3541 igb_set_loopback_mode(igb_t *igb, uint32_t mode) 3542 { 3543 struct e1000_hw *hw; 3544 3545 if (mode == igb->loopback_mode) 3546 return (B_TRUE); 3547 3548 hw = &igb->hw; 3549 3550 igb->loopback_mode = mode; 3551 3552 if (mode == IGB_LB_NONE) { 3553 /* Reset the chip */ 3554 hw->phy.autoneg_wait_to_complete = B_TRUE; 3555 (void) igb_reset(igb); 3556 hw->phy.autoneg_wait_to_complete = B_FALSE; 3557 return (B_TRUE); 3558 } 3559 3560 mutex_enter(&igb->gen_lock); 3561 3562 switch (mode) { 3563 default: 3564 mutex_exit(&igb->gen_lock); 3565 return (B_FALSE); 3566 3567 case IGB_LB_EXTERNAL: 3568 igb_set_external_loopback(igb); 3569 break; 3570 3571 case IGB_LB_INTERNAL_MAC: 3572 igb_set_internal_mac_loopback(igb); 3573 break; 3574 3575 case IGB_LB_INTERNAL_PHY: 3576 igb_set_internal_phy_loopback(igb); 3577 break; 3578 3579 case IGB_LB_INTERNAL_SERDES: 3580 igb_set_internal_serdes_loopback(igb); 3581 break; 3582 } 3583 3584 mutex_exit(&igb->gen_lock); 3585 3586 return (B_TRUE); 3587 } 3588 3589 /* 3590 * igb_set_external_loopback - Set the external loopback mode 3591 */ 3592 static void 3593 igb_set_external_loopback(igb_t *igb) 3594 { 3595 struct e1000_hw *hw; 3596 3597 hw = &igb->hw; 3598 3599 /* Set phy to known state */ 3600 (void) e1000_phy_hw_reset(hw); 3601 3602 (void) e1000_write_phy_reg(hw, 0x0, 0x0140); 3603 (void) e1000_write_phy_reg(hw, 0x9, 0x1b00); 3604 (void) e1000_write_phy_reg(hw, 0x12, 0x1610); 3605 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c); 3606 } 3607 3608 /* 3609 * igb_set_internal_mac_loopback - Set the internal MAC loopback mode 3610 */ 3611 static void 3612 igb_set_internal_mac_loopback(igb_t *igb) 3613 { 3614 struct e1000_hw *hw; 3615 uint32_t ctrl; 3616 uint32_t rctl; 3617 uint32_t ctrl_ext; 3618 uint16_t phy_ctrl; 3619 uint16_t phy_status; 3620 3621 hw = &igb->hw; 3622 3623 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 3624 phy_ctrl &= ~MII_CR_AUTO_NEG_EN; 3625 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3626 3627 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 3628 3629 /* Set link mode to PHY (00b) in the Extended Control register */ 3630 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3631 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3632 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3633 3634 /* Set the Device Control register */ 3635 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3636 if (!(phy_status & MII_SR_LINK_STATUS)) 3637 ctrl |= E1000_CTRL_ILOS; /* Set ILOS when the link is down */ 3638 ctrl &= ~E1000_CTRL_SPD_SEL; /* Clear the speed sel bits */ 3639 ctrl |= (E1000_CTRL_SLU | /* Force link up */ 3640 E1000_CTRL_FRCSPD | /* Force speed */ 3641 E1000_CTRL_FRCDPX | /* Force duplex */ 3642 E1000_CTRL_SPD_1000 | /* Force speed to 1000 */ 3643 E1000_CTRL_FD); /* Force full duplex */ 3644 3645 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3646 3647 /* Set the Receive Control register */ 3648 rctl = E1000_READ_REG(hw, E1000_RCTL); 3649 rctl &= ~E1000_RCTL_LBM_TCVR; 3650 rctl |= E1000_RCTL_LBM_MAC; 3651 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3652 } 3653 3654 /* 3655 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode 3656 */ 3657 static void 3658 igb_set_internal_phy_loopback(igb_t *igb) 3659 { 3660 struct e1000_hw *hw; 3661 uint32_t ctrl_ext; 3662 uint16_t phy_ctrl; 3663 uint16_t phy_pconf; 3664 3665 hw = &igb->hw; 3666 3667 /* Set link mode to PHY (00b) in the Extended Control register */ 3668 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3669 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3670 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3671 3672 /* 3673 * Set PHY control register (0x4140): 3674 * Set full duplex mode 3675 * Set loopback bit 3676 * Clear auto-neg enable bit 3677 * Set PHY speed 3678 */ 3679 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK; 3680 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3681 3682 /* Set the link disable bit in the Port Configuration register */ 3683 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf); 3684 phy_pconf |= (uint16_t)1 << 14; 3685 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf); 3686 } 3687 3688 /* 3689 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode 3690 */ 3691 static void 3692 igb_set_internal_serdes_loopback(igb_t *igb) 3693 { 3694 struct e1000_hw *hw; 3695 uint32_t ctrl_ext; 3696 uint32_t ctrl; 3697 uint32_t pcs_lctl; 3698 uint32_t connsw; 3699 3700 hw = &igb->hw; 3701 3702 /* Set link mode to SerDes (11b) in the Extended Control register */ 3703 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3704 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 3705 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3706 3707 /* Configure the SerDes to loopback */ 3708 E1000_WRITE_REG(hw, E1000_SCTL, 0x410); 3709 3710 /* Set Device Control register */ 3711 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3712 ctrl |= (E1000_CTRL_FD | /* Force full duplex */ 3713 E1000_CTRL_SLU); /* Force link up */ 3714 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */ 3715 E1000_CTRL_TFCE | /* Disable transmit flow control */ 3716 E1000_CTRL_LRST); /* Clear link reset */ 3717 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3718 3719 /* Set PCS Link Control register */ 3720 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL); 3721 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK | 3722 E1000_PCS_LCTL_FSD | 3723 E1000_PCS_LCTL_FDV_FULL | 3724 E1000_PCS_LCTL_FLV_LINK_UP); 3725 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE; 3726 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl); 3727 3728 /* Set the Copper/Fiber Switch Control - CONNSW register */ 3729 connsw = E1000_READ_REG(hw, E1000_CONNSW); 3730 connsw &= ~E1000_CONNSW_ENRGSRC; 3731 E1000_WRITE_REG(hw, E1000_CONNSW, connsw); 3732 } 3733 3734 #pragma inline(igb_intr_rx_work) 3735 /* 3736 * igb_intr_rx_work - rx processing of ISR 3737 */ 3738 static void 3739 igb_intr_rx_work(igb_rx_ring_t *rx_ring) 3740 { 3741 mblk_t *mp; 3742 3743 mutex_enter(&rx_ring->rx_lock); 3744 mp = igb_rx(rx_ring, IGB_NO_POLL); 3745 mutex_exit(&rx_ring->rx_lock); 3746 3747 if (mp != NULL) 3748 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp, 3749 rx_ring->ring_gen_num); 3750 } 3751 3752 #pragma inline(igb_intr_tx_work) 3753 /* 3754 * igb_intr_tx_work - tx processing of ISR 3755 */ 3756 static void 3757 igb_intr_tx_work(igb_tx_ring_t *tx_ring) 3758 { 3759 /* Recycle the tx descriptors */ 3760 tx_ring->tx_recycle(tx_ring); 3761 3762 /* Schedule the re-transmit */ 3763 if (tx_ring->reschedule && 3764 (tx_ring->tbd_free >= tx_ring->resched_thresh)) { 3765 tx_ring->reschedule = B_FALSE; 3766 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle); 3767 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3768 } 3769 } 3770 3771 #pragma inline(igb_intr_link_work) 3772 /* 3773 * igb_intr_link_work - link-status-change processing of ISR 3774 */ 3775 static void 3776 igb_intr_link_work(igb_t *igb) 3777 { 3778 boolean_t link_changed; 3779 3780 igb_stop_watchdog_timer(igb); 3781 3782 mutex_enter(&igb->gen_lock); 3783 3784 /* 3785 * Because we got a link-status-change interrupt, force 3786 * e1000_check_for_link() to look at phy 3787 */ 3788 igb->hw.mac.get_link_status = B_TRUE; 3789 3790 /* igb_link_check takes care of link status change */ 3791 link_changed = igb_link_check(igb); 3792 3793 /* Get new phy state */ 3794 igb_get_phy_state(igb); 3795 3796 mutex_exit(&igb->gen_lock); 3797 3798 if (link_changed) 3799 mac_link_update(igb->mac_hdl, igb->link_state); 3800 3801 igb_start_watchdog_timer(igb); 3802 } 3803 3804 /* 3805 * igb_intr_legacy - Interrupt handler for legacy interrupts 3806 */ 3807 static uint_t 3808 igb_intr_legacy(void *arg1, void *arg2) 3809 { 3810 igb_t *igb = (igb_t *)arg1; 3811 igb_tx_ring_t *tx_ring; 3812 uint32_t icr; 3813 mblk_t *mp; 3814 boolean_t tx_reschedule; 3815 boolean_t link_changed; 3816 uint_t result; 3817 3818 _NOTE(ARGUNUSED(arg2)); 3819 3820 mutex_enter(&igb->gen_lock); 3821 3822 if (igb->igb_state & IGB_SUSPENDED) { 3823 mutex_exit(&igb->gen_lock); 3824 return (DDI_INTR_UNCLAIMED); 3825 } 3826 3827 mp = NULL; 3828 tx_reschedule = B_FALSE; 3829 link_changed = B_FALSE; 3830 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3831 3832 if (icr & E1000_ICR_INT_ASSERTED) { 3833 /* 3834 * E1000_ICR_INT_ASSERTED bit was set: 3835 * Read(Clear) the ICR, claim this interrupt, 3836 * look for work to do. 3837 */ 3838 ASSERT(igb->num_rx_rings == 1); 3839 ASSERT(igb->num_tx_rings == 1); 3840 3841 /* Make sure all interrupt causes cleared */ 3842 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3843 3844 if (icr & E1000_ICR_RXT0) { 3845 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL); 3846 } 3847 3848 if (icr & E1000_ICR_TXDW) { 3849 tx_ring = &igb->tx_rings[0]; 3850 3851 /* Recycle the tx descriptors */ 3852 tx_ring->tx_recycle(tx_ring); 3853 3854 /* Schedule the re-transmit */ 3855 tx_reschedule = (tx_ring->reschedule && 3856 (tx_ring->tbd_free >= tx_ring->resched_thresh)); 3857 } 3858 3859 if (icr & E1000_ICR_LSC) { 3860 /* 3861 * Because we got a link-status-change interrupt, force 3862 * e1000_check_for_link() to look at phy 3863 */ 3864 igb->hw.mac.get_link_status = B_TRUE; 3865 3866 /* igb_link_check takes care of link status change */ 3867 link_changed = igb_link_check(igb); 3868 3869 /* Get new phy state */ 3870 igb_get_phy_state(igb); 3871 } 3872 3873 if (icr & E1000_ICR_DRSTA) { 3874 /* 82580 Full Device Reset needed */ 3875 igb->igb_state |= IGB_STALL; 3876 } 3877 3878 result = DDI_INTR_CLAIMED; 3879 } else { 3880 /* 3881 * E1000_ICR_INT_ASSERTED bit was not set: 3882 * Don't claim this interrupt. 3883 */ 3884 result = DDI_INTR_UNCLAIMED; 3885 } 3886 3887 mutex_exit(&igb->gen_lock); 3888 3889 /* 3890 * Do the following work outside of the gen_lock 3891 */ 3892 if (mp != NULL) 3893 mac_rx(igb->mac_hdl, NULL, mp); 3894 3895 if (tx_reschedule) { 3896 tx_ring->reschedule = B_FALSE; 3897 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle); 3898 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3899 } 3900 3901 if (link_changed) 3902 mac_link_update(igb->mac_hdl, igb->link_state); 3903 3904 return (result); 3905 } 3906 3907 /* 3908 * igb_intr_msi - Interrupt handler for MSI 3909 */ 3910 static uint_t 3911 igb_intr_msi(void *arg1, void *arg2) 3912 { 3913 igb_t *igb = (igb_t *)arg1; 3914 uint32_t icr; 3915 3916 _NOTE(ARGUNUSED(arg2)); 3917 3918 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3919 3920 /* Make sure all interrupt causes cleared */ 3921 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3922 3923 /* 3924 * For MSI interrupt, we have only one vector, 3925 * so we have only one rx ring and one tx ring enabled. 3926 */ 3927 ASSERT(igb->num_rx_rings == 1); 3928 ASSERT(igb->num_tx_rings == 1); 3929 3930 if (icr & E1000_ICR_RXT0) { 3931 igb_intr_rx_work(&igb->rx_rings[0]); 3932 } 3933 3934 if (icr & E1000_ICR_TXDW) { 3935 igb_intr_tx_work(&igb->tx_rings[0]); 3936 } 3937 3938 if (icr & E1000_ICR_LSC) { 3939 igb_intr_link_work(igb); 3940 } 3941 3942 if (icr & E1000_ICR_DRSTA) { 3943 /* 82580 Full Device Reset needed */ 3944 igb->igb_state |= IGB_STALL; 3945 } 3946 3947 return (DDI_INTR_CLAIMED); 3948 } 3949 3950 /* 3951 * igb_intr_rx - Interrupt handler for rx 3952 */ 3953 static uint_t 3954 igb_intr_rx(void *arg1, void *arg2) 3955 { 3956 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1; 3957 3958 _NOTE(ARGUNUSED(arg2)); 3959 3960 /* 3961 * Only used via MSI-X vector so don't check cause bits 3962 * and only clean the given ring. 3963 */ 3964 igb_intr_rx_work(rx_ring); 3965 3966 return (DDI_INTR_CLAIMED); 3967 } 3968 3969 /* 3970 * igb_intr_tx - Interrupt handler for tx 3971 */ 3972 static uint_t 3973 igb_intr_tx(void *arg1, void *arg2) 3974 { 3975 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1; 3976 3977 _NOTE(ARGUNUSED(arg2)); 3978 3979 /* 3980 * Only used via MSI-X vector so don't check cause bits 3981 * and only clean the given ring. 3982 */ 3983 igb_intr_tx_work(tx_ring); 3984 3985 return (DDI_INTR_CLAIMED); 3986 } 3987 3988 /* 3989 * igb_intr_tx_other - Interrupt handler for both tx and other 3990 * 3991 */ 3992 static uint_t 3993 igb_intr_tx_other(void *arg1, void *arg2) 3994 { 3995 igb_t *igb = (igb_t *)arg1; 3996 uint32_t icr; 3997 3998 _NOTE(ARGUNUSED(arg2)); 3999 4000 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4001 4002 /* 4003 * Look for tx reclaiming work first. Remember, in the 4004 * case of only interrupt sharing, only one tx ring is 4005 * used 4006 */ 4007 igb_intr_tx_work(&igb->tx_rings[0]); 4008 4009 /* 4010 * Check for "other" causes. 4011 */ 4012 if (icr & E1000_ICR_LSC) { 4013 igb_intr_link_work(igb); 4014 } 4015 4016 /* 4017 * The DOUTSYNC bit indicates a tx packet dropped because 4018 * DMA engine gets "out of sync". There isn't a real fix 4019 * for this. The Intel recommendation is to count the number 4020 * of occurrences so user can detect when it is happening. 4021 * The issue is non-fatal and there's no recovery action 4022 * available. 4023 */ 4024 if (icr & E1000_ICR_DOUTSYNC) { 4025 IGB_STAT(igb->dout_sync); 4026 } 4027 4028 if (icr & E1000_ICR_DRSTA) { 4029 /* 82580 Full Device Reset needed */ 4030 igb->igb_state |= IGB_STALL; 4031 } 4032 4033 return (DDI_INTR_CLAIMED); 4034 } 4035 4036 /* 4037 * igb_alloc_intrs - Allocate interrupts for the driver 4038 * 4039 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4040 * if not successful, try Legacy. 4041 * igb->intr_force can be used to force sequence to start with 4042 * any of the 3 types. 4043 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4044 */ 4045 static int 4046 igb_alloc_intrs(igb_t *igb) 4047 { 4048 dev_info_t *devinfo; 4049 int intr_types; 4050 int rc; 4051 4052 devinfo = igb->dip; 4053 4054 /* Get supported interrupt types */ 4055 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4056 4057 if (rc != DDI_SUCCESS) { 4058 igb_log(igb, 4059 "Get supported interrupt types failed: %d", rc); 4060 return (IGB_FAILURE); 4061 } 4062 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types); 4063 4064 igb->intr_type = 0; 4065 4066 /* Install MSI-X interrupts */ 4067 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4068 (igb->intr_force <= IGB_INTR_MSIX)) { 4069 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX); 4070 4071 if (rc == IGB_SUCCESS) 4072 return (IGB_SUCCESS); 4073 4074 igb_log(igb, 4075 "Allocate MSI-X failed, trying MSI interrupts..."); 4076 } 4077 4078 /* MSI-X not used, force rings to 1 */ 4079 igb->num_rx_rings = 1; 4080 igb->num_tx_rings = 1; 4081 igb_log(igb, 4082 "MSI-X not used, force rx and tx queue number to 1"); 4083 4084 /* Install MSI interrupts */ 4085 if ((intr_types & DDI_INTR_TYPE_MSI) && 4086 (igb->intr_force <= IGB_INTR_MSI)) { 4087 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI); 4088 4089 if (rc == IGB_SUCCESS) 4090 return (IGB_SUCCESS); 4091 4092 igb_log(igb, 4093 "Allocate MSI failed, trying Legacy interrupts..."); 4094 } 4095 4096 /* Install legacy interrupts */ 4097 if (intr_types & DDI_INTR_TYPE_FIXED) { 4098 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED); 4099 4100 if (rc == IGB_SUCCESS) 4101 return (IGB_SUCCESS); 4102 4103 igb_log(igb, 4104 "Allocate Legacy interrupts failed"); 4105 } 4106 4107 /* If none of the 3 types succeeded, return failure */ 4108 return (IGB_FAILURE); 4109 } 4110 4111 /* 4112 * igb_alloc_intr_handles - Allocate interrupt handles. 4113 * 4114 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4115 * if fewer than 2 handles are available, return failure. 4116 * Upon success, this sets the number of Rx rings to a number that 4117 * matches the handles available for Rx interrupts. 4118 */ 4119 static int 4120 igb_alloc_intr_handles(igb_t *igb, int intr_type) 4121 { 4122 dev_info_t *devinfo; 4123 int orig, request, count, avail, actual; 4124 int diff, minimum; 4125 int rc; 4126 4127 devinfo = igb->dip; 4128 4129 switch (intr_type) { 4130 case DDI_INTR_TYPE_FIXED: 4131 request = 1; /* Request 1 legacy interrupt handle */ 4132 minimum = 1; 4133 IGB_DEBUGLOG_0(igb, "interrupt type: legacy"); 4134 break; 4135 4136 case DDI_INTR_TYPE_MSI: 4137 request = 1; /* Request 1 MSI interrupt handle */ 4138 minimum = 1; 4139 IGB_DEBUGLOG_0(igb, "interrupt type: MSI"); 4140 break; 4141 4142 case DDI_INTR_TYPE_MSIX: 4143 /* 4144 * Number of vectors for the adapter is 4145 * # rx rings + # tx rings 4146 * One of tx vectors is for tx & other 4147 */ 4148 request = igb->num_rx_rings + igb->num_tx_rings; 4149 orig = request; 4150 minimum = 2; 4151 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X"); 4152 break; 4153 4154 default: 4155 igb_log(igb, 4156 "invalid call to igb_alloc_intr_handles(): %d\n", 4157 intr_type); 4158 return (IGB_FAILURE); 4159 } 4160 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d", 4161 request, minimum); 4162 4163 /* 4164 * Get number of supported interrupts 4165 */ 4166 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4167 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4168 igb_log(igb, 4169 "Get supported interrupt number failed. " 4170 "Return: %d, count: %d", rc, count); 4171 return (IGB_FAILURE); 4172 } 4173 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count); 4174 4175 /* 4176 * Get number of available interrupts 4177 */ 4178 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 4179 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 4180 igb_log(igb, 4181 "Get available interrupt number failed. " 4182 "Return: %d, available: %d", rc, avail); 4183 return (IGB_FAILURE); 4184 } 4185 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail); 4186 4187 if (avail < request) { 4188 igb_log(igb, "Request %d handles, %d available", 4189 request, avail); 4190 request = avail; 4191 } 4192 4193 actual = 0; 4194 igb->intr_cnt = 0; 4195 4196 /* 4197 * Allocate an array of interrupt handles 4198 */ 4199 igb->intr_size = request * sizeof (ddi_intr_handle_t); 4200 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP); 4201 4202 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0, 4203 request, &actual, DDI_INTR_ALLOC_NORMAL); 4204 if (rc != DDI_SUCCESS) { 4205 igb_log(igb, "Allocate interrupts failed. " 4206 "return: %d, request: %d, actual: %d", 4207 rc, request, actual); 4208 goto alloc_handle_fail; 4209 } 4210 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual); 4211 4212 igb->intr_cnt = actual; 4213 4214 if (actual < minimum) { 4215 igb_log(igb, "Insufficient interrupt handles allocated: %d", 4216 actual); 4217 goto alloc_handle_fail; 4218 } 4219 4220 /* 4221 * For MSI-X, actual might force us to reduce number of tx & rx rings 4222 */ 4223 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { 4224 diff = orig - actual; 4225 if (diff < igb->num_tx_rings) { 4226 igb_log(igb, 4227 "MSI-X vectors force Tx queue number to %d", 4228 igb->num_tx_rings - diff); 4229 igb->num_tx_rings -= diff; 4230 } else { 4231 igb_log(igb, 4232 "MSI-X vectors force Tx queue number to 1"); 4233 igb->num_tx_rings = 1; 4234 4235 igb_log(igb, 4236 "MSI-X vectors force Rx queue number to %d", 4237 actual - 1); 4238 igb->num_rx_rings = actual - 1; 4239 } 4240 } 4241 4242 /* 4243 * Get priority for first vector, assume remaining are all the same 4244 */ 4245 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri); 4246 if (rc != DDI_SUCCESS) { 4247 igb_log(igb, 4248 "Get interrupt priority failed: %d", rc); 4249 goto alloc_handle_fail; 4250 } 4251 4252 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap); 4253 if (rc != DDI_SUCCESS) { 4254 igb_log(igb, 4255 "Get interrupt cap failed: %d", rc); 4256 goto alloc_handle_fail; 4257 } 4258 4259 igb->intr_type = intr_type; 4260 4261 return (IGB_SUCCESS); 4262 4263 alloc_handle_fail: 4264 igb_rem_intrs(igb); 4265 4266 return (IGB_FAILURE); 4267 } 4268 4269 /* 4270 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type 4271 * 4272 * Before adding the interrupt handlers, the interrupt vectors have 4273 * been allocated, and the rx/tx rings have also been allocated. 4274 */ 4275 static int 4276 igb_add_intr_handlers(igb_t *igb) 4277 { 4278 igb_rx_ring_t *rx_ring; 4279 igb_tx_ring_t *tx_ring; 4280 int vector; 4281 int rc; 4282 int i; 4283 4284 vector = 0; 4285 4286 switch (igb->intr_type) { 4287 case DDI_INTR_TYPE_MSIX: 4288 /* Add interrupt handler for tx + other */ 4289 tx_ring = &igb->tx_rings[0]; 4290 rc = ddi_intr_add_handler(igb->htable[vector], 4291 (ddi_intr_handler_t *)igb_intr_tx_other, 4292 (void *)igb, NULL); 4293 4294 if (rc != DDI_SUCCESS) { 4295 igb_log(igb, 4296 "Add tx/other interrupt handler failed: %d", rc); 4297 return (IGB_FAILURE); 4298 } 4299 tx_ring->intr_vector = vector; 4300 vector++; 4301 4302 /* Add interrupt handler for each rx ring */ 4303 for (i = 0; i < igb->num_rx_rings; i++) { 4304 rx_ring = &igb->rx_rings[i]; 4305 4306 rc = ddi_intr_add_handler(igb->htable[vector], 4307 (ddi_intr_handler_t *)igb_intr_rx, 4308 (void *)rx_ring, NULL); 4309 4310 if (rc != DDI_SUCCESS) { 4311 igb_log(igb, 4312 "Add rx interrupt handler failed. " 4313 "return: %d, rx ring: %d", rc, i); 4314 for (vector--; vector >= 0; vector--) { 4315 (void) ddi_intr_remove_handler( 4316 igb->htable[vector]); 4317 } 4318 return (IGB_FAILURE); 4319 } 4320 4321 rx_ring->intr_vector = vector; 4322 4323 vector++; 4324 } 4325 4326 /* Add interrupt handler for each tx ring from 2nd ring */ 4327 for (i = 1; i < igb->num_tx_rings; i++) { 4328 tx_ring = &igb->tx_rings[i]; 4329 4330 rc = ddi_intr_add_handler(igb->htable[vector], 4331 (ddi_intr_handler_t *)igb_intr_tx, 4332 (void *)tx_ring, NULL); 4333 4334 if (rc != DDI_SUCCESS) { 4335 igb_log(igb, 4336 "Add tx interrupt handler failed. " 4337 "return: %d, tx ring: %d", rc, i); 4338 for (vector--; vector >= 0; vector--) { 4339 (void) ddi_intr_remove_handler( 4340 igb->htable[vector]); 4341 } 4342 return (IGB_FAILURE); 4343 } 4344 4345 tx_ring->intr_vector = vector; 4346 4347 vector++; 4348 } 4349 4350 break; 4351 4352 case DDI_INTR_TYPE_MSI: 4353 /* Add interrupt handlers for the only vector */ 4354 rc = ddi_intr_add_handler(igb->htable[vector], 4355 (ddi_intr_handler_t *)igb_intr_msi, 4356 (void *)igb, NULL); 4357 4358 if (rc != DDI_SUCCESS) { 4359 igb_log(igb, 4360 "Add MSI interrupt handler failed: %d", rc); 4361 return (IGB_FAILURE); 4362 } 4363 4364 rx_ring = &igb->rx_rings[0]; 4365 rx_ring->intr_vector = vector; 4366 4367 vector++; 4368 break; 4369 4370 case DDI_INTR_TYPE_FIXED: 4371 /* Add interrupt handlers for the only vector */ 4372 rc = ddi_intr_add_handler(igb->htable[vector], 4373 (ddi_intr_handler_t *)igb_intr_legacy, 4374 (void *)igb, NULL); 4375 4376 if (rc != DDI_SUCCESS) { 4377 igb_log(igb, 4378 "Add legacy interrupt handler failed: %d", rc); 4379 return (IGB_FAILURE); 4380 } 4381 4382 rx_ring = &igb->rx_rings[0]; 4383 rx_ring->intr_vector = vector; 4384 4385 vector++; 4386 break; 4387 4388 default: 4389 return (IGB_FAILURE); 4390 } 4391 4392 ASSERT(vector == igb->intr_cnt); 4393 4394 return (IGB_SUCCESS); 4395 } 4396 4397 /* 4398 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts 4399 * 4400 * For each vector enabled on the adapter, Set the MSIXBM register accordingly 4401 */ 4402 static void 4403 igb_setup_msix_82575(igb_t *igb) 4404 { 4405 uint32_t eims = 0; 4406 int i, vector; 4407 struct e1000_hw *hw = &igb->hw; 4408 4409 /* 4410 * Set vector for tx ring 0 and other causes. 4411 * NOTE assumption that it is vector 0. 4412 */ 4413 vector = 0; 4414 4415 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER; 4416 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask); 4417 vector++; 4418 4419 for (i = 0; i < igb->num_rx_rings; i++) { 4420 /* 4421 * Set vector for each rx ring 4422 */ 4423 eims = (E1000_EICR_RX_QUEUE0 << i); 4424 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4425 4426 /* 4427 * Accumulate bits to enable in 4428 * igb_enable_adapter_interrupts_82575() 4429 */ 4430 igb->eims_mask |= eims; 4431 4432 vector++; 4433 } 4434 4435 for (i = 1; i < igb->num_tx_rings; i++) { 4436 /* 4437 * Set vector for each tx ring from 2nd tx ring 4438 */ 4439 eims = (E1000_EICR_TX_QUEUE0 << i); 4440 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4441 4442 /* 4443 * Accumulate bits to enable in 4444 * igb_enable_adapter_interrupts_82575() 4445 */ 4446 igb->eims_mask |= eims; 4447 4448 vector++; 4449 } 4450 4451 ASSERT(vector == igb->intr_cnt); 4452 4453 /* 4454 * Disable IAM for ICR interrupt bits 4455 */ 4456 E1000_WRITE_REG(hw, E1000_IAM, 0); 4457 E1000_WRITE_FLUSH(hw); 4458 } 4459 4460 /* 4461 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts 4462 * 4463 * 82576 uses a table based method for assigning vectors. Each queue has a 4464 * single entry in the table to which we write a vector number along with a 4465 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors 4466 * take a different position in the 4-byte register depending on whether 4467 * they are numbered above or below 8. 4468 */ 4469 static void 4470 igb_setup_msix_82576(igb_t *igb) 4471 { 4472 struct e1000_hw *hw = &igb->hw; 4473 uint32_t ivar, index, vector; 4474 int i; 4475 4476 /* must enable msi-x capability before IVAR settings */ 4477 E1000_WRITE_REG(hw, E1000_GPIE, 4478 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR)); 4479 4480 /* 4481 * Set vector for tx ring 0 and other causes. 4482 * NOTE assumption that it is vector 0. 4483 * This is also interdependent with installation of interrupt service 4484 * routines in igb_add_intr_handlers(). 4485 */ 4486 4487 /* assign "other" causes to vector 0 */ 4488 vector = 0; 4489 ivar = ((vector | E1000_IVAR_VALID) << 8); 4490 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4491 4492 /* assign tx ring 0 to vector 0 */ 4493 ivar = ((vector | E1000_IVAR_VALID) << 8); 4494 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4495 4496 /* prepare to enable tx & other interrupt causes */ 4497 igb->eims_mask = (1 << vector); 4498 4499 vector ++; 4500 for (i = 0; i < igb->num_rx_rings; i++) { 4501 /* 4502 * Set vector for each rx ring 4503 */ 4504 index = (i & 0x7); 4505 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4506 4507 if (i < 8) { 4508 /* vector goes into low byte of register */ 4509 ivar = ivar & 0xFFFFFF00; 4510 ivar |= (vector | E1000_IVAR_VALID); 4511 } else { 4512 /* vector goes into third byte of register */ 4513 ivar = ivar & 0xFF00FFFF; 4514 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4515 } 4516 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4517 4518 /* Accumulate interrupt-cause bits to enable */ 4519 igb->eims_mask |= (1 << vector); 4520 4521 vector ++; 4522 } 4523 4524 for (i = 1; i < igb->num_tx_rings; i++) { 4525 /* 4526 * Set vector for each tx ring from 2nd tx ring. 4527 * Note assumption that tx vectors numericall follow rx vectors. 4528 */ 4529 index = (i & 0x7); 4530 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4531 4532 if (i < 8) { 4533 /* vector goes into second byte of register */ 4534 ivar = ivar & 0xFFFF00FF; 4535 ivar |= ((vector | E1000_IVAR_VALID) << 8); 4536 } else { 4537 /* vector goes into fourth byte of register */ 4538 ivar = ivar & 0x00FFFFFF; 4539 ivar |= (vector | E1000_IVAR_VALID) << 24; 4540 } 4541 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4542 4543 /* Accumulate interrupt-cause bits to enable */ 4544 igb->eims_mask |= (1 << vector); 4545 4546 vector ++; 4547 } 4548 4549 ASSERT(vector == igb->intr_cnt); 4550 } 4551 4552 /* 4553 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts 4554 * 4555 * 82580 uses same table approach at 82576 but has fewer entries. Each 4556 * queue has a single entry in the table to which we write a vector number 4557 * along with a "valid" bit. Vectors take a different position in the 4558 * register depending on * whether * they are numbered above or below 4. 4559 */ 4560 static void 4561 igb_setup_msix_82580(igb_t *igb) 4562 { 4563 struct e1000_hw *hw = &igb->hw; 4564 uint32_t ivar, index, vector; 4565 int i; 4566 4567 /* must enable msi-x capability before IVAR settings */ 4568 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE | 4569 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME)); 4570 /* 4571 * Set vector for tx ring 0 and other causes. 4572 * NOTE assumption that it is vector 0. 4573 * This is also interdependent with installation of interrupt service 4574 * routines in igb_add_intr_handlers(). 4575 */ 4576 4577 /* assign "other" causes to vector 0 */ 4578 vector = 0; 4579 ivar = ((vector | E1000_IVAR_VALID) << 8); 4580 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4581 4582 /* assign tx ring 0 to vector 0 */ 4583 ivar = ((vector | E1000_IVAR_VALID) << 8); 4584 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4585 4586 /* prepare to enable tx & other interrupt causes */ 4587 igb->eims_mask = (1 << vector); 4588 4589 vector ++; 4590 4591 for (i = 0; i < igb->num_rx_rings; i++) { 4592 /* 4593 * Set vector for each rx ring 4594 */ 4595 index = (i >> 1); 4596 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4597 4598 if (i & 1) { 4599 /* vector goes into third byte of register */ 4600 ivar = ivar & 0xFF00FFFF; 4601 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4602 } else { 4603 /* vector goes into low byte of register */ 4604 ivar = ivar & 0xFFFFFF00; 4605 ivar |= (vector | E1000_IVAR_VALID); 4606 } 4607 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4608 4609 /* Accumulate interrupt-cause bits to enable */ 4610 igb->eims_mask |= (1 << vector); 4611 4612 vector ++; 4613 } 4614 4615 for (i = 1; i < igb->num_tx_rings; i++) { 4616 /* 4617 * Set vector for each tx ring from 2nd tx ring. 4618 * Note assumption that tx vectors numericall follow rx vectors. 4619 */ 4620 index = (i >> 1); 4621 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4622 4623 if (i & 1) { 4624 /* vector goes into high byte of register */ 4625 ivar = ivar & 0x00FFFFFF; 4626 ivar |= ((vector | E1000_IVAR_VALID) << 24); 4627 } else { 4628 /* vector goes into second byte of register */ 4629 ivar = ivar & 0xFFFF00FF; 4630 ivar |= (vector | E1000_IVAR_VALID) << 8; 4631 } 4632 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4633 4634 /* Accumulate interrupt-cause bits to enable */ 4635 igb->eims_mask |= (1 << vector); 4636 4637 vector ++; 4638 } 4639 ASSERT(vector == igb->intr_cnt); 4640 } 4641 4642 /* 4643 * igb_rem_intr_handlers - remove the interrupt handlers 4644 */ 4645 static void 4646 igb_rem_intr_handlers(igb_t *igb) 4647 { 4648 int i; 4649 int rc; 4650 4651 for (i = 0; i < igb->intr_cnt; i++) { 4652 rc = ddi_intr_remove_handler(igb->htable[i]); 4653 if (rc != DDI_SUCCESS) { 4654 IGB_DEBUGLOG_1(igb, 4655 "Remove intr handler failed: %d", rc); 4656 } 4657 } 4658 } 4659 4660 /* 4661 * igb_rem_intrs - remove the allocated interrupts 4662 */ 4663 static void 4664 igb_rem_intrs(igb_t *igb) 4665 { 4666 int i; 4667 int rc; 4668 4669 for (i = 0; i < igb->intr_cnt; i++) { 4670 rc = ddi_intr_free(igb->htable[i]); 4671 if (rc != DDI_SUCCESS) { 4672 IGB_DEBUGLOG_1(igb, 4673 "Free intr failed: %d", rc); 4674 } 4675 } 4676 4677 kmem_free(igb->htable, igb->intr_size); 4678 igb->htable = NULL; 4679 } 4680 4681 /* 4682 * igb_enable_intrs - enable all the ddi interrupts 4683 */ 4684 static int 4685 igb_enable_intrs(igb_t *igb) 4686 { 4687 int i; 4688 int rc; 4689 4690 /* Enable interrupts */ 4691 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4692 /* Call ddi_intr_block_enable() for MSI */ 4693 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt); 4694 if (rc != DDI_SUCCESS) { 4695 igb_log(igb, 4696 "Enable block intr failed: %d", rc); 4697 return (IGB_FAILURE); 4698 } 4699 } else { 4700 /* Call ddi_intr_enable() for Legacy/MSI non block enable */ 4701 for (i = 0; i < igb->intr_cnt; i++) { 4702 rc = ddi_intr_enable(igb->htable[i]); 4703 if (rc != DDI_SUCCESS) { 4704 igb_log(igb, 4705 "Enable intr failed: %d", rc); 4706 return (IGB_FAILURE); 4707 } 4708 } 4709 } 4710 4711 return (IGB_SUCCESS); 4712 } 4713 4714 /* 4715 * igb_disable_intrs - disable all the ddi interrupts 4716 */ 4717 static int 4718 igb_disable_intrs(igb_t *igb) 4719 { 4720 int i; 4721 int rc; 4722 4723 /* Disable all interrupts */ 4724 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4725 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt); 4726 if (rc != DDI_SUCCESS) { 4727 igb_log(igb, 4728 "Disable block intr failed: %d", rc); 4729 return (IGB_FAILURE); 4730 } 4731 } else { 4732 for (i = 0; i < igb->intr_cnt; i++) { 4733 rc = ddi_intr_disable(igb->htable[i]); 4734 if (rc != DDI_SUCCESS) { 4735 igb_log(igb, 4736 "Disable intr failed: %d", rc); 4737 return (IGB_FAILURE); 4738 } 4739 } 4740 } 4741 4742 return (IGB_SUCCESS); 4743 } 4744 4745 /* 4746 * igb_get_phy_state - Get and save the parameters read from PHY registers 4747 */ 4748 static void 4749 igb_get_phy_state(igb_t *igb) 4750 { 4751 struct e1000_hw *hw = &igb->hw; 4752 uint16_t phy_ctrl; 4753 uint16_t phy_status; 4754 uint16_t phy_an_adv; 4755 uint16_t phy_an_exp; 4756 uint16_t phy_ext_status; 4757 uint16_t phy_1000t_ctrl; 4758 uint16_t phy_1000t_status; 4759 uint16_t phy_lp_able; 4760 4761 ASSERT(mutex_owned(&igb->gen_lock)); 4762 4763 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 4764 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 4765 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv); 4766 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp); 4767 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status); 4768 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl); 4769 (void) e1000_read_phy_reg(hw, PHY_1000T_STATUS, &phy_1000t_status); 4770 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able); 4771 4772 igb->param_autoneg_cap = 4773 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0; 4774 igb->param_pause_cap = 4775 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4776 igb->param_asym_pause_cap = 4777 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4778 igb->param_1000fdx_cap = ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) || 4779 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0; 4780 igb->param_1000hdx_cap = ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) || 4781 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0; 4782 igb->param_100t4_cap = 4783 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0; 4784 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) || 4785 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0; 4786 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) || 4787 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0; 4788 igb->param_10fdx_cap = 4789 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0; 4790 igb->param_10hdx_cap = 4791 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0; 4792 igb->param_rem_fault = 4793 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0; 4794 4795 igb->param_adv_autoneg_cap = hw->mac.autoneg; 4796 igb->param_adv_pause_cap = 4797 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4798 igb->param_adv_asym_pause_cap = 4799 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4800 igb->param_adv_1000hdx_cap = 4801 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0; 4802 igb->param_adv_100t4_cap = 4803 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0; 4804 igb->param_adv_rem_fault = 4805 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0; 4806 if (igb->param_adv_autoneg_cap == 1) { 4807 igb->param_adv_1000fdx_cap = 4808 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0; 4809 igb->param_adv_100fdx_cap = 4810 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0; 4811 igb->param_adv_100hdx_cap = 4812 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0; 4813 igb->param_adv_10fdx_cap = 4814 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0; 4815 igb->param_adv_10hdx_cap = 4816 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0; 4817 } 4818 4819 igb->param_lp_autoneg_cap = 4820 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0; 4821 igb->param_lp_pause_cap = 4822 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0; 4823 igb->param_lp_asym_pause_cap = 4824 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0; 4825 igb->param_lp_1000fdx_cap = 4826 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0; 4827 igb->param_lp_1000hdx_cap = 4828 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0; 4829 igb->param_lp_100t4_cap = 4830 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0; 4831 igb->param_lp_100fdx_cap = 4832 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0; 4833 igb->param_lp_100hdx_cap = 4834 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0; 4835 igb->param_lp_10fdx_cap = 4836 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0; 4837 igb->param_lp_10hdx_cap = 4838 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0; 4839 igb->param_lp_rem_fault = 4840 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0; 4841 } 4842 4843 /* 4844 * igb_get_driver_control 4845 */ 4846 static void 4847 igb_get_driver_control(struct e1000_hw *hw) 4848 { 4849 uint32_t ctrl_ext; 4850 4851 /* Notify firmware that driver is in control of device */ 4852 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 4853 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD; 4854 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 4855 } 4856 4857 /* 4858 * igb_release_driver_control 4859 */ 4860 static void 4861 igb_release_driver_control(struct e1000_hw *hw) 4862 { 4863 uint32_t ctrl_ext; 4864 4865 /* Notify firmware that driver is no longer in control of device */ 4866 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 4867 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD; 4868 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 4869 } 4870 4871 /* 4872 * igb_atomic_reserve - Atomic decrease operation 4873 */ 4874 int 4875 igb_atomic_reserve(uint32_t *count_p, uint32_t n) 4876 { 4877 uint32_t oldval; 4878 uint32_t newval; 4879 4880 /* ATOMICALLY */ 4881 do { 4882 oldval = *count_p; 4883 if (oldval < n) 4884 return (-1); 4885 newval = oldval - n; 4886 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 4887 4888 return (newval); 4889 } 4890 4891 /* 4892 * FMA support 4893 */ 4894 4895 int 4896 igb_check_acc_handle(ddi_acc_handle_t handle) 4897 { 4898 ddi_fm_error_t de; 4899 4900 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 4901 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 4902 return (de.fme_status); 4903 } 4904 4905 int 4906 igb_check_dma_handle(ddi_dma_handle_t handle) 4907 { 4908 ddi_fm_error_t de; 4909 4910 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 4911 return (de.fme_status); 4912 } 4913 4914 /* 4915 * The IO fault service error handling callback function 4916 */ 4917 /*ARGSUSED*/ 4918 static int 4919 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 4920 { 4921 /* 4922 * as the driver can always deal with an error in any dma or 4923 * access handle, we can just return the fme_status value. 4924 */ 4925 pci_ereport_post(dip, err, NULL); 4926 return (err->fme_status); 4927 } 4928 4929 static void 4930 igb_fm_init(igb_t *igb) 4931 { 4932 ddi_iblock_cookie_t iblk; 4933 int fma_acc_flag, fma_dma_flag; 4934 4935 /* Only register with IO Fault Services if we have some capability */ 4936 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 4937 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 4938 fma_acc_flag = 1; 4939 } else { 4940 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 4941 fma_acc_flag = 0; 4942 } 4943 4944 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 4945 fma_dma_flag = 1; 4946 } else { 4947 fma_dma_flag = 0; 4948 } 4949 4950 (void) igb_set_fma_flags(fma_acc_flag, fma_dma_flag); 4951 4952 if (igb->fm_capabilities) { 4953 4954 /* Register capabilities with IO Fault Services */ 4955 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk); 4956 4957 /* 4958 * Initialize pci ereport capabilities if ereport capable 4959 */ 4960 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 4961 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 4962 pci_ereport_setup(igb->dip); 4963 4964 /* 4965 * Register error callback if error callback capable 4966 */ 4967 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 4968 ddi_fm_handler_register(igb->dip, 4969 igb_fm_error_cb, (void*) igb); 4970 } 4971 } 4972 4973 static void 4974 igb_fm_fini(igb_t *igb) 4975 { 4976 /* Only unregister FMA capabilities if we registered some */ 4977 if (igb->fm_capabilities) { 4978 4979 /* 4980 * Release any resources allocated by pci_ereport_setup() 4981 */ 4982 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 4983 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 4984 pci_ereport_teardown(igb->dip); 4985 4986 /* 4987 * Un-register error callback if error callback capable 4988 */ 4989 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 4990 ddi_fm_handler_unregister(igb->dip); 4991 4992 /* Unregister from IO Fault Services */ 4993 ddi_fm_fini(igb->dip); 4994 } 4995 } 4996 4997 void 4998 igb_fm_ereport(igb_t *igb, char *detail) 4999 { 5000 uint64_t ena; 5001 char buf[FM_MAX_CLASS]; 5002 5003 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5004 ena = fm_ena_generate(0, FM_ENA_FMT1); 5005 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) { 5006 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP, 5007 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5008 } 5009 } 5010