1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 23 /* 24 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 25 * Use is subject to license terms. 26 */ 27 28 #include "ixgbe_sw.h" 29 30 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 31 static char ixgbe_version[] = "ixgbe 1.1.4"; 32 33 /* 34 * Local function protoypes 35 */ 36 static int ixgbe_register_mac(ixgbe_t *); 37 static int ixgbe_identify_hardware(ixgbe_t *); 38 static int ixgbe_regs_map(ixgbe_t *); 39 static void ixgbe_init_properties(ixgbe_t *); 40 static int ixgbe_init_driver_settings(ixgbe_t *); 41 static void ixgbe_init_locks(ixgbe_t *); 42 static void ixgbe_destroy_locks(ixgbe_t *); 43 static int ixgbe_init(ixgbe_t *); 44 static int ixgbe_chip_start(ixgbe_t *); 45 static void ixgbe_chip_stop(ixgbe_t *); 46 static int ixgbe_reset(ixgbe_t *); 47 static void ixgbe_tx_clean(ixgbe_t *); 48 static boolean_t ixgbe_tx_drain(ixgbe_t *); 49 static boolean_t ixgbe_rx_drain(ixgbe_t *); 50 static int ixgbe_alloc_rings(ixgbe_t *); 51 static void ixgbe_free_rings(ixgbe_t *); 52 static int ixgbe_alloc_rx_data(ixgbe_t *); 53 static void ixgbe_free_rx_data(ixgbe_t *); 54 static void ixgbe_setup_rings(ixgbe_t *); 55 static void ixgbe_setup_rx(ixgbe_t *); 56 static void ixgbe_setup_tx(ixgbe_t *); 57 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 58 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 59 static void ixgbe_setup_rss(ixgbe_t *); 60 static void ixgbe_setup_vmdq(ixgbe_t *); 61 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 62 static void ixgbe_init_unicst(ixgbe_t *); 63 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 64 static void ixgbe_setup_multicst(ixgbe_t *); 65 static void ixgbe_get_hw_state(ixgbe_t *); 66 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 67 static void ixgbe_get_conf(ixgbe_t *); 68 static void ixgbe_init_params(ixgbe_t *); 69 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 70 static void ixgbe_driver_link_check(ixgbe_t *); 71 static void ixgbe_sfp_check(void *); 72 static void ixgbe_link_timer(void *); 73 static void ixgbe_local_timer(void *); 74 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 75 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 76 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 77 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 78 static boolean_t is_valid_mac_addr(uint8_t *); 79 static boolean_t ixgbe_stall_check(ixgbe_t *); 80 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 81 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 82 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 83 static int ixgbe_alloc_intrs(ixgbe_t *); 84 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 85 static int ixgbe_add_intr_handlers(ixgbe_t *); 86 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 87 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 88 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 89 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 90 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 91 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 92 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 93 static void ixgbe_setup_adapter_vector(ixgbe_t *); 94 static void ixgbe_rem_intr_handlers(ixgbe_t *); 95 static void ixgbe_rem_intrs(ixgbe_t *); 96 static int ixgbe_enable_intrs(ixgbe_t *); 97 static int ixgbe_disable_intrs(ixgbe_t *); 98 static uint_t ixgbe_intr_legacy(void *, void *); 99 static uint_t ixgbe_intr_msi(void *, void *); 100 static uint_t ixgbe_intr_msix(void *, void *); 101 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 102 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 103 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 104 static void ixgbe_get_driver_control(struct ixgbe_hw *); 105 static int ixgbe_addmac(void *, const uint8_t *); 106 static int ixgbe_remmac(void *, const uint8_t *); 107 static void ixgbe_release_driver_control(struct ixgbe_hw *); 108 109 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 110 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 111 static int ixgbe_resume(dev_info_t *); 112 static int ixgbe_suspend(dev_info_t *); 113 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 114 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 115 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 116 static int ixgbe_intr_cb_register(ixgbe_t *); 117 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 118 119 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 120 const void *impl_data); 121 static void ixgbe_fm_init(ixgbe_t *); 122 static void ixgbe_fm_fini(ixgbe_t *); 123 124 char *ixgbe_priv_props[] = { 125 "_tx_copy_thresh", 126 "_tx_recycle_thresh", 127 "_tx_overload_thresh", 128 "_tx_resched_thresh", 129 "_rx_copy_thresh", 130 "_rx_limit_per_intr", 131 "_intr_throttling", 132 "_adv_pause_cap", 133 "_adv_asym_pause_cap", 134 NULL 135 }; 136 137 #define IXGBE_MAX_PRIV_PROPS \ 138 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 139 140 static struct cb_ops ixgbe_cb_ops = { 141 nulldev, /* cb_open */ 142 nulldev, /* cb_close */ 143 nodev, /* cb_strategy */ 144 nodev, /* cb_print */ 145 nodev, /* cb_dump */ 146 nodev, /* cb_read */ 147 nodev, /* cb_write */ 148 nodev, /* cb_ioctl */ 149 nodev, /* cb_devmap */ 150 nodev, /* cb_mmap */ 151 nodev, /* cb_segmap */ 152 nochpoll, /* cb_chpoll */ 153 ddi_prop_op, /* cb_prop_op */ 154 NULL, /* cb_stream */ 155 D_MP | D_HOTPLUG, /* cb_flag */ 156 CB_REV, /* cb_rev */ 157 nodev, /* cb_aread */ 158 nodev /* cb_awrite */ 159 }; 160 161 static struct dev_ops ixgbe_dev_ops = { 162 DEVO_REV, /* devo_rev */ 163 0, /* devo_refcnt */ 164 NULL, /* devo_getinfo */ 165 nulldev, /* devo_identify */ 166 nulldev, /* devo_probe */ 167 ixgbe_attach, /* devo_attach */ 168 ixgbe_detach, /* devo_detach */ 169 nodev, /* devo_reset */ 170 &ixgbe_cb_ops, /* devo_cb_ops */ 171 NULL, /* devo_bus_ops */ 172 ddi_power, /* devo_power */ 173 ddi_quiesce_not_supported, /* devo_quiesce */ 174 }; 175 176 static struct modldrv ixgbe_modldrv = { 177 &mod_driverops, /* Type of module. This one is a driver */ 178 ixgbe_ident, /* Discription string */ 179 &ixgbe_dev_ops /* driver ops */ 180 }; 181 182 static struct modlinkage ixgbe_modlinkage = { 183 MODREV_1, &ixgbe_modldrv, NULL 184 }; 185 186 /* 187 * Access attributes for register mapping 188 */ 189 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 190 DDI_DEVICE_ATTR_V1, 191 DDI_STRUCTURE_LE_ACC, 192 DDI_STRICTORDER_ACC, 193 DDI_FLAGERR_ACC 194 }; 195 196 /* 197 * Loopback property 198 */ 199 static lb_property_t lb_normal = { 200 normal, "normal", IXGBE_LB_NONE 201 }; 202 203 static lb_property_t lb_mac = { 204 internal, "MAC", IXGBE_LB_INTERNAL_MAC 205 }; 206 207 static lb_property_t lb_external = { 208 external, "External", IXGBE_LB_EXTERNAL 209 }; 210 211 #define IXGBE_M_CALLBACK_FLAGS \ 212 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 213 214 static mac_callbacks_t ixgbe_m_callbacks = { 215 IXGBE_M_CALLBACK_FLAGS, 216 ixgbe_m_stat, 217 ixgbe_m_start, 218 ixgbe_m_stop, 219 ixgbe_m_promisc, 220 ixgbe_m_multicst, 221 NULL, 222 NULL, 223 NULL, 224 ixgbe_m_ioctl, 225 ixgbe_m_getcapab, 226 NULL, 227 NULL, 228 ixgbe_m_setprop, 229 ixgbe_m_getprop, 230 ixgbe_m_propinfo 231 }; 232 233 /* 234 * Initialize capabilities of each supported adapter type 235 */ 236 static adapter_info_t ixgbe_82598eb_cap = { 237 64, /* maximum number of rx queues */ 238 1, /* minimum number of rx queues */ 239 64, /* default number of rx queues */ 240 16, /* maximum number of rx groups */ 241 1, /* minimum number of rx groups */ 242 1, /* default number of rx groups */ 243 32, /* maximum number of tx queues */ 244 1, /* minimum number of tx queues */ 245 8, /* default number of tx queues */ 246 16366, /* maximum MTU size */ 247 0xFFFF, /* maximum interrupt throttle rate */ 248 0, /* minimum interrupt throttle rate */ 249 200, /* default interrupt throttle rate */ 250 18, /* maximum total msix vectors */ 251 16, /* maximum number of ring vectors */ 252 2, /* maximum number of other vectors */ 253 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 254 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 255 | IXGBE_FLAG_RSS_CAPABLE 256 | IXGBE_FLAG_VMDQ_CAPABLE) 257 }; 258 259 static adapter_info_t ixgbe_82599eb_cap = { 260 128, /* maximum number of rx queues */ 261 1, /* minimum number of rx queues */ 262 128, /* default number of rx queues */ 263 64, /* maximum number of rx groups */ 264 1, /* minimum number of rx groups */ 265 1, /* default number of rx groups */ 266 128, /* maximum number of tx queues */ 267 1, /* minimum number of tx queues */ 268 8, /* default number of tx queues */ 269 15500, /* maximum MTU size */ 270 0xFF8, /* maximum interrupt throttle rate */ 271 0, /* minimum interrupt throttle rate */ 272 200, /* default interrupt throttle rate */ 273 64, /* maximum total msix vectors */ 274 16, /* maximum number of ring vectors */ 275 2, /* maximum number of other vectors */ 276 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 277 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 278 | IXGBE_FLAG_RSS_CAPABLE 279 | IXGBE_FLAG_VMDQ_CAPABLE 280 | IXGBE_FLAG_RSC_CAPABLE) 281 }; 282 283 /* 284 * Module Initialization Functions. 285 */ 286 287 int 288 _init(void) 289 { 290 int status; 291 292 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 293 294 status = mod_install(&ixgbe_modlinkage); 295 296 if (status != DDI_SUCCESS) { 297 mac_fini_ops(&ixgbe_dev_ops); 298 } 299 300 return (status); 301 } 302 303 int 304 _fini(void) 305 { 306 int status; 307 308 status = mod_remove(&ixgbe_modlinkage); 309 310 if (status == DDI_SUCCESS) { 311 mac_fini_ops(&ixgbe_dev_ops); 312 } 313 314 return (status); 315 } 316 317 int 318 _info(struct modinfo *modinfop) 319 { 320 int status; 321 322 status = mod_info(&ixgbe_modlinkage, modinfop); 323 324 return (status); 325 } 326 327 /* 328 * ixgbe_attach - Driver attach. 329 * 330 * This function is the device specific initialization entry 331 * point. This entry point is required and must be written. 332 * The DDI_ATTACH command must be provided in the attach entry 333 * point. When attach() is called with cmd set to DDI_ATTACH, 334 * all normal kernel services (such as kmem_alloc(9F)) are 335 * available for use by the driver. 336 * 337 * The attach() function will be called once for each instance 338 * of the device on the system with cmd set to DDI_ATTACH. 339 * Until attach() succeeds, the only driver entry points which 340 * may be called are open(9E) and getinfo(9E). 341 */ 342 static int 343 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 344 { 345 ixgbe_t *ixgbe; 346 struct ixgbe_osdep *osdep; 347 struct ixgbe_hw *hw; 348 int instance; 349 char taskqname[32]; 350 351 /* 352 * Check the command and perform corresponding operations 353 */ 354 switch (cmd) { 355 default: 356 return (DDI_FAILURE); 357 358 case DDI_RESUME: 359 return (ixgbe_resume(devinfo)); 360 361 case DDI_ATTACH: 362 break; 363 } 364 365 /* Get the device instance */ 366 instance = ddi_get_instance(devinfo); 367 368 /* Allocate memory for the instance data structure */ 369 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 370 371 ixgbe->dip = devinfo; 372 ixgbe->instance = instance; 373 374 hw = &ixgbe->hw; 375 osdep = &ixgbe->osdep; 376 hw->back = osdep; 377 osdep->ixgbe = ixgbe; 378 379 /* Attach the instance pointer to the dev_info data structure */ 380 ddi_set_driver_private(devinfo, ixgbe); 381 382 /* 383 * Initialize for fma support 384 */ 385 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 386 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 387 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 388 ixgbe_fm_init(ixgbe); 389 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 390 391 /* 392 * Map PCI config space registers 393 */ 394 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 395 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 396 goto attach_fail; 397 } 398 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 399 400 /* 401 * Identify the chipset family 402 */ 403 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 404 ixgbe_error(ixgbe, "Failed to identify hardware"); 405 goto attach_fail; 406 } 407 408 /* 409 * Map device registers 410 */ 411 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 412 ixgbe_error(ixgbe, "Failed to map device registers"); 413 goto attach_fail; 414 } 415 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 416 417 /* 418 * Initialize driver parameters 419 */ 420 ixgbe_init_properties(ixgbe); 421 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 422 423 /* 424 * Register interrupt callback 425 */ 426 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 427 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 428 goto attach_fail; 429 } 430 431 /* 432 * Allocate interrupts 433 */ 434 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 435 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 436 goto attach_fail; 437 } 438 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 439 440 /* 441 * Allocate rx/tx rings based on the ring numbers. 442 * The actual numbers of rx/tx rings are decided by the number of 443 * allocated interrupt vectors, so we should allocate the rings after 444 * interrupts are allocated. 445 */ 446 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 447 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 448 goto attach_fail; 449 } 450 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 451 452 /* 453 * Map rings to interrupt vectors 454 */ 455 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 456 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 457 goto attach_fail; 458 } 459 460 /* 461 * Add interrupt handlers 462 */ 463 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 464 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 465 goto attach_fail; 466 } 467 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 468 469 /* 470 * Create a taskq for sfp-change 471 */ 472 (void) sprintf(taskqname, "ixgbe%d_taskq", instance); 473 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 474 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 475 ixgbe_error(ixgbe, "taskq_create failed"); 476 goto attach_fail; 477 } 478 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 479 480 /* 481 * Initialize driver parameters 482 */ 483 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 484 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 485 goto attach_fail; 486 } 487 488 /* 489 * Initialize mutexes for this device. 490 * Do this before enabling the interrupt handler and 491 * register the softint to avoid the condition where 492 * interrupt handler can try using uninitialized mutex. 493 */ 494 ixgbe_init_locks(ixgbe); 495 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 496 497 /* 498 * Initialize chipset hardware 499 */ 500 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 501 ixgbe_error(ixgbe, "Failed to initialize adapter"); 502 goto attach_fail; 503 } 504 ixgbe->link_check_complete = B_FALSE; 505 ixgbe->link_check_hrtime = gethrtime() + 506 (IXGBE_LINK_UP_TIME * 100000000ULL); 507 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 508 509 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 510 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 511 goto attach_fail; 512 } 513 514 /* 515 * Initialize statistics 516 */ 517 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 518 ixgbe_error(ixgbe, "Failed to initialize statistics"); 519 goto attach_fail; 520 } 521 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 522 523 /* 524 * Register the driver to the MAC 525 */ 526 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 527 ixgbe_error(ixgbe, "Failed to register MAC"); 528 goto attach_fail; 529 } 530 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 531 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 532 533 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 534 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 535 if (ixgbe->periodic_id == 0) { 536 ixgbe_error(ixgbe, "Failed to add the link check timer"); 537 goto attach_fail; 538 } 539 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 540 541 /* 542 * Now that mutex locks are initialized, and the chip is also 543 * initialized, enable interrupts. 544 */ 545 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 546 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 547 goto attach_fail; 548 } 549 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 550 551 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version); 552 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 553 554 return (DDI_SUCCESS); 555 556 attach_fail: 557 ixgbe_unconfigure(devinfo, ixgbe); 558 return (DDI_FAILURE); 559 } 560 561 /* 562 * ixgbe_detach - Driver detach. 563 * 564 * The detach() function is the complement of the attach routine. 565 * If cmd is set to DDI_DETACH, detach() is used to remove the 566 * state associated with a given instance of a device node 567 * prior to the removal of that instance from the system. 568 * 569 * The detach() function will be called once for each instance 570 * of the device for which there has been a successful attach() 571 * once there are no longer any opens on the device. 572 * 573 * Interrupts routine are disabled, All memory allocated by this 574 * driver are freed. 575 */ 576 static int 577 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 578 { 579 ixgbe_t *ixgbe; 580 581 /* 582 * Check detach command 583 */ 584 switch (cmd) { 585 default: 586 return (DDI_FAILURE); 587 588 case DDI_SUSPEND: 589 return (ixgbe_suspend(devinfo)); 590 591 case DDI_DETACH: 592 break; 593 } 594 595 /* 596 * Get the pointer to the driver private data structure 597 */ 598 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 599 if (ixgbe == NULL) 600 return (DDI_FAILURE); 601 602 /* 603 * If the device is still running, it needs to be stopped first. 604 * This check is necessary because under some specific circumstances, 605 * the detach routine can be called without stopping the interface 606 * first. 607 */ 608 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 609 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 610 mutex_enter(&ixgbe->gen_lock); 611 ixgbe_stop(ixgbe, B_TRUE); 612 mutex_exit(&ixgbe->gen_lock); 613 /* Disable and stop the watchdog timer */ 614 ixgbe_disable_watchdog_timer(ixgbe); 615 } 616 617 /* 618 * Check if there are still rx buffers held by the upper layer. 619 * If so, fail the detach. 620 */ 621 if (!ixgbe_rx_drain(ixgbe)) 622 return (DDI_FAILURE); 623 624 /* 625 * Do the remaining unconfigure routines 626 */ 627 ixgbe_unconfigure(devinfo, ixgbe); 628 629 return (DDI_SUCCESS); 630 } 631 632 static void 633 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 634 { 635 /* 636 * Disable interrupt 637 */ 638 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 639 (void) ixgbe_disable_intrs(ixgbe); 640 } 641 642 /* 643 * remove the link check timer 644 */ 645 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 646 if (ixgbe->periodic_id != NULL) { 647 ddi_periodic_delete(ixgbe->periodic_id); 648 ixgbe->periodic_id = NULL; 649 } 650 } 651 652 /* 653 * Unregister MAC 654 */ 655 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 656 (void) mac_unregister(ixgbe->mac_hdl); 657 } 658 659 /* 660 * Free statistics 661 */ 662 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 663 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 664 } 665 666 /* 667 * Remove interrupt handlers 668 */ 669 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 670 ixgbe_rem_intr_handlers(ixgbe); 671 } 672 673 /* 674 * Remove taskq for sfp-status-change 675 */ 676 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 677 ddi_taskq_destroy(ixgbe->sfp_taskq); 678 } 679 680 /* 681 * Remove interrupts 682 */ 683 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 684 ixgbe_rem_intrs(ixgbe); 685 } 686 687 /* 688 * Unregister interrupt callback handler 689 */ 690 (void) ddi_cb_unregister(ixgbe->cb_hdl); 691 692 /* 693 * Remove driver properties 694 */ 695 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 696 (void) ddi_prop_remove_all(devinfo); 697 } 698 699 /* 700 * Stop the chipset 701 */ 702 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 703 mutex_enter(&ixgbe->gen_lock); 704 ixgbe_chip_stop(ixgbe); 705 mutex_exit(&ixgbe->gen_lock); 706 } 707 708 /* 709 * Free register handle 710 */ 711 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 712 if (ixgbe->osdep.reg_handle != NULL) 713 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 714 } 715 716 /* 717 * Free PCI config handle 718 */ 719 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 720 if (ixgbe->osdep.cfg_handle != NULL) 721 pci_config_teardown(&ixgbe->osdep.cfg_handle); 722 } 723 724 /* 725 * Free locks 726 */ 727 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 728 ixgbe_destroy_locks(ixgbe); 729 } 730 731 /* 732 * Free the rx/tx rings 733 */ 734 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 735 ixgbe_free_rings(ixgbe); 736 } 737 738 /* 739 * Unregister FMA capabilities 740 */ 741 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 742 ixgbe_fm_fini(ixgbe); 743 } 744 745 /* 746 * Free the driver data structure 747 */ 748 kmem_free(ixgbe, sizeof (ixgbe_t)); 749 750 ddi_set_driver_private(devinfo, NULL); 751 } 752 753 /* 754 * ixgbe_register_mac - Register the driver and its function pointers with 755 * the GLD interface. 756 */ 757 static int 758 ixgbe_register_mac(ixgbe_t *ixgbe) 759 { 760 struct ixgbe_hw *hw = &ixgbe->hw; 761 mac_register_t *mac; 762 int status; 763 764 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 765 return (IXGBE_FAILURE); 766 767 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 768 mac->m_driver = ixgbe; 769 mac->m_dip = ixgbe->dip; 770 mac->m_src_addr = hw->mac.addr; 771 mac->m_callbacks = &ixgbe_m_callbacks; 772 mac->m_min_sdu = 0; 773 mac->m_max_sdu = ixgbe->default_mtu; 774 mac->m_margin = VLAN_TAGSZ; 775 mac->m_priv_props = ixgbe_priv_props; 776 mac->m_v12n = MAC_VIRT_LEVEL1; 777 778 status = mac_register(mac, &ixgbe->mac_hdl); 779 780 mac_free(mac); 781 782 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 783 } 784 785 /* 786 * ixgbe_identify_hardware - Identify the type of the chipset. 787 */ 788 static int 789 ixgbe_identify_hardware(ixgbe_t *ixgbe) 790 { 791 struct ixgbe_hw *hw = &ixgbe->hw; 792 struct ixgbe_osdep *osdep = &ixgbe->osdep; 793 794 /* 795 * Get the device id 796 */ 797 hw->vendor_id = 798 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 799 hw->device_id = 800 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 801 hw->revision_id = 802 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 803 hw->subsystem_device_id = 804 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 805 hw->subsystem_vendor_id = 806 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 807 808 /* 809 * Set the mac type of the adapter based on the device id 810 */ 811 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 812 return (IXGBE_FAILURE); 813 } 814 815 /* 816 * Install adapter capabilities 817 */ 818 switch (hw->mac.type) { 819 case ixgbe_mac_82598EB: 820 ixgbe_log(ixgbe, "identify 82598 adapter\n"); 821 ixgbe->capab = &ixgbe_82598eb_cap; 822 823 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 824 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 825 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 826 } 827 ixgbe->capab->other_intr |= IXGBE_EICR_LSC; 828 829 break; 830 case ixgbe_mac_82599EB: 831 ixgbe_log(ixgbe, "identify 82599 adapter\n"); 832 ixgbe->capab = &ixgbe_82599eb_cap; 833 834 ixgbe->capab->other_intr = (IXGBE_EICR_GPI_SDP1 | 835 IXGBE_EICR_GPI_SDP2 | IXGBE_EICR_LSC); 836 837 break; 838 default: 839 ixgbe_log(ixgbe, 840 "adapter not supported in ixgbe_identify_hardware(): %d\n", 841 hw->mac.type); 842 return (IXGBE_FAILURE); 843 } 844 845 return (IXGBE_SUCCESS); 846 } 847 848 /* 849 * ixgbe_regs_map - Map the device registers. 850 * 851 */ 852 static int 853 ixgbe_regs_map(ixgbe_t *ixgbe) 854 { 855 dev_info_t *devinfo = ixgbe->dip; 856 struct ixgbe_hw *hw = &ixgbe->hw; 857 struct ixgbe_osdep *osdep = &ixgbe->osdep; 858 off_t mem_size; 859 860 /* 861 * First get the size of device registers to be mapped. 862 */ 863 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 864 != DDI_SUCCESS) { 865 return (IXGBE_FAILURE); 866 } 867 868 /* 869 * Call ddi_regs_map_setup() to map registers 870 */ 871 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 872 (caddr_t *)&hw->hw_addr, 0, 873 mem_size, &ixgbe_regs_acc_attr, 874 &osdep->reg_handle)) != DDI_SUCCESS) { 875 return (IXGBE_FAILURE); 876 } 877 878 return (IXGBE_SUCCESS); 879 } 880 881 /* 882 * ixgbe_init_properties - Initialize driver properties. 883 */ 884 static void 885 ixgbe_init_properties(ixgbe_t *ixgbe) 886 { 887 /* 888 * Get conf file properties, including link settings 889 * jumbo frames, ring number, descriptor number, etc. 890 */ 891 ixgbe_get_conf(ixgbe); 892 893 ixgbe_init_params(ixgbe); 894 } 895 896 /* 897 * ixgbe_init_driver_settings - Initialize driver settings. 898 * 899 * The settings include hardware function pointers, bus information, 900 * rx/tx rings settings, link state, and any other parameters that 901 * need to be setup during driver initialization. 902 */ 903 static int 904 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 905 { 906 struct ixgbe_hw *hw = &ixgbe->hw; 907 dev_info_t *devinfo = ixgbe->dip; 908 ixgbe_rx_ring_t *rx_ring; 909 ixgbe_rx_group_t *rx_group; 910 ixgbe_tx_ring_t *tx_ring; 911 uint32_t rx_size; 912 uint32_t tx_size; 913 uint32_t ring_per_group; 914 int i; 915 916 /* 917 * Initialize chipset specific hardware function pointers 918 */ 919 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 920 return (IXGBE_FAILURE); 921 } 922 923 /* 924 * Get the system page size 925 */ 926 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 927 928 /* 929 * Set rx buffer size 930 * 931 * The IP header alignment room is counted in the calculation. 932 * The rx buffer size is in unit of 1K that is required by the 933 * chipset hardware. 934 */ 935 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 936 ixgbe->rx_buf_size = ((rx_size >> 10) + 937 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 938 939 /* 940 * Set tx buffer size 941 */ 942 tx_size = ixgbe->max_frame_size; 943 ixgbe->tx_buf_size = ((tx_size >> 10) + 944 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 945 946 /* 947 * Initialize rx/tx rings/groups parameters 948 */ 949 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 950 for (i = 0; i < ixgbe->num_rx_rings; i++) { 951 rx_ring = &ixgbe->rx_rings[i]; 952 rx_ring->index = i; 953 rx_ring->ixgbe = ixgbe; 954 rx_ring->group_index = i / ring_per_group; 955 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 956 } 957 958 for (i = 0; i < ixgbe->num_rx_groups; i++) { 959 rx_group = &ixgbe->rx_groups[i]; 960 rx_group->index = i; 961 rx_group->ixgbe = ixgbe; 962 } 963 964 for (i = 0; i < ixgbe->num_tx_rings; i++) { 965 tx_ring = &ixgbe->tx_rings[i]; 966 tx_ring->index = i; 967 tx_ring->ixgbe = ixgbe; 968 if (ixgbe->tx_head_wb_enable) 969 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 970 else 971 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 972 973 tx_ring->ring_size = ixgbe->tx_ring_size; 974 tx_ring->free_list_size = ixgbe->tx_ring_size + 975 (ixgbe->tx_ring_size >> 1); 976 } 977 978 /* 979 * Initialize values of interrupt throttling rate 980 */ 981 for (i = 1; i < MAX_INTR_VECTOR; i++) 982 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 983 984 /* 985 * The initial link state should be "unknown" 986 */ 987 ixgbe->link_state = LINK_STATE_UNKNOWN; 988 989 return (IXGBE_SUCCESS); 990 } 991 992 /* 993 * ixgbe_init_locks - Initialize locks. 994 */ 995 static void 996 ixgbe_init_locks(ixgbe_t *ixgbe) 997 { 998 ixgbe_rx_ring_t *rx_ring; 999 ixgbe_tx_ring_t *tx_ring; 1000 int i; 1001 1002 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1003 rx_ring = &ixgbe->rx_rings[i]; 1004 mutex_init(&rx_ring->rx_lock, NULL, 1005 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1006 } 1007 1008 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1009 tx_ring = &ixgbe->tx_rings[i]; 1010 mutex_init(&tx_ring->tx_lock, NULL, 1011 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1012 mutex_init(&tx_ring->recycle_lock, NULL, 1013 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1014 mutex_init(&tx_ring->tcb_head_lock, NULL, 1015 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1016 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1017 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1018 } 1019 1020 mutex_init(&ixgbe->gen_lock, NULL, 1021 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1022 1023 mutex_init(&ixgbe->watchdog_lock, NULL, 1024 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1025 } 1026 1027 /* 1028 * ixgbe_destroy_locks - Destroy locks. 1029 */ 1030 static void 1031 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1032 { 1033 ixgbe_rx_ring_t *rx_ring; 1034 ixgbe_tx_ring_t *tx_ring; 1035 int i; 1036 1037 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1038 rx_ring = &ixgbe->rx_rings[i]; 1039 mutex_destroy(&rx_ring->rx_lock); 1040 } 1041 1042 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1043 tx_ring = &ixgbe->tx_rings[i]; 1044 mutex_destroy(&tx_ring->tx_lock); 1045 mutex_destroy(&tx_ring->recycle_lock); 1046 mutex_destroy(&tx_ring->tcb_head_lock); 1047 mutex_destroy(&tx_ring->tcb_tail_lock); 1048 } 1049 1050 mutex_destroy(&ixgbe->gen_lock); 1051 mutex_destroy(&ixgbe->watchdog_lock); 1052 } 1053 1054 static int 1055 ixgbe_resume(dev_info_t *devinfo) 1056 { 1057 ixgbe_t *ixgbe; 1058 int i; 1059 1060 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1061 if (ixgbe == NULL) 1062 return (DDI_FAILURE); 1063 1064 mutex_enter(&ixgbe->gen_lock); 1065 1066 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1067 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1068 mutex_exit(&ixgbe->gen_lock); 1069 return (DDI_FAILURE); 1070 } 1071 1072 /* 1073 * Enable and start the watchdog timer 1074 */ 1075 ixgbe_enable_watchdog_timer(ixgbe); 1076 } 1077 1078 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1079 1080 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1081 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1082 mac_tx_ring_update(ixgbe->mac_hdl, 1083 ixgbe->tx_rings[i].ring_handle); 1084 } 1085 } 1086 1087 mutex_exit(&ixgbe->gen_lock); 1088 1089 return (DDI_SUCCESS); 1090 } 1091 1092 static int 1093 ixgbe_suspend(dev_info_t *devinfo) 1094 { 1095 ixgbe_t *ixgbe; 1096 1097 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1098 if (ixgbe == NULL) 1099 return (DDI_FAILURE); 1100 1101 mutex_enter(&ixgbe->gen_lock); 1102 1103 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1104 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1105 mutex_exit(&ixgbe->gen_lock); 1106 return (DDI_SUCCESS); 1107 } 1108 ixgbe_stop(ixgbe, B_FALSE); 1109 1110 mutex_exit(&ixgbe->gen_lock); 1111 1112 /* 1113 * Disable and stop the watchdog timer 1114 */ 1115 ixgbe_disable_watchdog_timer(ixgbe); 1116 1117 return (DDI_SUCCESS); 1118 } 1119 1120 /* 1121 * ixgbe_init - Initialize the device. 1122 */ 1123 static int 1124 ixgbe_init(ixgbe_t *ixgbe) 1125 { 1126 struct ixgbe_hw *hw = &ixgbe->hw; 1127 1128 mutex_enter(&ixgbe->gen_lock); 1129 1130 /* 1131 * Reset chipset to put the hardware in a known state 1132 * before we try to do anything with the eeprom. 1133 */ 1134 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1135 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1136 goto init_fail; 1137 } 1138 1139 /* 1140 * Need to init eeprom before validating the checksum. 1141 */ 1142 if (ixgbe_init_eeprom_params(hw) < 0) { 1143 ixgbe_error(ixgbe, 1144 "Unable to intitialize the eeprom interface."); 1145 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1146 goto init_fail; 1147 } 1148 1149 /* 1150 * NVM validation 1151 */ 1152 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1153 /* 1154 * Some PCI-E parts fail the first check due to 1155 * the link being in sleep state. Call it again, 1156 * if it fails a second time it's a real issue. 1157 */ 1158 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1159 ixgbe_error(ixgbe, 1160 "Invalid NVM checksum. Please contact " 1161 "the vendor to update the NVM."); 1162 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1163 goto init_fail; 1164 } 1165 } 1166 1167 /* 1168 * Setup default flow control thresholds - enable/disable 1169 * & flow control type is controlled by ixgbe.conf 1170 */ 1171 hw->fc.high_water = DEFAULT_FCRTH; 1172 hw->fc.low_water = DEFAULT_FCRTL; 1173 hw->fc.pause_time = DEFAULT_FCPAUSE; 1174 hw->fc.send_xon = B_TRUE; 1175 1176 /* 1177 * Initialize link settings 1178 */ 1179 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1180 1181 /* 1182 * Initialize the chipset hardware 1183 */ 1184 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1185 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1186 goto init_fail; 1187 } 1188 1189 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1190 goto init_fail; 1191 } 1192 1193 mutex_exit(&ixgbe->gen_lock); 1194 return (IXGBE_SUCCESS); 1195 1196 init_fail: 1197 /* 1198 * Reset PHY 1199 */ 1200 (void) ixgbe_reset_phy(hw); 1201 1202 mutex_exit(&ixgbe->gen_lock); 1203 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1204 return (IXGBE_FAILURE); 1205 } 1206 1207 /* 1208 * ixgbe_chip_start - Initialize and start the chipset hardware. 1209 */ 1210 static int 1211 ixgbe_chip_start(ixgbe_t *ixgbe) 1212 { 1213 struct ixgbe_hw *hw = &ixgbe->hw; 1214 int ret_val, i; 1215 1216 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1217 1218 /* 1219 * Get the mac address 1220 * This function should handle SPARC case correctly. 1221 */ 1222 if (!ixgbe_find_mac_address(ixgbe)) { 1223 ixgbe_error(ixgbe, "Failed to get the mac address"); 1224 return (IXGBE_FAILURE); 1225 } 1226 1227 /* 1228 * Validate the mac address 1229 */ 1230 (void) ixgbe_init_rx_addrs(hw); 1231 if (!is_valid_mac_addr(hw->mac.addr)) { 1232 ixgbe_error(ixgbe, "Invalid mac address"); 1233 return (IXGBE_FAILURE); 1234 } 1235 1236 /* 1237 * Configure/Initialize hardware 1238 */ 1239 ret_val = ixgbe_init_hw(hw); 1240 if (ret_val != IXGBE_SUCCESS) { 1241 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1242 ixgbe_error(ixgbe, 1243 "This 82599 device is pre-release and contains" 1244 " outdated firmware, please contact your hardware" 1245 " vendor for a replacement."); 1246 } else { 1247 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1248 return (IXGBE_FAILURE); 1249 } 1250 } 1251 1252 /* 1253 * Setup adapter interrupt vectors 1254 */ 1255 ixgbe_setup_adapter_vector(ixgbe); 1256 1257 /* 1258 * Initialize unicast addresses. 1259 */ 1260 ixgbe_init_unicst(ixgbe); 1261 1262 /* 1263 * Setup and initialize the mctable structures. 1264 */ 1265 ixgbe_setup_multicst(ixgbe); 1266 1267 /* 1268 * Set interrupt throttling rate 1269 */ 1270 for (i = 0; i < ixgbe->intr_cnt; i++) { 1271 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1272 } 1273 1274 /* 1275 * Save the state of the phy 1276 */ 1277 ixgbe_get_hw_state(ixgbe); 1278 1279 /* 1280 * Make sure driver has control 1281 */ 1282 ixgbe_get_driver_control(hw); 1283 1284 return (IXGBE_SUCCESS); 1285 } 1286 1287 /* 1288 * ixgbe_chip_stop - Stop the chipset hardware 1289 */ 1290 static void 1291 ixgbe_chip_stop(ixgbe_t *ixgbe) 1292 { 1293 struct ixgbe_hw *hw = &ixgbe->hw; 1294 1295 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1296 1297 /* 1298 * Tell firmware driver is no longer in control 1299 */ 1300 ixgbe_release_driver_control(hw); 1301 1302 /* 1303 * Reset the chipset 1304 */ 1305 (void) ixgbe_reset_hw(hw); 1306 1307 /* 1308 * Reset PHY 1309 */ 1310 (void) ixgbe_reset_phy(hw); 1311 } 1312 1313 /* 1314 * ixgbe_reset - Reset the chipset and re-start the driver. 1315 * 1316 * It involves stopping and re-starting the chipset, 1317 * and re-configuring the rx/tx rings. 1318 */ 1319 static int 1320 ixgbe_reset(ixgbe_t *ixgbe) 1321 { 1322 int i; 1323 1324 /* 1325 * Disable and stop the watchdog timer 1326 */ 1327 ixgbe_disable_watchdog_timer(ixgbe); 1328 1329 mutex_enter(&ixgbe->gen_lock); 1330 1331 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1332 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1333 1334 ixgbe_stop(ixgbe, B_FALSE); 1335 1336 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1337 mutex_exit(&ixgbe->gen_lock); 1338 return (IXGBE_FAILURE); 1339 } 1340 1341 /* 1342 * After resetting, need to recheck the link status. 1343 */ 1344 ixgbe->link_check_complete = B_FALSE; 1345 ixgbe->link_check_hrtime = gethrtime() + 1346 (IXGBE_LINK_UP_TIME * 100000000ULL); 1347 1348 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1349 1350 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1351 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1352 mac_tx_ring_update(ixgbe->mac_hdl, 1353 ixgbe->tx_rings[i].ring_handle); 1354 } 1355 } 1356 1357 mutex_exit(&ixgbe->gen_lock); 1358 1359 /* 1360 * Enable and start the watchdog timer 1361 */ 1362 ixgbe_enable_watchdog_timer(ixgbe); 1363 1364 return (IXGBE_SUCCESS); 1365 } 1366 1367 /* 1368 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1369 */ 1370 static void 1371 ixgbe_tx_clean(ixgbe_t *ixgbe) 1372 { 1373 ixgbe_tx_ring_t *tx_ring; 1374 tx_control_block_t *tcb; 1375 link_list_t pending_list; 1376 uint32_t desc_num; 1377 int i, j; 1378 1379 LINK_LIST_INIT(&pending_list); 1380 1381 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1382 tx_ring = &ixgbe->tx_rings[i]; 1383 1384 mutex_enter(&tx_ring->recycle_lock); 1385 1386 /* 1387 * Clean the pending tx data - the pending packets in the 1388 * work_list that have no chances to be transmitted again. 1389 * 1390 * We must ensure the chipset is stopped or the link is down 1391 * before cleaning the transmit packets. 1392 */ 1393 desc_num = 0; 1394 for (j = 0; j < tx_ring->ring_size; j++) { 1395 tcb = tx_ring->work_list[j]; 1396 if (tcb != NULL) { 1397 desc_num += tcb->desc_num; 1398 1399 tx_ring->work_list[j] = NULL; 1400 1401 ixgbe_free_tcb(tcb); 1402 1403 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1404 } 1405 } 1406 1407 if (desc_num > 0) { 1408 atomic_add_32(&tx_ring->tbd_free, desc_num); 1409 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1410 1411 /* 1412 * Reset the head and tail pointers of the tbd ring; 1413 * Reset the writeback head if it's enable. 1414 */ 1415 tx_ring->tbd_head = 0; 1416 tx_ring->tbd_tail = 0; 1417 if (ixgbe->tx_head_wb_enable) 1418 *tx_ring->tbd_head_wb = 0; 1419 1420 IXGBE_WRITE_REG(&ixgbe->hw, 1421 IXGBE_TDH(tx_ring->index), 0); 1422 IXGBE_WRITE_REG(&ixgbe->hw, 1423 IXGBE_TDT(tx_ring->index), 0); 1424 } 1425 1426 mutex_exit(&tx_ring->recycle_lock); 1427 1428 /* 1429 * Add the tx control blocks in the pending list to 1430 * the free list. 1431 */ 1432 ixgbe_put_free_list(tx_ring, &pending_list); 1433 } 1434 } 1435 1436 /* 1437 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1438 * transmitted. 1439 */ 1440 static boolean_t 1441 ixgbe_tx_drain(ixgbe_t *ixgbe) 1442 { 1443 ixgbe_tx_ring_t *tx_ring; 1444 boolean_t done; 1445 int i, j; 1446 1447 /* 1448 * Wait for a specific time to allow pending tx packets 1449 * to be transmitted. 1450 * 1451 * Check the counter tbd_free to see if transmission is done. 1452 * No lock protection is needed here. 1453 * 1454 * Return B_TRUE if all pending packets have been transmitted; 1455 * Otherwise return B_FALSE; 1456 */ 1457 for (i = 0; i < TX_DRAIN_TIME; i++) { 1458 1459 done = B_TRUE; 1460 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1461 tx_ring = &ixgbe->tx_rings[j]; 1462 done = done && 1463 (tx_ring->tbd_free == tx_ring->ring_size); 1464 } 1465 1466 if (done) 1467 break; 1468 1469 msec_delay(1); 1470 } 1471 1472 return (done); 1473 } 1474 1475 /* 1476 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1477 */ 1478 static boolean_t 1479 ixgbe_rx_drain(ixgbe_t *ixgbe) 1480 { 1481 boolean_t done = B_TRUE; 1482 int i; 1483 1484 /* 1485 * Polling the rx free list to check if those rx buffers held by 1486 * the upper layer are released. 1487 * 1488 * Check the counter rcb_free to see if all pending buffers are 1489 * released. No lock protection is needed here. 1490 * 1491 * Return B_TRUE if all pending buffers have been released; 1492 * Otherwise return B_FALSE; 1493 */ 1494 for (i = 0; i < RX_DRAIN_TIME; i++) { 1495 done = (ixgbe->rcb_pending == 0); 1496 1497 if (done) 1498 break; 1499 1500 msec_delay(1); 1501 } 1502 1503 return (done); 1504 } 1505 1506 /* 1507 * ixgbe_start - Start the driver/chipset. 1508 */ 1509 int 1510 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1511 { 1512 int i; 1513 1514 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1515 1516 if (alloc_buffer) { 1517 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1518 ixgbe_error(ixgbe, 1519 "Failed to allocate software receive rings"); 1520 return (IXGBE_FAILURE); 1521 } 1522 1523 /* Allocate buffers for all the rx/tx rings */ 1524 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1525 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1526 return (IXGBE_FAILURE); 1527 } 1528 1529 ixgbe->tx_ring_init = B_TRUE; 1530 } else { 1531 ixgbe->tx_ring_init = B_FALSE; 1532 } 1533 1534 for (i = 0; i < ixgbe->num_rx_rings; i++) 1535 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1536 for (i = 0; i < ixgbe->num_tx_rings; i++) 1537 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1538 1539 /* 1540 * Start the chipset hardware 1541 */ 1542 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1543 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1544 goto start_failure; 1545 } 1546 1547 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1548 goto start_failure; 1549 } 1550 1551 /* 1552 * Setup the rx/tx rings 1553 */ 1554 ixgbe_setup_rings(ixgbe); 1555 1556 /* 1557 * ixgbe_start() will be called when resetting, however if reset 1558 * happens, we need to clear the ERROR and STALL flags before 1559 * enabling the interrupts. 1560 */ 1561 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR | IXGBE_STALL)); 1562 1563 /* 1564 * Enable adapter interrupts 1565 * The interrupts must be enabled after the driver state is START 1566 */ 1567 ixgbe_enable_adapter_interrupts(ixgbe); 1568 1569 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1570 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1571 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1572 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1573 1574 return (IXGBE_SUCCESS); 1575 1576 start_failure: 1577 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1578 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1579 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1580 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1581 1582 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1583 1584 return (IXGBE_FAILURE); 1585 } 1586 1587 /* 1588 * ixgbe_stop - Stop the driver/chipset. 1589 */ 1590 void 1591 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1592 { 1593 int i; 1594 1595 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1596 1597 /* 1598 * Disable the adapter interrupts 1599 */ 1600 ixgbe_disable_adapter_interrupts(ixgbe); 1601 1602 /* 1603 * Drain the pending tx packets 1604 */ 1605 (void) ixgbe_tx_drain(ixgbe); 1606 1607 for (i = 0; i < ixgbe->num_rx_rings; i++) 1608 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1609 for (i = 0; i < ixgbe->num_tx_rings; i++) 1610 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1611 1612 /* 1613 * Stop the chipset hardware 1614 */ 1615 ixgbe_chip_stop(ixgbe); 1616 1617 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1618 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1619 } 1620 1621 /* 1622 * Clean the pending tx data/resources 1623 */ 1624 ixgbe_tx_clean(ixgbe); 1625 1626 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1627 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1628 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1629 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1630 1631 if (ixgbe->link_state == LINK_STATE_UP) { 1632 ixgbe->link_state = LINK_STATE_UNKNOWN; 1633 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1634 } 1635 1636 if (free_buffer) { 1637 /* 1638 * Release the DMA/memory resources of rx/tx rings 1639 */ 1640 ixgbe_free_dma(ixgbe); 1641 ixgbe_free_rx_data(ixgbe); 1642 } 1643 } 1644 1645 /* 1646 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 1647 */ 1648 /* ARGSUSED */ 1649 static int 1650 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 1651 void *arg1, void *arg2) 1652 { 1653 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 1654 1655 switch (cbaction) { 1656 /* IRM callback */ 1657 int count; 1658 case DDI_CB_INTR_ADD: 1659 case DDI_CB_INTR_REMOVE: 1660 count = (int)(uintptr_t)cbarg; 1661 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 1662 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 1663 int, ixgbe->intr_cnt); 1664 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 1665 DDI_SUCCESS) { 1666 ixgbe_error(ixgbe, 1667 "IRM CB: Failed to adjust interrupts"); 1668 goto cb_fail; 1669 } 1670 break; 1671 default: 1672 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 1673 cbaction); 1674 return (DDI_ENOTSUP); 1675 } 1676 return (DDI_SUCCESS); 1677 cb_fail: 1678 return (DDI_FAILURE); 1679 } 1680 1681 /* 1682 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 1683 */ 1684 static int 1685 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 1686 { 1687 int i, rc, actual; 1688 1689 if (count == 0) 1690 return (DDI_SUCCESS); 1691 1692 if ((cbaction == DDI_CB_INTR_ADD && 1693 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 1694 (cbaction == DDI_CB_INTR_REMOVE && 1695 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 1696 return (DDI_FAILURE); 1697 1698 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1699 return (DDI_FAILURE); 1700 } 1701 1702 for (i = 0; i < ixgbe->num_rx_rings; i++) 1703 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 1704 for (i = 0; i < ixgbe->num_tx_rings; i++) 1705 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 1706 1707 mutex_enter(&ixgbe->gen_lock); 1708 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1709 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 1710 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1711 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 1712 1713 ixgbe_stop(ixgbe, B_FALSE); 1714 /* 1715 * Disable interrupts 1716 */ 1717 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1718 rc = ixgbe_disable_intrs(ixgbe); 1719 ASSERT(rc == IXGBE_SUCCESS); 1720 } 1721 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 1722 1723 /* 1724 * Remove interrupt handlers 1725 */ 1726 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 1727 ixgbe_rem_intr_handlers(ixgbe); 1728 } 1729 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 1730 1731 /* 1732 * Clear vect_map 1733 */ 1734 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 1735 switch (cbaction) { 1736 case DDI_CB_INTR_ADD: 1737 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 1738 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 1739 DDI_INTR_ALLOC_NORMAL); 1740 if (rc != DDI_SUCCESS || actual != count) { 1741 ixgbe_log(ixgbe, "Adjust interrupts failed." 1742 "return: %d, irm cb size: %d, actual: %d", 1743 rc, count, actual); 1744 goto intr_adjust_fail; 1745 } 1746 ixgbe->intr_cnt += count; 1747 break; 1748 1749 case DDI_CB_INTR_REMOVE: 1750 for (i = ixgbe->intr_cnt - count; 1751 i < ixgbe->intr_cnt; i ++) { 1752 rc = ddi_intr_free(ixgbe->htable[i]); 1753 ixgbe->htable[i] = NULL; 1754 if (rc != DDI_SUCCESS) { 1755 ixgbe_log(ixgbe, "Adjust interrupts failed." 1756 "return: %d, irm cb size: %d, actual: %d", 1757 rc, count, actual); 1758 goto intr_adjust_fail; 1759 } 1760 } 1761 ixgbe->intr_cnt -= count; 1762 break; 1763 } 1764 1765 /* 1766 * Get priority for first vector, assume remaining are all the same 1767 */ 1768 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 1769 if (rc != DDI_SUCCESS) { 1770 ixgbe_log(ixgbe, 1771 "Get interrupt priority failed: %d", rc); 1772 goto intr_adjust_fail; 1773 } 1774 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 1775 if (rc != DDI_SUCCESS) { 1776 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 1777 goto intr_adjust_fail; 1778 } 1779 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 1780 1781 /* 1782 * Map rings to interrupt vectors 1783 */ 1784 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 1785 ixgbe_error(ixgbe, 1786 "IRM CB: Failed to map interrupts to vectors"); 1787 goto intr_adjust_fail; 1788 } 1789 1790 /* 1791 * Add interrupt handlers 1792 */ 1793 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 1794 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 1795 goto intr_adjust_fail; 1796 } 1797 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 1798 1799 /* 1800 * Now that mutex locks are initialized, and the chip is also 1801 * initialized, enable interrupts. 1802 */ 1803 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 1804 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 1805 goto intr_adjust_fail; 1806 } 1807 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 1808 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1809 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 1810 goto intr_adjust_fail; 1811 } 1812 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 1813 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1814 ixgbe->ixgbe_state |= IXGBE_STARTED; 1815 mutex_exit(&ixgbe->gen_lock); 1816 1817 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1818 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 1819 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 1820 } 1821 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1822 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 1823 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 1824 } 1825 1826 /* Wakeup all Tx rings */ 1827 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1828 mac_tx_ring_update(ixgbe->mac_hdl, 1829 ixgbe->tx_rings[i].ring_handle); 1830 } 1831 1832 IXGBE_DEBUGLOG_3(ixgbe, 1833 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 1834 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 1835 return (DDI_SUCCESS); 1836 1837 intr_adjust_fail: 1838 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1839 mutex_exit(&ixgbe->gen_lock); 1840 return (DDI_FAILURE); 1841 } 1842 1843 /* 1844 * ixgbe_intr_cb_register - Register interrupt callback function. 1845 */ 1846 static int 1847 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 1848 { 1849 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 1850 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 1851 return (IXGBE_FAILURE); 1852 } 1853 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 1854 return (IXGBE_SUCCESS); 1855 } 1856 1857 /* 1858 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 1859 */ 1860 static int 1861 ixgbe_alloc_rings(ixgbe_t *ixgbe) 1862 { 1863 /* 1864 * Allocate memory space for rx rings 1865 */ 1866 ixgbe->rx_rings = kmem_zalloc( 1867 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 1868 KM_NOSLEEP); 1869 1870 if (ixgbe->rx_rings == NULL) { 1871 return (IXGBE_FAILURE); 1872 } 1873 1874 /* 1875 * Allocate memory space for tx rings 1876 */ 1877 ixgbe->tx_rings = kmem_zalloc( 1878 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 1879 KM_NOSLEEP); 1880 1881 if (ixgbe->tx_rings == NULL) { 1882 kmem_free(ixgbe->rx_rings, 1883 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1884 ixgbe->rx_rings = NULL; 1885 return (IXGBE_FAILURE); 1886 } 1887 1888 /* 1889 * Allocate memory space for rx ring groups 1890 */ 1891 ixgbe->rx_groups = kmem_zalloc( 1892 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 1893 KM_NOSLEEP); 1894 1895 if (ixgbe->rx_groups == NULL) { 1896 kmem_free(ixgbe->rx_rings, 1897 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1898 kmem_free(ixgbe->tx_rings, 1899 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1900 ixgbe->rx_rings = NULL; 1901 ixgbe->tx_rings = NULL; 1902 return (IXGBE_FAILURE); 1903 } 1904 1905 return (IXGBE_SUCCESS); 1906 } 1907 1908 /* 1909 * ixgbe_free_rings - Free the memory space of rx/tx rings. 1910 */ 1911 static void 1912 ixgbe_free_rings(ixgbe_t *ixgbe) 1913 { 1914 if (ixgbe->rx_rings != NULL) { 1915 kmem_free(ixgbe->rx_rings, 1916 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1917 ixgbe->rx_rings = NULL; 1918 } 1919 1920 if (ixgbe->tx_rings != NULL) { 1921 kmem_free(ixgbe->tx_rings, 1922 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1923 ixgbe->tx_rings = NULL; 1924 } 1925 1926 if (ixgbe->rx_groups != NULL) { 1927 kmem_free(ixgbe->rx_groups, 1928 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 1929 ixgbe->rx_groups = NULL; 1930 } 1931 } 1932 1933 static int 1934 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 1935 { 1936 ixgbe_rx_ring_t *rx_ring; 1937 int i; 1938 1939 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1940 rx_ring = &ixgbe->rx_rings[i]; 1941 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 1942 goto alloc_rx_rings_failure; 1943 } 1944 return (IXGBE_SUCCESS); 1945 1946 alloc_rx_rings_failure: 1947 ixgbe_free_rx_data(ixgbe); 1948 return (IXGBE_FAILURE); 1949 } 1950 1951 static void 1952 ixgbe_free_rx_data(ixgbe_t *ixgbe) 1953 { 1954 ixgbe_rx_ring_t *rx_ring; 1955 ixgbe_rx_data_t *rx_data; 1956 int i; 1957 1958 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1959 rx_ring = &ixgbe->rx_rings[i]; 1960 1961 mutex_enter(&ixgbe->rx_pending_lock); 1962 rx_data = rx_ring->rx_data; 1963 1964 if (rx_data != NULL) { 1965 rx_data->flag |= IXGBE_RX_STOPPED; 1966 1967 if (rx_data->rcb_pending == 0) { 1968 ixgbe_free_rx_ring_data(rx_data); 1969 rx_ring->rx_data = NULL; 1970 } 1971 } 1972 1973 mutex_exit(&ixgbe->rx_pending_lock); 1974 } 1975 } 1976 1977 /* 1978 * ixgbe_setup_rings - Setup rx/tx rings. 1979 */ 1980 static void 1981 ixgbe_setup_rings(ixgbe_t *ixgbe) 1982 { 1983 /* 1984 * Setup the rx/tx rings, including the following: 1985 * 1986 * 1. Setup the descriptor ring and the control block buffers; 1987 * 2. Initialize necessary registers for receive/transmit; 1988 * 3. Initialize software pointers/parameters for receive/transmit; 1989 */ 1990 ixgbe_setup_rx(ixgbe); 1991 1992 ixgbe_setup_tx(ixgbe); 1993 } 1994 1995 static void 1996 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 1997 { 1998 ixgbe_t *ixgbe = rx_ring->ixgbe; 1999 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2000 struct ixgbe_hw *hw = &ixgbe->hw; 2001 rx_control_block_t *rcb; 2002 union ixgbe_adv_rx_desc *rbd; 2003 uint32_t size; 2004 uint32_t buf_low; 2005 uint32_t buf_high; 2006 uint32_t reg_val; 2007 int i; 2008 2009 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2010 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2011 2012 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2013 rcb = rx_data->work_list[i]; 2014 rbd = &rx_data->rbd_ring[i]; 2015 2016 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2017 rbd->read.hdr_addr = NULL; 2018 } 2019 2020 /* 2021 * Initialize the length register 2022 */ 2023 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2024 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2025 2026 /* 2027 * Initialize the base address registers 2028 */ 2029 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2030 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2031 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2032 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2033 2034 /* 2035 * Setup head & tail pointers 2036 */ 2037 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2038 rx_data->ring_size - 1); 2039 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2040 2041 rx_data->rbd_next = 0; 2042 rx_data->lro_first = 0; 2043 2044 /* 2045 * Setup the Receive Descriptor Control Register (RXDCTL) 2046 * PTHRESH=32 descriptors (half the internal cache) 2047 * HTHRESH=0 descriptors (to minimize latency on fetch) 2048 * WTHRESH defaults to 1 (writeback each descriptor) 2049 */ 2050 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2051 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2052 2053 /* Not a valid value for 82599 */ 2054 if (hw->mac.type < ixgbe_mac_82599EB) { 2055 reg_val |= 0x0020; /* pthresh */ 2056 } 2057 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2058 2059 if (hw->mac.type == ixgbe_mac_82599EB) { 2060 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2061 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2062 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2063 } 2064 2065 /* 2066 * Setup the Split and Replication Receive Control Register. 2067 * Set the rx buffer size and the advanced descriptor type. 2068 */ 2069 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2070 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2071 reg_val |= IXGBE_SRRCTL_DROP_EN; 2072 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2073 } 2074 2075 static void 2076 ixgbe_setup_rx(ixgbe_t *ixgbe) 2077 { 2078 ixgbe_rx_ring_t *rx_ring; 2079 struct ixgbe_hw *hw = &ixgbe->hw; 2080 uint32_t reg_val; 2081 uint32_t ring_mapping; 2082 uint32_t i, index; 2083 uint32_t psrtype_rss_bit; 2084 2085 /* PSRTYPE must be configured for 82599 */ 2086 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2087 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2088 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2089 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2090 reg_val |= IXGBE_PSRTYPE_L2HDR; 2091 reg_val |= 0x80000000; 2092 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2093 } else { 2094 if (ixgbe->num_rx_groups > 32) { 2095 psrtype_rss_bit = 0x20000000; 2096 } else { 2097 psrtype_rss_bit = 0x40000000; 2098 } 2099 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2100 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2101 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2102 reg_val |= IXGBE_PSRTYPE_L2HDR; 2103 reg_val |= psrtype_rss_bit; 2104 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2105 } 2106 } 2107 2108 /* 2109 * Set filter control in FCTRL to accept broadcast packets and do 2110 * not pass pause frames to host. Flow control settings are already 2111 * in this register, so preserve them. 2112 */ 2113 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2114 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 2115 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 2116 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2117 2118 /* 2119 * Hardware checksum settings 2120 */ 2121 if (ixgbe->rx_hcksum_enable) { 2122 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2123 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2124 } 2125 2126 /* 2127 * Setup VMDq and RSS for multiple receive queues 2128 */ 2129 switch (ixgbe->classify_mode) { 2130 case IXGBE_CLASSIFY_RSS: 2131 /* 2132 * One group, only RSS is needed when more than 2133 * one ring enabled. 2134 */ 2135 ixgbe_setup_rss(ixgbe); 2136 break; 2137 2138 case IXGBE_CLASSIFY_VMDQ: 2139 /* 2140 * Multiple groups, each group has one ring, 2141 * only VMDq is needed. 2142 */ 2143 ixgbe_setup_vmdq(ixgbe); 2144 break; 2145 2146 case IXGBE_CLASSIFY_VMDQ_RSS: 2147 /* 2148 * Multiple groups and multiple rings, both 2149 * VMDq and RSS are needed. 2150 */ 2151 ixgbe_setup_vmdq_rss(ixgbe); 2152 break; 2153 2154 default: 2155 break; 2156 } 2157 2158 /* 2159 * Enable the receive unit. This must be done after filter 2160 * control is set in FCTRL. 2161 */ 2162 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 2163 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 2164 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 2165 2166 /* 2167 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2168 */ 2169 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2170 rx_ring = &ixgbe->rx_rings[i]; 2171 ixgbe_setup_rx_ring(rx_ring); 2172 } 2173 2174 /* 2175 * Setup the per-ring statistics mapping. 2176 */ 2177 ring_mapping = 0; 2178 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2179 index = ixgbe->rx_rings[i].hw_index; 2180 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2181 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2182 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2183 } 2184 2185 /* 2186 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2187 * by four bytes if the packet has a VLAN field, so includes MTU, 2188 * ethernet header and frame check sequence. 2189 * Register is MAXFRS in 82599. 2190 */ 2191 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 2192 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2193 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2194 2195 /* 2196 * Setup Jumbo Frame enable bit 2197 */ 2198 if (ixgbe->default_mtu > ETHERMTU) { 2199 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2200 reg_val |= IXGBE_HLREG0_JUMBOEN; 2201 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2202 } 2203 } 2204 2205 static void 2206 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2207 { 2208 ixgbe_t *ixgbe = tx_ring->ixgbe; 2209 struct ixgbe_hw *hw = &ixgbe->hw; 2210 uint32_t size; 2211 uint32_t buf_low; 2212 uint32_t buf_high; 2213 uint32_t reg_val; 2214 2215 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2216 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2217 2218 /* 2219 * Initialize the length register 2220 */ 2221 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2222 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2223 2224 /* 2225 * Initialize the base address registers 2226 */ 2227 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2228 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2229 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2230 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2231 2232 /* 2233 * Setup head & tail pointers 2234 */ 2235 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2236 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2237 2238 /* 2239 * Setup head write-back 2240 */ 2241 if (ixgbe->tx_head_wb_enable) { 2242 /* 2243 * The memory of the head write-back is allocated using 2244 * the extra tbd beyond the tail of the tbd ring. 2245 */ 2246 tx_ring->tbd_head_wb = (uint32_t *) 2247 ((uintptr_t)tx_ring->tbd_area.address + size); 2248 *tx_ring->tbd_head_wb = 0; 2249 2250 buf_low = (uint32_t) 2251 (tx_ring->tbd_area.dma_address + size); 2252 buf_high = (uint32_t) 2253 ((tx_ring->tbd_area.dma_address + size) >> 32); 2254 2255 /* Set the head write-back enable bit */ 2256 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2257 2258 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2259 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2260 2261 /* 2262 * Turn off relaxed ordering for head write back or it will 2263 * cause problems with the tx recycling 2264 */ 2265 reg_val = IXGBE_READ_REG(hw, 2266 IXGBE_DCA_TXCTRL(tx_ring->index)); 2267 reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2268 IXGBE_WRITE_REG(hw, 2269 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2270 } else { 2271 tx_ring->tbd_head_wb = NULL; 2272 } 2273 2274 tx_ring->tbd_head = 0; 2275 tx_ring->tbd_tail = 0; 2276 tx_ring->tbd_free = tx_ring->ring_size; 2277 2278 if (ixgbe->tx_ring_init == B_TRUE) { 2279 tx_ring->tcb_head = 0; 2280 tx_ring->tcb_tail = 0; 2281 tx_ring->tcb_free = tx_ring->free_list_size; 2282 } 2283 2284 /* 2285 * Initialize the s/w context structure 2286 */ 2287 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2288 } 2289 2290 static void 2291 ixgbe_setup_tx(ixgbe_t *ixgbe) 2292 { 2293 struct ixgbe_hw *hw = &ixgbe->hw; 2294 ixgbe_tx_ring_t *tx_ring; 2295 uint32_t reg_val; 2296 uint32_t ring_mapping; 2297 int i; 2298 2299 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2300 tx_ring = &ixgbe->tx_rings[i]; 2301 ixgbe_setup_tx_ring(tx_ring); 2302 } 2303 2304 /* 2305 * Setup the per-ring statistics mapping. 2306 */ 2307 ring_mapping = 0; 2308 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2309 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2310 if ((i & 0x3) == 0x3) { 2311 if (hw->mac.type >= ixgbe_mac_82599EB) { 2312 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2313 ring_mapping); 2314 } else { 2315 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2316 ring_mapping); 2317 } 2318 ring_mapping = 0; 2319 } 2320 } 2321 if ((i & 0x3) != 0x3) 2322 if (hw->mac.type >= ixgbe_mac_82599EB) { 2323 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2324 } else { 2325 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2326 } 2327 2328 /* 2329 * Enable CRC appending and TX padding (for short tx frames) 2330 */ 2331 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2332 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2333 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2334 2335 /* 2336 * enable DMA for 82599 parts 2337 */ 2338 if (hw->mac.type == ixgbe_mac_82599EB) { 2339 /* DMATXCTL.TE must be set after all Tx config is complete */ 2340 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2341 reg_val |= IXGBE_DMATXCTL_TE; 2342 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2343 } 2344 2345 /* 2346 * Enabling tx queues .. 2347 * For 82599 must be done after DMATXCTL.TE is set 2348 */ 2349 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2350 tx_ring = &ixgbe->tx_rings[i]; 2351 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2352 reg_val |= IXGBE_TXDCTL_ENABLE; 2353 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2354 } 2355 } 2356 2357 /* 2358 * ixgbe_setup_rss - Setup receive-side scaling feature. 2359 */ 2360 static void 2361 ixgbe_setup_rss(ixgbe_t *ixgbe) 2362 { 2363 struct ixgbe_hw *hw = &ixgbe->hw; 2364 uint32_t i, mrqc, rxcsum; 2365 uint32_t random; 2366 uint32_t reta; 2367 uint32_t ring_per_group; 2368 2369 /* 2370 * Fill out redirection table 2371 */ 2372 reta = 0; 2373 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2374 2375 for (i = 0; i < 128; i++) { 2376 reta = (reta << 8) | (i % ring_per_group) | 2377 ((i % ring_per_group) << 4); 2378 if ((i & 3) == 3) 2379 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2380 } 2381 2382 /* 2383 * Fill out hash function seeds with a random constant 2384 */ 2385 for (i = 0; i < 10; i++) { 2386 (void) random_get_pseudo_bytes((uint8_t *)&random, 2387 sizeof (uint32_t)); 2388 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2389 } 2390 2391 /* 2392 * Enable RSS & perform hash on these packet types 2393 */ 2394 mrqc = IXGBE_MRQC_RSSEN | 2395 IXGBE_MRQC_RSS_FIELD_IPV4 | 2396 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2397 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2398 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2399 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2400 IXGBE_MRQC_RSS_FIELD_IPV6 | 2401 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2402 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2403 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2404 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2405 2406 /* 2407 * Disable Packet Checksum to enable RSS for multiple receive queues. 2408 * It is an adapter hardware limitation that Packet Checksum is 2409 * mutually exclusive with RSS. 2410 */ 2411 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2412 rxcsum |= IXGBE_RXCSUM_PCSD; 2413 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2414 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2415 } 2416 2417 /* 2418 * ixgbe_setup_vmdq - Setup MAC classification feature 2419 */ 2420 static void 2421 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2422 { 2423 struct ixgbe_hw *hw = &ixgbe->hw; 2424 uint32_t vmdctl, i, vtctl; 2425 2426 /* 2427 * Setup the VMDq Control register, enable VMDq based on 2428 * packet destination MAC address: 2429 */ 2430 switch (hw->mac.type) { 2431 case ixgbe_mac_82598EB: 2432 /* 2433 * VMDq Enable = 1; 2434 * VMDq Filter = 0; MAC filtering 2435 * Default VMDq output index = 0; 2436 */ 2437 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2438 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2439 break; 2440 2441 case ixgbe_mac_82599EB: 2442 /* 2443 * Enable VMDq-only. 2444 */ 2445 vmdctl = IXGBE_MRQC_VMDQEN; 2446 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2447 2448 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2449 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2450 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2451 } 2452 2453 /* 2454 * Enable Virtualization and Replication. 2455 */ 2456 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2457 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2458 2459 /* 2460 * Enable receiving packets to all VFs 2461 */ 2462 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2463 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2464 2465 break; 2466 2467 default: 2468 break; 2469 } 2470 } 2471 2472 /* 2473 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2474 */ 2475 static void 2476 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2477 { 2478 struct ixgbe_hw *hw = &ixgbe->hw; 2479 uint32_t i, mrqc, rxcsum; 2480 uint32_t random; 2481 uint32_t reta; 2482 uint32_t ring_per_group; 2483 uint32_t vmdctl, vtctl; 2484 2485 /* 2486 * Fill out redirection table 2487 */ 2488 reta = 0; 2489 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2490 for (i = 0; i < 128; i++) { 2491 reta = (reta << 8) | (i % ring_per_group) | 2492 ((i % ring_per_group) << 4); 2493 if ((i & 3) == 3) 2494 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2495 } 2496 2497 /* 2498 * Fill out hash function seeds with a random constant 2499 */ 2500 for (i = 0; i < 10; i++) { 2501 (void) random_get_pseudo_bytes((uint8_t *)&random, 2502 sizeof (uint32_t)); 2503 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2504 } 2505 2506 /* 2507 * Enable and setup RSS and VMDq 2508 */ 2509 switch (hw->mac.type) { 2510 case ixgbe_mac_82598EB: 2511 /* 2512 * Enable RSS & Setup RSS Hash functions 2513 */ 2514 mrqc = IXGBE_MRQC_RSSEN | 2515 IXGBE_MRQC_RSS_FIELD_IPV4 | 2516 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2517 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2518 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2519 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2520 IXGBE_MRQC_RSS_FIELD_IPV6 | 2521 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2522 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2523 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2524 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2525 2526 /* 2527 * Enable and Setup VMDq 2528 * VMDq Filter = 0; MAC filtering 2529 * Default VMDq output index = 0; 2530 */ 2531 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2532 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2533 break; 2534 2535 case ixgbe_mac_82599EB: 2536 /* 2537 * Enable RSS & Setup RSS Hash functions 2538 */ 2539 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2540 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2541 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2542 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2543 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2544 IXGBE_MRQC_RSS_FIELD_IPV6 | 2545 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2546 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2547 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2548 2549 /* 2550 * Enable VMDq+RSS. 2551 */ 2552 if (ixgbe->num_rx_groups > 32) { 2553 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2554 } else { 2555 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2556 } 2557 2558 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2559 2560 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2561 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2562 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2563 } 2564 break; 2565 2566 default: 2567 break; 2568 2569 } 2570 2571 /* 2572 * Disable Packet Checksum to enable RSS for multiple receive queues. 2573 * It is an adapter hardware limitation that Packet Checksum is 2574 * mutually exclusive with RSS. 2575 */ 2576 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2577 rxcsum |= IXGBE_RXCSUM_PCSD; 2578 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2579 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2580 2581 if (hw->mac.type == ixgbe_mac_82599EB) { 2582 /* 2583 * Enable Virtualization and Replication. 2584 */ 2585 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2586 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2587 2588 /* 2589 * Enable receiving packets to all VFs 2590 */ 2591 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2592 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2593 } 2594 } 2595 2596 /* 2597 * ixgbe_init_unicst - Initialize the unicast addresses. 2598 */ 2599 static void 2600 ixgbe_init_unicst(ixgbe_t *ixgbe) 2601 { 2602 struct ixgbe_hw *hw = &ixgbe->hw; 2603 uint8_t *mac_addr; 2604 int slot; 2605 /* 2606 * Here we should consider two situations: 2607 * 2608 * 1. Chipset is initialized at the first time, 2609 * Clear all the multiple unicast addresses. 2610 * 2611 * 2. Chipset is reset 2612 * Recover the multiple unicast addresses from the 2613 * software data structure to the RAR registers. 2614 */ 2615 if (!ixgbe->unicst_init) { 2616 /* 2617 * Initialize the multiple unicast addresses 2618 */ 2619 ixgbe->unicst_total = hw->mac.num_rar_entries; 2620 ixgbe->unicst_avail = ixgbe->unicst_total; 2621 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2622 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2623 bzero(mac_addr, ETHERADDRL); 2624 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2625 ixgbe->unicst_addr[slot].mac.set = 0; 2626 } 2627 ixgbe->unicst_init = B_TRUE; 2628 } else { 2629 /* Re-configure the RAR registers */ 2630 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2631 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2632 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2633 (void) ixgbe_set_rar(hw, slot, mac_addr, 2634 ixgbe->unicst_addr[slot].mac.group_index, 2635 IXGBE_RAH_AV); 2636 } else { 2637 bzero(mac_addr, ETHERADDRL); 2638 (void) ixgbe_set_rar(hw, slot, mac_addr, 2639 NULL, NULL); 2640 } 2641 } 2642 } 2643 } 2644 2645 /* 2646 * ixgbe_unicst_find - Find the slot for the specified unicast address 2647 */ 2648 int 2649 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2650 { 2651 int slot; 2652 2653 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2654 2655 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2656 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2657 mac_addr, ETHERADDRL) == 0) 2658 return (slot); 2659 } 2660 2661 return (-1); 2662 } 2663 2664 /* 2665 * ixgbe_multicst_add - Add a multicst address. 2666 */ 2667 int 2668 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2669 { 2670 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2671 2672 if ((multiaddr[0] & 01) == 0) { 2673 return (EINVAL); 2674 } 2675 2676 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2677 return (ENOENT); 2678 } 2679 2680 bcopy(multiaddr, 2681 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2682 ixgbe->mcast_count++; 2683 2684 /* 2685 * Update the multicast table in the hardware 2686 */ 2687 ixgbe_setup_multicst(ixgbe); 2688 2689 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2690 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2691 return (EIO); 2692 } 2693 2694 return (0); 2695 } 2696 2697 /* 2698 * ixgbe_multicst_remove - Remove a multicst address. 2699 */ 2700 int 2701 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2702 { 2703 int i; 2704 2705 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2706 2707 for (i = 0; i < ixgbe->mcast_count; i++) { 2708 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2709 ETHERADDRL) == 0) { 2710 for (i++; i < ixgbe->mcast_count; i++) { 2711 ixgbe->mcast_table[i - 1] = 2712 ixgbe->mcast_table[i]; 2713 } 2714 ixgbe->mcast_count--; 2715 break; 2716 } 2717 } 2718 2719 /* 2720 * Update the multicast table in the hardware 2721 */ 2722 ixgbe_setup_multicst(ixgbe); 2723 2724 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2725 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2726 return (EIO); 2727 } 2728 2729 return (0); 2730 } 2731 2732 /* 2733 * ixgbe_setup_multicast - Setup multicast data structures. 2734 * 2735 * This routine initializes all of the multicast related structures 2736 * and save them in the hardware registers. 2737 */ 2738 static void 2739 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2740 { 2741 uint8_t *mc_addr_list; 2742 uint32_t mc_addr_count; 2743 struct ixgbe_hw *hw = &ixgbe->hw; 2744 2745 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2746 2747 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2748 2749 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2750 mc_addr_count = ixgbe->mcast_count; 2751 2752 /* 2753 * Update the multicast addresses to the MTA registers 2754 */ 2755 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2756 ixgbe_mc_table_itr); 2757 } 2758 2759 /* 2760 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 2761 * 2762 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 2763 * Different chipsets may have different allowed configuration of vmdq and rss. 2764 */ 2765 static void 2766 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 2767 { 2768 struct ixgbe_hw *hw = &ixgbe->hw; 2769 uint32_t ring_per_group; 2770 2771 switch (hw->mac.type) { 2772 case ixgbe_mac_82598EB: 2773 /* 2774 * 82598 supports the following combination: 2775 * vmdq no. x rss no. 2776 * [5..16] x 1 2777 * [1..4] x [1..16] 2778 * However 8 rss queue per pool (vmdq) is sufficient for 2779 * most cases. 2780 */ 2781 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2782 if (ixgbe->num_rx_groups > 4) { 2783 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 2784 } else { 2785 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2786 min(8, ring_per_group); 2787 } 2788 2789 break; 2790 2791 case ixgbe_mac_82599EB: 2792 /* 2793 * 82599 supports the following combination: 2794 * vmdq no. x rss no. 2795 * [33..64] x [1..2] 2796 * [2..32] x [1..4] 2797 * 1 x [1..16] 2798 * However 8 rss queue per pool (vmdq) is sufficient for 2799 * most cases. 2800 */ 2801 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2802 if (ixgbe->num_rx_groups == 1) { 2803 ixgbe->num_rx_rings = min(8, ring_per_group); 2804 } else if (ixgbe->num_rx_groups <= 32) { 2805 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2806 min(4, ring_per_group); 2807 } else if (ixgbe->num_rx_groups <= 64) { 2808 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2809 min(2, ring_per_group); 2810 } 2811 2812 break; 2813 2814 default: 2815 break; 2816 } 2817 2818 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2819 2820 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 2821 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 2822 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 2823 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 2824 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 2825 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 2826 } else { 2827 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 2828 } 2829 2830 ixgbe_log(ixgbe, "rx group number:%d, rx ring number:%d", 2831 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 2832 } 2833 2834 /* 2835 * ixgbe_get_conf - Get driver configurations set in driver.conf. 2836 * 2837 * This routine gets user-configured values out of the configuration 2838 * file ixgbe.conf. 2839 * 2840 * For each configurable value, there is a minimum, a maximum, and a 2841 * default. 2842 * If user does not configure a value, use the default. 2843 * If user configures below the minimum, use the minumum. 2844 * If user configures above the maximum, use the maxumum. 2845 */ 2846 static void 2847 ixgbe_get_conf(ixgbe_t *ixgbe) 2848 { 2849 struct ixgbe_hw *hw = &ixgbe->hw; 2850 uint32_t flow_control; 2851 2852 /* 2853 * ixgbe driver supports the following user configurations: 2854 * 2855 * Jumbo frame configuration: 2856 * default_mtu 2857 * 2858 * Ethernet flow control configuration: 2859 * flow_control 2860 * 2861 * Multiple rings configurations: 2862 * tx_queue_number 2863 * tx_ring_size 2864 * rx_queue_number 2865 * rx_ring_size 2866 * 2867 * Call ixgbe_get_prop() to get the value for a specific 2868 * configuration parameter. 2869 */ 2870 2871 /* 2872 * Jumbo frame configuration - max_frame_size controls host buffer 2873 * allocation, so includes MTU, ethernet header, vlan tag and 2874 * frame check sequence. 2875 */ 2876 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 2877 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 2878 2879 ixgbe->max_frame_size = ixgbe->default_mtu + 2880 sizeof (struct ether_vlan_header) + ETHERFCSL; 2881 2882 /* 2883 * Ethernet flow control configuration 2884 */ 2885 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 2886 ixgbe_fc_none, 3, ixgbe_fc_none); 2887 if (flow_control == 3) 2888 flow_control = ixgbe_fc_default; 2889 2890 /* 2891 * fc.requested mode is what the user requests. After autoneg, 2892 * fc.current_mode will be the flow_control mode that was negotiated. 2893 */ 2894 hw->fc.requested_mode = flow_control; 2895 2896 /* 2897 * Multiple rings configurations 2898 */ 2899 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 2900 ixgbe->capab->min_tx_que_num, 2901 ixgbe->capab->max_tx_que_num, 2902 ixgbe->capab->def_tx_que_num); 2903 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 2904 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2905 2906 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 2907 ixgbe->capab->min_rx_que_num, 2908 ixgbe->capab->max_rx_que_num, 2909 ixgbe->capab->def_rx_que_num); 2910 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 2911 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2912 2913 /* 2914 * Multiple groups configuration 2915 */ 2916 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 2917 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 2918 ixgbe->capab->def_rx_grp_num); 2919 2920 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 2921 0, 1, DEFAULT_MR_ENABLE); 2922 2923 if (ixgbe->mr_enable == B_FALSE) { 2924 ixgbe->num_tx_rings = 1; 2925 ixgbe->num_rx_rings = 1; 2926 ixgbe->num_rx_groups = 1; 2927 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 2928 } else { 2929 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2930 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 2931 /* 2932 * The combination of num_rx_rings and num_rx_groups 2933 * may be not supported by h/w. We need to adjust 2934 * them to appropriate values. 2935 */ 2936 ixgbe_setup_vmdq_rss_conf(ixgbe); 2937 } 2938 2939 /* 2940 * Tunable used to force an interrupt type. The only use is 2941 * for testing of the lesser interrupt types. 2942 * 0 = don't force interrupt type 2943 * 1 = force interrupt type MSI-X 2944 * 2 = force interrupt type MSI 2945 * 3 = force interrupt type Legacy 2946 */ 2947 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 2948 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 2949 2950 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 2951 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 2952 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 2953 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 2954 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 2955 0, 1, DEFAULT_LSO_ENABLE); 2956 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 2957 0, 1, DEFAULT_LRO_ENABLE); 2958 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 2959 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 2960 2961 /* Head Write Back not recommended for 82599 */ 2962 if (hw->mac.type >= ixgbe_mac_82599EB) { 2963 ixgbe->tx_head_wb_enable = B_FALSE; 2964 } 2965 2966 /* 2967 * ixgbe LSO needs the tx h/w checksum support. 2968 * LSO will be disabled if tx h/w checksum is not 2969 * enabled. 2970 */ 2971 if (ixgbe->tx_hcksum_enable == B_FALSE) { 2972 ixgbe->lso_enable = B_FALSE; 2973 } 2974 2975 /* 2976 * ixgbe LRO needs the rx h/w checksum support. 2977 * LRO will be disabled if rx h/w checksum is not 2978 * enabled. 2979 */ 2980 if (ixgbe->rx_hcksum_enable == B_FALSE) { 2981 ixgbe->lro_enable = B_FALSE; 2982 } 2983 2984 /* 2985 * ixgbe LRO only been supported by 82599 now 2986 */ 2987 if (hw->mac.type != ixgbe_mac_82599EB) { 2988 ixgbe->lro_enable = B_FALSE; 2989 } 2990 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 2991 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2992 DEFAULT_TX_COPY_THRESHOLD); 2993 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 2994 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 2995 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 2996 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 2997 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 2998 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 2999 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3000 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3001 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3002 3003 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3004 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3005 DEFAULT_RX_COPY_THRESHOLD); 3006 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3007 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3008 DEFAULT_RX_LIMIT_PER_INTR); 3009 3010 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3011 ixgbe->capab->min_intr_throttle, 3012 ixgbe->capab->max_intr_throttle, 3013 ixgbe->capab->def_intr_throttle); 3014 /* 3015 * 82599 requires the interupt throttling rate is 3016 * a multiple of 8. This is enforced by the register 3017 * definiton. 3018 */ 3019 if (hw->mac.type == ixgbe_mac_82599EB) 3020 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3021 } 3022 3023 static void 3024 ixgbe_init_params(ixgbe_t *ixgbe) 3025 { 3026 ixgbe->param_en_10000fdx_cap = 1; 3027 ixgbe->param_en_1000fdx_cap = 1; 3028 ixgbe->param_en_100fdx_cap = 1; 3029 ixgbe->param_adv_10000fdx_cap = 1; 3030 ixgbe->param_adv_1000fdx_cap = 1; 3031 ixgbe->param_adv_100fdx_cap = 1; 3032 3033 ixgbe->param_pause_cap = 1; 3034 ixgbe->param_asym_pause_cap = 1; 3035 ixgbe->param_rem_fault = 0; 3036 3037 ixgbe->param_adv_autoneg_cap = 1; 3038 ixgbe->param_adv_pause_cap = 1; 3039 ixgbe->param_adv_asym_pause_cap = 1; 3040 ixgbe->param_adv_rem_fault = 0; 3041 3042 ixgbe->param_lp_10000fdx_cap = 0; 3043 ixgbe->param_lp_1000fdx_cap = 0; 3044 ixgbe->param_lp_100fdx_cap = 0; 3045 ixgbe->param_lp_autoneg_cap = 0; 3046 ixgbe->param_lp_pause_cap = 0; 3047 ixgbe->param_lp_asym_pause_cap = 0; 3048 ixgbe->param_lp_rem_fault = 0; 3049 } 3050 3051 /* 3052 * ixgbe_get_prop - Get a property value out of the configuration file 3053 * ixgbe.conf. 3054 * 3055 * Caller provides the name of the property, a default value, a minimum 3056 * value, and a maximum value. 3057 * 3058 * Return configured value of the property, with default, minimum and 3059 * maximum properly applied. 3060 */ 3061 static int 3062 ixgbe_get_prop(ixgbe_t *ixgbe, 3063 char *propname, /* name of the property */ 3064 int minval, /* minimum acceptable value */ 3065 int maxval, /* maximim acceptable value */ 3066 int defval) /* default value */ 3067 { 3068 int value; 3069 3070 /* 3071 * Call ddi_prop_get_int() to read the conf settings 3072 */ 3073 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3074 DDI_PROP_DONTPASS, propname, defval); 3075 if (value > maxval) 3076 value = maxval; 3077 3078 if (value < minval) 3079 value = minval; 3080 3081 return (value); 3082 } 3083 3084 /* 3085 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3086 */ 3087 int 3088 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3089 { 3090 u32 autoneg_advertised = 0; 3091 3092 /* 3093 * No half duplex support with 10Gb parts 3094 */ 3095 if (ixgbe->param_adv_10000fdx_cap == 1) 3096 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3097 3098 if (ixgbe->param_adv_1000fdx_cap == 1) 3099 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3100 3101 if (ixgbe->param_adv_100fdx_cap == 1) 3102 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 3103 3104 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) { 3105 ixgbe_notice(ixgbe, "Invalid link settings. Setup link " 3106 "to autonegotiation with full link capabilities."); 3107 3108 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL | 3109 IXGBE_LINK_SPEED_1GB_FULL | 3110 IXGBE_LINK_SPEED_100_FULL; 3111 } 3112 3113 if (setup_hw) { 3114 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised, 3115 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) { 3116 ixgbe_notice(ixgbe, "Setup link failed on this " 3117 "device."); 3118 return (IXGBE_FAILURE); 3119 } 3120 } 3121 3122 return (IXGBE_SUCCESS); 3123 } 3124 3125 /* 3126 * ixgbe_driver_link_check - Link status processing. 3127 * 3128 * This function can be called in both kernel context and interrupt context 3129 */ 3130 static void 3131 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3132 { 3133 struct ixgbe_hw *hw = &ixgbe->hw; 3134 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3135 boolean_t link_up = B_FALSE; 3136 boolean_t link_changed = B_FALSE; 3137 3138 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3139 3140 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3141 if (link_up) { 3142 ixgbe->link_check_complete = B_TRUE; 3143 3144 /* Link is up, enable flow control settings */ 3145 (void) ixgbe_fc_enable(hw, 0); 3146 3147 /* 3148 * The Link is up, check whether it was marked as down earlier 3149 */ 3150 if (ixgbe->link_state != LINK_STATE_UP) { 3151 switch (speed) { 3152 case IXGBE_LINK_SPEED_10GB_FULL: 3153 ixgbe->link_speed = SPEED_10GB; 3154 break; 3155 case IXGBE_LINK_SPEED_1GB_FULL: 3156 ixgbe->link_speed = SPEED_1GB; 3157 break; 3158 case IXGBE_LINK_SPEED_100_FULL: 3159 ixgbe->link_speed = SPEED_100; 3160 } 3161 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3162 ixgbe->link_state = LINK_STATE_UP; 3163 link_changed = B_TRUE; 3164 } 3165 } else { 3166 if (ixgbe->link_check_complete == B_TRUE || 3167 (ixgbe->link_check_complete == B_FALSE && 3168 gethrtime() >= ixgbe->link_check_hrtime)) { 3169 /* 3170 * The link is really down 3171 */ 3172 ixgbe->link_check_complete = B_TRUE; 3173 3174 if (ixgbe->link_state != LINK_STATE_DOWN) { 3175 ixgbe->link_speed = 0; 3176 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3177 ixgbe->link_state = LINK_STATE_DOWN; 3178 link_changed = B_TRUE; 3179 } 3180 } 3181 } 3182 3183 /* 3184 * this is only reached after a link-status-change interrupt 3185 * so always get new phy state 3186 */ 3187 ixgbe_get_hw_state(ixgbe); 3188 3189 /* 3190 * If we are in an interrupt context, need to re-enable the 3191 * interrupt, which was automasked 3192 */ 3193 if (servicing_interrupt() != 0) { 3194 ixgbe->eims |= IXGBE_EICR_LSC; 3195 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3196 } 3197 3198 if (link_changed) { 3199 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3200 } 3201 } 3202 3203 /* 3204 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3205 */ 3206 static void 3207 ixgbe_sfp_check(void *arg) 3208 { 3209 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3210 uint32_t eicr = ixgbe->eicr; 3211 struct ixgbe_hw *hw = &ixgbe->hw; 3212 3213 mutex_enter(&ixgbe->gen_lock); 3214 if (eicr & IXGBE_EICR_GPI_SDP1) { 3215 /* clear the interrupt */ 3216 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 3217 3218 /* if link up, do multispeed fiber setup */ 3219 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3220 B_TRUE, B_TRUE); 3221 ixgbe_driver_link_check(ixgbe); 3222 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 3223 /* clear the interrupt */ 3224 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 3225 3226 /* if link up, do sfp module setup */ 3227 (void) hw->mac.ops.setup_sfp(hw); 3228 3229 /* do multispeed fiber setup */ 3230 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3231 B_TRUE, B_TRUE); 3232 ixgbe_driver_link_check(ixgbe); 3233 } 3234 mutex_exit(&ixgbe->gen_lock); 3235 } 3236 3237 /* 3238 * ixgbe_link_timer - timer for link status detection 3239 */ 3240 static void 3241 ixgbe_link_timer(void *arg) 3242 { 3243 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3244 3245 mutex_enter(&ixgbe->gen_lock); 3246 ixgbe_driver_link_check(ixgbe); 3247 mutex_exit(&ixgbe->gen_lock); 3248 } 3249 3250 /* 3251 * ixgbe_local_timer - Driver watchdog function. 3252 * 3253 * This function will handle the transmit stall check and other routines. 3254 */ 3255 static void 3256 ixgbe_local_timer(void *arg) 3257 { 3258 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3259 3260 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3261 ixgbe->reset_count++; 3262 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3263 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3264 ixgbe_restart_watchdog_timer(ixgbe); 3265 return; 3266 } 3267 3268 if (ixgbe_stall_check(ixgbe)) { 3269 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3270 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3271 3272 ixgbe->reset_count++; 3273 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3274 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3275 } 3276 3277 ixgbe_restart_watchdog_timer(ixgbe); 3278 } 3279 3280 /* 3281 * ixgbe_stall_check - Check for transmit stall. 3282 * 3283 * This function checks if the adapter is stalled (in transmit). 3284 * 3285 * It is called each time the watchdog timeout is invoked. 3286 * If the transmit descriptor reclaim continuously fails, 3287 * the watchdog value will increment by 1. If the watchdog 3288 * value exceeds the threshold, the ixgbe is assumed to 3289 * have stalled and need to be reset. 3290 */ 3291 static boolean_t 3292 ixgbe_stall_check(ixgbe_t *ixgbe) 3293 { 3294 ixgbe_tx_ring_t *tx_ring; 3295 boolean_t result; 3296 int i; 3297 3298 if (ixgbe->link_state != LINK_STATE_UP) 3299 return (B_FALSE); 3300 3301 /* 3302 * If any tx ring is stalled, we'll reset the chipset 3303 */ 3304 result = B_FALSE; 3305 for (i = 0; i < ixgbe->num_tx_rings; i++) { 3306 tx_ring = &ixgbe->tx_rings[i]; 3307 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 3308 tx_ring->tx_recycle(tx_ring); 3309 } 3310 3311 if (tx_ring->recycle_fail > 0) 3312 tx_ring->stall_watchdog++; 3313 else 3314 tx_ring->stall_watchdog = 0; 3315 3316 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3317 result = B_TRUE; 3318 break; 3319 } 3320 } 3321 3322 if (result) { 3323 tx_ring->stall_watchdog = 0; 3324 tx_ring->recycle_fail = 0; 3325 } 3326 3327 return (result); 3328 } 3329 3330 3331 /* 3332 * is_valid_mac_addr - Check if the mac address is valid. 3333 */ 3334 static boolean_t 3335 is_valid_mac_addr(uint8_t *mac_addr) 3336 { 3337 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3338 const uint8_t addr_test2[6] = 3339 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3340 3341 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3342 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3343 return (B_FALSE); 3344 3345 return (B_TRUE); 3346 } 3347 3348 static boolean_t 3349 ixgbe_find_mac_address(ixgbe_t *ixgbe) 3350 { 3351 #ifdef __sparc 3352 struct ixgbe_hw *hw = &ixgbe->hw; 3353 uchar_t *bytes; 3354 struct ether_addr sysaddr; 3355 uint_t nelts; 3356 int err; 3357 boolean_t found = B_FALSE; 3358 3359 /* 3360 * The "vendor's factory-set address" may already have 3361 * been extracted from the chip, but if the property 3362 * "local-mac-address" is set we use that instead. 3363 * 3364 * We check whether it looks like an array of 6 3365 * bytes (which it should, if OBP set it). If we can't 3366 * make sense of it this way, we'll ignore it. 3367 */ 3368 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3369 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3370 if (err == DDI_PROP_SUCCESS) { 3371 if (nelts == ETHERADDRL) { 3372 while (nelts--) 3373 hw->mac.addr[nelts] = bytes[nelts]; 3374 found = B_TRUE; 3375 } 3376 ddi_prop_free(bytes); 3377 } 3378 3379 /* 3380 * Look up the OBP property "local-mac-address?". If the user has set 3381 * 'local-mac-address? = false', use "the system address" instead. 3382 */ 3383 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 3384 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3385 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3386 if (localetheraddr(NULL, &sysaddr) != 0) { 3387 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3388 found = B_TRUE; 3389 } 3390 } 3391 ddi_prop_free(bytes); 3392 } 3393 3394 /* 3395 * Finally(!), if there's a valid "mac-address" property (created 3396 * if we netbooted from this interface), we must use this instead 3397 * of any of the above to ensure that the NFS/install server doesn't 3398 * get confused by the address changing as Solaris takes over! 3399 */ 3400 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3401 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3402 if (err == DDI_PROP_SUCCESS) { 3403 if (nelts == ETHERADDRL) { 3404 while (nelts--) 3405 hw->mac.addr[nelts] = bytes[nelts]; 3406 found = B_TRUE; 3407 } 3408 ddi_prop_free(bytes); 3409 } 3410 3411 if (found) { 3412 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3413 return (B_TRUE); 3414 } 3415 #else 3416 _NOTE(ARGUNUSED(ixgbe)); 3417 #endif 3418 3419 return (B_TRUE); 3420 } 3421 3422 #pragma inline(ixgbe_arm_watchdog_timer) 3423 static void 3424 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 3425 { 3426 /* 3427 * Fire a watchdog timer 3428 */ 3429 ixgbe->watchdog_tid = 3430 timeout(ixgbe_local_timer, 3431 (void *)ixgbe, 1 * drv_usectohz(1000000)); 3432 3433 } 3434 3435 /* 3436 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 3437 */ 3438 void 3439 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 3440 { 3441 mutex_enter(&ixgbe->watchdog_lock); 3442 3443 if (!ixgbe->watchdog_enable) { 3444 ixgbe->watchdog_enable = B_TRUE; 3445 ixgbe->watchdog_start = B_TRUE; 3446 ixgbe_arm_watchdog_timer(ixgbe); 3447 } 3448 3449 mutex_exit(&ixgbe->watchdog_lock); 3450 } 3451 3452 /* 3453 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 3454 */ 3455 void 3456 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 3457 { 3458 timeout_id_t tid; 3459 3460 mutex_enter(&ixgbe->watchdog_lock); 3461 3462 ixgbe->watchdog_enable = B_FALSE; 3463 ixgbe->watchdog_start = B_FALSE; 3464 tid = ixgbe->watchdog_tid; 3465 ixgbe->watchdog_tid = 0; 3466 3467 mutex_exit(&ixgbe->watchdog_lock); 3468 3469 if (tid != 0) 3470 (void) untimeout(tid); 3471 } 3472 3473 /* 3474 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 3475 */ 3476 void 3477 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 3478 { 3479 mutex_enter(&ixgbe->watchdog_lock); 3480 3481 if (ixgbe->watchdog_enable) { 3482 if (!ixgbe->watchdog_start) { 3483 ixgbe->watchdog_start = B_TRUE; 3484 ixgbe_arm_watchdog_timer(ixgbe); 3485 } 3486 } 3487 3488 mutex_exit(&ixgbe->watchdog_lock); 3489 } 3490 3491 /* 3492 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 3493 */ 3494 static void 3495 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 3496 { 3497 mutex_enter(&ixgbe->watchdog_lock); 3498 3499 if (ixgbe->watchdog_start) 3500 ixgbe_arm_watchdog_timer(ixgbe); 3501 3502 mutex_exit(&ixgbe->watchdog_lock); 3503 } 3504 3505 /* 3506 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 3507 */ 3508 void 3509 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 3510 { 3511 timeout_id_t tid; 3512 3513 mutex_enter(&ixgbe->watchdog_lock); 3514 3515 ixgbe->watchdog_start = B_FALSE; 3516 tid = ixgbe->watchdog_tid; 3517 ixgbe->watchdog_tid = 0; 3518 3519 mutex_exit(&ixgbe->watchdog_lock); 3520 3521 if (tid != 0) 3522 (void) untimeout(tid); 3523 } 3524 3525 /* 3526 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 3527 */ 3528 static void 3529 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 3530 { 3531 struct ixgbe_hw *hw = &ixgbe->hw; 3532 3533 /* 3534 * mask all interrupts off 3535 */ 3536 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 3537 3538 /* 3539 * for MSI-X, also disable autoclear 3540 */ 3541 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3542 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 3543 } 3544 3545 IXGBE_WRITE_FLUSH(hw); 3546 } 3547 3548 /* 3549 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 3550 */ 3551 static void 3552 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 3553 { 3554 struct ixgbe_hw *hw = &ixgbe->hw; 3555 uint32_t eiac, eiam; 3556 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 3557 3558 /* interrupt types to enable */ 3559 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 3560 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 3561 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 3562 3563 /* enable automask on "other" causes that this adapter can generate */ 3564 eiam = ixgbe->capab->other_intr; 3565 3566 /* 3567 * msi-x mode 3568 */ 3569 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3570 /* enable autoclear but not on bits 29:20 */ 3571 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 3572 3573 /* general purpose interrupt enable */ 3574 gpie |= (IXGBE_GPIE_MSIX_MODE 3575 | IXGBE_GPIE_PBA_SUPPORT 3576 | IXGBE_GPIE_OCD 3577 | IXGBE_GPIE_EIAME); 3578 /* 3579 * non-msi-x mode 3580 */ 3581 } else { 3582 3583 /* disable autoclear, leave gpie at default */ 3584 eiac = 0; 3585 3586 /* 3587 * General purpose interrupt enable. 3588 * For 82599, extended interrupt automask enable 3589 * only in MSI or MSI-X mode 3590 */ 3591 if ((hw->mac.type < ixgbe_mac_82599EB) || 3592 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 3593 gpie |= IXGBE_GPIE_EIAME; 3594 } 3595 } 3596 /* Enable specific interrupts for 82599 */ 3597 if (hw->mac.type == ixgbe_mac_82599EB) { 3598 gpie |= IXGBE_SDP2_GPIEN; /* pluggable optics intr */ 3599 gpie |= IXGBE_SDP1_GPIEN; /* LSC interrupt */ 3600 } 3601 /* Enable RSC Dealy 8us for 82599 */ 3602 if (ixgbe->lro_enable) { 3603 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 3604 } 3605 /* write to interrupt control registers */ 3606 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3607 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3608 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3609 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3610 IXGBE_WRITE_FLUSH(hw); 3611 } 3612 3613 /* 3614 * ixgbe_loopback_ioctl - Loopback support. 3615 */ 3616 enum ioc_reply 3617 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3618 { 3619 lb_info_sz_t *lbsp; 3620 lb_property_t *lbpp; 3621 uint32_t *lbmp; 3622 uint32_t size; 3623 uint32_t value; 3624 3625 if (mp->b_cont == NULL) 3626 return (IOC_INVAL); 3627 3628 switch (iocp->ioc_cmd) { 3629 default: 3630 return (IOC_INVAL); 3631 3632 case LB_GET_INFO_SIZE: 3633 size = sizeof (lb_info_sz_t); 3634 if (iocp->ioc_count != size) 3635 return (IOC_INVAL); 3636 3637 value = sizeof (lb_normal); 3638 value += sizeof (lb_mac); 3639 value += sizeof (lb_external); 3640 3641 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3642 *lbsp = value; 3643 break; 3644 3645 case LB_GET_INFO: 3646 value = sizeof (lb_normal); 3647 value += sizeof (lb_mac); 3648 value += sizeof (lb_external); 3649 3650 size = value; 3651 if (iocp->ioc_count != size) 3652 return (IOC_INVAL); 3653 3654 value = 0; 3655 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3656 3657 lbpp[value++] = lb_normal; 3658 lbpp[value++] = lb_mac; 3659 lbpp[value++] = lb_external; 3660 break; 3661 3662 case LB_GET_MODE: 3663 size = sizeof (uint32_t); 3664 if (iocp->ioc_count != size) 3665 return (IOC_INVAL); 3666 3667 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3668 *lbmp = ixgbe->loopback_mode; 3669 break; 3670 3671 case LB_SET_MODE: 3672 size = 0; 3673 if (iocp->ioc_count != sizeof (uint32_t)) 3674 return (IOC_INVAL); 3675 3676 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3677 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3678 return (IOC_INVAL); 3679 break; 3680 } 3681 3682 iocp->ioc_count = size; 3683 iocp->ioc_error = 0; 3684 3685 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3686 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3687 return (IOC_INVAL); 3688 } 3689 3690 return (IOC_REPLY); 3691 } 3692 3693 /* 3694 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3695 */ 3696 static boolean_t 3697 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3698 { 3699 if (mode == ixgbe->loopback_mode) 3700 return (B_TRUE); 3701 3702 ixgbe->loopback_mode = mode; 3703 3704 if (mode == IXGBE_LB_NONE) { 3705 /* 3706 * Reset the chip 3707 */ 3708 (void) ixgbe_reset(ixgbe); 3709 return (B_TRUE); 3710 } 3711 3712 mutex_enter(&ixgbe->gen_lock); 3713 3714 switch (mode) { 3715 default: 3716 mutex_exit(&ixgbe->gen_lock); 3717 return (B_FALSE); 3718 3719 case IXGBE_LB_EXTERNAL: 3720 break; 3721 3722 case IXGBE_LB_INTERNAL_MAC: 3723 ixgbe_set_internal_mac_loopback(ixgbe); 3724 break; 3725 } 3726 3727 mutex_exit(&ixgbe->gen_lock); 3728 3729 return (B_TRUE); 3730 } 3731 3732 /* 3733 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 3734 */ 3735 static void 3736 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 3737 { 3738 struct ixgbe_hw *hw; 3739 uint32_t reg; 3740 uint8_t atlas; 3741 3742 hw = &ixgbe->hw; 3743 3744 /* 3745 * Setup MAC loopback 3746 */ 3747 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 3748 reg |= IXGBE_HLREG0_LPBK; 3749 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 3750 3751 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 3752 reg &= ~IXGBE_AUTOC_LMS_MASK; 3753 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 3754 3755 /* 3756 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 3757 */ 3758 if (hw->mac.type == ixgbe_mac_82598EB) { 3759 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3760 &atlas); 3761 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 3762 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3763 atlas); 3764 3765 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3766 &atlas); 3767 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 3768 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3769 atlas); 3770 3771 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3772 &atlas); 3773 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 3774 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3775 atlas); 3776 3777 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3778 &atlas); 3779 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 3780 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3781 atlas); 3782 } 3783 } 3784 3785 #pragma inline(ixgbe_intr_rx_work) 3786 /* 3787 * ixgbe_intr_rx_work - RX processing of ISR. 3788 */ 3789 static void 3790 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 3791 { 3792 mblk_t *mp; 3793 3794 mutex_enter(&rx_ring->rx_lock); 3795 3796 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 3797 mutex_exit(&rx_ring->rx_lock); 3798 3799 if (mp != NULL) 3800 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 3801 rx_ring->ring_gen_num); 3802 } 3803 3804 #pragma inline(ixgbe_intr_tx_work) 3805 /* 3806 * ixgbe_intr_tx_work - TX processing of ISR. 3807 */ 3808 static void 3809 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 3810 { 3811 ixgbe_t *ixgbe = tx_ring->ixgbe; 3812 3813 /* 3814 * Recycle the tx descriptors 3815 */ 3816 tx_ring->tx_recycle(tx_ring); 3817 3818 /* 3819 * Schedule the re-transmit 3820 */ 3821 if (tx_ring->reschedule && 3822 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 3823 tx_ring->reschedule = B_FALSE; 3824 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 3825 tx_ring->ring_handle); 3826 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 3827 } 3828 } 3829 3830 #pragma inline(ixgbe_intr_other_work) 3831 /* 3832 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 3833 */ 3834 static void 3835 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 3836 { 3837 struct ixgbe_hw *hw = &ixgbe->hw; 3838 3839 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3840 3841 /* 3842 * handle link status change 3843 */ 3844 if (eicr & IXGBE_EICR_LSC) { 3845 ixgbe_driver_link_check(ixgbe); 3846 } 3847 3848 /* 3849 * check for fan failure on adapters with fans 3850 */ 3851 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 3852 (eicr & IXGBE_EICR_GPI_SDP1)) { 3853 if (hw->mac.type < ixgbe_mac_82599EB) { 3854 ixgbe_log(ixgbe, 3855 "Fan has stopped, replace the adapter\n"); 3856 3857 /* re-enable the interrupt, which was automasked */ 3858 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 3859 } 3860 } 3861 3862 /* 3863 * Do SFP check for 82599 3864 */ 3865 if (hw->mac.type == ixgbe_mac_82599EB) { 3866 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 3867 ixgbe_sfp_check, (void *)ixgbe, 3868 DDI_NOSLEEP)) != DDI_SUCCESS) { 3869 ixgbe_log(ixgbe, "No memory available to dispatch " 3870 "taskq for SFP check"); 3871 } 3872 3873 /* 3874 * We need to fully re-check the link later. 3875 */ 3876 ixgbe->link_check_complete = B_FALSE; 3877 ixgbe->link_check_hrtime = gethrtime() + 3878 (IXGBE_LINK_UP_TIME * 100000000ULL); 3879 } 3880 } 3881 3882 /* 3883 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 3884 */ 3885 static uint_t 3886 ixgbe_intr_legacy(void *arg1, void *arg2) 3887 { 3888 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 3889 struct ixgbe_hw *hw = &ixgbe->hw; 3890 ixgbe_tx_ring_t *tx_ring; 3891 ixgbe_rx_ring_t *rx_ring; 3892 uint32_t eicr; 3893 mblk_t *mp; 3894 boolean_t tx_reschedule; 3895 uint_t result; 3896 3897 _NOTE(ARGUNUSED(arg2)); 3898 3899 mutex_enter(&ixgbe->gen_lock); 3900 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 3901 mutex_exit(&ixgbe->gen_lock); 3902 return (DDI_INTR_UNCLAIMED); 3903 } 3904 3905 mp = NULL; 3906 tx_reschedule = B_FALSE; 3907 3908 /* 3909 * Any bit set in eicr: claim this interrupt 3910 */ 3911 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 3912 3913 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3914 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3915 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 3916 return (DDI_INTR_CLAIMED); 3917 } 3918 3919 if (eicr) { 3920 /* 3921 * For legacy interrupt, we have only one interrupt, 3922 * so we have only one rx ring and one tx ring enabled. 3923 */ 3924 ASSERT(ixgbe->num_rx_rings == 1); 3925 ASSERT(ixgbe->num_tx_rings == 1); 3926 3927 /* 3928 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 3929 */ 3930 if (eicr & 0x1) { 3931 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 3932 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 3933 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 3934 /* 3935 * Clean the rx descriptors 3936 */ 3937 rx_ring = &ixgbe->rx_rings[0]; 3938 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 3939 } 3940 3941 /* 3942 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 3943 */ 3944 if (eicr & 0x2) { 3945 /* 3946 * Recycle the tx descriptors 3947 */ 3948 tx_ring = &ixgbe->tx_rings[0]; 3949 tx_ring->tx_recycle(tx_ring); 3950 3951 /* 3952 * Schedule the re-transmit 3953 */ 3954 tx_reschedule = (tx_ring->reschedule && 3955 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 3956 } 3957 3958 /* any interrupt type other than tx/rx */ 3959 if (eicr & ixgbe->capab->other_intr) { 3960 if (hw->mac.type < ixgbe_mac_82599EB) { 3961 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3962 } 3963 if (hw->mac.type == ixgbe_mac_82599EB) { 3964 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 3965 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 3966 } 3967 ixgbe_intr_other_work(ixgbe, eicr); 3968 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 3969 } 3970 3971 mutex_exit(&ixgbe->gen_lock); 3972 3973 result = DDI_INTR_CLAIMED; 3974 } else { 3975 mutex_exit(&ixgbe->gen_lock); 3976 3977 /* 3978 * No interrupt cause bits set: don't claim this interrupt. 3979 */ 3980 result = DDI_INTR_UNCLAIMED; 3981 } 3982 3983 /* re-enable the interrupts which were automasked */ 3984 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3985 3986 /* 3987 * Do the following work outside of the gen_lock 3988 */ 3989 if (mp != NULL) { 3990 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 3991 rx_ring->ring_gen_num); 3992 } 3993 3994 if (tx_reschedule) { 3995 tx_ring->reschedule = B_FALSE; 3996 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 3997 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 3998 } 3999 4000 return (result); 4001 } 4002 4003 /* 4004 * ixgbe_intr_msi - Interrupt handler for MSI. 4005 */ 4006 static uint_t 4007 ixgbe_intr_msi(void *arg1, void *arg2) 4008 { 4009 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4010 struct ixgbe_hw *hw = &ixgbe->hw; 4011 uint32_t eicr; 4012 4013 _NOTE(ARGUNUSED(arg2)); 4014 4015 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4016 4017 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4018 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4019 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4020 return (DDI_INTR_CLAIMED); 4021 } 4022 4023 /* 4024 * For MSI interrupt, we have only one vector, 4025 * so we have only one rx ring and one tx ring enabled. 4026 */ 4027 ASSERT(ixgbe->num_rx_rings == 1); 4028 ASSERT(ixgbe->num_tx_rings == 1); 4029 4030 /* 4031 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4032 */ 4033 if (eicr & 0x1) { 4034 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4035 } 4036 4037 /* 4038 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4039 */ 4040 if (eicr & 0x2) { 4041 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4042 } 4043 4044 /* any interrupt type other than tx/rx */ 4045 if (eicr & ixgbe->capab->other_intr) { 4046 mutex_enter(&ixgbe->gen_lock); 4047 if (hw->mac.type < ixgbe_mac_82599EB) { 4048 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4049 } 4050 if (hw->mac.type == ixgbe_mac_82599EB) { 4051 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4052 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4053 } 4054 ixgbe_intr_other_work(ixgbe, eicr); 4055 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4056 mutex_exit(&ixgbe->gen_lock); 4057 } 4058 4059 /* re-enable the interrupts which were automasked */ 4060 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4061 4062 return (DDI_INTR_CLAIMED); 4063 } 4064 4065 /* 4066 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4067 */ 4068 static uint_t 4069 ixgbe_intr_msix(void *arg1, void *arg2) 4070 { 4071 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4072 ixgbe_t *ixgbe = vect->ixgbe; 4073 struct ixgbe_hw *hw = &ixgbe->hw; 4074 uint32_t eicr; 4075 int r_idx = 0; 4076 4077 _NOTE(ARGUNUSED(arg2)); 4078 4079 /* 4080 * Clean each rx ring that has its bit set in the map 4081 */ 4082 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4083 while (r_idx >= 0) { 4084 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4085 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4086 (ixgbe->num_rx_rings - 1)); 4087 } 4088 4089 /* 4090 * Clean each tx ring that has its bit set in the map 4091 */ 4092 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4093 while (r_idx >= 0) { 4094 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4095 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4096 (ixgbe->num_tx_rings - 1)); 4097 } 4098 4099 4100 /* 4101 * Clean other interrupt (link change) that has its bit set in the map 4102 */ 4103 if (BT_TEST(vect->other_map, 0) == 1) { 4104 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4105 4106 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4107 DDI_FM_OK) { 4108 ddi_fm_service_impact(ixgbe->dip, 4109 DDI_SERVICE_DEGRADED); 4110 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4111 return (DDI_INTR_CLAIMED); 4112 } 4113 4114 /* 4115 * Need check cause bits and only other causes will 4116 * be processed 4117 */ 4118 /* any interrupt type other than tx/rx */ 4119 if (eicr & ixgbe->capab->other_intr) { 4120 if (hw->mac.type < ixgbe_mac_82599EB) { 4121 mutex_enter(&ixgbe->gen_lock); 4122 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4123 ixgbe_intr_other_work(ixgbe, eicr); 4124 mutex_exit(&ixgbe->gen_lock); 4125 } else { 4126 if (hw->mac.type == ixgbe_mac_82599EB) { 4127 mutex_enter(&ixgbe->gen_lock); 4128 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4129 ixgbe_intr_other_work(ixgbe, eicr); 4130 mutex_exit(&ixgbe->gen_lock); 4131 } 4132 } 4133 } 4134 4135 /* re-enable the interrupts which were automasked */ 4136 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4137 } 4138 4139 return (DDI_INTR_CLAIMED); 4140 } 4141 4142 /* 4143 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4144 * 4145 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4146 * if not successful, try Legacy. 4147 * ixgbe->intr_force can be used to force sequence to start with 4148 * any of the 3 types. 4149 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4150 */ 4151 static int 4152 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4153 { 4154 dev_info_t *devinfo; 4155 int intr_types; 4156 int rc; 4157 4158 devinfo = ixgbe->dip; 4159 4160 /* 4161 * Get supported interrupt types 4162 */ 4163 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4164 4165 if (rc != DDI_SUCCESS) { 4166 ixgbe_log(ixgbe, 4167 "Get supported interrupt types failed: %d", rc); 4168 return (IXGBE_FAILURE); 4169 } 4170 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4171 4172 ixgbe->intr_type = 0; 4173 4174 /* 4175 * Install MSI-X interrupts 4176 */ 4177 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4178 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4179 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4180 if (rc == IXGBE_SUCCESS) 4181 return (IXGBE_SUCCESS); 4182 4183 ixgbe_log(ixgbe, 4184 "Allocate MSI-X failed, trying MSI interrupts..."); 4185 } 4186 4187 /* 4188 * MSI-X not used, force rings and groups to 1 4189 */ 4190 ixgbe->num_rx_rings = 1; 4191 ixgbe->num_rx_groups = 1; 4192 ixgbe->num_tx_rings = 1; 4193 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4194 ixgbe_log(ixgbe, 4195 "MSI-X not used, force rings and groups number to 1"); 4196 4197 /* 4198 * Install MSI interrupts 4199 */ 4200 if ((intr_types & DDI_INTR_TYPE_MSI) && 4201 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 4202 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 4203 if (rc == IXGBE_SUCCESS) 4204 return (IXGBE_SUCCESS); 4205 4206 ixgbe_log(ixgbe, 4207 "Allocate MSI failed, trying Legacy interrupts..."); 4208 } 4209 4210 /* 4211 * Install legacy interrupts 4212 */ 4213 if (intr_types & DDI_INTR_TYPE_FIXED) { 4214 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 4215 if (rc == IXGBE_SUCCESS) 4216 return (IXGBE_SUCCESS); 4217 4218 ixgbe_log(ixgbe, 4219 "Allocate Legacy interrupts failed"); 4220 } 4221 4222 /* 4223 * If none of the 3 types succeeded, return failure 4224 */ 4225 return (IXGBE_FAILURE); 4226 } 4227 4228 /* 4229 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 4230 * 4231 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4232 * if fewer than 2 handles are available, return failure. 4233 * Upon success, this maps the vectors to rx and tx rings for 4234 * interrupts. 4235 */ 4236 static int 4237 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 4238 { 4239 dev_info_t *devinfo; 4240 int request, count, actual; 4241 int minimum; 4242 int rc; 4243 uint32_t ring_per_group; 4244 4245 devinfo = ixgbe->dip; 4246 4247 switch (intr_type) { 4248 case DDI_INTR_TYPE_FIXED: 4249 request = 1; /* Request 1 legacy interrupt handle */ 4250 minimum = 1; 4251 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 4252 break; 4253 4254 case DDI_INTR_TYPE_MSI: 4255 request = 1; /* Request 1 MSI interrupt handle */ 4256 minimum = 1; 4257 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 4258 break; 4259 4260 case DDI_INTR_TYPE_MSIX: 4261 /* 4262 * Best number of vectors for the adapter is 4263 * (# rx rings + # tx rings), however we will 4264 * limit the request number. 4265 */ 4266 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 4267 if (request > ixgbe->capab->max_ring_vect) 4268 request = ixgbe->capab->max_ring_vect; 4269 minimum = 1; 4270 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 4271 break; 4272 4273 default: 4274 ixgbe_log(ixgbe, 4275 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 4276 intr_type); 4277 return (IXGBE_FAILURE); 4278 } 4279 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 4280 request, minimum); 4281 4282 /* 4283 * Get number of supported interrupts 4284 */ 4285 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4286 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4287 ixgbe_log(ixgbe, 4288 "Get interrupt number failed. Return: %d, count: %d", 4289 rc, count); 4290 return (IXGBE_FAILURE); 4291 } 4292 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 4293 4294 actual = 0; 4295 ixgbe->intr_cnt = 0; 4296 ixgbe->intr_cnt_max = 0; 4297 ixgbe->intr_cnt_min = 0; 4298 4299 /* 4300 * Allocate an array of interrupt handles 4301 */ 4302 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 4303 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 4304 4305 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 4306 request, &actual, DDI_INTR_ALLOC_NORMAL); 4307 if (rc != DDI_SUCCESS) { 4308 ixgbe_log(ixgbe, "Allocate interrupts failed. " 4309 "return: %d, request: %d, actual: %d", 4310 rc, request, actual); 4311 goto alloc_handle_fail; 4312 } 4313 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 4314 4315 /* 4316 * upper/lower limit of interrupts 4317 */ 4318 ixgbe->intr_cnt = actual; 4319 ixgbe->intr_cnt_max = request; 4320 ixgbe->intr_cnt_min = minimum; 4321 4322 /* 4323 * rss number per group should not exceed the rx interrupt number, 4324 * else need to adjust rx ring number. 4325 */ 4326 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4327 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 4328 if (min(actual, ixgbe->num_rx_rings) < ring_per_group) { 4329 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 4330 min(actual, ixgbe->num_rx_rings); 4331 ixgbe_setup_vmdq_rss_conf(ixgbe); 4332 } 4333 4334 /* 4335 * Now we know the actual number of vectors. Here we map the vector 4336 * to other, rx rings and tx ring. 4337 */ 4338 if (actual < minimum) { 4339 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 4340 actual); 4341 goto alloc_handle_fail; 4342 } 4343 4344 /* 4345 * Get priority for first vector, assume remaining are all the same 4346 */ 4347 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 4348 if (rc != DDI_SUCCESS) { 4349 ixgbe_log(ixgbe, 4350 "Get interrupt priority failed: %d", rc); 4351 goto alloc_handle_fail; 4352 } 4353 4354 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 4355 if (rc != DDI_SUCCESS) { 4356 ixgbe_log(ixgbe, 4357 "Get interrupt cap failed: %d", rc); 4358 goto alloc_handle_fail; 4359 } 4360 4361 ixgbe->intr_type = intr_type; 4362 4363 return (IXGBE_SUCCESS); 4364 4365 alloc_handle_fail: 4366 ixgbe_rem_intrs(ixgbe); 4367 4368 return (IXGBE_FAILURE); 4369 } 4370 4371 /* 4372 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 4373 * 4374 * Before adding the interrupt handlers, the interrupt vectors have 4375 * been allocated, and the rx/tx rings have also been allocated. 4376 */ 4377 static int 4378 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 4379 { 4380 int vector = 0; 4381 int rc; 4382 4383 switch (ixgbe->intr_type) { 4384 case DDI_INTR_TYPE_MSIX: 4385 /* 4386 * Add interrupt handler for all vectors 4387 */ 4388 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 4389 /* 4390 * install pointer to vect_map[vector] 4391 */ 4392 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4393 (ddi_intr_handler_t *)ixgbe_intr_msix, 4394 (void *)&ixgbe->vect_map[vector], NULL); 4395 4396 if (rc != DDI_SUCCESS) { 4397 ixgbe_log(ixgbe, 4398 "Add rx interrupt handler failed. " 4399 "return: %d, vector: %d", rc, vector); 4400 for (vector--; vector >= 0; vector--) { 4401 (void) ddi_intr_remove_handler( 4402 ixgbe->htable[vector]); 4403 } 4404 return (IXGBE_FAILURE); 4405 } 4406 } 4407 4408 break; 4409 4410 case DDI_INTR_TYPE_MSI: 4411 /* 4412 * Add interrupt handlers for the only vector 4413 */ 4414 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4415 (ddi_intr_handler_t *)ixgbe_intr_msi, 4416 (void *)ixgbe, NULL); 4417 4418 if (rc != DDI_SUCCESS) { 4419 ixgbe_log(ixgbe, 4420 "Add MSI interrupt handler failed: %d", rc); 4421 return (IXGBE_FAILURE); 4422 } 4423 4424 break; 4425 4426 case DDI_INTR_TYPE_FIXED: 4427 /* 4428 * Add interrupt handlers for the only vector 4429 */ 4430 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4431 (ddi_intr_handler_t *)ixgbe_intr_legacy, 4432 (void *)ixgbe, NULL); 4433 4434 if (rc != DDI_SUCCESS) { 4435 ixgbe_log(ixgbe, 4436 "Add legacy interrupt handler failed: %d", rc); 4437 return (IXGBE_FAILURE); 4438 } 4439 4440 break; 4441 4442 default: 4443 return (IXGBE_FAILURE); 4444 } 4445 4446 return (IXGBE_SUCCESS); 4447 } 4448 4449 #pragma inline(ixgbe_map_rxring_to_vector) 4450 /* 4451 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 4452 */ 4453 static void 4454 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 4455 { 4456 /* 4457 * Set bit in map 4458 */ 4459 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4460 4461 /* 4462 * Count bits set 4463 */ 4464 ixgbe->vect_map[v_idx].rxr_cnt++; 4465 4466 /* 4467 * Remember bit position 4468 */ 4469 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 4470 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 4471 } 4472 4473 #pragma inline(ixgbe_map_txring_to_vector) 4474 /* 4475 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 4476 */ 4477 static void 4478 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 4479 { 4480 /* 4481 * Set bit in map 4482 */ 4483 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 4484 4485 /* 4486 * Count bits set 4487 */ 4488 ixgbe->vect_map[v_idx].txr_cnt++; 4489 4490 /* 4491 * Remember bit position 4492 */ 4493 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 4494 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 4495 } 4496 4497 /* 4498 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 4499 * allocation register (IVAR). 4500 * cause: 4501 * -1 : other cause 4502 * 0 : rx 4503 * 1 : tx 4504 */ 4505 static void 4506 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 4507 int8_t cause) 4508 { 4509 struct ixgbe_hw *hw = &ixgbe->hw; 4510 u32 ivar, index; 4511 4512 switch (hw->mac.type) { 4513 case ixgbe_mac_82598EB: 4514 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4515 if (cause == -1) { 4516 cause = 0; 4517 } 4518 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4519 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4520 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 4521 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 4522 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4523 break; 4524 case ixgbe_mac_82599EB: 4525 if (cause == -1) { 4526 /* other causes */ 4527 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4528 index = (intr_alloc_entry & 1) * 8; 4529 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4530 ivar &= ~(0xFF << index); 4531 ivar |= (msix_vector << index); 4532 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4533 } else { 4534 /* tx or rx causes */ 4535 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4536 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4537 ivar = IXGBE_READ_REG(hw, 4538 IXGBE_IVAR(intr_alloc_entry >> 1)); 4539 ivar &= ~(0xFF << index); 4540 ivar |= (msix_vector << index); 4541 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4542 ivar); 4543 } 4544 break; 4545 default: 4546 break; 4547 } 4548 } 4549 4550 /* 4551 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 4552 * given interrupt vector allocation register (IVAR). 4553 * cause: 4554 * -1 : other cause 4555 * 0 : rx 4556 * 1 : tx 4557 */ 4558 static void 4559 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4560 { 4561 struct ixgbe_hw *hw = &ixgbe->hw; 4562 u32 ivar, index; 4563 4564 switch (hw->mac.type) { 4565 case ixgbe_mac_82598EB: 4566 if (cause == -1) { 4567 cause = 0; 4568 } 4569 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4570 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4571 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 4572 (intr_alloc_entry & 0x3))); 4573 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4574 break; 4575 case ixgbe_mac_82599EB: 4576 if (cause == -1) { 4577 /* other causes */ 4578 index = (intr_alloc_entry & 1) * 8; 4579 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4580 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4581 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4582 } else { 4583 /* tx or rx causes */ 4584 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4585 ivar = IXGBE_READ_REG(hw, 4586 IXGBE_IVAR(intr_alloc_entry >> 1)); 4587 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4588 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4589 ivar); 4590 } 4591 break; 4592 default: 4593 break; 4594 } 4595 } 4596 4597 /* 4598 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 4599 * given interrupt vector allocation register (IVAR). 4600 * cause: 4601 * -1 : other cause 4602 * 0 : rx 4603 * 1 : tx 4604 */ 4605 static void 4606 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4607 { 4608 struct ixgbe_hw *hw = &ixgbe->hw; 4609 u32 ivar, index; 4610 4611 switch (hw->mac.type) { 4612 case ixgbe_mac_82598EB: 4613 if (cause == -1) { 4614 cause = 0; 4615 } 4616 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4617 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4618 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 4619 (intr_alloc_entry & 0x3))); 4620 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4621 break; 4622 case ixgbe_mac_82599EB: 4623 if (cause == -1) { 4624 /* other causes */ 4625 index = (intr_alloc_entry & 1) * 8; 4626 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4627 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4628 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4629 } else { 4630 /* tx or rx causes */ 4631 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4632 ivar = IXGBE_READ_REG(hw, 4633 IXGBE_IVAR(intr_alloc_entry >> 1)); 4634 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4635 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4636 ivar); 4637 } 4638 break; 4639 default: 4640 break; 4641 } 4642 } 4643 4644 /* 4645 * Convert the rx ring index driver maintained to the rx ring index 4646 * in h/w. 4647 */ 4648 static uint32_t 4649 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 4650 { 4651 4652 struct ixgbe_hw *hw = &ixgbe->hw; 4653 uint32_t rx_ring_per_group, hw_rx_index; 4654 4655 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 4656 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 4657 return (sw_rx_index); 4658 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 4659 if (hw->mac.type == ixgbe_mac_82598EB) { 4660 return (sw_rx_index); 4661 } else if (hw->mac.type == ixgbe_mac_82599EB) { 4662 return (sw_rx_index * 2); 4663 } 4664 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 4665 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4666 4667 if (hw->mac.type == ixgbe_mac_82598EB) { 4668 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 4669 16 + (sw_rx_index % rx_ring_per_group); 4670 return (hw_rx_index); 4671 } else if (hw->mac.type == ixgbe_mac_82599EB) { 4672 if (ixgbe->num_rx_groups > 32) { 4673 hw_rx_index = (sw_rx_index / 4674 rx_ring_per_group) * 2 + 4675 (sw_rx_index % rx_ring_per_group); 4676 } else { 4677 hw_rx_index = (sw_rx_index / 4678 rx_ring_per_group) * 4 + 4679 (sw_rx_index % rx_ring_per_group); 4680 } 4681 return (hw_rx_index); 4682 } 4683 } 4684 4685 /* 4686 * Should never reach. Just to make compiler happy. 4687 */ 4688 return (sw_rx_index); 4689 } 4690 4691 /* 4692 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 4693 * 4694 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 4695 * to vector[0 - (intr_cnt -1)]. 4696 */ 4697 static int 4698 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 4699 { 4700 int i, vector = 0; 4701 4702 /* initialize vector map */ 4703 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 4704 for (i = 0; i < ixgbe->intr_cnt; i++) { 4705 ixgbe->vect_map[i].ixgbe = ixgbe; 4706 } 4707 4708 /* 4709 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 4710 * tx rings[0] on RTxQ[1]. 4711 */ 4712 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 4713 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 4714 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 4715 return (IXGBE_SUCCESS); 4716 } 4717 4718 /* 4719 * Interrupts/vectors mapping for MSI-X 4720 */ 4721 4722 /* 4723 * Map other interrupt to vector 0, 4724 * Set bit in map and count the bits set. 4725 */ 4726 BT_SET(ixgbe->vect_map[vector].other_map, 0); 4727 ixgbe->vect_map[vector].other_cnt++; 4728 4729 /* 4730 * Map rx ring interrupts to vectors 4731 */ 4732 for (i = 0; i < ixgbe->num_rx_rings; i++) { 4733 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 4734 vector = (vector +1) % ixgbe->intr_cnt; 4735 } 4736 4737 /* 4738 * Map tx ring interrupts to vectors 4739 */ 4740 for (i = 0; i < ixgbe->num_tx_rings; i++) { 4741 ixgbe_map_txring_to_vector(ixgbe, i, vector); 4742 vector = (vector +1) % ixgbe->intr_cnt; 4743 } 4744 4745 return (IXGBE_SUCCESS); 4746 } 4747 4748 /* 4749 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 4750 * 4751 * This relies on ring/vector mapping already set up in the 4752 * vect_map[] structures 4753 */ 4754 static void 4755 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 4756 { 4757 struct ixgbe_hw *hw = &ixgbe->hw; 4758 ixgbe_intr_vector_t *vect; /* vector bitmap */ 4759 int r_idx; /* ring index */ 4760 int v_idx; /* vector index */ 4761 uint32_t hw_index; 4762 4763 /* 4764 * Clear any previous entries 4765 */ 4766 switch (hw->mac.type) { 4767 case ixgbe_mac_82598EB: 4768 for (v_idx = 0; v_idx < 25; v_idx++) 4769 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4770 4771 break; 4772 case ixgbe_mac_82599EB: 4773 for (v_idx = 0; v_idx < 64; v_idx++) 4774 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4775 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 4776 4777 break; 4778 default: 4779 break; 4780 } 4781 4782 /* 4783 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 4784 * tx rings[0] will use RTxQ[1]. 4785 */ 4786 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 4787 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 4788 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 4789 return; 4790 } 4791 4792 /* 4793 * For MSI-X interrupt, "Other" is always on vector[0]. 4794 */ 4795 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 4796 4797 /* 4798 * For each interrupt vector, populate the IVAR table 4799 */ 4800 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 4801 vect = &ixgbe->vect_map[v_idx]; 4802 4803 /* 4804 * For each rx ring bit set 4805 */ 4806 r_idx = bt_getlowbit(vect->rx_map, 0, 4807 (ixgbe->num_rx_rings - 1)); 4808 4809 while (r_idx >= 0) { 4810 hw_index = ixgbe->rx_rings[r_idx].hw_index; 4811 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 4812 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4813 (ixgbe->num_rx_rings - 1)); 4814 } 4815 4816 /* 4817 * For each tx ring bit set 4818 */ 4819 r_idx = bt_getlowbit(vect->tx_map, 0, 4820 (ixgbe->num_tx_rings - 1)); 4821 4822 while (r_idx >= 0) { 4823 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 4824 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4825 (ixgbe->num_tx_rings - 1)); 4826 } 4827 } 4828 } 4829 4830 /* 4831 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 4832 */ 4833 static void 4834 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 4835 { 4836 int i; 4837 int rc; 4838 4839 for (i = 0; i < ixgbe->intr_cnt; i++) { 4840 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 4841 if (rc != DDI_SUCCESS) { 4842 IXGBE_DEBUGLOG_1(ixgbe, 4843 "Remove intr handler failed: %d", rc); 4844 } 4845 } 4846 } 4847 4848 /* 4849 * ixgbe_rem_intrs - Remove the allocated interrupts. 4850 */ 4851 static void 4852 ixgbe_rem_intrs(ixgbe_t *ixgbe) 4853 { 4854 int i; 4855 int rc; 4856 4857 for (i = 0; i < ixgbe->intr_cnt; i++) { 4858 rc = ddi_intr_free(ixgbe->htable[i]); 4859 if (rc != DDI_SUCCESS) { 4860 IXGBE_DEBUGLOG_1(ixgbe, 4861 "Free intr failed: %d", rc); 4862 } 4863 } 4864 4865 kmem_free(ixgbe->htable, ixgbe->intr_size); 4866 ixgbe->htable = NULL; 4867 } 4868 4869 /* 4870 * ixgbe_enable_intrs - Enable all the ddi interrupts. 4871 */ 4872 static int 4873 ixgbe_enable_intrs(ixgbe_t *ixgbe) 4874 { 4875 int i; 4876 int rc; 4877 4878 /* 4879 * Enable interrupts 4880 */ 4881 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 4882 /* 4883 * Call ddi_intr_block_enable() for MSI 4884 */ 4885 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 4886 if (rc != DDI_SUCCESS) { 4887 ixgbe_log(ixgbe, 4888 "Enable block intr failed: %d", rc); 4889 return (IXGBE_FAILURE); 4890 } 4891 } else { 4892 /* 4893 * Call ddi_intr_enable() for Legacy/MSI non block enable 4894 */ 4895 for (i = 0; i < ixgbe->intr_cnt; i++) { 4896 rc = ddi_intr_enable(ixgbe->htable[i]); 4897 if (rc != DDI_SUCCESS) { 4898 ixgbe_log(ixgbe, 4899 "Enable intr failed: %d", rc); 4900 return (IXGBE_FAILURE); 4901 } 4902 } 4903 } 4904 4905 return (IXGBE_SUCCESS); 4906 } 4907 4908 /* 4909 * ixgbe_disable_intrs - Disable all the interrupts. 4910 */ 4911 static int 4912 ixgbe_disable_intrs(ixgbe_t *ixgbe) 4913 { 4914 int i; 4915 int rc; 4916 4917 /* 4918 * Disable all interrupts 4919 */ 4920 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 4921 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 4922 if (rc != DDI_SUCCESS) { 4923 ixgbe_log(ixgbe, 4924 "Disable block intr failed: %d", rc); 4925 return (IXGBE_FAILURE); 4926 } 4927 } else { 4928 for (i = 0; i < ixgbe->intr_cnt; i++) { 4929 rc = ddi_intr_disable(ixgbe->htable[i]); 4930 if (rc != DDI_SUCCESS) { 4931 ixgbe_log(ixgbe, 4932 "Disable intr failed: %d", rc); 4933 return (IXGBE_FAILURE); 4934 } 4935 } 4936 } 4937 4938 return (IXGBE_SUCCESS); 4939 } 4940 4941 /* 4942 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 4943 */ 4944 static void 4945 ixgbe_get_hw_state(ixgbe_t *ixgbe) 4946 { 4947 struct ixgbe_hw *hw = &ixgbe->hw; 4948 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 4949 boolean_t link_up = B_FALSE; 4950 uint32_t pcs1g_anlp = 0; 4951 uint32_t pcs1g_ana = 0; 4952 4953 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4954 ixgbe->param_lp_1000fdx_cap = 0; 4955 ixgbe->param_lp_100fdx_cap = 0; 4956 4957 /* check for link, don't wait */ 4958 (void) ixgbe_check_link(hw, &speed, &link_up, false); 4959 if (link_up) { 4960 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 4961 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 4962 4963 ixgbe->param_lp_1000fdx_cap = 4964 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 4965 ixgbe->param_lp_100fdx_cap = 4966 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 4967 } 4968 4969 ixgbe->param_adv_1000fdx_cap = 4970 (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; 4971 ixgbe->param_adv_100fdx_cap = (pcs1g_ana & IXGBE_PCS1GANA_FDC) ? 1 : 0; 4972 } 4973 4974 /* 4975 * ixgbe_get_driver_control - Notify that driver is in control of device. 4976 */ 4977 static void 4978 ixgbe_get_driver_control(struct ixgbe_hw *hw) 4979 { 4980 uint32_t ctrl_ext; 4981 4982 /* 4983 * Notify firmware that driver is in control of device 4984 */ 4985 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 4986 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 4987 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 4988 } 4989 4990 /* 4991 * ixgbe_release_driver_control - Notify that driver is no longer in control 4992 * of device. 4993 */ 4994 static void 4995 ixgbe_release_driver_control(struct ixgbe_hw *hw) 4996 { 4997 uint32_t ctrl_ext; 4998 4999 /* 5000 * Notify firmware that driver is no longer in control of device 5001 */ 5002 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5003 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5004 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5005 } 5006 5007 /* 5008 * ixgbe_atomic_reserve - Atomic decrease operation. 5009 */ 5010 int 5011 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5012 { 5013 uint32_t oldval; 5014 uint32_t newval; 5015 5016 /* 5017 * ATOMICALLY 5018 */ 5019 do { 5020 oldval = *count_p; 5021 if (oldval < n) 5022 return (-1); 5023 newval = oldval - n; 5024 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5025 5026 return (newval); 5027 } 5028 5029 /* 5030 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5031 */ 5032 static uint8_t * 5033 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5034 { 5035 uint8_t *addr = *upd_ptr; 5036 uint8_t *new_ptr; 5037 5038 _NOTE(ARGUNUSED(hw)); 5039 _NOTE(ARGUNUSED(vmdq)); 5040 5041 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5042 *upd_ptr = new_ptr; 5043 return (addr); 5044 } 5045 5046 /* 5047 * FMA support 5048 */ 5049 int 5050 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5051 { 5052 ddi_fm_error_t de; 5053 5054 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5055 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5056 return (de.fme_status); 5057 } 5058 5059 int 5060 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5061 { 5062 ddi_fm_error_t de; 5063 5064 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5065 return (de.fme_status); 5066 } 5067 5068 /* 5069 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5070 */ 5071 static int 5072 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5073 { 5074 _NOTE(ARGUNUSED(impl_data)); 5075 /* 5076 * as the driver can always deal with an error in any dma or 5077 * access handle, we can just return the fme_status value. 5078 */ 5079 pci_ereport_post(dip, err, NULL); 5080 return (err->fme_status); 5081 } 5082 5083 static void 5084 ixgbe_fm_init(ixgbe_t *ixgbe) 5085 { 5086 ddi_iblock_cookie_t iblk; 5087 int fma_dma_flag; 5088 5089 /* 5090 * Only register with IO Fault Services if we have some capability 5091 */ 5092 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5093 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5094 } else { 5095 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5096 } 5097 5098 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5099 fma_dma_flag = 1; 5100 } else { 5101 fma_dma_flag = 0; 5102 } 5103 5104 ixgbe_set_fma_flags(fma_dma_flag); 5105 5106 if (ixgbe->fm_capabilities) { 5107 5108 /* 5109 * Register capabilities with IO Fault Services 5110 */ 5111 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 5112 5113 /* 5114 * Initialize pci ereport capabilities if ereport capable 5115 */ 5116 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5117 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5118 pci_ereport_setup(ixgbe->dip); 5119 5120 /* 5121 * Register error callback if error callback capable 5122 */ 5123 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5124 ddi_fm_handler_register(ixgbe->dip, 5125 ixgbe_fm_error_cb, (void*) ixgbe); 5126 } 5127 } 5128 5129 static void 5130 ixgbe_fm_fini(ixgbe_t *ixgbe) 5131 { 5132 /* 5133 * Only unregister FMA capabilities if they are registered 5134 */ 5135 if (ixgbe->fm_capabilities) { 5136 5137 /* 5138 * Release any resources allocated by pci_ereport_setup() 5139 */ 5140 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5141 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5142 pci_ereport_teardown(ixgbe->dip); 5143 5144 /* 5145 * Un-register error callback if error callback capable 5146 */ 5147 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5148 ddi_fm_handler_unregister(ixgbe->dip); 5149 5150 /* 5151 * Unregister from IO Fault Service 5152 */ 5153 ddi_fm_fini(ixgbe->dip); 5154 } 5155 } 5156 5157 void 5158 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 5159 { 5160 uint64_t ena; 5161 char buf[FM_MAX_CLASS]; 5162 5163 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5164 ena = fm_ena_generate(0, FM_ENA_FMT1); 5165 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 5166 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 5167 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5168 } 5169 } 5170 5171 static int 5172 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 5173 { 5174 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 5175 5176 mutex_enter(&rx_ring->rx_lock); 5177 rx_ring->ring_gen_num = mr_gen_num; 5178 mutex_exit(&rx_ring->rx_lock); 5179 return (0); 5180 } 5181 5182 /* 5183 * Get the global ring index by a ring index within a group. 5184 */ 5185 static int 5186 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 5187 { 5188 ixgbe_rx_ring_t *rx_ring; 5189 int i; 5190 5191 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5192 rx_ring = &ixgbe->rx_rings[i]; 5193 if (rx_ring->group_index == gindex) 5194 rindex--; 5195 if (rindex < 0) 5196 return (i); 5197 } 5198 5199 return (-1); 5200 } 5201 5202 /* 5203 * Callback funtion for MAC layer to register all rings. 5204 */ 5205 /* ARGSUSED */ 5206 void 5207 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 5208 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 5209 { 5210 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5211 mac_intr_t *mintr = &infop->mri_intr; 5212 5213 switch (rtype) { 5214 case MAC_RING_TYPE_RX: { 5215 /* 5216 * 'index' is the ring index within the group. 5217 * Need to get the global ring index by searching in groups. 5218 */ 5219 int global_ring_index = ixgbe_get_rx_ring_index( 5220 ixgbe, group_index, ring_index); 5221 5222 ASSERT(global_ring_index >= 0); 5223 5224 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 5225 rx_ring->ring_handle = rh; 5226 5227 infop->mri_driver = (mac_ring_driver_t)rx_ring; 5228 infop->mri_start = ixgbe_ring_start; 5229 infop->mri_stop = NULL; 5230 infop->mri_poll = ixgbe_ring_rx_poll; 5231 infop->mri_stat = ixgbe_rx_ring_stat; 5232 5233 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 5234 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 5235 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 5236 if (ixgbe->intr_type & 5237 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5238 mintr->mi_ddi_handle = 5239 ixgbe->htable[rx_ring->intr_vector]; 5240 } 5241 5242 break; 5243 } 5244 case MAC_RING_TYPE_TX: { 5245 ASSERT(group_index == -1); 5246 ASSERT(ring_index < ixgbe->num_tx_rings); 5247 5248 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 5249 tx_ring->ring_handle = rh; 5250 5251 infop->mri_driver = (mac_ring_driver_t)tx_ring; 5252 infop->mri_start = NULL; 5253 infop->mri_stop = NULL; 5254 infop->mri_tx = ixgbe_ring_tx; 5255 infop->mri_stat = ixgbe_tx_ring_stat; 5256 if (ixgbe->intr_type & 5257 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5258 mintr->mi_ddi_handle = 5259 ixgbe->htable[tx_ring->intr_vector]; 5260 } 5261 break; 5262 } 5263 default: 5264 break; 5265 } 5266 } 5267 5268 /* 5269 * Callback funtion for MAC layer to register all groups. 5270 */ 5271 void 5272 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 5273 mac_group_info_t *infop, mac_group_handle_t gh) 5274 { 5275 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5276 5277 switch (rtype) { 5278 case MAC_RING_TYPE_RX: { 5279 ixgbe_rx_group_t *rx_group; 5280 5281 rx_group = &ixgbe->rx_groups[index]; 5282 rx_group->group_handle = gh; 5283 5284 infop->mgi_driver = (mac_group_driver_t)rx_group; 5285 infop->mgi_start = NULL; 5286 infop->mgi_stop = NULL; 5287 infop->mgi_addmac = ixgbe_addmac; 5288 infop->mgi_remmac = ixgbe_remmac; 5289 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 5290 5291 break; 5292 } 5293 case MAC_RING_TYPE_TX: 5294 break; 5295 default: 5296 break; 5297 } 5298 } 5299 5300 /* 5301 * Enable interrupt on the specificed rx ring. 5302 */ 5303 int 5304 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 5305 { 5306 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5307 ixgbe_t *ixgbe = rx_ring->ixgbe; 5308 int r_idx = rx_ring->index; 5309 int hw_r_idx = rx_ring->hw_index; 5310 int v_idx = rx_ring->intr_vector; 5311 5312 mutex_enter(&ixgbe->gen_lock); 5313 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5314 mutex_exit(&ixgbe->gen_lock); 5315 /* 5316 * Simply return 0. 5317 * Interrupts are being adjusted. ixgbe_intr_adjust() 5318 * will eventually re-enable the interrupt when it's 5319 * done with the adjustment. 5320 */ 5321 return (0); 5322 } 5323 5324 /* 5325 * To enable interrupt by setting the VAL bit of given interrupt 5326 * vector allocation register (IVAR). 5327 */ 5328 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 5329 5330 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5331 5332 /* 5333 * To trigger a Rx interrupt to on this ring 5334 */ 5335 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 5336 IXGBE_WRITE_FLUSH(&ixgbe->hw); 5337 5338 mutex_exit(&ixgbe->gen_lock); 5339 5340 return (0); 5341 } 5342 5343 /* 5344 * Disable interrupt on the specificed rx ring. 5345 */ 5346 int 5347 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 5348 { 5349 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5350 ixgbe_t *ixgbe = rx_ring->ixgbe; 5351 int r_idx = rx_ring->index; 5352 int hw_r_idx = rx_ring->hw_index; 5353 int v_idx = rx_ring->intr_vector; 5354 5355 mutex_enter(&ixgbe->gen_lock); 5356 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5357 mutex_exit(&ixgbe->gen_lock); 5358 /* 5359 * Simply return 0. 5360 * In the rare case where an interrupt is being 5361 * disabled while interrupts are being adjusted, 5362 * we don't fail the operation. No interrupts will 5363 * be generated while they are adjusted, and 5364 * ixgbe_intr_adjust() will cause the interrupts 5365 * to be re-enabled once it completes. Note that 5366 * in this case, packets may be delivered to the 5367 * stack via interrupts before xgbe_rx_ring_intr_enable() 5368 * is called again. This is acceptable since interrupt 5369 * adjustment is infrequent, and the stack will be 5370 * able to handle these packets. 5371 */ 5372 return (0); 5373 } 5374 5375 /* 5376 * To disable interrupt by clearing the VAL bit of given interrupt 5377 * vector allocation register (IVAR). 5378 */ 5379 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 5380 5381 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 5382 5383 mutex_exit(&ixgbe->gen_lock); 5384 5385 return (0); 5386 } 5387 5388 /* 5389 * Add a mac address. 5390 */ 5391 static int 5392 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 5393 { 5394 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5395 ixgbe_t *ixgbe = rx_group->ixgbe; 5396 struct ixgbe_hw *hw = &ixgbe->hw; 5397 int slot, i; 5398 5399 mutex_enter(&ixgbe->gen_lock); 5400 5401 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5402 mutex_exit(&ixgbe->gen_lock); 5403 return (ECANCELED); 5404 } 5405 5406 if (ixgbe->unicst_avail == 0) { 5407 /* no slots available */ 5408 mutex_exit(&ixgbe->gen_lock); 5409 return (ENOSPC); 5410 } 5411 5412 /* 5413 * The first ixgbe->num_rx_groups slots are reserved for each respective 5414 * group. The rest slots are shared by all groups. While adding a 5415 * MAC address, reserved slots are firstly checked then the shared 5416 * slots are searched. 5417 */ 5418 slot = -1; 5419 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 5420 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 5421 if (ixgbe->unicst_addr[i].mac.set == 0) { 5422 slot = i; 5423 break; 5424 } 5425 } 5426 } else { 5427 slot = rx_group->index; 5428 } 5429 5430 if (slot == -1) { 5431 /* no slots available */ 5432 mutex_exit(&ixgbe->gen_lock); 5433 return (ENOSPC); 5434 } 5435 5436 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5437 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 5438 rx_group->index, IXGBE_RAH_AV); 5439 ixgbe->unicst_addr[slot].mac.set = 1; 5440 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 5441 ixgbe->unicst_avail--; 5442 5443 mutex_exit(&ixgbe->gen_lock); 5444 5445 return (0); 5446 } 5447 5448 /* 5449 * Remove a mac address. 5450 */ 5451 static int 5452 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 5453 { 5454 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5455 ixgbe_t *ixgbe = rx_group->ixgbe; 5456 struct ixgbe_hw *hw = &ixgbe->hw; 5457 int slot; 5458 5459 mutex_enter(&ixgbe->gen_lock); 5460 5461 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5462 mutex_exit(&ixgbe->gen_lock); 5463 return (ECANCELED); 5464 } 5465 5466 slot = ixgbe_unicst_find(ixgbe, mac_addr); 5467 if (slot == -1) { 5468 mutex_exit(&ixgbe->gen_lock); 5469 return (EINVAL); 5470 } 5471 5472 if (ixgbe->unicst_addr[slot].mac.set == 0) { 5473 mutex_exit(&ixgbe->gen_lock); 5474 return (EINVAL); 5475 } 5476 5477 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5478 (void) ixgbe_clear_rar(hw, slot); 5479 ixgbe->unicst_addr[slot].mac.set = 0; 5480 ixgbe->unicst_avail++; 5481 5482 mutex_exit(&ixgbe->gen_lock); 5483 5484 return (0); 5485 } 5486