1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 */ 29 30 #include "ixgbe_sw.h" 31 32 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 33 static char ixgbe_version[] = "ixgbe 1.1.7"; 34 35 /* 36 * Local function protoypes 37 */ 38 static int ixgbe_register_mac(ixgbe_t *); 39 static int ixgbe_identify_hardware(ixgbe_t *); 40 static int ixgbe_regs_map(ixgbe_t *); 41 static void ixgbe_init_properties(ixgbe_t *); 42 static int ixgbe_init_driver_settings(ixgbe_t *); 43 static void ixgbe_init_locks(ixgbe_t *); 44 static void ixgbe_destroy_locks(ixgbe_t *); 45 static int ixgbe_init(ixgbe_t *); 46 static int ixgbe_chip_start(ixgbe_t *); 47 static void ixgbe_chip_stop(ixgbe_t *); 48 static int ixgbe_reset(ixgbe_t *); 49 static void ixgbe_tx_clean(ixgbe_t *); 50 static boolean_t ixgbe_tx_drain(ixgbe_t *); 51 static boolean_t ixgbe_rx_drain(ixgbe_t *); 52 static int ixgbe_alloc_rings(ixgbe_t *); 53 static void ixgbe_free_rings(ixgbe_t *); 54 static int ixgbe_alloc_rx_data(ixgbe_t *); 55 static void ixgbe_free_rx_data(ixgbe_t *); 56 static void ixgbe_setup_rings(ixgbe_t *); 57 static void ixgbe_setup_rx(ixgbe_t *); 58 static void ixgbe_setup_tx(ixgbe_t *); 59 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 60 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 61 static void ixgbe_setup_rss(ixgbe_t *); 62 static void ixgbe_setup_vmdq(ixgbe_t *); 63 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 64 static void ixgbe_init_unicst(ixgbe_t *); 65 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 66 static void ixgbe_setup_multicst(ixgbe_t *); 67 static void ixgbe_get_hw_state(ixgbe_t *); 68 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 69 static void ixgbe_get_conf(ixgbe_t *); 70 static void ixgbe_init_params(ixgbe_t *); 71 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 72 static void ixgbe_driver_link_check(ixgbe_t *); 73 static void ixgbe_sfp_check(void *); 74 static void ixgbe_overtemp_check(void *); 75 static void ixgbe_link_timer(void *); 76 static void ixgbe_local_timer(void *); 77 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 78 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 79 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 80 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 81 static boolean_t is_valid_mac_addr(uint8_t *); 82 static boolean_t ixgbe_stall_check(ixgbe_t *); 83 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 84 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 85 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 86 static int ixgbe_alloc_intrs(ixgbe_t *); 87 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 88 static int ixgbe_add_intr_handlers(ixgbe_t *); 89 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 90 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 91 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 92 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 93 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 94 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 95 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 96 static void ixgbe_setup_adapter_vector(ixgbe_t *); 97 static void ixgbe_rem_intr_handlers(ixgbe_t *); 98 static void ixgbe_rem_intrs(ixgbe_t *); 99 static int ixgbe_enable_intrs(ixgbe_t *); 100 static int ixgbe_disable_intrs(ixgbe_t *); 101 static uint_t ixgbe_intr_legacy(void *, void *); 102 static uint_t ixgbe_intr_msi(void *, void *); 103 static uint_t ixgbe_intr_msix(void *, void *); 104 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 105 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 106 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 107 static void ixgbe_get_driver_control(struct ixgbe_hw *); 108 static int ixgbe_addmac(void *, const uint8_t *); 109 static int ixgbe_remmac(void *, const uint8_t *); 110 static void ixgbe_release_driver_control(struct ixgbe_hw *); 111 112 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 113 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 114 static int ixgbe_resume(dev_info_t *); 115 static int ixgbe_suspend(dev_info_t *); 116 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 117 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 118 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 119 static int ixgbe_intr_cb_register(ixgbe_t *); 120 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 121 122 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 123 const void *impl_data); 124 static void ixgbe_fm_init(ixgbe_t *); 125 static void ixgbe_fm_fini(ixgbe_t *); 126 127 char *ixgbe_priv_props[] = { 128 "_tx_copy_thresh", 129 "_tx_recycle_thresh", 130 "_tx_overload_thresh", 131 "_tx_resched_thresh", 132 "_rx_copy_thresh", 133 "_rx_limit_per_intr", 134 "_intr_throttling", 135 "_adv_pause_cap", 136 "_adv_asym_pause_cap", 137 NULL 138 }; 139 140 #define IXGBE_MAX_PRIV_PROPS \ 141 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 142 143 static struct cb_ops ixgbe_cb_ops = { 144 nulldev, /* cb_open */ 145 nulldev, /* cb_close */ 146 nodev, /* cb_strategy */ 147 nodev, /* cb_print */ 148 nodev, /* cb_dump */ 149 nodev, /* cb_read */ 150 nodev, /* cb_write */ 151 nodev, /* cb_ioctl */ 152 nodev, /* cb_devmap */ 153 nodev, /* cb_mmap */ 154 nodev, /* cb_segmap */ 155 nochpoll, /* cb_chpoll */ 156 ddi_prop_op, /* cb_prop_op */ 157 NULL, /* cb_stream */ 158 D_MP | D_HOTPLUG, /* cb_flag */ 159 CB_REV, /* cb_rev */ 160 nodev, /* cb_aread */ 161 nodev /* cb_awrite */ 162 }; 163 164 static struct dev_ops ixgbe_dev_ops = { 165 DEVO_REV, /* devo_rev */ 166 0, /* devo_refcnt */ 167 NULL, /* devo_getinfo */ 168 nulldev, /* devo_identify */ 169 nulldev, /* devo_probe */ 170 ixgbe_attach, /* devo_attach */ 171 ixgbe_detach, /* devo_detach */ 172 nodev, /* devo_reset */ 173 &ixgbe_cb_ops, /* devo_cb_ops */ 174 NULL, /* devo_bus_ops */ 175 ddi_power, /* devo_power */ 176 ddi_quiesce_not_supported, /* devo_quiesce */ 177 }; 178 179 static struct modldrv ixgbe_modldrv = { 180 &mod_driverops, /* Type of module. This one is a driver */ 181 ixgbe_ident, /* Discription string */ 182 &ixgbe_dev_ops /* driver ops */ 183 }; 184 185 static struct modlinkage ixgbe_modlinkage = { 186 MODREV_1, &ixgbe_modldrv, NULL 187 }; 188 189 /* 190 * Access attributes for register mapping 191 */ 192 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 193 DDI_DEVICE_ATTR_V1, 194 DDI_STRUCTURE_LE_ACC, 195 DDI_STRICTORDER_ACC, 196 DDI_FLAGERR_ACC 197 }; 198 199 /* 200 * Loopback property 201 */ 202 static lb_property_t lb_normal = { 203 normal, "normal", IXGBE_LB_NONE 204 }; 205 206 static lb_property_t lb_mac = { 207 internal, "MAC", IXGBE_LB_INTERNAL_MAC 208 }; 209 210 static lb_property_t lb_external = { 211 external, "External", IXGBE_LB_EXTERNAL 212 }; 213 214 #define IXGBE_M_CALLBACK_FLAGS \ 215 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 216 217 static mac_callbacks_t ixgbe_m_callbacks = { 218 IXGBE_M_CALLBACK_FLAGS, 219 ixgbe_m_stat, 220 ixgbe_m_start, 221 ixgbe_m_stop, 222 ixgbe_m_promisc, 223 ixgbe_m_multicst, 224 NULL, 225 NULL, 226 NULL, 227 ixgbe_m_ioctl, 228 ixgbe_m_getcapab, 229 NULL, 230 NULL, 231 ixgbe_m_setprop, 232 ixgbe_m_getprop, 233 ixgbe_m_propinfo 234 }; 235 236 /* 237 * Initialize capabilities of each supported adapter type 238 */ 239 static adapter_info_t ixgbe_82598eb_cap = { 240 64, /* maximum number of rx queues */ 241 1, /* minimum number of rx queues */ 242 64, /* default number of rx queues */ 243 16, /* maximum number of rx groups */ 244 1, /* minimum number of rx groups */ 245 1, /* default number of rx groups */ 246 32, /* maximum number of tx queues */ 247 1, /* minimum number of tx queues */ 248 8, /* default number of tx queues */ 249 16366, /* maximum MTU size */ 250 0xFFFF, /* maximum interrupt throttle rate */ 251 0, /* minimum interrupt throttle rate */ 252 200, /* default interrupt throttle rate */ 253 18, /* maximum total msix vectors */ 254 16, /* maximum number of ring vectors */ 255 2, /* maximum number of other vectors */ 256 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 257 0, /* "other" interrupt types enable mask */ 258 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 259 | IXGBE_FLAG_RSS_CAPABLE 260 | IXGBE_FLAG_VMDQ_CAPABLE) 261 }; 262 263 static adapter_info_t ixgbe_82599eb_cap = { 264 128, /* maximum number of rx queues */ 265 1, /* minimum number of rx queues */ 266 128, /* default number of rx queues */ 267 64, /* maximum number of rx groups */ 268 1, /* minimum number of rx groups */ 269 1, /* default number of rx groups */ 270 128, /* maximum number of tx queues */ 271 1, /* minimum number of tx queues */ 272 8, /* default number of tx queues */ 273 15500, /* maximum MTU size */ 274 0xFF8, /* maximum interrupt throttle rate */ 275 0, /* minimum interrupt throttle rate */ 276 200, /* default interrupt throttle rate */ 277 64, /* maximum total msix vectors */ 278 16, /* maximum number of ring vectors */ 279 2, /* maximum number of other vectors */ 280 (IXGBE_EICR_LSC 281 | IXGBE_EICR_GPI_SDP1 282 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 283 284 (IXGBE_SDP1_GPIEN 285 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 286 287 (IXGBE_FLAG_DCA_CAPABLE 288 | IXGBE_FLAG_RSS_CAPABLE 289 | IXGBE_FLAG_VMDQ_CAPABLE 290 | IXGBE_FLAG_RSC_CAPABLE 291 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 292 }; 293 294 /* 295 * Module Initialization Functions. 296 */ 297 298 int 299 _init(void) 300 { 301 int status; 302 303 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 304 305 status = mod_install(&ixgbe_modlinkage); 306 307 if (status != DDI_SUCCESS) { 308 mac_fini_ops(&ixgbe_dev_ops); 309 } 310 311 return (status); 312 } 313 314 int 315 _fini(void) 316 { 317 int status; 318 319 status = mod_remove(&ixgbe_modlinkage); 320 321 if (status == DDI_SUCCESS) { 322 mac_fini_ops(&ixgbe_dev_ops); 323 } 324 325 return (status); 326 } 327 328 int 329 _info(struct modinfo *modinfop) 330 { 331 int status; 332 333 status = mod_info(&ixgbe_modlinkage, modinfop); 334 335 return (status); 336 } 337 338 /* 339 * ixgbe_attach - Driver attach. 340 * 341 * This function is the device specific initialization entry 342 * point. This entry point is required and must be written. 343 * The DDI_ATTACH command must be provided in the attach entry 344 * point. When attach() is called with cmd set to DDI_ATTACH, 345 * all normal kernel services (such as kmem_alloc(9F)) are 346 * available for use by the driver. 347 * 348 * The attach() function will be called once for each instance 349 * of the device on the system with cmd set to DDI_ATTACH. 350 * Until attach() succeeds, the only driver entry points which 351 * may be called are open(9E) and getinfo(9E). 352 */ 353 static int 354 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 355 { 356 ixgbe_t *ixgbe; 357 struct ixgbe_osdep *osdep; 358 struct ixgbe_hw *hw; 359 int instance; 360 char taskqname[32]; 361 362 /* 363 * Check the command and perform corresponding operations 364 */ 365 switch (cmd) { 366 default: 367 return (DDI_FAILURE); 368 369 case DDI_RESUME: 370 return (ixgbe_resume(devinfo)); 371 372 case DDI_ATTACH: 373 break; 374 } 375 376 /* Get the device instance */ 377 instance = ddi_get_instance(devinfo); 378 379 /* Allocate memory for the instance data structure */ 380 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 381 382 ixgbe->dip = devinfo; 383 ixgbe->instance = instance; 384 385 hw = &ixgbe->hw; 386 osdep = &ixgbe->osdep; 387 hw->back = osdep; 388 osdep->ixgbe = ixgbe; 389 390 /* Attach the instance pointer to the dev_info data structure */ 391 ddi_set_driver_private(devinfo, ixgbe); 392 393 /* 394 * Initialize for fma support 395 */ 396 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 397 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 398 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 399 ixgbe_fm_init(ixgbe); 400 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 401 402 /* 403 * Map PCI config space registers 404 */ 405 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 406 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 407 goto attach_fail; 408 } 409 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 410 411 /* 412 * Identify the chipset family 413 */ 414 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 415 ixgbe_error(ixgbe, "Failed to identify hardware"); 416 goto attach_fail; 417 } 418 419 /* 420 * Map device registers 421 */ 422 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 423 ixgbe_error(ixgbe, "Failed to map device registers"); 424 goto attach_fail; 425 } 426 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 427 428 /* 429 * Initialize driver parameters 430 */ 431 ixgbe_init_properties(ixgbe); 432 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 433 434 /* 435 * Register interrupt callback 436 */ 437 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 438 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 439 goto attach_fail; 440 } 441 442 /* 443 * Allocate interrupts 444 */ 445 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 446 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 447 goto attach_fail; 448 } 449 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 450 451 /* 452 * Allocate rx/tx rings based on the ring numbers. 453 * The actual numbers of rx/tx rings are decided by the number of 454 * allocated interrupt vectors, so we should allocate the rings after 455 * interrupts are allocated. 456 */ 457 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 458 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 459 goto attach_fail; 460 } 461 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 462 463 /* 464 * Map rings to interrupt vectors 465 */ 466 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 467 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 468 goto attach_fail; 469 } 470 471 /* 472 * Add interrupt handlers 473 */ 474 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 475 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 476 goto attach_fail; 477 } 478 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 479 480 /* 481 * Create a taskq for sfp-change 482 */ 483 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 484 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 485 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 486 ixgbe_error(ixgbe, "sfp_taskq create failed"); 487 goto attach_fail; 488 } 489 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 490 491 /* 492 * Create a taskq for over-temp 493 */ 494 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 495 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 496 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 497 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 498 goto attach_fail; 499 } 500 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 501 502 /* 503 * Initialize driver parameters 504 */ 505 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 506 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 507 goto attach_fail; 508 } 509 510 /* 511 * Initialize mutexes for this device. 512 * Do this before enabling the interrupt handler and 513 * register the softint to avoid the condition where 514 * interrupt handler can try using uninitialized mutex. 515 */ 516 ixgbe_init_locks(ixgbe); 517 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 518 519 /* 520 * Initialize chipset hardware 521 */ 522 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 523 ixgbe_error(ixgbe, "Failed to initialize adapter"); 524 goto attach_fail; 525 } 526 ixgbe->link_check_complete = B_FALSE; 527 ixgbe->link_check_hrtime = gethrtime() + 528 (IXGBE_LINK_UP_TIME * 100000000ULL); 529 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 530 531 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 532 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 533 goto attach_fail; 534 } 535 536 /* 537 * Initialize statistics 538 */ 539 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 540 ixgbe_error(ixgbe, "Failed to initialize statistics"); 541 goto attach_fail; 542 } 543 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 544 545 /* 546 * Register the driver to the MAC 547 */ 548 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 549 ixgbe_error(ixgbe, "Failed to register MAC"); 550 goto attach_fail; 551 } 552 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 553 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 554 555 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 556 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 557 if (ixgbe->periodic_id == 0) { 558 ixgbe_error(ixgbe, "Failed to add the link check timer"); 559 goto attach_fail; 560 } 561 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 562 563 /* 564 * Now that mutex locks are initialized, and the chip is also 565 * initialized, enable interrupts. 566 */ 567 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 568 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 569 goto attach_fail; 570 } 571 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 572 573 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version); 574 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 575 576 return (DDI_SUCCESS); 577 578 attach_fail: 579 ixgbe_unconfigure(devinfo, ixgbe); 580 return (DDI_FAILURE); 581 } 582 583 /* 584 * ixgbe_detach - Driver detach. 585 * 586 * The detach() function is the complement of the attach routine. 587 * If cmd is set to DDI_DETACH, detach() is used to remove the 588 * state associated with a given instance of a device node 589 * prior to the removal of that instance from the system. 590 * 591 * The detach() function will be called once for each instance 592 * of the device for which there has been a successful attach() 593 * once there are no longer any opens on the device. 594 * 595 * Interrupts routine are disabled, All memory allocated by this 596 * driver are freed. 597 */ 598 static int 599 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 600 { 601 ixgbe_t *ixgbe; 602 603 /* 604 * Check detach command 605 */ 606 switch (cmd) { 607 default: 608 return (DDI_FAILURE); 609 610 case DDI_SUSPEND: 611 return (ixgbe_suspend(devinfo)); 612 613 case DDI_DETACH: 614 break; 615 } 616 617 /* 618 * Get the pointer to the driver private data structure 619 */ 620 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 621 if (ixgbe == NULL) 622 return (DDI_FAILURE); 623 624 /* 625 * If the device is still running, it needs to be stopped first. 626 * This check is necessary because under some specific circumstances, 627 * the detach routine can be called without stopping the interface 628 * first. 629 */ 630 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 631 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 632 mutex_enter(&ixgbe->gen_lock); 633 ixgbe_stop(ixgbe, B_TRUE); 634 mutex_exit(&ixgbe->gen_lock); 635 /* Disable and stop the watchdog timer */ 636 ixgbe_disable_watchdog_timer(ixgbe); 637 } 638 639 /* 640 * Check if there are still rx buffers held by the upper layer. 641 * If so, fail the detach. 642 */ 643 if (!ixgbe_rx_drain(ixgbe)) 644 return (DDI_FAILURE); 645 646 /* 647 * Do the remaining unconfigure routines 648 */ 649 ixgbe_unconfigure(devinfo, ixgbe); 650 651 return (DDI_SUCCESS); 652 } 653 654 static void 655 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 656 { 657 /* 658 * Disable interrupt 659 */ 660 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 661 (void) ixgbe_disable_intrs(ixgbe); 662 } 663 664 /* 665 * remove the link check timer 666 */ 667 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 668 if (ixgbe->periodic_id != NULL) { 669 ddi_periodic_delete(ixgbe->periodic_id); 670 ixgbe->periodic_id = NULL; 671 } 672 } 673 674 /* 675 * Unregister MAC 676 */ 677 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 678 (void) mac_unregister(ixgbe->mac_hdl); 679 } 680 681 /* 682 * Free statistics 683 */ 684 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 685 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 686 } 687 688 /* 689 * Remove interrupt handlers 690 */ 691 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 692 ixgbe_rem_intr_handlers(ixgbe); 693 } 694 695 /* 696 * Remove taskq for sfp-status-change 697 */ 698 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 699 ddi_taskq_destroy(ixgbe->sfp_taskq); 700 } 701 702 /* 703 * Remove taskq for over-temp 704 */ 705 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 706 ddi_taskq_destroy(ixgbe->overtemp_taskq); 707 } 708 709 /* 710 * Remove interrupts 711 */ 712 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 713 ixgbe_rem_intrs(ixgbe); 714 } 715 716 /* 717 * Unregister interrupt callback handler 718 */ 719 (void) ddi_cb_unregister(ixgbe->cb_hdl); 720 721 /* 722 * Remove driver properties 723 */ 724 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 725 (void) ddi_prop_remove_all(devinfo); 726 } 727 728 /* 729 * Stop the chipset 730 */ 731 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 732 mutex_enter(&ixgbe->gen_lock); 733 ixgbe_chip_stop(ixgbe); 734 mutex_exit(&ixgbe->gen_lock); 735 } 736 737 /* 738 * Free register handle 739 */ 740 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 741 if (ixgbe->osdep.reg_handle != NULL) 742 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 743 } 744 745 /* 746 * Free PCI config handle 747 */ 748 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 749 if (ixgbe->osdep.cfg_handle != NULL) 750 pci_config_teardown(&ixgbe->osdep.cfg_handle); 751 } 752 753 /* 754 * Free locks 755 */ 756 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 757 ixgbe_destroy_locks(ixgbe); 758 } 759 760 /* 761 * Free the rx/tx rings 762 */ 763 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 764 ixgbe_free_rings(ixgbe); 765 } 766 767 /* 768 * Unregister FMA capabilities 769 */ 770 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 771 ixgbe_fm_fini(ixgbe); 772 } 773 774 /* 775 * Free the driver data structure 776 */ 777 kmem_free(ixgbe, sizeof (ixgbe_t)); 778 779 ddi_set_driver_private(devinfo, NULL); 780 } 781 782 /* 783 * ixgbe_register_mac - Register the driver and its function pointers with 784 * the GLD interface. 785 */ 786 static int 787 ixgbe_register_mac(ixgbe_t *ixgbe) 788 { 789 struct ixgbe_hw *hw = &ixgbe->hw; 790 mac_register_t *mac; 791 int status; 792 793 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 794 return (IXGBE_FAILURE); 795 796 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 797 mac->m_driver = ixgbe; 798 mac->m_dip = ixgbe->dip; 799 mac->m_src_addr = hw->mac.addr; 800 mac->m_callbacks = &ixgbe_m_callbacks; 801 mac->m_min_sdu = 0; 802 mac->m_max_sdu = ixgbe->default_mtu; 803 mac->m_margin = VLAN_TAGSZ; 804 mac->m_priv_props = ixgbe_priv_props; 805 mac->m_v12n = MAC_VIRT_LEVEL1; 806 807 status = mac_register(mac, &ixgbe->mac_hdl); 808 809 mac_free(mac); 810 811 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 812 } 813 814 /* 815 * ixgbe_identify_hardware - Identify the type of the chipset. 816 */ 817 static int 818 ixgbe_identify_hardware(ixgbe_t *ixgbe) 819 { 820 struct ixgbe_hw *hw = &ixgbe->hw; 821 struct ixgbe_osdep *osdep = &ixgbe->osdep; 822 823 /* 824 * Get the device id 825 */ 826 hw->vendor_id = 827 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 828 hw->device_id = 829 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 830 hw->revision_id = 831 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 832 hw->subsystem_device_id = 833 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 834 hw->subsystem_vendor_id = 835 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 836 837 /* 838 * Set the mac type of the adapter based on the device id 839 */ 840 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 841 return (IXGBE_FAILURE); 842 } 843 844 /* 845 * Install adapter capabilities 846 */ 847 switch (hw->mac.type) { 848 case ixgbe_mac_82598EB: 849 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 850 ixgbe->capab = &ixgbe_82598eb_cap; 851 852 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 853 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 854 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 855 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 856 } 857 break; 858 859 case ixgbe_mac_82599EB: 860 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 861 ixgbe->capab = &ixgbe_82599eb_cap; 862 863 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 864 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 865 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 866 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 867 } 868 break; 869 870 default: 871 IXGBE_DEBUGLOG_1(ixgbe, 872 "adapter not supported in ixgbe_identify_hardware(): %d\n", 873 hw->mac.type); 874 return (IXGBE_FAILURE); 875 } 876 877 return (IXGBE_SUCCESS); 878 } 879 880 /* 881 * ixgbe_regs_map - Map the device registers. 882 * 883 */ 884 static int 885 ixgbe_regs_map(ixgbe_t *ixgbe) 886 { 887 dev_info_t *devinfo = ixgbe->dip; 888 struct ixgbe_hw *hw = &ixgbe->hw; 889 struct ixgbe_osdep *osdep = &ixgbe->osdep; 890 off_t mem_size; 891 892 /* 893 * First get the size of device registers to be mapped. 894 */ 895 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 896 != DDI_SUCCESS) { 897 return (IXGBE_FAILURE); 898 } 899 900 /* 901 * Call ddi_regs_map_setup() to map registers 902 */ 903 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 904 (caddr_t *)&hw->hw_addr, 0, 905 mem_size, &ixgbe_regs_acc_attr, 906 &osdep->reg_handle)) != DDI_SUCCESS) { 907 return (IXGBE_FAILURE); 908 } 909 910 return (IXGBE_SUCCESS); 911 } 912 913 /* 914 * ixgbe_init_properties - Initialize driver properties. 915 */ 916 static void 917 ixgbe_init_properties(ixgbe_t *ixgbe) 918 { 919 /* 920 * Get conf file properties, including link settings 921 * jumbo frames, ring number, descriptor number, etc. 922 */ 923 ixgbe_get_conf(ixgbe); 924 925 ixgbe_init_params(ixgbe); 926 } 927 928 /* 929 * ixgbe_init_driver_settings - Initialize driver settings. 930 * 931 * The settings include hardware function pointers, bus information, 932 * rx/tx rings settings, link state, and any other parameters that 933 * need to be setup during driver initialization. 934 */ 935 static int 936 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 937 { 938 struct ixgbe_hw *hw = &ixgbe->hw; 939 dev_info_t *devinfo = ixgbe->dip; 940 ixgbe_rx_ring_t *rx_ring; 941 ixgbe_rx_group_t *rx_group; 942 ixgbe_tx_ring_t *tx_ring; 943 uint32_t rx_size; 944 uint32_t tx_size; 945 uint32_t ring_per_group; 946 int i; 947 948 /* 949 * Initialize chipset specific hardware function pointers 950 */ 951 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 952 return (IXGBE_FAILURE); 953 } 954 955 /* 956 * Get the system page size 957 */ 958 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 959 960 /* 961 * Set rx buffer size 962 * 963 * The IP header alignment room is counted in the calculation. 964 * The rx buffer size is in unit of 1K that is required by the 965 * chipset hardware. 966 */ 967 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 968 ixgbe->rx_buf_size = ((rx_size >> 10) + 969 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 970 971 /* 972 * Set tx buffer size 973 */ 974 tx_size = ixgbe->max_frame_size; 975 ixgbe->tx_buf_size = ((tx_size >> 10) + 976 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 977 978 /* 979 * Initialize rx/tx rings/groups parameters 980 */ 981 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 982 for (i = 0; i < ixgbe->num_rx_rings; i++) { 983 rx_ring = &ixgbe->rx_rings[i]; 984 rx_ring->index = i; 985 rx_ring->ixgbe = ixgbe; 986 rx_ring->group_index = i / ring_per_group; 987 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 988 } 989 990 for (i = 0; i < ixgbe->num_rx_groups; i++) { 991 rx_group = &ixgbe->rx_groups[i]; 992 rx_group->index = i; 993 rx_group->ixgbe = ixgbe; 994 } 995 996 for (i = 0; i < ixgbe->num_tx_rings; i++) { 997 tx_ring = &ixgbe->tx_rings[i]; 998 tx_ring->index = i; 999 tx_ring->ixgbe = ixgbe; 1000 if (ixgbe->tx_head_wb_enable) 1001 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1002 else 1003 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1004 1005 tx_ring->ring_size = ixgbe->tx_ring_size; 1006 tx_ring->free_list_size = ixgbe->tx_ring_size + 1007 (ixgbe->tx_ring_size >> 1); 1008 } 1009 1010 /* 1011 * Initialize values of interrupt throttling rate 1012 */ 1013 for (i = 1; i < MAX_INTR_VECTOR; i++) 1014 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1015 1016 /* 1017 * The initial link state should be "unknown" 1018 */ 1019 ixgbe->link_state = LINK_STATE_UNKNOWN; 1020 1021 return (IXGBE_SUCCESS); 1022 } 1023 1024 /* 1025 * ixgbe_init_locks - Initialize locks. 1026 */ 1027 static void 1028 ixgbe_init_locks(ixgbe_t *ixgbe) 1029 { 1030 ixgbe_rx_ring_t *rx_ring; 1031 ixgbe_tx_ring_t *tx_ring; 1032 int i; 1033 1034 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1035 rx_ring = &ixgbe->rx_rings[i]; 1036 mutex_init(&rx_ring->rx_lock, NULL, 1037 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1038 } 1039 1040 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1041 tx_ring = &ixgbe->tx_rings[i]; 1042 mutex_init(&tx_ring->tx_lock, NULL, 1043 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1044 mutex_init(&tx_ring->recycle_lock, NULL, 1045 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1046 mutex_init(&tx_ring->tcb_head_lock, NULL, 1047 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1048 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1049 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1050 } 1051 1052 mutex_init(&ixgbe->gen_lock, NULL, 1053 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1054 1055 mutex_init(&ixgbe->watchdog_lock, NULL, 1056 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1057 } 1058 1059 /* 1060 * ixgbe_destroy_locks - Destroy locks. 1061 */ 1062 static void 1063 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1064 { 1065 ixgbe_rx_ring_t *rx_ring; 1066 ixgbe_tx_ring_t *tx_ring; 1067 int i; 1068 1069 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1070 rx_ring = &ixgbe->rx_rings[i]; 1071 mutex_destroy(&rx_ring->rx_lock); 1072 } 1073 1074 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1075 tx_ring = &ixgbe->tx_rings[i]; 1076 mutex_destroy(&tx_ring->tx_lock); 1077 mutex_destroy(&tx_ring->recycle_lock); 1078 mutex_destroy(&tx_ring->tcb_head_lock); 1079 mutex_destroy(&tx_ring->tcb_tail_lock); 1080 } 1081 1082 mutex_destroy(&ixgbe->gen_lock); 1083 mutex_destroy(&ixgbe->watchdog_lock); 1084 } 1085 1086 static int 1087 ixgbe_resume(dev_info_t *devinfo) 1088 { 1089 ixgbe_t *ixgbe; 1090 int i; 1091 1092 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1093 if (ixgbe == NULL) 1094 return (DDI_FAILURE); 1095 1096 mutex_enter(&ixgbe->gen_lock); 1097 1098 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1099 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1100 mutex_exit(&ixgbe->gen_lock); 1101 return (DDI_FAILURE); 1102 } 1103 1104 /* 1105 * Enable and start the watchdog timer 1106 */ 1107 ixgbe_enable_watchdog_timer(ixgbe); 1108 } 1109 1110 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1111 1112 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1113 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1114 mac_tx_ring_update(ixgbe->mac_hdl, 1115 ixgbe->tx_rings[i].ring_handle); 1116 } 1117 } 1118 1119 mutex_exit(&ixgbe->gen_lock); 1120 1121 return (DDI_SUCCESS); 1122 } 1123 1124 static int 1125 ixgbe_suspend(dev_info_t *devinfo) 1126 { 1127 ixgbe_t *ixgbe; 1128 1129 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1130 if (ixgbe == NULL) 1131 return (DDI_FAILURE); 1132 1133 mutex_enter(&ixgbe->gen_lock); 1134 1135 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1136 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1137 mutex_exit(&ixgbe->gen_lock); 1138 return (DDI_SUCCESS); 1139 } 1140 ixgbe_stop(ixgbe, B_FALSE); 1141 1142 mutex_exit(&ixgbe->gen_lock); 1143 1144 /* 1145 * Disable and stop the watchdog timer 1146 */ 1147 ixgbe_disable_watchdog_timer(ixgbe); 1148 1149 return (DDI_SUCCESS); 1150 } 1151 1152 /* 1153 * ixgbe_init - Initialize the device. 1154 */ 1155 static int 1156 ixgbe_init(ixgbe_t *ixgbe) 1157 { 1158 struct ixgbe_hw *hw = &ixgbe->hw; 1159 1160 mutex_enter(&ixgbe->gen_lock); 1161 1162 /* 1163 * Reset chipset to put the hardware in a known state 1164 * before we try to do anything with the eeprom. 1165 */ 1166 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1167 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1168 goto init_fail; 1169 } 1170 1171 /* 1172 * Need to init eeprom before validating the checksum. 1173 */ 1174 if (ixgbe_init_eeprom_params(hw) < 0) { 1175 ixgbe_error(ixgbe, 1176 "Unable to intitialize the eeprom interface."); 1177 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1178 goto init_fail; 1179 } 1180 1181 /* 1182 * NVM validation 1183 */ 1184 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1185 /* 1186 * Some PCI-E parts fail the first check due to 1187 * the link being in sleep state. Call it again, 1188 * if it fails a second time it's a real issue. 1189 */ 1190 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1191 ixgbe_error(ixgbe, 1192 "Invalid NVM checksum. Please contact " 1193 "the vendor to update the NVM."); 1194 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1195 goto init_fail; 1196 } 1197 } 1198 1199 /* 1200 * Setup default flow control thresholds - enable/disable 1201 * & flow control type is controlled by ixgbe.conf 1202 */ 1203 hw->fc.high_water = DEFAULT_FCRTH; 1204 hw->fc.low_water = DEFAULT_FCRTL; 1205 hw->fc.pause_time = DEFAULT_FCPAUSE; 1206 hw->fc.send_xon = B_TRUE; 1207 1208 /* 1209 * Initialize link settings 1210 */ 1211 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1212 1213 /* 1214 * Initialize the chipset hardware 1215 */ 1216 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1217 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1218 goto init_fail; 1219 } 1220 1221 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1222 goto init_fail; 1223 } 1224 1225 mutex_exit(&ixgbe->gen_lock); 1226 return (IXGBE_SUCCESS); 1227 1228 init_fail: 1229 /* 1230 * Reset PHY 1231 */ 1232 (void) ixgbe_reset_phy(hw); 1233 1234 mutex_exit(&ixgbe->gen_lock); 1235 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1236 return (IXGBE_FAILURE); 1237 } 1238 1239 /* 1240 * ixgbe_chip_start - Initialize and start the chipset hardware. 1241 */ 1242 static int 1243 ixgbe_chip_start(ixgbe_t *ixgbe) 1244 { 1245 struct ixgbe_hw *hw = &ixgbe->hw; 1246 int ret_val, i; 1247 1248 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1249 1250 /* 1251 * Get the mac address 1252 * This function should handle SPARC case correctly. 1253 */ 1254 if (!ixgbe_find_mac_address(ixgbe)) { 1255 ixgbe_error(ixgbe, "Failed to get the mac address"); 1256 return (IXGBE_FAILURE); 1257 } 1258 1259 /* 1260 * Validate the mac address 1261 */ 1262 (void) ixgbe_init_rx_addrs(hw); 1263 if (!is_valid_mac_addr(hw->mac.addr)) { 1264 ixgbe_error(ixgbe, "Invalid mac address"); 1265 return (IXGBE_FAILURE); 1266 } 1267 1268 /* 1269 * Configure/Initialize hardware 1270 */ 1271 ret_val = ixgbe_init_hw(hw); 1272 if (ret_val != IXGBE_SUCCESS) { 1273 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1274 ixgbe_error(ixgbe, 1275 "This 82599 device is pre-release and contains" 1276 " outdated firmware, please contact your hardware" 1277 " vendor for a replacement."); 1278 } else { 1279 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1280 return (IXGBE_FAILURE); 1281 } 1282 } 1283 1284 /* 1285 * Re-enable relaxed ordering for performance. It is disabled 1286 * by default in the hardware init. 1287 */ 1288 if (ixgbe->relax_order_enable == B_TRUE) 1289 ixgbe_enable_relaxed_ordering(hw); 1290 1291 /* 1292 * Setup adapter interrupt vectors 1293 */ 1294 ixgbe_setup_adapter_vector(ixgbe); 1295 1296 /* 1297 * Initialize unicast addresses. 1298 */ 1299 ixgbe_init_unicst(ixgbe); 1300 1301 /* 1302 * Setup and initialize the mctable structures. 1303 */ 1304 ixgbe_setup_multicst(ixgbe); 1305 1306 /* 1307 * Set interrupt throttling rate 1308 */ 1309 for (i = 0; i < ixgbe->intr_cnt; i++) { 1310 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1311 } 1312 1313 /* 1314 * Save the state of the phy 1315 */ 1316 ixgbe_get_hw_state(ixgbe); 1317 1318 /* 1319 * Make sure driver has control 1320 */ 1321 ixgbe_get_driver_control(hw); 1322 1323 return (IXGBE_SUCCESS); 1324 } 1325 1326 /* 1327 * ixgbe_chip_stop - Stop the chipset hardware 1328 */ 1329 static void 1330 ixgbe_chip_stop(ixgbe_t *ixgbe) 1331 { 1332 struct ixgbe_hw *hw = &ixgbe->hw; 1333 1334 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1335 1336 /* 1337 * Tell firmware driver is no longer in control 1338 */ 1339 ixgbe_release_driver_control(hw); 1340 1341 /* 1342 * Reset the chipset 1343 */ 1344 (void) ixgbe_reset_hw(hw); 1345 1346 /* 1347 * Reset PHY 1348 */ 1349 (void) ixgbe_reset_phy(hw); 1350 } 1351 1352 /* 1353 * ixgbe_reset - Reset the chipset and re-start the driver. 1354 * 1355 * It involves stopping and re-starting the chipset, 1356 * and re-configuring the rx/tx rings. 1357 */ 1358 static int 1359 ixgbe_reset(ixgbe_t *ixgbe) 1360 { 1361 int i; 1362 1363 /* 1364 * Disable and stop the watchdog timer 1365 */ 1366 ixgbe_disable_watchdog_timer(ixgbe); 1367 1368 mutex_enter(&ixgbe->gen_lock); 1369 1370 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1371 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1372 1373 ixgbe_stop(ixgbe, B_FALSE); 1374 1375 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1376 mutex_exit(&ixgbe->gen_lock); 1377 return (IXGBE_FAILURE); 1378 } 1379 1380 /* 1381 * After resetting, need to recheck the link status. 1382 */ 1383 ixgbe->link_check_complete = B_FALSE; 1384 ixgbe->link_check_hrtime = gethrtime() + 1385 (IXGBE_LINK_UP_TIME * 100000000ULL); 1386 1387 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1388 1389 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1390 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1391 mac_tx_ring_update(ixgbe->mac_hdl, 1392 ixgbe->tx_rings[i].ring_handle); 1393 } 1394 } 1395 1396 mutex_exit(&ixgbe->gen_lock); 1397 1398 /* 1399 * Enable and start the watchdog timer 1400 */ 1401 ixgbe_enable_watchdog_timer(ixgbe); 1402 1403 return (IXGBE_SUCCESS); 1404 } 1405 1406 /* 1407 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1408 */ 1409 static void 1410 ixgbe_tx_clean(ixgbe_t *ixgbe) 1411 { 1412 ixgbe_tx_ring_t *tx_ring; 1413 tx_control_block_t *tcb; 1414 link_list_t pending_list; 1415 uint32_t desc_num; 1416 int i, j; 1417 1418 LINK_LIST_INIT(&pending_list); 1419 1420 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1421 tx_ring = &ixgbe->tx_rings[i]; 1422 1423 mutex_enter(&tx_ring->recycle_lock); 1424 1425 /* 1426 * Clean the pending tx data - the pending packets in the 1427 * work_list that have no chances to be transmitted again. 1428 * 1429 * We must ensure the chipset is stopped or the link is down 1430 * before cleaning the transmit packets. 1431 */ 1432 desc_num = 0; 1433 for (j = 0; j < tx_ring->ring_size; j++) { 1434 tcb = tx_ring->work_list[j]; 1435 if (tcb != NULL) { 1436 desc_num += tcb->desc_num; 1437 1438 tx_ring->work_list[j] = NULL; 1439 1440 ixgbe_free_tcb(tcb); 1441 1442 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1443 } 1444 } 1445 1446 if (desc_num > 0) { 1447 atomic_add_32(&tx_ring->tbd_free, desc_num); 1448 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1449 1450 /* 1451 * Reset the head and tail pointers of the tbd ring; 1452 * Reset the writeback head if it's enable. 1453 */ 1454 tx_ring->tbd_head = 0; 1455 tx_ring->tbd_tail = 0; 1456 if (ixgbe->tx_head_wb_enable) 1457 *tx_ring->tbd_head_wb = 0; 1458 1459 IXGBE_WRITE_REG(&ixgbe->hw, 1460 IXGBE_TDH(tx_ring->index), 0); 1461 IXGBE_WRITE_REG(&ixgbe->hw, 1462 IXGBE_TDT(tx_ring->index), 0); 1463 } 1464 1465 mutex_exit(&tx_ring->recycle_lock); 1466 1467 /* 1468 * Add the tx control blocks in the pending list to 1469 * the free list. 1470 */ 1471 ixgbe_put_free_list(tx_ring, &pending_list); 1472 } 1473 } 1474 1475 /* 1476 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1477 * transmitted. 1478 */ 1479 static boolean_t 1480 ixgbe_tx_drain(ixgbe_t *ixgbe) 1481 { 1482 ixgbe_tx_ring_t *tx_ring; 1483 boolean_t done; 1484 int i, j; 1485 1486 /* 1487 * Wait for a specific time to allow pending tx packets 1488 * to be transmitted. 1489 * 1490 * Check the counter tbd_free to see if transmission is done. 1491 * No lock protection is needed here. 1492 * 1493 * Return B_TRUE if all pending packets have been transmitted; 1494 * Otherwise return B_FALSE; 1495 */ 1496 for (i = 0; i < TX_DRAIN_TIME; i++) { 1497 1498 done = B_TRUE; 1499 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1500 tx_ring = &ixgbe->tx_rings[j]; 1501 done = done && 1502 (tx_ring->tbd_free == tx_ring->ring_size); 1503 } 1504 1505 if (done) 1506 break; 1507 1508 msec_delay(1); 1509 } 1510 1511 return (done); 1512 } 1513 1514 /* 1515 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1516 */ 1517 static boolean_t 1518 ixgbe_rx_drain(ixgbe_t *ixgbe) 1519 { 1520 boolean_t done = B_TRUE; 1521 int i; 1522 1523 /* 1524 * Polling the rx free list to check if those rx buffers held by 1525 * the upper layer are released. 1526 * 1527 * Check the counter rcb_free to see if all pending buffers are 1528 * released. No lock protection is needed here. 1529 * 1530 * Return B_TRUE if all pending buffers have been released; 1531 * Otherwise return B_FALSE; 1532 */ 1533 for (i = 0; i < RX_DRAIN_TIME; i++) { 1534 done = (ixgbe->rcb_pending == 0); 1535 1536 if (done) 1537 break; 1538 1539 msec_delay(1); 1540 } 1541 1542 return (done); 1543 } 1544 1545 /* 1546 * ixgbe_start - Start the driver/chipset. 1547 */ 1548 int 1549 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1550 { 1551 int i; 1552 1553 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1554 1555 if (alloc_buffer) { 1556 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1557 ixgbe_error(ixgbe, 1558 "Failed to allocate software receive rings"); 1559 return (IXGBE_FAILURE); 1560 } 1561 1562 /* Allocate buffers for all the rx/tx rings */ 1563 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1564 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1565 return (IXGBE_FAILURE); 1566 } 1567 1568 ixgbe->tx_ring_init = B_TRUE; 1569 } else { 1570 ixgbe->tx_ring_init = B_FALSE; 1571 } 1572 1573 for (i = 0; i < ixgbe->num_rx_rings; i++) 1574 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1575 for (i = 0; i < ixgbe->num_tx_rings; i++) 1576 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1577 1578 /* 1579 * Start the chipset hardware 1580 */ 1581 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1582 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1583 goto start_failure; 1584 } 1585 1586 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1587 goto start_failure; 1588 } 1589 1590 /* 1591 * Setup the rx/tx rings 1592 */ 1593 ixgbe_setup_rings(ixgbe); 1594 1595 /* 1596 * ixgbe_start() will be called when resetting, however if reset 1597 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1598 * before enabling the interrupts. 1599 */ 1600 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1601 | IXGBE_STALL| IXGBE_OVERTEMP)); 1602 1603 /* 1604 * Enable adapter interrupts 1605 * The interrupts must be enabled after the driver state is START 1606 */ 1607 ixgbe_enable_adapter_interrupts(ixgbe); 1608 1609 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1610 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1611 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1612 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1613 1614 return (IXGBE_SUCCESS); 1615 1616 start_failure: 1617 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1618 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1619 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1620 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1621 1622 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1623 1624 return (IXGBE_FAILURE); 1625 } 1626 1627 /* 1628 * ixgbe_stop - Stop the driver/chipset. 1629 */ 1630 void 1631 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1632 { 1633 int i; 1634 1635 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1636 1637 /* 1638 * Disable the adapter interrupts 1639 */ 1640 ixgbe_disable_adapter_interrupts(ixgbe); 1641 1642 /* 1643 * Drain the pending tx packets 1644 */ 1645 (void) ixgbe_tx_drain(ixgbe); 1646 1647 for (i = 0; i < ixgbe->num_rx_rings; i++) 1648 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1649 for (i = 0; i < ixgbe->num_tx_rings; i++) 1650 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1651 1652 /* 1653 * Stop the chipset hardware 1654 */ 1655 ixgbe_chip_stop(ixgbe); 1656 1657 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1658 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1659 } 1660 1661 /* 1662 * Clean the pending tx data/resources 1663 */ 1664 ixgbe_tx_clean(ixgbe); 1665 1666 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1667 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1668 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1669 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1670 1671 if (ixgbe->link_state == LINK_STATE_UP) { 1672 ixgbe->link_state = LINK_STATE_UNKNOWN; 1673 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1674 } 1675 1676 if (free_buffer) { 1677 /* 1678 * Release the DMA/memory resources of rx/tx rings 1679 */ 1680 ixgbe_free_dma(ixgbe); 1681 ixgbe_free_rx_data(ixgbe); 1682 } 1683 } 1684 1685 /* 1686 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 1687 */ 1688 /* ARGSUSED */ 1689 static int 1690 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 1691 void *arg1, void *arg2) 1692 { 1693 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 1694 1695 switch (cbaction) { 1696 /* IRM callback */ 1697 int count; 1698 case DDI_CB_INTR_ADD: 1699 case DDI_CB_INTR_REMOVE: 1700 count = (int)(uintptr_t)cbarg; 1701 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 1702 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 1703 int, ixgbe->intr_cnt); 1704 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 1705 DDI_SUCCESS) { 1706 ixgbe_error(ixgbe, 1707 "IRM CB: Failed to adjust interrupts"); 1708 goto cb_fail; 1709 } 1710 break; 1711 default: 1712 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 1713 cbaction); 1714 return (DDI_ENOTSUP); 1715 } 1716 return (DDI_SUCCESS); 1717 cb_fail: 1718 return (DDI_FAILURE); 1719 } 1720 1721 /* 1722 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 1723 */ 1724 static int 1725 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 1726 { 1727 int i, rc, actual; 1728 1729 if (count == 0) 1730 return (DDI_SUCCESS); 1731 1732 if ((cbaction == DDI_CB_INTR_ADD && 1733 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 1734 (cbaction == DDI_CB_INTR_REMOVE && 1735 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 1736 return (DDI_FAILURE); 1737 1738 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1739 return (DDI_FAILURE); 1740 } 1741 1742 for (i = 0; i < ixgbe->num_rx_rings; i++) 1743 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 1744 for (i = 0; i < ixgbe->num_tx_rings; i++) 1745 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 1746 1747 mutex_enter(&ixgbe->gen_lock); 1748 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1749 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 1750 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1751 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 1752 1753 ixgbe_stop(ixgbe, B_FALSE); 1754 /* 1755 * Disable interrupts 1756 */ 1757 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1758 rc = ixgbe_disable_intrs(ixgbe); 1759 ASSERT(rc == IXGBE_SUCCESS); 1760 } 1761 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 1762 1763 /* 1764 * Remove interrupt handlers 1765 */ 1766 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 1767 ixgbe_rem_intr_handlers(ixgbe); 1768 } 1769 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 1770 1771 /* 1772 * Clear vect_map 1773 */ 1774 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 1775 switch (cbaction) { 1776 case DDI_CB_INTR_ADD: 1777 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 1778 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 1779 DDI_INTR_ALLOC_NORMAL); 1780 if (rc != DDI_SUCCESS || actual != count) { 1781 ixgbe_log(ixgbe, "Adjust interrupts failed." 1782 "return: %d, irm cb size: %d, actual: %d", 1783 rc, count, actual); 1784 goto intr_adjust_fail; 1785 } 1786 ixgbe->intr_cnt += count; 1787 break; 1788 1789 case DDI_CB_INTR_REMOVE: 1790 for (i = ixgbe->intr_cnt - count; 1791 i < ixgbe->intr_cnt; i ++) { 1792 rc = ddi_intr_free(ixgbe->htable[i]); 1793 ixgbe->htable[i] = NULL; 1794 if (rc != DDI_SUCCESS) { 1795 ixgbe_log(ixgbe, "Adjust interrupts failed." 1796 "return: %d, irm cb size: %d, actual: %d", 1797 rc, count, actual); 1798 goto intr_adjust_fail; 1799 } 1800 } 1801 ixgbe->intr_cnt -= count; 1802 break; 1803 } 1804 1805 /* 1806 * Get priority for first vector, assume remaining are all the same 1807 */ 1808 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 1809 if (rc != DDI_SUCCESS) { 1810 ixgbe_log(ixgbe, 1811 "Get interrupt priority failed: %d", rc); 1812 goto intr_adjust_fail; 1813 } 1814 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 1815 if (rc != DDI_SUCCESS) { 1816 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 1817 goto intr_adjust_fail; 1818 } 1819 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 1820 1821 /* 1822 * Map rings to interrupt vectors 1823 */ 1824 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 1825 ixgbe_error(ixgbe, 1826 "IRM CB: Failed to map interrupts to vectors"); 1827 goto intr_adjust_fail; 1828 } 1829 1830 /* 1831 * Add interrupt handlers 1832 */ 1833 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 1834 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 1835 goto intr_adjust_fail; 1836 } 1837 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 1838 1839 /* 1840 * Now that mutex locks are initialized, and the chip is also 1841 * initialized, enable interrupts. 1842 */ 1843 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 1844 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 1845 goto intr_adjust_fail; 1846 } 1847 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 1848 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1849 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 1850 goto intr_adjust_fail; 1851 } 1852 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 1853 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1854 ixgbe->ixgbe_state |= IXGBE_STARTED; 1855 mutex_exit(&ixgbe->gen_lock); 1856 1857 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1858 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 1859 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 1860 } 1861 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1862 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 1863 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 1864 } 1865 1866 /* Wakeup all Tx rings */ 1867 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1868 mac_tx_ring_update(ixgbe->mac_hdl, 1869 ixgbe->tx_rings[i].ring_handle); 1870 } 1871 1872 IXGBE_DEBUGLOG_3(ixgbe, 1873 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 1874 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 1875 return (DDI_SUCCESS); 1876 1877 intr_adjust_fail: 1878 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1879 mutex_exit(&ixgbe->gen_lock); 1880 return (DDI_FAILURE); 1881 } 1882 1883 /* 1884 * ixgbe_intr_cb_register - Register interrupt callback function. 1885 */ 1886 static int 1887 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 1888 { 1889 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 1890 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 1891 return (IXGBE_FAILURE); 1892 } 1893 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 1894 return (IXGBE_SUCCESS); 1895 } 1896 1897 /* 1898 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 1899 */ 1900 static int 1901 ixgbe_alloc_rings(ixgbe_t *ixgbe) 1902 { 1903 /* 1904 * Allocate memory space for rx rings 1905 */ 1906 ixgbe->rx_rings = kmem_zalloc( 1907 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 1908 KM_NOSLEEP); 1909 1910 if (ixgbe->rx_rings == NULL) { 1911 return (IXGBE_FAILURE); 1912 } 1913 1914 /* 1915 * Allocate memory space for tx rings 1916 */ 1917 ixgbe->tx_rings = kmem_zalloc( 1918 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 1919 KM_NOSLEEP); 1920 1921 if (ixgbe->tx_rings == NULL) { 1922 kmem_free(ixgbe->rx_rings, 1923 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1924 ixgbe->rx_rings = NULL; 1925 return (IXGBE_FAILURE); 1926 } 1927 1928 /* 1929 * Allocate memory space for rx ring groups 1930 */ 1931 ixgbe->rx_groups = kmem_zalloc( 1932 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 1933 KM_NOSLEEP); 1934 1935 if (ixgbe->rx_groups == NULL) { 1936 kmem_free(ixgbe->rx_rings, 1937 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1938 kmem_free(ixgbe->tx_rings, 1939 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1940 ixgbe->rx_rings = NULL; 1941 ixgbe->tx_rings = NULL; 1942 return (IXGBE_FAILURE); 1943 } 1944 1945 return (IXGBE_SUCCESS); 1946 } 1947 1948 /* 1949 * ixgbe_free_rings - Free the memory space of rx/tx rings. 1950 */ 1951 static void 1952 ixgbe_free_rings(ixgbe_t *ixgbe) 1953 { 1954 if (ixgbe->rx_rings != NULL) { 1955 kmem_free(ixgbe->rx_rings, 1956 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1957 ixgbe->rx_rings = NULL; 1958 } 1959 1960 if (ixgbe->tx_rings != NULL) { 1961 kmem_free(ixgbe->tx_rings, 1962 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1963 ixgbe->tx_rings = NULL; 1964 } 1965 1966 if (ixgbe->rx_groups != NULL) { 1967 kmem_free(ixgbe->rx_groups, 1968 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 1969 ixgbe->rx_groups = NULL; 1970 } 1971 } 1972 1973 static int 1974 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 1975 { 1976 ixgbe_rx_ring_t *rx_ring; 1977 int i; 1978 1979 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1980 rx_ring = &ixgbe->rx_rings[i]; 1981 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 1982 goto alloc_rx_rings_failure; 1983 } 1984 return (IXGBE_SUCCESS); 1985 1986 alloc_rx_rings_failure: 1987 ixgbe_free_rx_data(ixgbe); 1988 return (IXGBE_FAILURE); 1989 } 1990 1991 static void 1992 ixgbe_free_rx_data(ixgbe_t *ixgbe) 1993 { 1994 ixgbe_rx_ring_t *rx_ring; 1995 ixgbe_rx_data_t *rx_data; 1996 int i; 1997 1998 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1999 rx_ring = &ixgbe->rx_rings[i]; 2000 2001 mutex_enter(&ixgbe->rx_pending_lock); 2002 rx_data = rx_ring->rx_data; 2003 2004 if (rx_data != NULL) { 2005 rx_data->flag |= IXGBE_RX_STOPPED; 2006 2007 if (rx_data->rcb_pending == 0) { 2008 ixgbe_free_rx_ring_data(rx_data); 2009 rx_ring->rx_data = NULL; 2010 } 2011 } 2012 2013 mutex_exit(&ixgbe->rx_pending_lock); 2014 } 2015 } 2016 2017 /* 2018 * ixgbe_setup_rings - Setup rx/tx rings. 2019 */ 2020 static void 2021 ixgbe_setup_rings(ixgbe_t *ixgbe) 2022 { 2023 /* 2024 * Setup the rx/tx rings, including the following: 2025 * 2026 * 1. Setup the descriptor ring and the control block buffers; 2027 * 2. Initialize necessary registers for receive/transmit; 2028 * 3. Initialize software pointers/parameters for receive/transmit; 2029 */ 2030 ixgbe_setup_rx(ixgbe); 2031 2032 ixgbe_setup_tx(ixgbe); 2033 } 2034 2035 static void 2036 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2037 { 2038 ixgbe_t *ixgbe = rx_ring->ixgbe; 2039 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2040 struct ixgbe_hw *hw = &ixgbe->hw; 2041 rx_control_block_t *rcb; 2042 union ixgbe_adv_rx_desc *rbd; 2043 uint32_t size; 2044 uint32_t buf_low; 2045 uint32_t buf_high; 2046 uint32_t reg_val; 2047 int i; 2048 2049 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2050 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2051 2052 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2053 rcb = rx_data->work_list[i]; 2054 rbd = &rx_data->rbd_ring[i]; 2055 2056 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2057 rbd->read.hdr_addr = NULL; 2058 } 2059 2060 /* 2061 * Initialize the length register 2062 */ 2063 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2064 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2065 2066 /* 2067 * Initialize the base address registers 2068 */ 2069 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2070 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2071 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2072 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2073 2074 /* 2075 * Setup head & tail pointers 2076 */ 2077 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2078 rx_data->ring_size - 1); 2079 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2080 2081 rx_data->rbd_next = 0; 2082 rx_data->lro_first = 0; 2083 2084 /* 2085 * Setup the Receive Descriptor Control Register (RXDCTL) 2086 * PTHRESH=32 descriptors (half the internal cache) 2087 * HTHRESH=0 descriptors (to minimize latency on fetch) 2088 * WTHRESH defaults to 1 (writeback each descriptor) 2089 */ 2090 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2091 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2092 2093 /* Not a valid value for 82599 */ 2094 if (hw->mac.type < ixgbe_mac_82599EB) { 2095 reg_val |= 0x0020; /* pthresh */ 2096 } 2097 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2098 2099 if (hw->mac.type == ixgbe_mac_82599EB) { 2100 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2101 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2102 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2103 } 2104 2105 /* 2106 * Setup the Split and Replication Receive Control Register. 2107 * Set the rx buffer size and the advanced descriptor type. 2108 */ 2109 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2110 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2111 reg_val |= IXGBE_SRRCTL_DROP_EN; 2112 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2113 } 2114 2115 static void 2116 ixgbe_setup_rx(ixgbe_t *ixgbe) 2117 { 2118 ixgbe_rx_ring_t *rx_ring; 2119 struct ixgbe_hw *hw = &ixgbe->hw; 2120 uint32_t reg_val; 2121 uint32_t ring_mapping; 2122 uint32_t i, index; 2123 uint32_t psrtype_rss_bit; 2124 2125 /* PSRTYPE must be configured for 82599 */ 2126 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2127 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2128 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2129 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2130 reg_val |= IXGBE_PSRTYPE_L2HDR; 2131 reg_val |= 0x80000000; 2132 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2133 } else { 2134 if (ixgbe->num_rx_groups > 32) { 2135 psrtype_rss_bit = 0x20000000; 2136 } else { 2137 psrtype_rss_bit = 0x40000000; 2138 } 2139 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2140 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2141 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2142 reg_val |= IXGBE_PSRTYPE_L2HDR; 2143 reg_val |= psrtype_rss_bit; 2144 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2145 } 2146 } 2147 2148 /* 2149 * Set filter control in FCTRL to accept broadcast packets and do 2150 * not pass pause frames to host. Flow control settings are already 2151 * in this register, so preserve them. 2152 */ 2153 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2154 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 2155 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 2156 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2157 2158 /* 2159 * Hardware checksum settings 2160 */ 2161 if (ixgbe->rx_hcksum_enable) { 2162 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2163 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2164 } 2165 2166 /* 2167 * Setup VMDq and RSS for multiple receive queues 2168 */ 2169 switch (ixgbe->classify_mode) { 2170 case IXGBE_CLASSIFY_RSS: 2171 /* 2172 * One group, only RSS is needed when more than 2173 * one ring enabled. 2174 */ 2175 ixgbe_setup_rss(ixgbe); 2176 break; 2177 2178 case IXGBE_CLASSIFY_VMDQ: 2179 /* 2180 * Multiple groups, each group has one ring, 2181 * only VMDq is needed. 2182 */ 2183 ixgbe_setup_vmdq(ixgbe); 2184 break; 2185 2186 case IXGBE_CLASSIFY_VMDQ_RSS: 2187 /* 2188 * Multiple groups and multiple rings, both 2189 * VMDq and RSS are needed. 2190 */ 2191 ixgbe_setup_vmdq_rss(ixgbe); 2192 break; 2193 2194 default: 2195 break; 2196 } 2197 2198 /* 2199 * Enable the receive unit. This must be done after filter 2200 * control is set in FCTRL. 2201 */ 2202 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 2203 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 2204 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 2205 2206 /* 2207 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2208 */ 2209 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2210 rx_ring = &ixgbe->rx_rings[i]; 2211 ixgbe_setup_rx_ring(rx_ring); 2212 } 2213 2214 /* 2215 * Setup the per-ring statistics mapping. 2216 */ 2217 ring_mapping = 0; 2218 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2219 index = ixgbe->rx_rings[i].hw_index; 2220 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2221 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2222 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2223 } 2224 2225 /* 2226 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2227 * by four bytes if the packet has a VLAN field, so includes MTU, 2228 * ethernet header and frame check sequence. 2229 * Register is MAXFRS in 82599. 2230 */ 2231 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 2232 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2233 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2234 2235 /* 2236 * Setup Jumbo Frame enable bit 2237 */ 2238 if (ixgbe->default_mtu > ETHERMTU) { 2239 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2240 reg_val |= IXGBE_HLREG0_JUMBOEN; 2241 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2242 } 2243 2244 /* 2245 * Setup RSC for multiple receive queues. 2246 */ 2247 if (ixgbe->lro_enable) { 2248 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2249 /* 2250 * Make sure rx_buf_size * MAXDESC not greater 2251 * than 65535. 2252 * Intel recommends 4 for MAXDESC field value. 2253 */ 2254 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2255 reg_val |= IXGBE_RSCCTL_RSCEN; 2256 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2257 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2258 else 2259 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2260 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2261 } 2262 2263 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2264 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2265 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2266 2267 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2268 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2269 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2270 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2271 2272 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2273 } 2274 } 2275 2276 static void 2277 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2278 { 2279 ixgbe_t *ixgbe = tx_ring->ixgbe; 2280 struct ixgbe_hw *hw = &ixgbe->hw; 2281 uint32_t size; 2282 uint32_t buf_low; 2283 uint32_t buf_high; 2284 uint32_t reg_val; 2285 2286 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2287 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2288 2289 /* 2290 * Initialize the length register 2291 */ 2292 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2293 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2294 2295 /* 2296 * Initialize the base address registers 2297 */ 2298 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2299 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2300 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2301 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2302 2303 /* 2304 * Setup head & tail pointers 2305 */ 2306 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2307 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2308 2309 /* 2310 * Setup head write-back 2311 */ 2312 if (ixgbe->tx_head_wb_enable) { 2313 /* 2314 * The memory of the head write-back is allocated using 2315 * the extra tbd beyond the tail of the tbd ring. 2316 */ 2317 tx_ring->tbd_head_wb = (uint32_t *) 2318 ((uintptr_t)tx_ring->tbd_area.address + size); 2319 *tx_ring->tbd_head_wb = 0; 2320 2321 buf_low = (uint32_t) 2322 (tx_ring->tbd_area.dma_address + size); 2323 buf_high = (uint32_t) 2324 ((tx_ring->tbd_area.dma_address + size) >> 32); 2325 2326 /* Set the head write-back enable bit */ 2327 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2328 2329 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2330 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2331 2332 /* 2333 * Turn off relaxed ordering for head write back or it will 2334 * cause problems with the tx recycling 2335 */ 2336 reg_val = IXGBE_READ_REG(hw, 2337 IXGBE_DCA_TXCTRL(tx_ring->index)); 2338 reg_val &= ~IXGBE_DCA_TXCTRL_TX_WB_RO_EN; 2339 IXGBE_WRITE_REG(hw, 2340 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2341 } else { 2342 tx_ring->tbd_head_wb = NULL; 2343 } 2344 2345 tx_ring->tbd_head = 0; 2346 tx_ring->tbd_tail = 0; 2347 tx_ring->tbd_free = tx_ring->ring_size; 2348 2349 if (ixgbe->tx_ring_init == B_TRUE) { 2350 tx_ring->tcb_head = 0; 2351 tx_ring->tcb_tail = 0; 2352 tx_ring->tcb_free = tx_ring->free_list_size; 2353 } 2354 2355 /* 2356 * Initialize the s/w context structure 2357 */ 2358 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2359 } 2360 2361 static void 2362 ixgbe_setup_tx(ixgbe_t *ixgbe) 2363 { 2364 struct ixgbe_hw *hw = &ixgbe->hw; 2365 ixgbe_tx_ring_t *tx_ring; 2366 uint32_t reg_val; 2367 uint32_t ring_mapping; 2368 int i; 2369 2370 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2371 tx_ring = &ixgbe->tx_rings[i]; 2372 ixgbe_setup_tx_ring(tx_ring); 2373 } 2374 2375 /* 2376 * Setup the per-ring statistics mapping. 2377 */ 2378 ring_mapping = 0; 2379 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2380 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2381 if ((i & 0x3) == 0x3) { 2382 switch (hw->mac.type) { 2383 case ixgbe_mac_82598EB: 2384 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2385 ring_mapping); 2386 break; 2387 2388 case ixgbe_mac_82599EB: 2389 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2390 ring_mapping); 2391 break; 2392 2393 default: 2394 break; 2395 } 2396 2397 ring_mapping = 0; 2398 } 2399 } 2400 if (i & 0x3) { 2401 switch (hw->mac.type) { 2402 case ixgbe_mac_82598EB: 2403 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2404 break; 2405 2406 case ixgbe_mac_82599EB: 2407 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2408 break; 2409 2410 default: 2411 break; 2412 } 2413 } 2414 2415 /* 2416 * Enable CRC appending and TX padding (for short tx frames) 2417 */ 2418 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2419 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2420 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2421 2422 /* 2423 * enable DMA for 82599 parts 2424 */ 2425 if (hw->mac.type == ixgbe_mac_82599EB) { 2426 /* DMATXCTL.TE must be set after all Tx config is complete */ 2427 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2428 reg_val |= IXGBE_DMATXCTL_TE; 2429 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2430 } 2431 2432 /* 2433 * Enabling tx queues .. 2434 * For 82599 must be done after DMATXCTL.TE is set 2435 */ 2436 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2437 tx_ring = &ixgbe->tx_rings[i]; 2438 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2439 reg_val |= IXGBE_TXDCTL_ENABLE; 2440 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2441 } 2442 } 2443 2444 /* 2445 * ixgbe_setup_rss - Setup receive-side scaling feature. 2446 */ 2447 static void 2448 ixgbe_setup_rss(ixgbe_t *ixgbe) 2449 { 2450 struct ixgbe_hw *hw = &ixgbe->hw; 2451 uint32_t i, mrqc, rxcsum; 2452 uint32_t random; 2453 uint32_t reta; 2454 uint32_t ring_per_group; 2455 2456 /* 2457 * Fill out redirection table 2458 */ 2459 reta = 0; 2460 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2461 2462 for (i = 0; i < 128; i++) { 2463 reta = (reta << 8) | (i % ring_per_group) | 2464 ((i % ring_per_group) << 4); 2465 if ((i & 3) == 3) 2466 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2467 } 2468 2469 /* 2470 * Fill out hash function seeds with a random constant 2471 */ 2472 for (i = 0; i < 10; i++) { 2473 (void) random_get_pseudo_bytes((uint8_t *)&random, 2474 sizeof (uint32_t)); 2475 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2476 } 2477 2478 /* 2479 * Enable RSS & perform hash on these packet types 2480 */ 2481 mrqc = IXGBE_MRQC_RSSEN | 2482 IXGBE_MRQC_RSS_FIELD_IPV4 | 2483 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2484 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2485 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2486 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2487 IXGBE_MRQC_RSS_FIELD_IPV6 | 2488 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2489 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2490 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2491 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2492 2493 /* 2494 * Disable Packet Checksum to enable RSS for multiple receive queues. 2495 * It is an adapter hardware limitation that Packet Checksum is 2496 * mutually exclusive with RSS. 2497 */ 2498 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2499 rxcsum |= IXGBE_RXCSUM_PCSD; 2500 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2501 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2502 } 2503 2504 /* 2505 * ixgbe_setup_vmdq - Setup MAC classification feature 2506 */ 2507 static void 2508 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2509 { 2510 struct ixgbe_hw *hw = &ixgbe->hw; 2511 uint32_t vmdctl, i, vtctl; 2512 2513 /* 2514 * Setup the VMDq Control register, enable VMDq based on 2515 * packet destination MAC address: 2516 */ 2517 switch (hw->mac.type) { 2518 case ixgbe_mac_82598EB: 2519 /* 2520 * VMDq Enable = 1; 2521 * VMDq Filter = 0; MAC filtering 2522 * Default VMDq output index = 0; 2523 */ 2524 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2525 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2526 break; 2527 2528 case ixgbe_mac_82599EB: 2529 /* 2530 * Enable VMDq-only. 2531 */ 2532 vmdctl = IXGBE_MRQC_VMDQEN; 2533 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2534 2535 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2536 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2537 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2538 } 2539 2540 /* 2541 * Enable Virtualization and Replication. 2542 */ 2543 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2544 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2545 2546 /* 2547 * Enable receiving packets to all VFs 2548 */ 2549 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2550 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2551 break; 2552 2553 default: 2554 break; 2555 } 2556 } 2557 2558 /* 2559 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2560 */ 2561 static void 2562 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2563 { 2564 struct ixgbe_hw *hw = &ixgbe->hw; 2565 uint32_t i, mrqc, rxcsum; 2566 uint32_t random; 2567 uint32_t reta; 2568 uint32_t ring_per_group; 2569 uint32_t vmdctl, vtctl; 2570 2571 /* 2572 * Fill out redirection table 2573 */ 2574 reta = 0; 2575 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2576 for (i = 0; i < 128; i++) { 2577 reta = (reta << 8) | (i % ring_per_group) | 2578 ((i % ring_per_group) << 4); 2579 if ((i & 3) == 3) 2580 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2581 } 2582 2583 /* 2584 * Fill out hash function seeds with a random constant 2585 */ 2586 for (i = 0; i < 10; i++) { 2587 (void) random_get_pseudo_bytes((uint8_t *)&random, 2588 sizeof (uint32_t)); 2589 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2590 } 2591 2592 /* 2593 * Enable and setup RSS and VMDq 2594 */ 2595 switch (hw->mac.type) { 2596 case ixgbe_mac_82598EB: 2597 /* 2598 * Enable RSS & Setup RSS Hash functions 2599 */ 2600 mrqc = IXGBE_MRQC_RSSEN | 2601 IXGBE_MRQC_RSS_FIELD_IPV4 | 2602 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2603 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2604 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2605 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2606 IXGBE_MRQC_RSS_FIELD_IPV6 | 2607 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2608 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2609 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2610 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2611 2612 /* 2613 * Enable and Setup VMDq 2614 * VMDq Filter = 0; MAC filtering 2615 * Default VMDq output index = 0; 2616 */ 2617 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2618 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2619 break; 2620 2621 case ixgbe_mac_82599EB: 2622 /* 2623 * Enable RSS & Setup RSS Hash functions 2624 */ 2625 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2626 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2627 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2628 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2629 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2630 IXGBE_MRQC_RSS_FIELD_IPV6 | 2631 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2632 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2633 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2634 2635 /* 2636 * Enable VMDq+RSS. 2637 */ 2638 if (ixgbe->num_rx_groups > 32) { 2639 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2640 } else { 2641 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2642 } 2643 2644 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2645 2646 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2647 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2648 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2649 } 2650 break; 2651 2652 default: 2653 break; 2654 2655 } 2656 2657 /* 2658 * Disable Packet Checksum to enable RSS for multiple receive queues. 2659 * It is an adapter hardware limitation that Packet Checksum is 2660 * mutually exclusive with RSS. 2661 */ 2662 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2663 rxcsum |= IXGBE_RXCSUM_PCSD; 2664 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2665 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2666 2667 if (hw->mac.type == ixgbe_mac_82599EB) { 2668 /* 2669 * Enable Virtualization and Replication. 2670 */ 2671 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2672 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2673 2674 /* 2675 * Enable receiving packets to all VFs 2676 */ 2677 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2678 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2679 } 2680 } 2681 2682 /* 2683 * ixgbe_init_unicst - Initialize the unicast addresses. 2684 */ 2685 static void 2686 ixgbe_init_unicst(ixgbe_t *ixgbe) 2687 { 2688 struct ixgbe_hw *hw = &ixgbe->hw; 2689 uint8_t *mac_addr; 2690 int slot; 2691 /* 2692 * Here we should consider two situations: 2693 * 2694 * 1. Chipset is initialized at the first time, 2695 * Clear all the multiple unicast addresses. 2696 * 2697 * 2. Chipset is reset 2698 * Recover the multiple unicast addresses from the 2699 * software data structure to the RAR registers. 2700 */ 2701 if (!ixgbe->unicst_init) { 2702 /* 2703 * Initialize the multiple unicast addresses 2704 */ 2705 ixgbe->unicst_total = hw->mac.num_rar_entries; 2706 ixgbe->unicst_avail = ixgbe->unicst_total; 2707 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2708 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2709 bzero(mac_addr, ETHERADDRL); 2710 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2711 ixgbe->unicst_addr[slot].mac.set = 0; 2712 } 2713 ixgbe->unicst_init = B_TRUE; 2714 } else { 2715 /* Re-configure the RAR registers */ 2716 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2717 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2718 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2719 (void) ixgbe_set_rar(hw, slot, mac_addr, 2720 ixgbe->unicst_addr[slot].mac.group_index, 2721 IXGBE_RAH_AV); 2722 } else { 2723 bzero(mac_addr, ETHERADDRL); 2724 (void) ixgbe_set_rar(hw, slot, mac_addr, 2725 NULL, NULL); 2726 } 2727 } 2728 } 2729 } 2730 2731 /* 2732 * ixgbe_unicst_find - Find the slot for the specified unicast address 2733 */ 2734 int 2735 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2736 { 2737 int slot; 2738 2739 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2740 2741 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2742 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2743 mac_addr, ETHERADDRL) == 0) 2744 return (slot); 2745 } 2746 2747 return (-1); 2748 } 2749 2750 /* 2751 * ixgbe_multicst_add - Add a multicst address. 2752 */ 2753 int 2754 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2755 { 2756 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2757 2758 if ((multiaddr[0] & 01) == 0) { 2759 return (EINVAL); 2760 } 2761 2762 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2763 return (ENOENT); 2764 } 2765 2766 bcopy(multiaddr, 2767 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2768 ixgbe->mcast_count++; 2769 2770 /* 2771 * Update the multicast table in the hardware 2772 */ 2773 ixgbe_setup_multicst(ixgbe); 2774 2775 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2776 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2777 return (EIO); 2778 } 2779 2780 return (0); 2781 } 2782 2783 /* 2784 * ixgbe_multicst_remove - Remove a multicst address. 2785 */ 2786 int 2787 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2788 { 2789 int i; 2790 2791 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2792 2793 for (i = 0; i < ixgbe->mcast_count; i++) { 2794 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2795 ETHERADDRL) == 0) { 2796 for (i++; i < ixgbe->mcast_count; i++) { 2797 ixgbe->mcast_table[i - 1] = 2798 ixgbe->mcast_table[i]; 2799 } 2800 ixgbe->mcast_count--; 2801 break; 2802 } 2803 } 2804 2805 /* 2806 * Update the multicast table in the hardware 2807 */ 2808 ixgbe_setup_multicst(ixgbe); 2809 2810 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2811 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2812 return (EIO); 2813 } 2814 2815 return (0); 2816 } 2817 2818 /* 2819 * ixgbe_setup_multicast - Setup multicast data structures. 2820 * 2821 * This routine initializes all of the multicast related structures 2822 * and save them in the hardware registers. 2823 */ 2824 static void 2825 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2826 { 2827 uint8_t *mc_addr_list; 2828 uint32_t mc_addr_count; 2829 struct ixgbe_hw *hw = &ixgbe->hw; 2830 2831 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2832 2833 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2834 2835 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2836 mc_addr_count = ixgbe->mcast_count; 2837 2838 /* 2839 * Update the multicast addresses to the MTA registers 2840 */ 2841 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2842 ixgbe_mc_table_itr); 2843 } 2844 2845 /* 2846 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 2847 * 2848 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 2849 * Different chipsets may have different allowed configuration of vmdq and rss. 2850 */ 2851 static void 2852 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 2853 { 2854 struct ixgbe_hw *hw = &ixgbe->hw; 2855 uint32_t ring_per_group; 2856 2857 switch (hw->mac.type) { 2858 case ixgbe_mac_82598EB: 2859 /* 2860 * 82598 supports the following combination: 2861 * vmdq no. x rss no. 2862 * [5..16] x 1 2863 * [1..4] x [1..16] 2864 * However 8 rss queue per pool (vmdq) is sufficient for 2865 * most cases. 2866 */ 2867 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2868 if (ixgbe->num_rx_groups > 4) { 2869 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 2870 } else { 2871 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2872 min(8, ring_per_group); 2873 } 2874 2875 break; 2876 2877 case ixgbe_mac_82599EB: 2878 /* 2879 * 82599 supports the following combination: 2880 * vmdq no. x rss no. 2881 * [33..64] x [1..2] 2882 * [2..32] x [1..4] 2883 * 1 x [1..16] 2884 * However 8 rss queue per pool (vmdq) is sufficient for 2885 * most cases. 2886 */ 2887 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2888 if (ixgbe->num_rx_groups == 1) { 2889 ixgbe->num_rx_rings = min(8, ring_per_group); 2890 } else if (ixgbe->num_rx_groups <= 32) { 2891 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2892 min(4, ring_per_group); 2893 } else if (ixgbe->num_rx_groups <= 64) { 2894 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2895 min(2, ring_per_group); 2896 } 2897 break; 2898 2899 default: 2900 break; 2901 } 2902 2903 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2904 2905 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 2906 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 2907 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 2908 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 2909 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 2910 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 2911 } else { 2912 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 2913 } 2914 2915 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 2916 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 2917 } 2918 2919 /* 2920 * ixgbe_get_conf - Get driver configurations set in driver.conf. 2921 * 2922 * This routine gets user-configured values out of the configuration 2923 * file ixgbe.conf. 2924 * 2925 * For each configurable value, there is a minimum, a maximum, and a 2926 * default. 2927 * If user does not configure a value, use the default. 2928 * If user configures below the minimum, use the minumum. 2929 * If user configures above the maximum, use the maxumum. 2930 */ 2931 static void 2932 ixgbe_get_conf(ixgbe_t *ixgbe) 2933 { 2934 struct ixgbe_hw *hw = &ixgbe->hw; 2935 uint32_t flow_control; 2936 2937 /* 2938 * ixgbe driver supports the following user configurations: 2939 * 2940 * Jumbo frame configuration: 2941 * default_mtu 2942 * 2943 * Ethernet flow control configuration: 2944 * flow_control 2945 * 2946 * Multiple rings configurations: 2947 * tx_queue_number 2948 * tx_ring_size 2949 * rx_queue_number 2950 * rx_ring_size 2951 * 2952 * Call ixgbe_get_prop() to get the value for a specific 2953 * configuration parameter. 2954 */ 2955 2956 /* 2957 * Jumbo frame configuration - max_frame_size controls host buffer 2958 * allocation, so includes MTU, ethernet header, vlan tag and 2959 * frame check sequence. 2960 */ 2961 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 2962 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 2963 2964 ixgbe->max_frame_size = ixgbe->default_mtu + 2965 sizeof (struct ether_vlan_header) + ETHERFCSL; 2966 2967 /* 2968 * Ethernet flow control configuration 2969 */ 2970 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 2971 ixgbe_fc_none, 3, ixgbe_fc_none); 2972 if (flow_control == 3) 2973 flow_control = ixgbe_fc_default; 2974 2975 /* 2976 * fc.requested mode is what the user requests. After autoneg, 2977 * fc.current_mode will be the flow_control mode that was negotiated. 2978 */ 2979 hw->fc.requested_mode = flow_control; 2980 2981 /* 2982 * Multiple rings configurations 2983 */ 2984 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 2985 ixgbe->capab->min_tx_que_num, 2986 ixgbe->capab->max_tx_que_num, 2987 ixgbe->capab->def_tx_que_num); 2988 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 2989 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2990 2991 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 2992 ixgbe->capab->min_rx_que_num, 2993 ixgbe->capab->max_rx_que_num, 2994 ixgbe->capab->def_rx_que_num); 2995 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 2996 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2997 2998 /* 2999 * Multiple groups configuration 3000 */ 3001 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3002 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3003 ixgbe->capab->def_rx_grp_num); 3004 3005 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3006 0, 1, DEFAULT_MR_ENABLE); 3007 3008 if (ixgbe->mr_enable == B_FALSE) { 3009 ixgbe->num_tx_rings = 1; 3010 ixgbe->num_rx_rings = 1; 3011 ixgbe->num_rx_groups = 1; 3012 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3013 } else { 3014 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3015 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3016 /* 3017 * The combination of num_rx_rings and num_rx_groups 3018 * may be not supported by h/w. We need to adjust 3019 * them to appropriate values. 3020 */ 3021 ixgbe_setup_vmdq_rss_conf(ixgbe); 3022 } 3023 3024 /* 3025 * Tunable used to force an interrupt type. The only use is 3026 * for testing of the lesser interrupt types. 3027 * 0 = don't force interrupt type 3028 * 1 = force interrupt type MSI-X 3029 * 2 = force interrupt type MSI 3030 * 3 = force interrupt type Legacy 3031 */ 3032 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3033 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3034 3035 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3036 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3037 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3038 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3039 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3040 0, 1, DEFAULT_LSO_ENABLE); 3041 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3042 0, 1, DEFAULT_LRO_ENABLE); 3043 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3044 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3045 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3046 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3047 3048 /* Head Write Back not recommended for 82599 */ 3049 if (hw->mac.type >= ixgbe_mac_82599EB) { 3050 ixgbe->tx_head_wb_enable = B_FALSE; 3051 } 3052 3053 /* 3054 * ixgbe LSO needs the tx h/w checksum support. 3055 * LSO will be disabled if tx h/w checksum is not 3056 * enabled. 3057 */ 3058 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3059 ixgbe->lso_enable = B_FALSE; 3060 } 3061 3062 /* 3063 * ixgbe LRO needs the rx h/w checksum support. 3064 * LRO will be disabled if rx h/w checksum is not 3065 * enabled. 3066 */ 3067 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3068 ixgbe->lro_enable = B_FALSE; 3069 } 3070 3071 /* 3072 * ixgbe LRO only been supported by 82599 now 3073 */ 3074 if (hw->mac.type != ixgbe_mac_82599EB) { 3075 ixgbe->lro_enable = B_FALSE; 3076 } 3077 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3078 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3079 DEFAULT_TX_COPY_THRESHOLD); 3080 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3081 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3082 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3083 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3084 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3085 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3086 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3087 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3088 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3089 3090 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3091 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3092 DEFAULT_RX_COPY_THRESHOLD); 3093 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3094 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3095 DEFAULT_RX_LIMIT_PER_INTR); 3096 3097 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3098 ixgbe->capab->min_intr_throttle, 3099 ixgbe->capab->max_intr_throttle, 3100 ixgbe->capab->def_intr_throttle); 3101 /* 3102 * 82599 requires the interupt throttling rate is 3103 * a multiple of 8. This is enforced by the register 3104 * definiton. 3105 */ 3106 if (hw->mac.type == ixgbe_mac_82599EB) 3107 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3108 } 3109 3110 static void 3111 ixgbe_init_params(ixgbe_t *ixgbe) 3112 { 3113 ixgbe->param_en_10000fdx_cap = 1; 3114 ixgbe->param_en_1000fdx_cap = 1; 3115 ixgbe->param_en_100fdx_cap = 1; 3116 ixgbe->param_adv_10000fdx_cap = 1; 3117 ixgbe->param_adv_1000fdx_cap = 1; 3118 ixgbe->param_adv_100fdx_cap = 1; 3119 3120 ixgbe->param_pause_cap = 1; 3121 ixgbe->param_asym_pause_cap = 1; 3122 ixgbe->param_rem_fault = 0; 3123 3124 ixgbe->param_adv_autoneg_cap = 1; 3125 ixgbe->param_adv_pause_cap = 1; 3126 ixgbe->param_adv_asym_pause_cap = 1; 3127 ixgbe->param_adv_rem_fault = 0; 3128 3129 ixgbe->param_lp_10000fdx_cap = 0; 3130 ixgbe->param_lp_1000fdx_cap = 0; 3131 ixgbe->param_lp_100fdx_cap = 0; 3132 ixgbe->param_lp_autoneg_cap = 0; 3133 ixgbe->param_lp_pause_cap = 0; 3134 ixgbe->param_lp_asym_pause_cap = 0; 3135 ixgbe->param_lp_rem_fault = 0; 3136 } 3137 3138 /* 3139 * ixgbe_get_prop - Get a property value out of the configuration file 3140 * ixgbe.conf. 3141 * 3142 * Caller provides the name of the property, a default value, a minimum 3143 * value, and a maximum value. 3144 * 3145 * Return configured value of the property, with default, minimum and 3146 * maximum properly applied. 3147 */ 3148 static int 3149 ixgbe_get_prop(ixgbe_t *ixgbe, 3150 char *propname, /* name of the property */ 3151 int minval, /* minimum acceptable value */ 3152 int maxval, /* maximim acceptable value */ 3153 int defval) /* default value */ 3154 { 3155 int value; 3156 3157 /* 3158 * Call ddi_prop_get_int() to read the conf settings 3159 */ 3160 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3161 DDI_PROP_DONTPASS, propname, defval); 3162 if (value > maxval) 3163 value = maxval; 3164 3165 if (value < minval) 3166 value = minval; 3167 3168 return (value); 3169 } 3170 3171 /* 3172 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3173 */ 3174 int 3175 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3176 { 3177 u32 autoneg_advertised = 0; 3178 3179 /* 3180 * No half duplex support with 10Gb parts 3181 */ 3182 if (ixgbe->param_adv_10000fdx_cap == 1) 3183 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3184 3185 if (ixgbe->param_adv_1000fdx_cap == 1) 3186 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3187 3188 if (ixgbe->param_adv_100fdx_cap == 1) 3189 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 3190 3191 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) { 3192 ixgbe_notice(ixgbe, "Invalid link settings. Setup link " 3193 "to autonegotiation with full link capabilities."); 3194 3195 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL | 3196 IXGBE_LINK_SPEED_1GB_FULL | 3197 IXGBE_LINK_SPEED_100_FULL; 3198 } 3199 3200 if (setup_hw) { 3201 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised, 3202 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) { 3203 ixgbe_notice(ixgbe, "Setup link failed on this " 3204 "device."); 3205 return (IXGBE_FAILURE); 3206 } 3207 } 3208 3209 return (IXGBE_SUCCESS); 3210 } 3211 3212 /* 3213 * ixgbe_driver_link_check - Link status processing. 3214 * 3215 * This function can be called in both kernel context and interrupt context 3216 */ 3217 static void 3218 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3219 { 3220 struct ixgbe_hw *hw = &ixgbe->hw; 3221 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3222 boolean_t link_up = B_FALSE; 3223 boolean_t link_changed = B_FALSE; 3224 3225 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3226 3227 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3228 if (link_up) { 3229 ixgbe->link_check_complete = B_TRUE; 3230 3231 /* Link is up, enable flow control settings */ 3232 (void) ixgbe_fc_enable(hw, 0); 3233 3234 /* 3235 * The Link is up, check whether it was marked as down earlier 3236 */ 3237 if (ixgbe->link_state != LINK_STATE_UP) { 3238 switch (speed) { 3239 case IXGBE_LINK_SPEED_10GB_FULL: 3240 ixgbe->link_speed = SPEED_10GB; 3241 break; 3242 case IXGBE_LINK_SPEED_1GB_FULL: 3243 ixgbe->link_speed = SPEED_1GB; 3244 break; 3245 case IXGBE_LINK_SPEED_100_FULL: 3246 ixgbe->link_speed = SPEED_100; 3247 } 3248 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3249 ixgbe->link_state = LINK_STATE_UP; 3250 link_changed = B_TRUE; 3251 } 3252 } else { 3253 if (ixgbe->link_check_complete == B_TRUE || 3254 (ixgbe->link_check_complete == B_FALSE && 3255 gethrtime() >= ixgbe->link_check_hrtime)) { 3256 /* 3257 * The link is really down 3258 */ 3259 ixgbe->link_check_complete = B_TRUE; 3260 3261 if (ixgbe->link_state != LINK_STATE_DOWN) { 3262 ixgbe->link_speed = 0; 3263 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3264 ixgbe->link_state = LINK_STATE_DOWN; 3265 link_changed = B_TRUE; 3266 } 3267 } 3268 } 3269 3270 /* 3271 * If we are in an interrupt context, need to re-enable the 3272 * interrupt, which was automasked 3273 */ 3274 if (servicing_interrupt() != 0) { 3275 ixgbe->eims |= IXGBE_EICR_LSC; 3276 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3277 } 3278 3279 if (link_changed) { 3280 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3281 } 3282 } 3283 3284 /* 3285 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3286 */ 3287 static void 3288 ixgbe_sfp_check(void *arg) 3289 { 3290 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3291 uint32_t eicr = ixgbe->eicr; 3292 struct ixgbe_hw *hw = &ixgbe->hw; 3293 3294 mutex_enter(&ixgbe->gen_lock); 3295 if (eicr & IXGBE_EICR_GPI_SDP1) { 3296 /* clear the interrupt */ 3297 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 3298 3299 /* if link up, do multispeed fiber setup */ 3300 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3301 B_TRUE, B_TRUE); 3302 ixgbe_driver_link_check(ixgbe); 3303 ixgbe_get_hw_state(ixgbe); 3304 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 3305 /* clear the interrupt */ 3306 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 3307 3308 /* if link up, do sfp module setup */ 3309 (void) hw->mac.ops.setup_sfp(hw); 3310 3311 /* do multispeed fiber setup */ 3312 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3313 B_TRUE, B_TRUE); 3314 ixgbe_driver_link_check(ixgbe); 3315 ixgbe_get_hw_state(ixgbe); 3316 } 3317 mutex_exit(&ixgbe->gen_lock); 3318 3319 /* 3320 * We need to fully re-check the link later. 3321 */ 3322 ixgbe->link_check_complete = B_FALSE; 3323 ixgbe->link_check_hrtime = gethrtime() + 3324 (IXGBE_LINK_UP_TIME * 100000000ULL); 3325 } 3326 3327 /* 3328 * ixgbe_overtemp_check - overtemp module processing done in taskq 3329 * 3330 * This routine will only be called on adapters with temperature sensor. 3331 * The indication of over-temperature can be either SDP0 interrupt or the link 3332 * status change interrupt. 3333 */ 3334 static void 3335 ixgbe_overtemp_check(void *arg) 3336 { 3337 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3338 struct ixgbe_hw *hw = &ixgbe->hw; 3339 uint32_t eicr = ixgbe->eicr; 3340 ixgbe_link_speed speed; 3341 boolean_t link_up; 3342 3343 mutex_enter(&ixgbe->gen_lock); 3344 3345 /* make sure we know current state of link */ 3346 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3347 3348 /* check over-temp condition */ 3349 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) || 3350 (eicr & IXGBE_EICR_LSC)) { 3351 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3352 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3353 3354 /* 3355 * Disable the adapter interrupts 3356 */ 3357 ixgbe_disable_adapter_interrupts(ixgbe); 3358 3359 /* 3360 * Disable Rx/Tx units 3361 */ 3362 (void) ixgbe_stop_adapter(hw); 3363 3364 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3365 ixgbe_error(ixgbe, 3366 "Problem: Network adapter has been stopped " 3367 "because it has overheated"); 3368 ixgbe_error(ixgbe, 3369 "Action: Restart the computer. " 3370 "If the problem persists, power off the system " 3371 "and replace the adapter"); 3372 } 3373 } 3374 3375 /* write to clear the interrupt */ 3376 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3377 3378 mutex_exit(&ixgbe->gen_lock); 3379 } 3380 3381 /* 3382 * ixgbe_link_timer - timer for link status detection 3383 */ 3384 static void 3385 ixgbe_link_timer(void *arg) 3386 { 3387 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3388 3389 mutex_enter(&ixgbe->gen_lock); 3390 ixgbe_driver_link_check(ixgbe); 3391 mutex_exit(&ixgbe->gen_lock); 3392 } 3393 3394 /* 3395 * ixgbe_local_timer - Driver watchdog function. 3396 * 3397 * This function will handle the transmit stall check and other routines. 3398 */ 3399 static void 3400 ixgbe_local_timer(void *arg) 3401 { 3402 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3403 3404 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3405 goto out; 3406 3407 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3408 ixgbe->reset_count++; 3409 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3410 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3411 goto out; 3412 } 3413 3414 if (ixgbe_stall_check(ixgbe)) { 3415 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3416 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3417 3418 ixgbe->reset_count++; 3419 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3420 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3421 } 3422 3423 out: 3424 ixgbe_restart_watchdog_timer(ixgbe); 3425 } 3426 3427 /* 3428 * ixgbe_stall_check - Check for transmit stall. 3429 * 3430 * This function checks if the adapter is stalled (in transmit). 3431 * 3432 * It is called each time the watchdog timeout is invoked. 3433 * If the transmit descriptor reclaim continuously fails, 3434 * the watchdog value will increment by 1. If the watchdog 3435 * value exceeds the threshold, the ixgbe is assumed to 3436 * have stalled and need to be reset. 3437 */ 3438 static boolean_t 3439 ixgbe_stall_check(ixgbe_t *ixgbe) 3440 { 3441 ixgbe_tx_ring_t *tx_ring; 3442 boolean_t result; 3443 int i; 3444 3445 if (ixgbe->link_state != LINK_STATE_UP) 3446 return (B_FALSE); 3447 3448 /* 3449 * If any tx ring is stalled, we'll reset the chipset 3450 */ 3451 result = B_FALSE; 3452 for (i = 0; i < ixgbe->num_tx_rings; i++) { 3453 tx_ring = &ixgbe->tx_rings[i]; 3454 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 3455 tx_ring->tx_recycle(tx_ring); 3456 } 3457 3458 if (tx_ring->recycle_fail > 0) 3459 tx_ring->stall_watchdog++; 3460 else 3461 tx_ring->stall_watchdog = 0; 3462 3463 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3464 result = B_TRUE; 3465 break; 3466 } 3467 } 3468 3469 if (result) { 3470 tx_ring->stall_watchdog = 0; 3471 tx_ring->recycle_fail = 0; 3472 } 3473 3474 return (result); 3475 } 3476 3477 3478 /* 3479 * is_valid_mac_addr - Check if the mac address is valid. 3480 */ 3481 static boolean_t 3482 is_valid_mac_addr(uint8_t *mac_addr) 3483 { 3484 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3485 const uint8_t addr_test2[6] = 3486 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3487 3488 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3489 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3490 return (B_FALSE); 3491 3492 return (B_TRUE); 3493 } 3494 3495 static boolean_t 3496 ixgbe_find_mac_address(ixgbe_t *ixgbe) 3497 { 3498 #ifdef __sparc 3499 struct ixgbe_hw *hw = &ixgbe->hw; 3500 uchar_t *bytes; 3501 struct ether_addr sysaddr; 3502 uint_t nelts; 3503 int err; 3504 boolean_t found = B_FALSE; 3505 3506 /* 3507 * The "vendor's factory-set address" may already have 3508 * been extracted from the chip, but if the property 3509 * "local-mac-address" is set we use that instead. 3510 * 3511 * We check whether it looks like an array of 6 3512 * bytes (which it should, if OBP set it). If we can't 3513 * make sense of it this way, we'll ignore it. 3514 */ 3515 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3516 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3517 if (err == DDI_PROP_SUCCESS) { 3518 if (nelts == ETHERADDRL) { 3519 while (nelts--) 3520 hw->mac.addr[nelts] = bytes[nelts]; 3521 found = B_TRUE; 3522 } 3523 ddi_prop_free(bytes); 3524 } 3525 3526 /* 3527 * Look up the OBP property "local-mac-address?". If the user has set 3528 * 'local-mac-address? = false', use "the system address" instead. 3529 */ 3530 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 3531 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3532 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3533 if (localetheraddr(NULL, &sysaddr) != 0) { 3534 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3535 found = B_TRUE; 3536 } 3537 } 3538 ddi_prop_free(bytes); 3539 } 3540 3541 /* 3542 * Finally(!), if there's a valid "mac-address" property (created 3543 * if we netbooted from this interface), we must use this instead 3544 * of any of the above to ensure that the NFS/install server doesn't 3545 * get confused by the address changing as Solaris takes over! 3546 */ 3547 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3548 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3549 if (err == DDI_PROP_SUCCESS) { 3550 if (nelts == ETHERADDRL) { 3551 while (nelts--) 3552 hw->mac.addr[nelts] = bytes[nelts]; 3553 found = B_TRUE; 3554 } 3555 ddi_prop_free(bytes); 3556 } 3557 3558 if (found) { 3559 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3560 return (B_TRUE); 3561 } 3562 #else 3563 _NOTE(ARGUNUSED(ixgbe)); 3564 #endif 3565 3566 return (B_TRUE); 3567 } 3568 3569 #pragma inline(ixgbe_arm_watchdog_timer) 3570 static void 3571 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 3572 { 3573 /* 3574 * Fire a watchdog timer 3575 */ 3576 ixgbe->watchdog_tid = 3577 timeout(ixgbe_local_timer, 3578 (void *)ixgbe, 1 * drv_usectohz(1000000)); 3579 3580 } 3581 3582 /* 3583 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 3584 */ 3585 void 3586 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 3587 { 3588 mutex_enter(&ixgbe->watchdog_lock); 3589 3590 if (!ixgbe->watchdog_enable) { 3591 ixgbe->watchdog_enable = B_TRUE; 3592 ixgbe->watchdog_start = B_TRUE; 3593 ixgbe_arm_watchdog_timer(ixgbe); 3594 } 3595 3596 mutex_exit(&ixgbe->watchdog_lock); 3597 } 3598 3599 /* 3600 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 3601 */ 3602 void 3603 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 3604 { 3605 timeout_id_t tid; 3606 3607 mutex_enter(&ixgbe->watchdog_lock); 3608 3609 ixgbe->watchdog_enable = B_FALSE; 3610 ixgbe->watchdog_start = B_FALSE; 3611 tid = ixgbe->watchdog_tid; 3612 ixgbe->watchdog_tid = 0; 3613 3614 mutex_exit(&ixgbe->watchdog_lock); 3615 3616 if (tid != 0) 3617 (void) untimeout(tid); 3618 } 3619 3620 /* 3621 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 3622 */ 3623 void 3624 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 3625 { 3626 mutex_enter(&ixgbe->watchdog_lock); 3627 3628 if (ixgbe->watchdog_enable) { 3629 if (!ixgbe->watchdog_start) { 3630 ixgbe->watchdog_start = B_TRUE; 3631 ixgbe_arm_watchdog_timer(ixgbe); 3632 } 3633 } 3634 3635 mutex_exit(&ixgbe->watchdog_lock); 3636 } 3637 3638 /* 3639 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 3640 */ 3641 static void 3642 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 3643 { 3644 mutex_enter(&ixgbe->watchdog_lock); 3645 3646 if (ixgbe->watchdog_start) 3647 ixgbe_arm_watchdog_timer(ixgbe); 3648 3649 mutex_exit(&ixgbe->watchdog_lock); 3650 } 3651 3652 /* 3653 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 3654 */ 3655 void 3656 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 3657 { 3658 timeout_id_t tid; 3659 3660 mutex_enter(&ixgbe->watchdog_lock); 3661 3662 ixgbe->watchdog_start = B_FALSE; 3663 tid = ixgbe->watchdog_tid; 3664 ixgbe->watchdog_tid = 0; 3665 3666 mutex_exit(&ixgbe->watchdog_lock); 3667 3668 if (tid != 0) 3669 (void) untimeout(tid); 3670 } 3671 3672 /* 3673 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 3674 */ 3675 static void 3676 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 3677 { 3678 struct ixgbe_hw *hw = &ixgbe->hw; 3679 3680 /* 3681 * mask all interrupts off 3682 */ 3683 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 3684 3685 /* 3686 * for MSI-X, also disable autoclear 3687 */ 3688 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3689 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 3690 } 3691 3692 IXGBE_WRITE_FLUSH(hw); 3693 } 3694 3695 /* 3696 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 3697 */ 3698 static void 3699 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 3700 { 3701 struct ixgbe_hw *hw = &ixgbe->hw; 3702 uint32_t eiac, eiam; 3703 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 3704 3705 /* interrupt types to enable */ 3706 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 3707 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 3708 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 3709 3710 /* enable automask on "other" causes that this adapter can generate */ 3711 eiam = ixgbe->capab->other_intr; 3712 3713 /* 3714 * msi-x mode 3715 */ 3716 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3717 /* enable autoclear but not on bits 29:20 */ 3718 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 3719 3720 /* general purpose interrupt enable */ 3721 gpie |= (IXGBE_GPIE_MSIX_MODE 3722 | IXGBE_GPIE_PBA_SUPPORT 3723 | IXGBE_GPIE_OCD 3724 | IXGBE_GPIE_EIAME); 3725 /* 3726 * non-msi-x mode 3727 */ 3728 } else { 3729 3730 /* disable autoclear, leave gpie at default */ 3731 eiac = 0; 3732 3733 /* 3734 * General purpose interrupt enable. 3735 * For 82599, extended interrupt automask enable 3736 * only in MSI or MSI-X mode 3737 */ 3738 if ((hw->mac.type < ixgbe_mac_82599EB) || 3739 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 3740 gpie |= IXGBE_GPIE_EIAME; 3741 } 3742 } 3743 3744 /* Enable specific "other" interrupt types */ 3745 switch (hw->mac.type) { 3746 case ixgbe_mac_82598EB: 3747 gpie |= ixgbe->capab->other_gpie; 3748 break; 3749 3750 case ixgbe_mac_82599EB: 3751 gpie |= ixgbe->capab->other_gpie; 3752 3753 /* Enable RSC Delay 8us when LRO enabled */ 3754 if (ixgbe->lro_enable) { 3755 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 3756 } 3757 break; 3758 3759 default: 3760 break; 3761 } 3762 3763 /* write to interrupt control registers */ 3764 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3765 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3766 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3767 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3768 IXGBE_WRITE_FLUSH(hw); 3769 } 3770 3771 /* 3772 * ixgbe_loopback_ioctl - Loopback support. 3773 */ 3774 enum ioc_reply 3775 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3776 { 3777 lb_info_sz_t *lbsp; 3778 lb_property_t *lbpp; 3779 uint32_t *lbmp; 3780 uint32_t size; 3781 uint32_t value; 3782 3783 if (mp->b_cont == NULL) 3784 return (IOC_INVAL); 3785 3786 switch (iocp->ioc_cmd) { 3787 default: 3788 return (IOC_INVAL); 3789 3790 case LB_GET_INFO_SIZE: 3791 size = sizeof (lb_info_sz_t); 3792 if (iocp->ioc_count != size) 3793 return (IOC_INVAL); 3794 3795 value = sizeof (lb_normal); 3796 value += sizeof (lb_mac); 3797 value += sizeof (lb_external); 3798 3799 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3800 *lbsp = value; 3801 break; 3802 3803 case LB_GET_INFO: 3804 value = sizeof (lb_normal); 3805 value += sizeof (lb_mac); 3806 value += sizeof (lb_external); 3807 3808 size = value; 3809 if (iocp->ioc_count != size) 3810 return (IOC_INVAL); 3811 3812 value = 0; 3813 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3814 3815 lbpp[value++] = lb_normal; 3816 lbpp[value++] = lb_mac; 3817 lbpp[value++] = lb_external; 3818 break; 3819 3820 case LB_GET_MODE: 3821 size = sizeof (uint32_t); 3822 if (iocp->ioc_count != size) 3823 return (IOC_INVAL); 3824 3825 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3826 *lbmp = ixgbe->loopback_mode; 3827 break; 3828 3829 case LB_SET_MODE: 3830 size = 0; 3831 if (iocp->ioc_count != sizeof (uint32_t)) 3832 return (IOC_INVAL); 3833 3834 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3835 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3836 return (IOC_INVAL); 3837 break; 3838 } 3839 3840 iocp->ioc_count = size; 3841 iocp->ioc_error = 0; 3842 3843 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3844 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3845 return (IOC_INVAL); 3846 } 3847 3848 return (IOC_REPLY); 3849 } 3850 3851 /* 3852 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3853 */ 3854 static boolean_t 3855 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3856 { 3857 if (mode == ixgbe->loopback_mode) 3858 return (B_TRUE); 3859 3860 ixgbe->loopback_mode = mode; 3861 3862 if (mode == IXGBE_LB_NONE) { 3863 /* 3864 * Reset the chip 3865 */ 3866 (void) ixgbe_reset(ixgbe); 3867 return (B_TRUE); 3868 } 3869 3870 mutex_enter(&ixgbe->gen_lock); 3871 3872 switch (mode) { 3873 default: 3874 mutex_exit(&ixgbe->gen_lock); 3875 return (B_FALSE); 3876 3877 case IXGBE_LB_EXTERNAL: 3878 break; 3879 3880 case IXGBE_LB_INTERNAL_MAC: 3881 ixgbe_set_internal_mac_loopback(ixgbe); 3882 break; 3883 } 3884 3885 mutex_exit(&ixgbe->gen_lock); 3886 3887 return (B_TRUE); 3888 } 3889 3890 /* 3891 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 3892 */ 3893 static void 3894 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 3895 { 3896 struct ixgbe_hw *hw; 3897 uint32_t reg; 3898 uint8_t atlas; 3899 3900 hw = &ixgbe->hw; 3901 3902 /* 3903 * Setup MAC loopback 3904 */ 3905 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 3906 reg |= IXGBE_HLREG0_LPBK; 3907 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 3908 3909 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 3910 reg &= ~IXGBE_AUTOC_LMS_MASK; 3911 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 3912 3913 /* 3914 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 3915 */ 3916 switch (hw->mac.type) { 3917 case ixgbe_mac_82598EB: 3918 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3919 &atlas); 3920 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 3921 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3922 atlas); 3923 3924 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3925 &atlas); 3926 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 3927 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3928 atlas); 3929 3930 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3931 &atlas); 3932 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 3933 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3934 atlas); 3935 3936 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3937 &atlas); 3938 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 3939 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 3940 atlas); 3941 break; 3942 3943 case ixgbe_mac_82599EB: 3944 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 3945 reg |= (IXGBE_AUTOC_FLU | 3946 IXGBE_AUTOC_10G_KX4); 3947 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 3948 3949 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 3950 B_FALSE, B_TRUE); 3951 break; 3952 3953 default: 3954 break; 3955 } 3956 } 3957 3958 #pragma inline(ixgbe_intr_rx_work) 3959 /* 3960 * ixgbe_intr_rx_work - RX processing of ISR. 3961 */ 3962 static void 3963 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 3964 { 3965 mblk_t *mp; 3966 3967 mutex_enter(&rx_ring->rx_lock); 3968 3969 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 3970 mutex_exit(&rx_ring->rx_lock); 3971 3972 if (mp != NULL) 3973 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 3974 rx_ring->ring_gen_num); 3975 } 3976 3977 #pragma inline(ixgbe_intr_tx_work) 3978 /* 3979 * ixgbe_intr_tx_work - TX processing of ISR. 3980 */ 3981 static void 3982 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 3983 { 3984 ixgbe_t *ixgbe = tx_ring->ixgbe; 3985 3986 /* 3987 * Recycle the tx descriptors 3988 */ 3989 tx_ring->tx_recycle(tx_ring); 3990 3991 /* 3992 * Schedule the re-transmit 3993 */ 3994 if (tx_ring->reschedule && 3995 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 3996 tx_ring->reschedule = B_FALSE; 3997 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 3998 tx_ring->ring_handle); 3999 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4000 } 4001 } 4002 4003 #pragma inline(ixgbe_intr_other_work) 4004 /* 4005 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4006 */ 4007 static void 4008 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4009 { 4010 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4011 4012 /* 4013 * handle link status change 4014 */ 4015 if (eicr & IXGBE_EICR_LSC) { 4016 ixgbe_driver_link_check(ixgbe); 4017 ixgbe_get_hw_state(ixgbe); 4018 } 4019 4020 /* 4021 * check for fan failure on adapters with fans 4022 */ 4023 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4024 (eicr & IXGBE_EICR_GPI_SDP1)) { 4025 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4026 4027 /* 4028 * Disable the adapter interrupts 4029 */ 4030 ixgbe_disable_adapter_interrupts(ixgbe); 4031 4032 /* 4033 * Disable Rx/Tx units 4034 */ 4035 (void) ixgbe_stop_adapter(&ixgbe->hw); 4036 4037 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4038 ixgbe_error(ixgbe, 4039 "Problem: Network adapter has been stopped " 4040 "because the fan has stopped.\n"); 4041 ixgbe_error(ixgbe, 4042 "Action: Replace the adapter.\n"); 4043 4044 /* re-enable the interrupt, which was automasked */ 4045 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4046 } 4047 4048 /* 4049 * Do SFP check for adapters with hot-plug capability 4050 */ 4051 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4052 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) { 4053 ixgbe->eicr = eicr; 4054 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4055 ixgbe_sfp_check, (void *)ixgbe, 4056 DDI_NOSLEEP)) != DDI_SUCCESS) { 4057 ixgbe_log(ixgbe, "No memory available to dispatch " 4058 "taskq for SFP check"); 4059 } 4060 } 4061 4062 /* 4063 * Do over-temperature check for adapters with temp sensor 4064 */ 4065 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4066 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) { 4067 ixgbe->eicr = eicr; 4068 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4069 ixgbe_overtemp_check, (void *)ixgbe, 4070 DDI_NOSLEEP)) != DDI_SUCCESS) { 4071 ixgbe_log(ixgbe, "No memory available to dispatch " 4072 "taskq for overtemp check"); 4073 } 4074 } 4075 } 4076 4077 /* 4078 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4079 */ 4080 static uint_t 4081 ixgbe_intr_legacy(void *arg1, void *arg2) 4082 { 4083 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4084 struct ixgbe_hw *hw = &ixgbe->hw; 4085 ixgbe_tx_ring_t *tx_ring; 4086 ixgbe_rx_ring_t *rx_ring; 4087 uint32_t eicr; 4088 mblk_t *mp; 4089 boolean_t tx_reschedule; 4090 uint_t result; 4091 4092 _NOTE(ARGUNUSED(arg2)); 4093 4094 mutex_enter(&ixgbe->gen_lock); 4095 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4096 mutex_exit(&ixgbe->gen_lock); 4097 return (DDI_INTR_UNCLAIMED); 4098 } 4099 4100 mp = NULL; 4101 tx_reschedule = B_FALSE; 4102 4103 /* 4104 * Any bit set in eicr: claim this interrupt 4105 */ 4106 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4107 4108 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4109 mutex_exit(&ixgbe->gen_lock); 4110 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4111 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4112 return (DDI_INTR_CLAIMED); 4113 } 4114 4115 if (eicr) { 4116 /* 4117 * For legacy interrupt, we have only one interrupt, 4118 * so we have only one rx ring and one tx ring enabled. 4119 */ 4120 ASSERT(ixgbe->num_rx_rings == 1); 4121 ASSERT(ixgbe->num_tx_rings == 1); 4122 4123 /* 4124 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4125 */ 4126 if (eicr & 0x1) { 4127 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4128 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4129 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4130 /* 4131 * Clean the rx descriptors 4132 */ 4133 rx_ring = &ixgbe->rx_rings[0]; 4134 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4135 } 4136 4137 /* 4138 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4139 */ 4140 if (eicr & 0x2) { 4141 /* 4142 * Recycle the tx descriptors 4143 */ 4144 tx_ring = &ixgbe->tx_rings[0]; 4145 tx_ring->tx_recycle(tx_ring); 4146 4147 /* 4148 * Schedule the re-transmit 4149 */ 4150 tx_reschedule = (tx_ring->reschedule && 4151 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4152 } 4153 4154 /* any interrupt type other than tx/rx */ 4155 if (eicr & ixgbe->capab->other_intr) { 4156 switch (hw->mac.type) { 4157 case ixgbe_mac_82598EB: 4158 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4159 break; 4160 4161 case ixgbe_mac_82599EB: 4162 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4163 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4164 break; 4165 4166 default: 4167 break; 4168 } 4169 ixgbe_intr_other_work(ixgbe, eicr); 4170 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4171 } 4172 4173 mutex_exit(&ixgbe->gen_lock); 4174 4175 result = DDI_INTR_CLAIMED; 4176 } else { 4177 mutex_exit(&ixgbe->gen_lock); 4178 4179 /* 4180 * No interrupt cause bits set: don't claim this interrupt. 4181 */ 4182 result = DDI_INTR_UNCLAIMED; 4183 } 4184 4185 /* re-enable the interrupts which were automasked */ 4186 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4187 4188 /* 4189 * Do the following work outside of the gen_lock 4190 */ 4191 if (mp != NULL) { 4192 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4193 rx_ring->ring_gen_num); 4194 } 4195 4196 if (tx_reschedule) { 4197 tx_ring->reschedule = B_FALSE; 4198 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4199 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4200 } 4201 4202 return (result); 4203 } 4204 4205 /* 4206 * ixgbe_intr_msi - Interrupt handler for MSI. 4207 */ 4208 static uint_t 4209 ixgbe_intr_msi(void *arg1, void *arg2) 4210 { 4211 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4212 struct ixgbe_hw *hw = &ixgbe->hw; 4213 uint32_t eicr; 4214 4215 _NOTE(ARGUNUSED(arg2)); 4216 4217 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4218 4219 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4220 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4221 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4222 return (DDI_INTR_CLAIMED); 4223 } 4224 4225 /* 4226 * For MSI interrupt, we have only one vector, 4227 * so we have only one rx ring and one tx ring enabled. 4228 */ 4229 ASSERT(ixgbe->num_rx_rings == 1); 4230 ASSERT(ixgbe->num_tx_rings == 1); 4231 4232 /* 4233 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4234 */ 4235 if (eicr & 0x1) { 4236 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4237 } 4238 4239 /* 4240 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4241 */ 4242 if (eicr & 0x2) { 4243 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4244 } 4245 4246 /* any interrupt type other than tx/rx */ 4247 if (eicr & ixgbe->capab->other_intr) { 4248 mutex_enter(&ixgbe->gen_lock); 4249 switch (hw->mac.type) { 4250 case ixgbe_mac_82598EB: 4251 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4252 break; 4253 4254 case ixgbe_mac_82599EB: 4255 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4256 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4257 break; 4258 4259 default: 4260 break; 4261 } 4262 ixgbe_intr_other_work(ixgbe, eicr); 4263 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4264 mutex_exit(&ixgbe->gen_lock); 4265 } 4266 4267 /* re-enable the interrupts which were automasked */ 4268 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4269 4270 return (DDI_INTR_CLAIMED); 4271 } 4272 4273 /* 4274 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4275 */ 4276 static uint_t 4277 ixgbe_intr_msix(void *arg1, void *arg2) 4278 { 4279 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4280 ixgbe_t *ixgbe = vect->ixgbe; 4281 struct ixgbe_hw *hw = &ixgbe->hw; 4282 uint32_t eicr; 4283 int r_idx = 0; 4284 4285 _NOTE(ARGUNUSED(arg2)); 4286 4287 /* 4288 * Clean each rx ring that has its bit set in the map 4289 */ 4290 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4291 while (r_idx >= 0) { 4292 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4293 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4294 (ixgbe->num_rx_rings - 1)); 4295 } 4296 4297 /* 4298 * Clean each tx ring that has its bit set in the map 4299 */ 4300 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4301 while (r_idx >= 0) { 4302 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4303 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4304 (ixgbe->num_tx_rings - 1)); 4305 } 4306 4307 4308 /* 4309 * Clean other interrupt (link change) that has its bit set in the map 4310 */ 4311 if (BT_TEST(vect->other_map, 0) == 1) { 4312 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4313 4314 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4315 DDI_FM_OK) { 4316 ddi_fm_service_impact(ixgbe->dip, 4317 DDI_SERVICE_DEGRADED); 4318 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4319 return (DDI_INTR_CLAIMED); 4320 } 4321 4322 /* 4323 * Check "other" cause bits: any interrupt type other than tx/rx 4324 */ 4325 if (eicr & ixgbe->capab->other_intr) { 4326 mutex_enter(&ixgbe->gen_lock); 4327 switch (hw->mac.type) { 4328 case ixgbe_mac_82598EB: 4329 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4330 ixgbe_intr_other_work(ixgbe, eicr); 4331 break; 4332 4333 case ixgbe_mac_82599EB: 4334 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4335 ixgbe_intr_other_work(ixgbe, eicr); 4336 break; 4337 4338 default: 4339 break; 4340 } 4341 mutex_exit(&ixgbe->gen_lock); 4342 } 4343 4344 /* re-enable the interrupts which were automasked */ 4345 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4346 } 4347 4348 return (DDI_INTR_CLAIMED); 4349 } 4350 4351 /* 4352 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4353 * 4354 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4355 * if not successful, try Legacy. 4356 * ixgbe->intr_force can be used to force sequence to start with 4357 * any of the 3 types. 4358 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4359 */ 4360 static int 4361 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4362 { 4363 dev_info_t *devinfo; 4364 int intr_types; 4365 int rc; 4366 4367 devinfo = ixgbe->dip; 4368 4369 /* 4370 * Get supported interrupt types 4371 */ 4372 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4373 4374 if (rc != DDI_SUCCESS) { 4375 ixgbe_log(ixgbe, 4376 "Get supported interrupt types failed: %d", rc); 4377 return (IXGBE_FAILURE); 4378 } 4379 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4380 4381 ixgbe->intr_type = 0; 4382 4383 /* 4384 * Install MSI-X interrupts 4385 */ 4386 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4387 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4388 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4389 if (rc == IXGBE_SUCCESS) 4390 return (IXGBE_SUCCESS); 4391 4392 ixgbe_log(ixgbe, 4393 "Allocate MSI-X failed, trying MSI interrupts..."); 4394 } 4395 4396 /* 4397 * MSI-X not used, force rings and groups to 1 4398 */ 4399 ixgbe->num_rx_rings = 1; 4400 ixgbe->num_rx_groups = 1; 4401 ixgbe->num_tx_rings = 1; 4402 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4403 ixgbe_log(ixgbe, 4404 "MSI-X not used, force rings and groups number to 1"); 4405 4406 /* 4407 * Install MSI interrupts 4408 */ 4409 if ((intr_types & DDI_INTR_TYPE_MSI) && 4410 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 4411 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 4412 if (rc == IXGBE_SUCCESS) 4413 return (IXGBE_SUCCESS); 4414 4415 ixgbe_log(ixgbe, 4416 "Allocate MSI failed, trying Legacy interrupts..."); 4417 } 4418 4419 /* 4420 * Install legacy interrupts 4421 */ 4422 if (intr_types & DDI_INTR_TYPE_FIXED) { 4423 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 4424 if (rc == IXGBE_SUCCESS) 4425 return (IXGBE_SUCCESS); 4426 4427 ixgbe_log(ixgbe, 4428 "Allocate Legacy interrupts failed"); 4429 } 4430 4431 /* 4432 * If none of the 3 types succeeded, return failure 4433 */ 4434 return (IXGBE_FAILURE); 4435 } 4436 4437 /* 4438 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 4439 * 4440 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4441 * if fewer than 2 handles are available, return failure. 4442 * Upon success, this maps the vectors to rx and tx rings for 4443 * interrupts. 4444 */ 4445 static int 4446 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 4447 { 4448 dev_info_t *devinfo; 4449 int request, count, actual; 4450 int minimum; 4451 int rc; 4452 uint32_t ring_per_group; 4453 4454 devinfo = ixgbe->dip; 4455 4456 switch (intr_type) { 4457 case DDI_INTR_TYPE_FIXED: 4458 request = 1; /* Request 1 legacy interrupt handle */ 4459 minimum = 1; 4460 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 4461 break; 4462 4463 case DDI_INTR_TYPE_MSI: 4464 request = 1; /* Request 1 MSI interrupt handle */ 4465 minimum = 1; 4466 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 4467 break; 4468 4469 case DDI_INTR_TYPE_MSIX: 4470 /* 4471 * Best number of vectors for the adapter is 4472 * (# rx rings + # tx rings), however we will 4473 * limit the request number. 4474 */ 4475 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 4476 if (request > ixgbe->capab->max_ring_vect) 4477 request = ixgbe->capab->max_ring_vect; 4478 minimum = 1; 4479 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 4480 break; 4481 4482 default: 4483 ixgbe_log(ixgbe, 4484 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 4485 intr_type); 4486 return (IXGBE_FAILURE); 4487 } 4488 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 4489 request, minimum); 4490 4491 /* 4492 * Get number of supported interrupts 4493 */ 4494 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4495 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4496 ixgbe_log(ixgbe, 4497 "Get interrupt number failed. Return: %d, count: %d", 4498 rc, count); 4499 return (IXGBE_FAILURE); 4500 } 4501 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 4502 4503 actual = 0; 4504 ixgbe->intr_cnt = 0; 4505 ixgbe->intr_cnt_max = 0; 4506 ixgbe->intr_cnt_min = 0; 4507 4508 /* 4509 * Allocate an array of interrupt handles 4510 */ 4511 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 4512 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 4513 4514 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 4515 request, &actual, DDI_INTR_ALLOC_NORMAL); 4516 if (rc != DDI_SUCCESS) { 4517 ixgbe_log(ixgbe, "Allocate interrupts failed. " 4518 "return: %d, request: %d, actual: %d", 4519 rc, request, actual); 4520 goto alloc_handle_fail; 4521 } 4522 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 4523 4524 /* 4525 * upper/lower limit of interrupts 4526 */ 4527 ixgbe->intr_cnt = actual; 4528 ixgbe->intr_cnt_max = request; 4529 ixgbe->intr_cnt_min = minimum; 4530 4531 /* 4532 * rss number per group should not exceed the rx interrupt number, 4533 * else need to adjust rx ring number. 4534 */ 4535 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4536 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 4537 if (actual < ring_per_group) { 4538 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 4539 ixgbe_setup_vmdq_rss_conf(ixgbe); 4540 } 4541 4542 /* 4543 * Now we know the actual number of vectors. Here we map the vector 4544 * to other, rx rings and tx ring. 4545 */ 4546 if (actual < minimum) { 4547 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 4548 actual); 4549 goto alloc_handle_fail; 4550 } 4551 4552 /* 4553 * Get priority for first vector, assume remaining are all the same 4554 */ 4555 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 4556 if (rc != DDI_SUCCESS) { 4557 ixgbe_log(ixgbe, 4558 "Get interrupt priority failed: %d", rc); 4559 goto alloc_handle_fail; 4560 } 4561 4562 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 4563 if (rc != DDI_SUCCESS) { 4564 ixgbe_log(ixgbe, 4565 "Get interrupt cap failed: %d", rc); 4566 goto alloc_handle_fail; 4567 } 4568 4569 ixgbe->intr_type = intr_type; 4570 4571 return (IXGBE_SUCCESS); 4572 4573 alloc_handle_fail: 4574 ixgbe_rem_intrs(ixgbe); 4575 4576 return (IXGBE_FAILURE); 4577 } 4578 4579 /* 4580 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 4581 * 4582 * Before adding the interrupt handlers, the interrupt vectors have 4583 * been allocated, and the rx/tx rings have also been allocated. 4584 */ 4585 static int 4586 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 4587 { 4588 int vector = 0; 4589 int rc; 4590 4591 switch (ixgbe->intr_type) { 4592 case DDI_INTR_TYPE_MSIX: 4593 /* 4594 * Add interrupt handler for all vectors 4595 */ 4596 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 4597 /* 4598 * install pointer to vect_map[vector] 4599 */ 4600 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4601 (ddi_intr_handler_t *)ixgbe_intr_msix, 4602 (void *)&ixgbe->vect_map[vector], NULL); 4603 4604 if (rc != DDI_SUCCESS) { 4605 ixgbe_log(ixgbe, 4606 "Add interrupt handler failed. " 4607 "return: %d, vector: %d", rc, vector); 4608 for (vector--; vector >= 0; vector--) { 4609 (void) ddi_intr_remove_handler( 4610 ixgbe->htable[vector]); 4611 } 4612 return (IXGBE_FAILURE); 4613 } 4614 } 4615 4616 break; 4617 4618 case DDI_INTR_TYPE_MSI: 4619 /* 4620 * Add interrupt handlers for the only vector 4621 */ 4622 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4623 (ddi_intr_handler_t *)ixgbe_intr_msi, 4624 (void *)ixgbe, NULL); 4625 4626 if (rc != DDI_SUCCESS) { 4627 ixgbe_log(ixgbe, 4628 "Add MSI interrupt handler failed: %d", rc); 4629 return (IXGBE_FAILURE); 4630 } 4631 4632 break; 4633 4634 case DDI_INTR_TYPE_FIXED: 4635 /* 4636 * Add interrupt handlers for the only vector 4637 */ 4638 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4639 (ddi_intr_handler_t *)ixgbe_intr_legacy, 4640 (void *)ixgbe, NULL); 4641 4642 if (rc != DDI_SUCCESS) { 4643 ixgbe_log(ixgbe, 4644 "Add legacy interrupt handler failed: %d", rc); 4645 return (IXGBE_FAILURE); 4646 } 4647 4648 break; 4649 4650 default: 4651 return (IXGBE_FAILURE); 4652 } 4653 4654 return (IXGBE_SUCCESS); 4655 } 4656 4657 #pragma inline(ixgbe_map_rxring_to_vector) 4658 /* 4659 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 4660 */ 4661 static void 4662 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 4663 { 4664 /* 4665 * Set bit in map 4666 */ 4667 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4668 4669 /* 4670 * Count bits set 4671 */ 4672 ixgbe->vect_map[v_idx].rxr_cnt++; 4673 4674 /* 4675 * Remember bit position 4676 */ 4677 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 4678 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 4679 } 4680 4681 #pragma inline(ixgbe_map_txring_to_vector) 4682 /* 4683 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 4684 */ 4685 static void 4686 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 4687 { 4688 /* 4689 * Set bit in map 4690 */ 4691 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 4692 4693 /* 4694 * Count bits set 4695 */ 4696 ixgbe->vect_map[v_idx].txr_cnt++; 4697 4698 /* 4699 * Remember bit position 4700 */ 4701 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 4702 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 4703 } 4704 4705 /* 4706 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 4707 * allocation register (IVAR). 4708 * cause: 4709 * -1 : other cause 4710 * 0 : rx 4711 * 1 : tx 4712 */ 4713 static void 4714 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 4715 int8_t cause) 4716 { 4717 struct ixgbe_hw *hw = &ixgbe->hw; 4718 u32 ivar, index; 4719 4720 switch (hw->mac.type) { 4721 case ixgbe_mac_82598EB: 4722 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4723 if (cause == -1) { 4724 cause = 0; 4725 } 4726 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4727 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4728 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 4729 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 4730 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4731 break; 4732 4733 case ixgbe_mac_82599EB: 4734 if (cause == -1) { 4735 /* other causes */ 4736 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4737 index = (intr_alloc_entry & 1) * 8; 4738 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4739 ivar &= ~(0xFF << index); 4740 ivar |= (msix_vector << index); 4741 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4742 } else { 4743 /* tx or rx causes */ 4744 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4745 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4746 ivar = IXGBE_READ_REG(hw, 4747 IXGBE_IVAR(intr_alloc_entry >> 1)); 4748 ivar &= ~(0xFF << index); 4749 ivar |= (msix_vector << index); 4750 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4751 ivar); 4752 } 4753 break; 4754 4755 default: 4756 break; 4757 } 4758 } 4759 4760 /* 4761 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 4762 * given interrupt vector allocation register (IVAR). 4763 * cause: 4764 * -1 : other cause 4765 * 0 : rx 4766 * 1 : tx 4767 */ 4768 static void 4769 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4770 { 4771 struct ixgbe_hw *hw = &ixgbe->hw; 4772 u32 ivar, index; 4773 4774 switch (hw->mac.type) { 4775 case ixgbe_mac_82598EB: 4776 if (cause == -1) { 4777 cause = 0; 4778 } 4779 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4780 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4781 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 4782 (intr_alloc_entry & 0x3))); 4783 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4784 break; 4785 4786 case ixgbe_mac_82599EB: 4787 if (cause == -1) { 4788 /* other causes */ 4789 index = (intr_alloc_entry & 1) * 8; 4790 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4791 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4792 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4793 } else { 4794 /* tx or rx causes */ 4795 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4796 ivar = IXGBE_READ_REG(hw, 4797 IXGBE_IVAR(intr_alloc_entry >> 1)); 4798 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4799 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4800 ivar); 4801 } 4802 break; 4803 4804 default: 4805 break; 4806 } 4807 } 4808 4809 /* 4810 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 4811 * given interrupt vector allocation register (IVAR). 4812 * cause: 4813 * -1 : other cause 4814 * 0 : rx 4815 * 1 : tx 4816 */ 4817 static void 4818 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4819 { 4820 struct ixgbe_hw *hw = &ixgbe->hw; 4821 u32 ivar, index; 4822 4823 switch (hw->mac.type) { 4824 case ixgbe_mac_82598EB: 4825 if (cause == -1) { 4826 cause = 0; 4827 } 4828 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4829 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4830 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 4831 (intr_alloc_entry & 0x3))); 4832 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4833 break; 4834 4835 case ixgbe_mac_82599EB: 4836 if (cause == -1) { 4837 /* other causes */ 4838 index = (intr_alloc_entry & 1) * 8; 4839 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4840 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4841 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4842 } else { 4843 /* tx or rx causes */ 4844 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4845 ivar = IXGBE_READ_REG(hw, 4846 IXGBE_IVAR(intr_alloc_entry >> 1)); 4847 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4848 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4849 ivar); 4850 } 4851 break; 4852 4853 default: 4854 break; 4855 } 4856 } 4857 4858 /* 4859 * Convert the rx ring index driver maintained to the rx ring index 4860 * in h/w. 4861 */ 4862 static uint32_t 4863 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 4864 { 4865 4866 struct ixgbe_hw *hw = &ixgbe->hw; 4867 uint32_t rx_ring_per_group, hw_rx_index; 4868 4869 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 4870 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 4871 return (sw_rx_index); 4872 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 4873 switch (hw->mac.type) { 4874 case ixgbe_mac_82598EB: 4875 return (sw_rx_index); 4876 4877 case ixgbe_mac_82599EB: 4878 return (sw_rx_index * 2); 4879 4880 default: 4881 break; 4882 } 4883 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 4884 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4885 4886 switch (hw->mac.type) { 4887 case ixgbe_mac_82598EB: 4888 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 4889 16 + (sw_rx_index % rx_ring_per_group); 4890 return (hw_rx_index); 4891 4892 case ixgbe_mac_82599EB: 4893 if (ixgbe->num_rx_groups > 32) { 4894 hw_rx_index = (sw_rx_index / 4895 rx_ring_per_group) * 2 + 4896 (sw_rx_index % rx_ring_per_group); 4897 } else { 4898 hw_rx_index = (sw_rx_index / 4899 rx_ring_per_group) * 4 + 4900 (sw_rx_index % rx_ring_per_group); 4901 } 4902 return (hw_rx_index); 4903 4904 default: 4905 break; 4906 } 4907 } 4908 4909 /* 4910 * Should never reach. Just to make compiler happy. 4911 */ 4912 return (sw_rx_index); 4913 } 4914 4915 /* 4916 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 4917 * 4918 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 4919 * to vector[0 - (intr_cnt -1)]. 4920 */ 4921 static int 4922 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 4923 { 4924 int i, vector = 0; 4925 4926 /* initialize vector map */ 4927 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 4928 for (i = 0; i < ixgbe->intr_cnt; i++) { 4929 ixgbe->vect_map[i].ixgbe = ixgbe; 4930 } 4931 4932 /* 4933 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 4934 * tx rings[0] on RTxQ[1]. 4935 */ 4936 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 4937 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 4938 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 4939 return (IXGBE_SUCCESS); 4940 } 4941 4942 /* 4943 * Interrupts/vectors mapping for MSI-X 4944 */ 4945 4946 /* 4947 * Map other interrupt to vector 0, 4948 * Set bit in map and count the bits set. 4949 */ 4950 BT_SET(ixgbe->vect_map[vector].other_map, 0); 4951 ixgbe->vect_map[vector].other_cnt++; 4952 4953 /* 4954 * Map rx ring interrupts to vectors 4955 */ 4956 for (i = 0; i < ixgbe->num_rx_rings; i++) { 4957 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 4958 vector = (vector +1) % ixgbe->intr_cnt; 4959 } 4960 4961 /* 4962 * Map tx ring interrupts to vectors 4963 */ 4964 for (i = 0; i < ixgbe->num_tx_rings; i++) { 4965 ixgbe_map_txring_to_vector(ixgbe, i, vector); 4966 vector = (vector +1) % ixgbe->intr_cnt; 4967 } 4968 4969 return (IXGBE_SUCCESS); 4970 } 4971 4972 /* 4973 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 4974 * 4975 * This relies on ring/vector mapping already set up in the 4976 * vect_map[] structures 4977 */ 4978 static void 4979 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 4980 { 4981 struct ixgbe_hw *hw = &ixgbe->hw; 4982 ixgbe_intr_vector_t *vect; /* vector bitmap */ 4983 int r_idx; /* ring index */ 4984 int v_idx; /* vector index */ 4985 uint32_t hw_index; 4986 4987 /* 4988 * Clear any previous entries 4989 */ 4990 switch (hw->mac.type) { 4991 case ixgbe_mac_82598EB: 4992 for (v_idx = 0; v_idx < 25; v_idx++) 4993 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4994 break; 4995 4996 case ixgbe_mac_82599EB: 4997 for (v_idx = 0; v_idx < 64; v_idx++) 4998 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 4999 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5000 break; 5001 5002 default: 5003 break; 5004 } 5005 5006 /* 5007 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5008 * tx rings[0] will use RTxQ[1]. 5009 */ 5010 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5011 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5012 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5013 return; 5014 } 5015 5016 /* 5017 * For MSI-X interrupt, "Other" is always on vector[0]. 5018 */ 5019 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5020 5021 /* 5022 * For each interrupt vector, populate the IVAR table 5023 */ 5024 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5025 vect = &ixgbe->vect_map[v_idx]; 5026 5027 /* 5028 * For each rx ring bit set 5029 */ 5030 r_idx = bt_getlowbit(vect->rx_map, 0, 5031 (ixgbe->num_rx_rings - 1)); 5032 5033 while (r_idx >= 0) { 5034 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5035 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5036 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5037 (ixgbe->num_rx_rings - 1)); 5038 } 5039 5040 /* 5041 * For each tx ring bit set 5042 */ 5043 r_idx = bt_getlowbit(vect->tx_map, 0, 5044 (ixgbe->num_tx_rings - 1)); 5045 5046 while (r_idx >= 0) { 5047 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5048 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5049 (ixgbe->num_tx_rings - 1)); 5050 } 5051 } 5052 } 5053 5054 /* 5055 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5056 */ 5057 static void 5058 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5059 { 5060 int i; 5061 int rc; 5062 5063 for (i = 0; i < ixgbe->intr_cnt; i++) { 5064 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5065 if (rc != DDI_SUCCESS) { 5066 IXGBE_DEBUGLOG_1(ixgbe, 5067 "Remove intr handler failed: %d", rc); 5068 } 5069 } 5070 } 5071 5072 /* 5073 * ixgbe_rem_intrs - Remove the allocated interrupts. 5074 */ 5075 static void 5076 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5077 { 5078 int i; 5079 int rc; 5080 5081 for (i = 0; i < ixgbe->intr_cnt; i++) { 5082 rc = ddi_intr_free(ixgbe->htable[i]); 5083 if (rc != DDI_SUCCESS) { 5084 IXGBE_DEBUGLOG_1(ixgbe, 5085 "Free intr failed: %d", rc); 5086 } 5087 } 5088 5089 kmem_free(ixgbe->htable, ixgbe->intr_size); 5090 ixgbe->htable = NULL; 5091 } 5092 5093 /* 5094 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5095 */ 5096 static int 5097 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5098 { 5099 int i; 5100 int rc; 5101 5102 /* 5103 * Enable interrupts 5104 */ 5105 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5106 /* 5107 * Call ddi_intr_block_enable() for MSI 5108 */ 5109 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5110 if (rc != DDI_SUCCESS) { 5111 ixgbe_log(ixgbe, 5112 "Enable block intr failed: %d", rc); 5113 return (IXGBE_FAILURE); 5114 } 5115 } else { 5116 /* 5117 * Call ddi_intr_enable() for Legacy/MSI non block enable 5118 */ 5119 for (i = 0; i < ixgbe->intr_cnt; i++) { 5120 rc = ddi_intr_enable(ixgbe->htable[i]); 5121 if (rc != DDI_SUCCESS) { 5122 ixgbe_log(ixgbe, 5123 "Enable intr failed: %d", rc); 5124 return (IXGBE_FAILURE); 5125 } 5126 } 5127 } 5128 5129 return (IXGBE_SUCCESS); 5130 } 5131 5132 /* 5133 * ixgbe_disable_intrs - Disable all the interrupts. 5134 */ 5135 static int 5136 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5137 { 5138 int i; 5139 int rc; 5140 5141 /* 5142 * Disable all interrupts 5143 */ 5144 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5145 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5146 if (rc != DDI_SUCCESS) { 5147 ixgbe_log(ixgbe, 5148 "Disable block intr failed: %d", rc); 5149 return (IXGBE_FAILURE); 5150 } 5151 } else { 5152 for (i = 0; i < ixgbe->intr_cnt; i++) { 5153 rc = ddi_intr_disable(ixgbe->htable[i]); 5154 if (rc != DDI_SUCCESS) { 5155 ixgbe_log(ixgbe, 5156 "Disable intr failed: %d", rc); 5157 return (IXGBE_FAILURE); 5158 } 5159 } 5160 } 5161 5162 return (IXGBE_SUCCESS); 5163 } 5164 5165 /* 5166 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5167 */ 5168 static void 5169 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5170 { 5171 struct ixgbe_hw *hw = &ixgbe->hw; 5172 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 5173 boolean_t link_up = B_FALSE; 5174 uint32_t pcs1g_anlp = 0; 5175 uint32_t pcs1g_ana = 0; 5176 boolean_t autoneg = B_FALSE; 5177 5178 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5179 ixgbe->param_lp_1000fdx_cap = 0; 5180 ixgbe->param_lp_100fdx_cap = 0; 5181 5182 /* check for link, don't wait */ 5183 (void) ixgbe_check_link(hw, &speed, &link_up, false); 5184 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 5185 5186 if (link_up) { 5187 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5188 5189 ixgbe->param_lp_1000fdx_cap = 5190 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5191 ixgbe->param_lp_100fdx_cap = 5192 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5193 } 5194 5195 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg); 5196 5197 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5198 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0; 5199 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5200 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0; 5201 } 5202 5203 /* 5204 * ixgbe_get_driver_control - Notify that driver is in control of device. 5205 */ 5206 static void 5207 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5208 { 5209 uint32_t ctrl_ext; 5210 5211 /* 5212 * Notify firmware that driver is in control of device 5213 */ 5214 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5215 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5216 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5217 } 5218 5219 /* 5220 * ixgbe_release_driver_control - Notify that driver is no longer in control 5221 * of device. 5222 */ 5223 static void 5224 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5225 { 5226 uint32_t ctrl_ext; 5227 5228 /* 5229 * Notify firmware that driver is no longer in control of device 5230 */ 5231 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5232 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5233 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5234 } 5235 5236 /* 5237 * ixgbe_atomic_reserve - Atomic decrease operation. 5238 */ 5239 int 5240 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5241 { 5242 uint32_t oldval; 5243 uint32_t newval; 5244 5245 /* 5246 * ATOMICALLY 5247 */ 5248 do { 5249 oldval = *count_p; 5250 if (oldval < n) 5251 return (-1); 5252 newval = oldval - n; 5253 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5254 5255 return (newval); 5256 } 5257 5258 /* 5259 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5260 */ 5261 static uint8_t * 5262 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5263 { 5264 uint8_t *addr = *upd_ptr; 5265 uint8_t *new_ptr; 5266 5267 _NOTE(ARGUNUSED(hw)); 5268 _NOTE(ARGUNUSED(vmdq)); 5269 5270 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5271 *upd_ptr = new_ptr; 5272 return (addr); 5273 } 5274 5275 /* 5276 * FMA support 5277 */ 5278 int 5279 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5280 { 5281 ddi_fm_error_t de; 5282 5283 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5284 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5285 return (de.fme_status); 5286 } 5287 5288 int 5289 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5290 { 5291 ddi_fm_error_t de; 5292 5293 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5294 return (de.fme_status); 5295 } 5296 5297 /* 5298 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5299 */ 5300 static int 5301 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5302 { 5303 _NOTE(ARGUNUSED(impl_data)); 5304 /* 5305 * as the driver can always deal with an error in any dma or 5306 * access handle, we can just return the fme_status value. 5307 */ 5308 pci_ereport_post(dip, err, NULL); 5309 return (err->fme_status); 5310 } 5311 5312 static void 5313 ixgbe_fm_init(ixgbe_t *ixgbe) 5314 { 5315 ddi_iblock_cookie_t iblk; 5316 int fma_dma_flag; 5317 5318 /* 5319 * Only register with IO Fault Services if we have some capability 5320 */ 5321 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5322 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5323 } else { 5324 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5325 } 5326 5327 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5328 fma_dma_flag = 1; 5329 } else { 5330 fma_dma_flag = 0; 5331 } 5332 5333 ixgbe_set_fma_flags(fma_dma_flag); 5334 5335 if (ixgbe->fm_capabilities) { 5336 5337 /* 5338 * Register capabilities with IO Fault Services 5339 */ 5340 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 5341 5342 /* 5343 * Initialize pci ereport capabilities if ereport capable 5344 */ 5345 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5346 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5347 pci_ereport_setup(ixgbe->dip); 5348 5349 /* 5350 * Register error callback if error callback capable 5351 */ 5352 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5353 ddi_fm_handler_register(ixgbe->dip, 5354 ixgbe_fm_error_cb, (void*) ixgbe); 5355 } 5356 } 5357 5358 static void 5359 ixgbe_fm_fini(ixgbe_t *ixgbe) 5360 { 5361 /* 5362 * Only unregister FMA capabilities if they are registered 5363 */ 5364 if (ixgbe->fm_capabilities) { 5365 5366 /* 5367 * Release any resources allocated by pci_ereport_setup() 5368 */ 5369 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5370 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5371 pci_ereport_teardown(ixgbe->dip); 5372 5373 /* 5374 * Un-register error callback if error callback capable 5375 */ 5376 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5377 ddi_fm_handler_unregister(ixgbe->dip); 5378 5379 /* 5380 * Unregister from IO Fault Service 5381 */ 5382 ddi_fm_fini(ixgbe->dip); 5383 } 5384 } 5385 5386 void 5387 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 5388 { 5389 uint64_t ena; 5390 char buf[FM_MAX_CLASS]; 5391 5392 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5393 ena = fm_ena_generate(0, FM_ENA_FMT1); 5394 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 5395 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 5396 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5397 } 5398 } 5399 5400 static int 5401 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 5402 { 5403 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 5404 5405 mutex_enter(&rx_ring->rx_lock); 5406 rx_ring->ring_gen_num = mr_gen_num; 5407 mutex_exit(&rx_ring->rx_lock); 5408 return (0); 5409 } 5410 5411 /* 5412 * Get the global ring index by a ring index within a group. 5413 */ 5414 static int 5415 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 5416 { 5417 ixgbe_rx_ring_t *rx_ring; 5418 int i; 5419 5420 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5421 rx_ring = &ixgbe->rx_rings[i]; 5422 if (rx_ring->group_index == gindex) 5423 rindex--; 5424 if (rindex < 0) 5425 return (i); 5426 } 5427 5428 return (-1); 5429 } 5430 5431 /* 5432 * Callback funtion for MAC layer to register all rings. 5433 */ 5434 /* ARGSUSED */ 5435 void 5436 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 5437 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 5438 { 5439 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5440 mac_intr_t *mintr = &infop->mri_intr; 5441 5442 switch (rtype) { 5443 case MAC_RING_TYPE_RX: { 5444 /* 5445 * 'index' is the ring index within the group. 5446 * Need to get the global ring index by searching in groups. 5447 */ 5448 int global_ring_index = ixgbe_get_rx_ring_index( 5449 ixgbe, group_index, ring_index); 5450 5451 ASSERT(global_ring_index >= 0); 5452 5453 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 5454 rx_ring->ring_handle = rh; 5455 5456 infop->mri_driver = (mac_ring_driver_t)rx_ring; 5457 infop->mri_start = ixgbe_ring_start; 5458 infop->mri_stop = NULL; 5459 infop->mri_poll = ixgbe_ring_rx_poll; 5460 infop->mri_stat = ixgbe_rx_ring_stat; 5461 5462 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 5463 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 5464 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 5465 if (ixgbe->intr_type & 5466 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5467 mintr->mi_ddi_handle = 5468 ixgbe->htable[rx_ring->intr_vector]; 5469 } 5470 5471 break; 5472 } 5473 case MAC_RING_TYPE_TX: { 5474 ASSERT(group_index == -1); 5475 ASSERT(ring_index < ixgbe->num_tx_rings); 5476 5477 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 5478 tx_ring->ring_handle = rh; 5479 5480 infop->mri_driver = (mac_ring_driver_t)tx_ring; 5481 infop->mri_start = NULL; 5482 infop->mri_stop = NULL; 5483 infop->mri_tx = ixgbe_ring_tx; 5484 infop->mri_stat = ixgbe_tx_ring_stat; 5485 if (ixgbe->intr_type & 5486 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5487 mintr->mi_ddi_handle = 5488 ixgbe->htable[tx_ring->intr_vector]; 5489 } 5490 break; 5491 } 5492 default: 5493 break; 5494 } 5495 } 5496 5497 /* 5498 * Callback funtion for MAC layer to register all groups. 5499 */ 5500 void 5501 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 5502 mac_group_info_t *infop, mac_group_handle_t gh) 5503 { 5504 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5505 5506 switch (rtype) { 5507 case MAC_RING_TYPE_RX: { 5508 ixgbe_rx_group_t *rx_group; 5509 5510 rx_group = &ixgbe->rx_groups[index]; 5511 rx_group->group_handle = gh; 5512 5513 infop->mgi_driver = (mac_group_driver_t)rx_group; 5514 infop->mgi_start = NULL; 5515 infop->mgi_stop = NULL; 5516 infop->mgi_addmac = ixgbe_addmac; 5517 infop->mgi_remmac = ixgbe_remmac; 5518 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 5519 5520 break; 5521 } 5522 case MAC_RING_TYPE_TX: 5523 break; 5524 default: 5525 break; 5526 } 5527 } 5528 5529 /* 5530 * Enable interrupt on the specificed rx ring. 5531 */ 5532 int 5533 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 5534 { 5535 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5536 ixgbe_t *ixgbe = rx_ring->ixgbe; 5537 int r_idx = rx_ring->index; 5538 int hw_r_idx = rx_ring->hw_index; 5539 int v_idx = rx_ring->intr_vector; 5540 5541 mutex_enter(&ixgbe->gen_lock); 5542 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5543 mutex_exit(&ixgbe->gen_lock); 5544 /* 5545 * Simply return 0. 5546 * Interrupts are being adjusted. ixgbe_intr_adjust() 5547 * will eventually re-enable the interrupt when it's 5548 * done with the adjustment. 5549 */ 5550 return (0); 5551 } 5552 5553 /* 5554 * To enable interrupt by setting the VAL bit of given interrupt 5555 * vector allocation register (IVAR). 5556 */ 5557 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 5558 5559 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5560 5561 /* 5562 * Trigger a Rx interrupt on this ring 5563 */ 5564 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 5565 IXGBE_WRITE_FLUSH(&ixgbe->hw); 5566 5567 mutex_exit(&ixgbe->gen_lock); 5568 5569 return (0); 5570 } 5571 5572 /* 5573 * Disable interrupt on the specificed rx ring. 5574 */ 5575 int 5576 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 5577 { 5578 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5579 ixgbe_t *ixgbe = rx_ring->ixgbe; 5580 int r_idx = rx_ring->index; 5581 int hw_r_idx = rx_ring->hw_index; 5582 int v_idx = rx_ring->intr_vector; 5583 5584 mutex_enter(&ixgbe->gen_lock); 5585 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5586 mutex_exit(&ixgbe->gen_lock); 5587 /* 5588 * Simply return 0. 5589 * In the rare case where an interrupt is being 5590 * disabled while interrupts are being adjusted, 5591 * we don't fail the operation. No interrupts will 5592 * be generated while they are adjusted, and 5593 * ixgbe_intr_adjust() will cause the interrupts 5594 * to be re-enabled once it completes. Note that 5595 * in this case, packets may be delivered to the 5596 * stack via interrupts before xgbe_rx_ring_intr_enable() 5597 * is called again. This is acceptable since interrupt 5598 * adjustment is infrequent, and the stack will be 5599 * able to handle these packets. 5600 */ 5601 return (0); 5602 } 5603 5604 /* 5605 * To disable interrupt by clearing the VAL bit of given interrupt 5606 * vector allocation register (IVAR). 5607 */ 5608 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 5609 5610 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 5611 5612 mutex_exit(&ixgbe->gen_lock); 5613 5614 return (0); 5615 } 5616 5617 /* 5618 * Add a mac address. 5619 */ 5620 static int 5621 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 5622 { 5623 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5624 ixgbe_t *ixgbe = rx_group->ixgbe; 5625 struct ixgbe_hw *hw = &ixgbe->hw; 5626 int slot, i; 5627 5628 mutex_enter(&ixgbe->gen_lock); 5629 5630 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5631 mutex_exit(&ixgbe->gen_lock); 5632 return (ECANCELED); 5633 } 5634 5635 if (ixgbe->unicst_avail == 0) { 5636 /* no slots available */ 5637 mutex_exit(&ixgbe->gen_lock); 5638 return (ENOSPC); 5639 } 5640 5641 /* 5642 * The first ixgbe->num_rx_groups slots are reserved for each respective 5643 * group. The rest slots are shared by all groups. While adding a 5644 * MAC address, reserved slots are firstly checked then the shared 5645 * slots are searched. 5646 */ 5647 slot = -1; 5648 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 5649 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 5650 if (ixgbe->unicst_addr[i].mac.set == 0) { 5651 slot = i; 5652 break; 5653 } 5654 } 5655 } else { 5656 slot = rx_group->index; 5657 } 5658 5659 if (slot == -1) { 5660 /* no slots available */ 5661 mutex_exit(&ixgbe->gen_lock); 5662 return (ENOSPC); 5663 } 5664 5665 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5666 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 5667 rx_group->index, IXGBE_RAH_AV); 5668 ixgbe->unicst_addr[slot].mac.set = 1; 5669 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 5670 ixgbe->unicst_avail--; 5671 5672 mutex_exit(&ixgbe->gen_lock); 5673 5674 return (0); 5675 } 5676 5677 /* 5678 * Remove a mac address. 5679 */ 5680 static int 5681 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 5682 { 5683 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5684 ixgbe_t *ixgbe = rx_group->ixgbe; 5685 struct ixgbe_hw *hw = &ixgbe->hw; 5686 int slot; 5687 5688 mutex_enter(&ixgbe->gen_lock); 5689 5690 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5691 mutex_exit(&ixgbe->gen_lock); 5692 return (ECANCELED); 5693 } 5694 5695 slot = ixgbe_unicst_find(ixgbe, mac_addr); 5696 if (slot == -1) { 5697 mutex_exit(&ixgbe->gen_lock); 5698 return (EINVAL); 5699 } 5700 5701 if (ixgbe->unicst_addr[slot].mac.set == 0) { 5702 mutex_exit(&ixgbe->gen_lock); 5703 return (EINVAL); 5704 } 5705 5706 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5707 (void) ixgbe_clear_rar(hw, slot); 5708 ixgbe->unicst_addr[slot].mac.set = 0; 5709 ixgbe->unicst_avail++; 5710 5711 mutex_exit(&ixgbe->gen_lock); 5712 5713 return (0); 5714 } 5715