1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright 2019 Joyent, Inc. 29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved. 31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved. 32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. 33 */ 34 35 #include "ixgbe_sw.h" 36 37 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 38 39 /* 40 * Local function protoypes 41 */ 42 static int ixgbe_register_mac(ixgbe_t *); 43 static int ixgbe_identify_hardware(ixgbe_t *); 44 static int ixgbe_regs_map(ixgbe_t *); 45 static void ixgbe_init_properties(ixgbe_t *); 46 static int ixgbe_init_driver_settings(ixgbe_t *); 47 static void ixgbe_init_locks(ixgbe_t *); 48 static void ixgbe_destroy_locks(ixgbe_t *); 49 static int ixgbe_init(ixgbe_t *); 50 static int ixgbe_chip_start(ixgbe_t *); 51 static void ixgbe_chip_stop(ixgbe_t *); 52 static int ixgbe_reset(ixgbe_t *); 53 static void ixgbe_tx_clean(ixgbe_t *); 54 static boolean_t ixgbe_tx_drain(ixgbe_t *); 55 static boolean_t ixgbe_rx_drain(ixgbe_t *); 56 static int ixgbe_alloc_rings(ixgbe_t *); 57 static void ixgbe_free_rings(ixgbe_t *); 58 static int ixgbe_alloc_rx_data(ixgbe_t *); 59 static void ixgbe_free_rx_data(ixgbe_t *); 60 static void ixgbe_setup_rings(ixgbe_t *); 61 static void ixgbe_setup_rx(ixgbe_t *); 62 static void ixgbe_setup_tx(ixgbe_t *); 63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 65 static void ixgbe_setup_rss(ixgbe_t *); 66 static void ixgbe_setup_vmdq(ixgbe_t *); 67 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 68 static void ixgbe_setup_rss_table(ixgbe_t *); 69 static void ixgbe_init_unicst(ixgbe_t *); 70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 71 static void ixgbe_setup_multicst(ixgbe_t *); 72 static void ixgbe_get_hw_state(ixgbe_t *); 73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 74 static void ixgbe_get_conf(ixgbe_t *); 75 static void ixgbe_init_params(ixgbe_t *); 76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 77 static void ixgbe_driver_link_check(ixgbe_t *); 78 static void ixgbe_sfp_check(void *); 79 static void ixgbe_overtemp_check(void *); 80 static void ixgbe_phy_check(void *); 81 static void ixgbe_link_timer(void *); 82 static void ixgbe_local_timer(void *); 83 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 84 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 87 static boolean_t is_valid_mac_addr(uint8_t *); 88 static boolean_t ixgbe_stall_check(ixgbe_t *); 89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 91 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 92 static int ixgbe_alloc_intrs(ixgbe_t *); 93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 94 static int ixgbe_add_intr_handlers(ixgbe_t *); 95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 102 static void ixgbe_setup_adapter_vector(ixgbe_t *); 103 static void ixgbe_rem_intr_handlers(ixgbe_t *); 104 static void ixgbe_rem_intrs(ixgbe_t *); 105 static int ixgbe_enable_intrs(ixgbe_t *); 106 static int ixgbe_disable_intrs(ixgbe_t *); 107 static uint_t ixgbe_intr_legacy(void *, void *); 108 static uint_t ixgbe_intr_msi(void *, void *); 109 static uint_t ixgbe_intr_msix(void *, void *); 110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 113 static void ixgbe_get_driver_control(struct ixgbe_hw *); 114 static int ixgbe_addmac(void *, const uint8_t *); 115 static int ixgbe_remmac(void *, const uint8_t *); 116 static void ixgbe_release_driver_control(struct ixgbe_hw *); 117 118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 120 static int ixgbe_resume(dev_info_t *); 121 static int ixgbe_suspend(dev_info_t *); 122 static int ixgbe_quiesce(dev_info_t *); 123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 126 static int ixgbe_intr_cb_register(ixgbe_t *); 127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 128 129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 130 const void *impl_data); 131 static void ixgbe_fm_init(ixgbe_t *); 132 static void ixgbe_fm_fini(ixgbe_t *); 133 134 char *ixgbe_priv_props[] = { 135 "_tx_copy_thresh", 136 "_tx_recycle_thresh", 137 "_tx_overload_thresh", 138 "_tx_resched_thresh", 139 "_rx_copy_thresh", 140 "_rx_limit_per_intr", 141 "_intr_throttling", 142 "_adv_pause_cap", 143 "_adv_asym_pause_cap", 144 NULL 145 }; 146 147 #define IXGBE_MAX_PRIV_PROPS \ 148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 149 150 static struct cb_ops ixgbe_cb_ops = { 151 nulldev, /* cb_open */ 152 nulldev, /* cb_close */ 153 nodev, /* cb_strategy */ 154 nodev, /* cb_print */ 155 nodev, /* cb_dump */ 156 nodev, /* cb_read */ 157 nodev, /* cb_write */ 158 nodev, /* cb_ioctl */ 159 nodev, /* cb_devmap */ 160 nodev, /* cb_mmap */ 161 nodev, /* cb_segmap */ 162 nochpoll, /* cb_chpoll */ 163 ddi_prop_op, /* cb_prop_op */ 164 NULL, /* cb_stream */ 165 D_MP | D_HOTPLUG, /* cb_flag */ 166 CB_REV, /* cb_rev */ 167 nodev, /* cb_aread */ 168 nodev /* cb_awrite */ 169 }; 170 171 static struct dev_ops ixgbe_dev_ops = { 172 DEVO_REV, /* devo_rev */ 173 0, /* devo_refcnt */ 174 NULL, /* devo_getinfo */ 175 nulldev, /* devo_identify */ 176 nulldev, /* devo_probe */ 177 ixgbe_attach, /* devo_attach */ 178 ixgbe_detach, /* devo_detach */ 179 nodev, /* devo_reset */ 180 &ixgbe_cb_ops, /* devo_cb_ops */ 181 NULL, /* devo_bus_ops */ 182 ddi_power, /* devo_power */ 183 ixgbe_quiesce, /* devo_quiesce */ 184 }; 185 186 static struct modldrv ixgbe_modldrv = { 187 &mod_driverops, /* Type of module. This one is a driver */ 188 ixgbe_ident, /* Discription string */ 189 &ixgbe_dev_ops /* driver ops */ 190 }; 191 192 static struct modlinkage ixgbe_modlinkage = { 193 MODREV_1, &ixgbe_modldrv, NULL 194 }; 195 196 /* 197 * Access attributes for register mapping 198 */ 199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 200 DDI_DEVICE_ATTR_V1, 201 DDI_STRUCTURE_LE_ACC, 202 DDI_STRICTORDER_ACC, 203 DDI_FLAGERR_ACC 204 }; 205 206 /* 207 * Loopback property 208 */ 209 static lb_property_t lb_normal = { 210 normal, "normal", IXGBE_LB_NONE 211 }; 212 213 static lb_property_t lb_mac = { 214 internal, "MAC", IXGBE_LB_INTERNAL_MAC 215 }; 216 217 static lb_property_t lb_external = { 218 external, "External", IXGBE_LB_EXTERNAL 219 }; 220 221 #define IXGBE_M_CALLBACK_FLAGS \ 222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 223 224 static mac_callbacks_t ixgbe_m_callbacks = { 225 IXGBE_M_CALLBACK_FLAGS, 226 ixgbe_m_stat, 227 ixgbe_m_start, 228 ixgbe_m_stop, 229 ixgbe_m_promisc, 230 ixgbe_m_multicst, 231 NULL, 232 NULL, 233 NULL, 234 ixgbe_m_ioctl, 235 ixgbe_m_getcapab, 236 NULL, 237 NULL, 238 ixgbe_m_setprop, 239 ixgbe_m_getprop, 240 ixgbe_m_propinfo 241 }; 242 243 /* 244 * Initialize capabilities of each supported adapter type 245 */ 246 static adapter_info_t ixgbe_82598eb_cap = { 247 64, /* maximum number of rx queues */ 248 1, /* minimum number of rx queues */ 249 64, /* default number of rx queues */ 250 16, /* maximum number of rx groups */ 251 1, /* minimum number of rx groups */ 252 1, /* default number of rx groups */ 253 32, /* maximum number of tx queues */ 254 1, /* minimum number of tx queues */ 255 8, /* default number of tx queues */ 256 16366, /* maximum MTU size */ 257 0xFFFF, /* maximum interrupt throttle rate */ 258 0, /* minimum interrupt throttle rate */ 259 200, /* default interrupt throttle rate */ 260 18, /* maximum total msix vectors */ 261 16, /* maximum number of ring vectors */ 262 2, /* maximum number of other vectors */ 263 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 264 0, /* "other" interrupt types enable mask */ 265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 266 | IXGBE_FLAG_RSS_CAPABLE 267 | IXGBE_FLAG_VMDQ_CAPABLE) 268 }; 269 270 static adapter_info_t ixgbe_82599eb_cap = { 271 128, /* maximum number of rx queues */ 272 1, /* minimum number of rx queues */ 273 128, /* default number of rx queues */ 274 64, /* maximum number of rx groups */ 275 1, /* minimum number of rx groups */ 276 1, /* default number of rx groups */ 277 128, /* maximum number of tx queues */ 278 1, /* minimum number of tx queues */ 279 8, /* default number of tx queues */ 280 15500, /* maximum MTU size */ 281 0xFF8, /* maximum interrupt throttle rate */ 282 0, /* minimum interrupt throttle rate */ 283 200, /* default interrupt throttle rate */ 284 64, /* maximum total msix vectors */ 285 16, /* maximum number of ring vectors */ 286 2, /* maximum number of other vectors */ 287 (IXGBE_EICR_LSC 288 | IXGBE_EICR_GPI_SDP1 289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 290 291 (IXGBE_SDP1_GPIEN 292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 293 294 (IXGBE_FLAG_DCA_CAPABLE 295 | IXGBE_FLAG_RSS_CAPABLE 296 | IXGBE_FLAG_VMDQ_CAPABLE 297 | IXGBE_FLAG_RSC_CAPABLE 298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 299 }; 300 301 static adapter_info_t ixgbe_X540_cap = { 302 128, /* maximum number of rx queues */ 303 1, /* minimum number of rx queues */ 304 128, /* default number of rx queues */ 305 64, /* maximum number of rx groups */ 306 1, /* minimum number of rx groups */ 307 1, /* default number of rx groups */ 308 128, /* maximum number of tx queues */ 309 1, /* minimum number of tx queues */ 310 8, /* default number of tx queues */ 311 15500, /* maximum MTU size */ 312 0xFF8, /* maximum interrupt throttle rate */ 313 0, /* minimum interrupt throttle rate */ 314 200, /* default interrupt throttle rate */ 315 64, /* maximum total msix vectors */ 316 16, /* maximum number of ring vectors */ 317 2, /* maximum number of other vectors */ 318 (IXGBE_EICR_LSC 319 | IXGBE_EICR_GPI_SDP1_X540 320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */ 321 322 (IXGBE_SDP1_GPIEN_X540 323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */ 324 325 (IXGBE_FLAG_DCA_CAPABLE 326 | IXGBE_FLAG_RSS_CAPABLE 327 | IXGBE_FLAG_VMDQ_CAPABLE 328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 329 }; 330 331 static adapter_info_t ixgbe_X550_cap = { 332 128, /* maximum number of rx queues */ 333 1, /* minimum number of rx queues */ 334 128, /* default number of rx queues */ 335 64, /* maximum number of rx groups */ 336 1, /* minimum number of rx groups */ 337 1, /* default number of rx groups */ 338 128, /* maximum number of tx queues */ 339 1, /* minimum number of tx queues */ 340 8, /* default number of tx queues */ 341 15500, /* maximum MTU size */ 342 0xFF8, /* maximum interrupt throttle rate */ 343 0, /* minimum interrupt throttle rate */ 344 0x200, /* default interrupt throttle rate */ 345 64, /* maximum total msix vectors */ 346 16, /* maximum number of ring vectors */ 347 2, /* maximum number of other vectors */ 348 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 349 0, /* "other" interrupt types enable mask */ 350 (IXGBE_FLAG_RSS_CAPABLE 351 | IXGBE_FLAG_VMDQ_CAPABLE 352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 353 }; 354 355 /* 356 * Module Initialization Functions. 357 */ 358 359 int 360 _init(void) 361 { 362 int status; 363 364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 365 366 status = mod_install(&ixgbe_modlinkage); 367 368 if (status != DDI_SUCCESS) { 369 mac_fini_ops(&ixgbe_dev_ops); 370 } 371 372 return (status); 373 } 374 375 int 376 _fini(void) 377 { 378 int status; 379 380 status = mod_remove(&ixgbe_modlinkage); 381 382 if (status == DDI_SUCCESS) { 383 mac_fini_ops(&ixgbe_dev_ops); 384 } 385 386 return (status); 387 } 388 389 int 390 _info(struct modinfo *modinfop) 391 { 392 int status; 393 394 status = mod_info(&ixgbe_modlinkage, modinfop); 395 396 return (status); 397 } 398 399 /* 400 * ixgbe_attach - Driver attach. 401 * 402 * This function is the device specific initialization entry 403 * point. This entry point is required and must be written. 404 * The DDI_ATTACH command must be provided in the attach entry 405 * point. When attach() is called with cmd set to DDI_ATTACH, 406 * all normal kernel services (such as kmem_alloc(9F)) are 407 * available for use by the driver. 408 * 409 * The attach() function will be called once for each instance 410 * of the device on the system with cmd set to DDI_ATTACH. 411 * Until attach() succeeds, the only driver entry points which 412 * may be called are open(9E) and getinfo(9E). 413 */ 414 static int 415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 416 { 417 ixgbe_t *ixgbe; 418 struct ixgbe_osdep *osdep; 419 struct ixgbe_hw *hw; 420 int instance; 421 char taskqname[32]; 422 423 /* 424 * Check the command and perform corresponding operations 425 */ 426 switch (cmd) { 427 default: 428 return (DDI_FAILURE); 429 430 case DDI_RESUME: 431 return (ixgbe_resume(devinfo)); 432 433 case DDI_ATTACH: 434 break; 435 } 436 437 /* Get the device instance */ 438 instance = ddi_get_instance(devinfo); 439 440 /* Allocate memory for the instance data structure */ 441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 442 443 ixgbe->dip = devinfo; 444 ixgbe->instance = instance; 445 446 hw = &ixgbe->hw; 447 osdep = &ixgbe->osdep; 448 hw->back = osdep; 449 osdep->ixgbe = ixgbe; 450 451 /* Attach the instance pointer to the dev_info data structure */ 452 ddi_set_driver_private(devinfo, ixgbe); 453 454 /* 455 * Initialize for FMA support 456 */ 457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 460 ixgbe_fm_init(ixgbe); 461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 462 463 /* 464 * Map PCI config space registers 465 */ 466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 467 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 468 goto attach_fail; 469 } 470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 471 472 /* 473 * Identify the chipset family 474 */ 475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 476 ixgbe_error(ixgbe, "Failed to identify hardware"); 477 goto attach_fail; 478 } 479 480 /* 481 * Map device registers 482 */ 483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 484 ixgbe_error(ixgbe, "Failed to map device registers"); 485 goto attach_fail; 486 } 487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 488 489 /* 490 * Initialize driver parameters 491 */ 492 ixgbe_init_properties(ixgbe); 493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 494 495 /* 496 * Register interrupt callback 497 */ 498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 499 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 500 goto attach_fail; 501 } 502 503 /* 504 * Allocate interrupts 505 */ 506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 507 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 508 goto attach_fail; 509 } 510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 511 512 /* 513 * Allocate rx/tx rings based on the ring numbers. 514 * The actual numbers of rx/tx rings are decided by the number of 515 * allocated interrupt vectors, so we should allocate the rings after 516 * interrupts are allocated. 517 */ 518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 520 goto attach_fail; 521 } 522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 523 524 /* 525 * Map rings to interrupt vectors 526 */ 527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 529 goto attach_fail; 530 } 531 532 /* 533 * Add interrupt handlers 534 */ 535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 536 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 537 goto attach_fail; 538 } 539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 540 541 /* 542 * Create a taskq for sfp-change 543 */ 544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 546 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 547 ixgbe_error(ixgbe, "sfp_taskq create failed"); 548 goto attach_fail; 549 } 550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 551 552 /* 553 * Create a taskq for over-temp 554 */ 555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 557 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 558 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 559 goto attach_fail; 560 } 561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 562 563 /* 564 * Create a taskq for processing external PHY interrupts 565 */ 566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance); 567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname, 568 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 569 ixgbe_error(ixgbe, "phy_taskq create failed"); 570 goto attach_fail; 571 } 572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ; 573 574 /* 575 * Initialize driver parameters 576 */ 577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 578 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 579 goto attach_fail; 580 } 581 582 /* 583 * Initialize mutexes for this device. 584 * Do this before enabling the interrupt handler and 585 * register the softint to avoid the condition where 586 * interrupt handler can try using uninitialized mutex. 587 */ 588 ixgbe_init_locks(ixgbe); 589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 590 591 /* 592 * Initialize chipset hardware 593 */ 594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 595 ixgbe_error(ixgbe, "Failed to initialize adapter"); 596 goto attach_fail; 597 } 598 ixgbe->link_check_complete = B_FALSE; 599 ixgbe->link_check_hrtime = gethrtime() + 600 (IXGBE_LINK_UP_TIME * 100000000ULL); 601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 602 603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 605 goto attach_fail; 606 } 607 608 /* 609 * Initialize adapter capabilities 610 */ 611 ixgbe_init_params(ixgbe); 612 613 /* 614 * Initialize statistics 615 */ 616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 617 ixgbe_error(ixgbe, "Failed to initialize statistics"); 618 goto attach_fail; 619 } 620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 621 622 /* 623 * Register the driver to the MAC 624 */ 625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 626 ixgbe_error(ixgbe, "Failed to register MAC"); 627 goto attach_fail; 628 } 629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 631 632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 634 if (ixgbe->periodic_id == 0) { 635 ixgbe_error(ixgbe, "Failed to add the link check timer"); 636 goto attach_fail; 637 } 638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 639 640 /* 641 * Now that mutex locks are initialized, and the chip is also 642 * initialized, enable interrupts. 643 */ 644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 646 goto attach_fail; 647 } 648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 649 650 ixgbe_log(ixgbe, "%s", ixgbe_ident); 651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 652 653 return (DDI_SUCCESS); 654 655 attach_fail: 656 ixgbe_unconfigure(devinfo, ixgbe); 657 return (DDI_FAILURE); 658 } 659 660 /* 661 * ixgbe_detach - Driver detach. 662 * 663 * The detach() function is the complement of the attach routine. 664 * If cmd is set to DDI_DETACH, detach() is used to remove the 665 * state associated with a given instance of a device node 666 * prior to the removal of that instance from the system. 667 * 668 * The detach() function will be called once for each instance 669 * of the device for which there has been a successful attach() 670 * once there are no longer any opens on the device. 671 * 672 * Interrupts routine are disabled, All memory allocated by this 673 * driver are freed. 674 */ 675 static int 676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 677 { 678 ixgbe_t *ixgbe; 679 680 /* 681 * Check detach command 682 */ 683 switch (cmd) { 684 default: 685 return (DDI_FAILURE); 686 687 case DDI_SUSPEND: 688 return (ixgbe_suspend(devinfo)); 689 690 case DDI_DETACH: 691 break; 692 } 693 694 /* 695 * Get the pointer to the driver private data structure 696 */ 697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 698 if (ixgbe == NULL) 699 return (DDI_FAILURE); 700 701 /* 702 * If the device is still running, it needs to be stopped first. 703 * This check is necessary because under some specific circumstances, 704 * the detach routine can be called without stopping the interface 705 * first. 706 */ 707 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 709 mutex_enter(&ixgbe->gen_lock); 710 ixgbe_stop(ixgbe, B_TRUE); 711 mutex_exit(&ixgbe->gen_lock); 712 /* Disable and stop the watchdog timer */ 713 ixgbe_disable_watchdog_timer(ixgbe); 714 } 715 716 /* 717 * Check if there are still rx buffers held by the upper layer. 718 * If so, fail the detach. 719 */ 720 if (!ixgbe_rx_drain(ixgbe)) 721 return (DDI_FAILURE); 722 723 /* 724 * Do the remaining unconfigure routines 725 */ 726 ixgbe_unconfigure(devinfo, ixgbe); 727 728 return (DDI_SUCCESS); 729 } 730 731 /* 732 * quiesce(9E) entry point. 733 * 734 * This function is called when the system is single-threaded at high 735 * PIL with preemption disabled. Therefore, this function must not be 736 * blocked. 737 * 738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 739 * DDI_FAILURE indicates an error condition and should almost never happen. 740 */ 741 static int 742 ixgbe_quiesce(dev_info_t *devinfo) 743 { 744 ixgbe_t *ixgbe; 745 struct ixgbe_hw *hw; 746 747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 748 749 if (ixgbe == NULL) 750 return (DDI_FAILURE); 751 752 hw = &ixgbe->hw; 753 754 /* 755 * Disable the adapter interrupts 756 */ 757 ixgbe_disable_adapter_interrupts(ixgbe); 758 759 /* 760 * Tell firmware driver is no longer in control 761 */ 762 ixgbe_release_driver_control(hw); 763 764 /* 765 * Reset the chipset 766 */ 767 (void) ixgbe_reset_hw(hw); 768 769 /* 770 * Reset PHY 771 */ 772 (void) ixgbe_reset_phy(hw); 773 774 return (DDI_SUCCESS); 775 } 776 777 static void 778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 779 { 780 /* 781 * Disable interrupt 782 */ 783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 784 (void) ixgbe_disable_intrs(ixgbe); 785 } 786 787 /* 788 * remove the link check timer 789 */ 790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 791 if (ixgbe->periodic_id != NULL) { 792 ddi_periodic_delete(ixgbe->periodic_id); 793 ixgbe->periodic_id = NULL; 794 } 795 } 796 797 /* 798 * Unregister MAC 799 */ 800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 801 (void) mac_unregister(ixgbe->mac_hdl); 802 } 803 804 /* 805 * Free statistics 806 */ 807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 809 } 810 811 /* 812 * Remove interrupt handlers 813 */ 814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 815 ixgbe_rem_intr_handlers(ixgbe); 816 } 817 818 /* 819 * Remove taskq for sfp-status-change 820 */ 821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 822 ddi_taskq_destroy(ixgbe->sfp_taskq); 823 } 824 825 /* 826 * Remove taskq for over-temp 827 */ 828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 829 ddi_taskq_destroy(ixgbe->overtemp_taskq); 830 } 831 832 /* 833 * Remove taskq for external PHYs 834 */ 835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) { 836 ddi_taskq_destroy(ixgbe->phy_taskq); 837 } 838 839 /* 840 * Remove interrupts 841 */ 842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 843 ixgbe_rem_intrs(ixgbe); 844 } 845 846 /* 847 * Unregister interrupt callback handler 848 */ 849 if (ixgbe->cb_hdl != NULL) { 850 (void) ddi_cb_unregister(ixgbe->cb_hdl); 851 } 852 853 /* 854 * Remove driver properties 855 */ 856 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 857 (void) ddi_prop_remove_all(devinfo); 858 } 859 860 /* 861 * Stop the chipset 862 */ 863 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 864 mutex_enter(&ixgbe->gen_lock); 865 ixgbe_chip_stop(ixgbe); 866 mutex_exit(&ixgbe->gen_lock); 867 } 868 869 /* 870 * Free register handle 871 */ 872 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 873 if (ixgbe->osdep.reg_handle != NULL) 874 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 875 } 876 877 /* 878 * Free PCI config handle 879 */ 880 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 881 if (ixgbe->osdep.cfg_handle != NULL) 882 pci_config_teardown(&ixgbe->osdep.cfg_handle); 883 } 884 885 /* 886 * Free locks 887 */ 888 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 889 ixgbe_destroy_locks(ixgbe); 890 } 891 892 /* 893 * Free the rx/tx rings 894 */ 895 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 896 ixgbe_free_rings(ixgbe); 897 } 898 899 /* 900 * Unregister FMA capabilities 901 */ 902 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 903 ixgbe_fm_fini(ixgbe); 904 } 905 906 /* 907 * Free the driver data structure 908 */ 909 kmem_free(ixgbe, sizeof (ixgbe_t)); 910 911 ddi_set_driver_private(devinfo, NULL); 912 } 913 914 /* 915 * ixgbe_register_mac - Register the driver and its function pointers with 916 * the GLD interface. 917 */ 918 static int 919 ixgbe_register_mac(ixgbe_t *ixgbe) 920 { 921 struct ixgbe_hw *hw = &ixgbe->hw; 922 mac_register_t *mac; 923 int status; 924 925 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 926 return (IXGBE_FAILURE); 927 928 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 929 mac->m_driver = ixgbe; 930 mac->m_dip = ixgbe->dip; 931 mac->m_src_addr = hw->mac.addr; 932 mac->m_callbacks = &ixgbe_m_callbacks; 933 mac->m_min_sdu = 0; 934 mac->m_max_sdu = ixgbe->default_mtu; 935 mac->m_margin = VLAN_TAGSZ; 936 mac->m_priv_props = ixgbe_priv_props; 937 mac->m_v12n = MAC_VIRT_LEVEL1; 938 939 status = mac_register(mac, &ixgbe->mac_hdl); 940 941 mac_free(mac); 942 943 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 944 } 945 946 /* 947 * ixgbe_identify_hardware - Identify the type of the chipset. 948 */ 949 static int 950 ixgbe_identify_hardware(ixgbe_t *ixgbe) 951 { 952 struct ixgbe_hw *hw = &ixgbe->hw; 953 struct ixgbe_osdep *osdep = &ixgbe->osdep; 954 955 /* 956 * Get the device id 957 */ 958 hw->vendor_id = 959 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 960 hw->device_id = 961 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 962 hw->revision_id = 963 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 964 hw->subsystem_device_id = 965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 966 hw->subsystem_vendor_id = 967 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 968 969 /* 970 * Set the mac type of the adapter based on the device id 971 */ 972 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 973 return (IXGBE_FAILURE); 974 } 975 976 /* 977 * Install adapter capabilities 978 */ 979 switch (hw->mac.type) { 980 case ixgbe_mac_82598EB: 981 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 982 ixgbe->capab = &ixgbe_82598eb_cap; 983 984 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 985 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 986 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 987 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 988 } 989 break; 990 991 case ixgbe_mac_82599EB: 992 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 993 ixgbe->capab = &ixgbe_82599eb_cap; 994 995 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 996 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 997 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 998 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 999 } 1000 break; 1001 1002 case ixgbe_mac_X540: 1003 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n"); 1004 ixgbe->capab = &ixgbe_X540_cap; 1005 /* 1006 * For now, X540 is all set in its capab structure. 1007 * As other X540 variants show up, things can change here. 1008 */ 1009 break; 1010 1011 case ixgbe_mac_X550: 1012 case ixgbe_mac_X550EM_x: 1013 case ixgbe_mac_X550EM_a: 1014 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n"); 1015 ixgbe->capab = &ixgbe_X550_cap; 1016 1017 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 1018 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 1019 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 1020 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP || 1021 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) { 1022 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE; 1023 } 1024 1025 /* 1026 * Link detection on X552 SFP+ and X552/X557-AT 1027 */ 1028 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 1029 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 1030 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 1031 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 1032 ixgbe->capab->other_intr |= 1033 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 1034 } 1035 if (hw->phy.type == ixgbe_phy_x550em_ext_t) { 1036 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540; 1037 } 1038 break; 1039 1040 default: 1041 IXGBE_DEBUGLOG_1(ixgbe, 1042 "adapter not supported in ixgbe_identify_hardware(): %d\n", 1043 hw->mac.type); 1044 return (IXGBE_FAILURE); 1045 } 1046 1047 return (IXGBE_SUCCESS); 1048 } 1049 1050 /* 1051 * ixgbe_regs_map - Map the device registers. 1052 * 1053 */ 1054 static int 1055 ixgbe_regs_map(ixgbe_t *ixgbe) 1056 { 1057 dev_info_t *devinfo = ixgbe->dip; 1058 struct ixgbe_hw *hw = &ixgbe->hw; 1059 struct ixgbe_osdep *osdep = &ixgbe->osdep; 1060 off_t mem_size; 1061 1062 /* 1063 * First get the size of device registers to be mapped. 1064 */ 1065 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 1066 != DDI_SUCCESS) { 1067 return (IXGBE_FAILURE); 1068 } 1069 1070 /* 1071 * Call ddi_regs_map_setup() to map registers 1072 */ 1073 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 1074 (caddr_t *)&hw->hw_addr, 0, 1075 mem_size, &ixgbe_regs_acc_attr, 1076 &osdep->reg_handle)) != DDI_SUCCESS) { 1077 return (IXGBE_FAILURE); 1078 } 1079 1080 return (IXGBE_SUCCESS); 1081 } 1082 1083 /* 1084 * ixgbe_init_properties - Initialize driver properties. 1085 */ 1086 static void 1087 ixgbe_init_properties(ixgbe_t *ixgbe) 1088 { 1089 /* 1090 * Get conf file properties, including link settings 1091 * jumbo frames, ring number, descriptor number, etc. 1092 */ 1093 ixgbe_get_conf(ixgbe); 1094 } 1095 1096 /* 1097 * ixgbe_init_driver_settings - Initialize driver settings. 1098 * 1099 * The settings include hardware function pointers, bus information, 1100 * rx/tx rings settings, link state, and any other parameters that 1101 * need to be setup during driver initialization. 1102 */ 1103 static int 1104 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 1105 { 1106 struct ixgbe_hw *hw = &ixgbe->hw; 1107 dev_info_t *devinfo = ixgbe->dip; 1108 ixgbe_rx_ring_t *rx_ring; 1109 ixgbe_rx_group_t *rx_group; 1110 ixgbe_tx_ring_t *tx_ring; 1111 uint32_t rx_size; 1112 uint32_t tx_size; 1113 uint32_t ring_per_group; 1114 int i; 1115 1116 /* 1117 * Initialize chipset specific hardware function pointers 1118 */ 1119 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 1120 return (IXGBE_FAILURE); 1121 } 1122 1123 /* 1124 * Get the system page size 1125 */ 1126 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 1127 1128 /* 1129 * Set rx buffer size 1130 * 1131 * The IP header alignment room is counted in the calculation. 1132 * The rx buffer size is in unit of 1K that is required by the 1133 * chipset hardware. 1134 */ 1135 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 1136 ixgbe->rx_buf_size = ((rx_size >> 10) + 1137 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1138 1139 /* 1140 * Set tx buffer size 1141 */ 1142 tx_size = ixgbe->max_frame_size; 1143 ixgbe->tx_buf_size = ((tx_size >> 10) + 1144 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1145 1146 /* 1147 * Initialize rx/tx rings/groups parameters 1148 */ 1149 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 1150 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1151 rx_ring = &ixgbe->rx_rings[i]; 1152 rx_ring->index = i; 1153 rx_ring->ixgbe = ixgbe; 1154 rx_ring->group_index = i / ring_per_group; 1155 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 1156 } 1157 1158 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1159 rx_group = &ixgbe->rx_groups[i]; 1160 rx_group->index = i; 1161 rx_group->ixgbe = ixgbe; 1162 } 1163 1164 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1165 tx_ring = &ixgbe->tx_rings[i]; 1166 tx_ring->index = i; 1167 tx_ring->ixgbe = ixgbe; 1168 if (ixgbe->tx_head_wb_enable) 1169 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1170 else 1171 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1172 1173 tx_ring->ring_size = ixgbe->tx_ring_size; 1174 tx_ring->free_list_size = ixgbe->tx_ring_size + 1175 (ixgbe->tx_ring_size >> 1); 1176 } 1177 1178 /* 1179 * Initialize values of interrupt throttling rate 1180 */ 1181 for (i = 1; i < MAX_INTR_VECTOR; i++) 1182 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1183 1184 /* 1185 * The initial link state should be "unknown" 1186 */ 1187 ixgbe->link_state = LINK_STATE_UNKNOWN; 1188 1189 return (IXGBE_SUCCESS); 1190 } 1191 1192 /* 1193 * ixgbe_init_locks - Initialize locks. 1194 */ 1195 static void 1196 ixgbe_init_locks(ixgbe_t *ixgbe) 1197 { 1198 ixgbe_rx_ring_t *rx_ring; 1199 ixgbe_tx_ring_t *tx_ring; 1200 int i; 1201 1202 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1203 rx_ring = &ixgbe->rx_rings[i]; 1204 mutex_init(&rx_ring->rx_lock, NULL, 1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1206 } 1207 1208 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1209 tx_ring = &ixgbe->tx_rings[i]; 1210 mutex_init(&tx_ring->tx_lock, NULL, 1211 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1212 mutex_init(&tx_ring->recycle_lock, NULL, 1213 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1214 mutex_init(&tx_ring->tcb_head_lock, NULL, 1215 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1216 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1217 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1218 } 1219 1220 mutex_init(&ixgbe->gen_lock, NULL, 1221 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1222 1223 mutex_init(&ixgbe->watchdog_lock, NULL, 1224 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1225 } 1226 1227 /* 1228 * ixgbe_destroy_locks - Destroy locks. 1229 */ 1230 static void 1231 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1232 { 1233 ixgbe_rx_ring_t *rx_ring; 1234 ixgbe_tx_ring_t *tx_ring; 1235 int i; 1236 1237 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1238 rx_ring = &ixgbe->rx_rings[i]; 1239 mutex_destroy(&rx_ring->rx_lock); 1240 } 1241 1242 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1243 tx_ring = &ixgbe->tx_rings[i]; 1244 mutex_destroy(&tx_ring->tx_lock); 1245 mutex_destroy(&tx_ring->recycle_lock); 1246 mutex_destroy(&tx_ring->tcb_head_lock); 1247 mutex_destroy(&tx_ring->tcb_tail_lock); 1248 } 1249 1250 mutex_destroy(&ixgbe->gen_lock); 1251 mutex_destroy(&ixgbe->watchdog_lock); 1252 } 1253 1254 /* 1255 * We need to try and determine which LED index in hardware corresponds to the 1256 * link/activity LED. This is the one that'll be overwritten when we perform 1257 * GLDv3 LED activity. 1258 */ 1259 static void 1260 ixgbe_led_init(ixgbe_t *ixgbe) 1261 { 1262 uint32_t reg, i; 1263 struct ixgbe_hw *hw = &ixgbe->hw; 1264 1265 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL); 1266 for (i = 0; i < 4; i++) { 1267 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) & 1268 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) { 1269 ixgbe->ixgbe_led_index = i; 1270 return; 1271 } 1272 } 1273 1274 /* 1275 * If we couldn't determine this, we use the default for various MACs 1276 * based on information Intel has inserted into other drivers over the 1277 * years. 1278 */ 1279 switch (hw->mac.type) { 1280 case ixgbe_mac_X550EM_a: 1281 ixgbe->ixgbe_led_index = 0; 1282 break; 1283 case ixgbe_mac_X550EM_x: 1284 ixgbe->ixgbe_led_index = 1; 1285 break; 1286 default: 1287 ixgbe->ixgbe_led_index = 2; 1288 break; 1289 } 1290 } 1291 1292 static int 1293 ixgbe_resume(dev_info_t *devinfo) 1294 { 1295 ixgbe_t *ixgbe; 1296 int i; 1297 1298 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1299 if (ixgbe == NULL) 1300 return (DDI_FAILURE); 1301 1302 mutex_enter(&ixgbe->gen_lock); 1303 1304 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1305 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1306 mutex_exit(&ixgbe->gen_lock); 1307 return (DDI_FAILURE); 1308 } 1309 1310 /* 1311 * Enable and start the watchdog timer 1312 */ 1313 ixgbe_enable_watchdog_timer(ixgbe); 1314 } 1315 1316 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1317 1318 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1319 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1320 mac_tx_ring_update(ixgbe->mac_hdl, 1321 ixgbe->tx_rings[i].ring_handle); 1322 } 1323 } 1324 1325 mutex_exit(&ixgbe->gen_lock); 1326 1327 return (DDI_SUCCESS); 1328 } 1329 1330 static int 1331 ixgbe_suspend(dev_info_t *devinfo) 1332 { 1333 ixgbe_t *ixgbe; 1334 1335 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1336 if (ixgbe == NULL) 1337 return (DDI_FAILURE); 1338 1339 mutex_enter(&ixgbe->gen_lock); 1340 1341 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1342 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1343 mutex_exit(&ixgbe->gen_lock); 1344 return (DDI_SUCCESS); 1345 } 1346 ixgbe_stop(ixgbe, B_FALSE); 1347 1348 mutex_exit(&ixgbe->gen_lock); 1349 1350 /* 1351 * Disable and stop the watchdog timer 1352 */ 1353 ixgbe_disable_watchdog_timer(ixgbe); 1354 1355 return (DDI_SUCCESS); 1356 } 1357 1358 /* 1359 * ixgbe_init - Initialize the device. 1360 */ 1361 static int 1362 ixgbe_init(ixgbe_t *ixgbe) 1363 { 1364 struct ixgbe_hw *hw = &ixgbe->hw; 1365 u8 pbanum[IXGBE_PBANUM_LENGTH]; 1366 int rv; 1367 1368 mutex_enter(&ixgbe->gen_lock); 1369 1370 /* 1371 * Configure/Initialize hardware 1372 */ 1373 rv = ixgbe_init_hw(hw); 1374 if (rv != IXGBE_SUCCESS) { 1375 switch (rv) { 1376 1377 /* 1378 * The first three errors are not prohibitive to us progressing 1379 * further, and are maily advisory in nature. In the case of a 1380 * SFP module not being present or not deemed supported by the 1381 * common code, we adivse the operator of this fact but carry on 1382 * instead of failing hard, as SFPs can be inserted or replaced 1383 * while the driver is running. In the case of a unknown error, 1384 * we fail-hard, logging the reason and emitting a FMA event. 1385 */ 1386 case IXGBE_ERR_EEPROM_VERSION: 1387 ixgbe_error(ixgbe, 1388 "This Intel 10Gb Ethernet device is pre-release and" 1389 " contains outdated firmware. Please contact your" 1390 " hardware vendor for a replacement."); 1391 break; 1392 case IXGBE_ERR_SFP_NOT_PRESENT: 1393 ixgbe_error(ixgbe, 1394 "No SFP+ module detected on this interface. Please " 1395 "install a supported SFP+ module for this " 1396 "interface to become operational."); 1397 break; 1398 case IXGBE_ERR_SFP_NOT_SUPPORTED: 1399 ixgbe_error(ixgbe, 1400 "Unsupported SFP+ module detected. Please replace " 1401 "it with a supported SFP+ module per Intel " 1402 "documentation, or bypass this check with " 1403 "allow_unsupported_sfp=1 in ixgbe.conf."); 1404 break; 1405 default: 1406 ixgbe_error(ixgbe, 1407 "Failed to initialize hardware. ixgbe_init_hw " 1408 "returned %d", rv); 1409 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1410 goto init_fail; 1411 } 1412 } 1413 1414 /* 1415 * Need to init eeprom before validating the checksum. 1416 */ 1417 if (ixgbe_init_eeprom_params(hw) < 0) { 1418 ixgbe_error(ixgbe, 1419 "Unable to intitialize the eeprom interface."); 1420 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1421 goto init_fail; 1422 } 1423 1424 /* 1425 * NVM validation 1426 */ 1427 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1428 /* 1429 * Some PCI-E parts fail the first check due to 1430 * the link being in sleep state. Call it again, 1431 * if it fails a second time it's a real issue. 1432 */ 1433 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1434 ixgbe_error(ixgbe, 1435 "Invalid NVM checksum. Please contact " 1436 "the vendor to update the NVM."); 1437 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1438 goto init_fail; 1439 } 1440 } 1441 1442 /* 1443 * Setup default flow control thresholds - enable/disable 1444 * & flow control type is controlled by ixgbe.conf 1445 */ 1446 hw->fc.high_water[0] = DEFAULT_FCRTH; 1447 hw->fc.low_water[0] = DEFAULT_FCRTL; 1448 hw->fc.pause_time = DEFAULT_FCPAUSE; 1449 hw->fc.send_xon = B_TRUE; 1450 1451 /* 1452 * Initialize flow control 1453 */ 1454 (void) ixgbe_start_hw(hw); 1455 1456 /* 1457 * Initialize link settings 1458 */ 1459 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1460 1461 /* 1462 * Initialize the chipset hardware 1463 */ 1464 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1465 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1466 goto init_fail; 1467 } 1468 1469 /* 1470 * Read identifying information and place in devinfo. 1471 */ 1472 pbanum[0] = '\0'; 1473 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum)); 1474 if (*pbanum != '\0') { 1475 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip, 1476 "printed-board-assembly", (char *)pbanum); 1477 } 1478 1479 /* 1480 * Determine LED index. 1481 */ 1482 ixgbe_led_init(ixgbe); 1483 1484 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1485 goto init_fail; 1486 } 1487 1488 mutex_exit(&ixgbe->gen_lock); 1489 return (IXGBE_SUCCESS); 1490 1491 init_fail: 1492 /* 1493 * Reset PHY 1494 */ 1495 (void) ixgbe_reset_phy(hw); 1496 1497 mutex_exit(&ixgbe->gen_lock); 1498 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1499 return (IXGBE_FAILURE); 1500 } 1501 1502 /* 1503 * ixgbe_chip_start - Initialize and start the chipset hardware. 1504 */ 1505 static int 1506 ixgbe_chip_start(ixgbe_t *ixgbe) 1507 { 1508 struct ixgbe_hw *hw = &ixgbe->hw; 1509 int i; 1510 1511 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1512 1513 /* 1514 * Get the mac address 1515 * This function should handle SPARC case correctly. 1516 */ 1517 if (!ixgbe_find_mac_address(ixgbe)) { 1518 ixgbe_error(ixgbe, "Failed to get the mac address"); 1519 return (IXGBE_FAILURE); 1520 } 1521 1522 /* 1523 * Validate the mac address 1524 */ 1525 (void) ixgbe_init_rx_addrs(hw); 1526 if (!is_valid_mac_addr(hw->mac.addr)) { 1527 ixgbe_error(ixgbe, "Invalid mac address"); 1528 return (IXGBE_FAILURE); 1529 } 1530 1531 /* 1532 * Re-enable relaxed ordering for performance. It is disabled 1533 * by default in the hardware init. 1534 */ 1535 if (ixgbe->relax_order_enable == B_TRUE) 1536 ixgbe_enable_relaxed_ordering(hw); 1537 1538 /* 1539 * Setup adapter interrupt vectors 1540 */ 1541 ixgbe_setup_adapter_vector(ixgbe); 1542 1543 /* 1544 * Initialize unicast addresses. 1545 */ 1546 ixgbe_init_unicst(ixgbe); 1547 1548 /* 1549 * Setup and initialize the mctable structures. 1550 */ 1551 ixgbe_setup_multicst(ixgbe); 1552 1553 /* 1554 * Set interrupt throttling rate 1555 */ 1556 for (i = 0; i < ixgbe->intr_cnt; i++) { 1557 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1558 } 1559 1560 /* 1561 * Disable Wake-on-LAN 1562 */ 1563 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 1564 1565 /* 1566 * Some adapters offer Energy Efficient Ethernet (EEE) support. 1567 * Due to issues with EEE in e1000g/igb, we disable this by default 1568 * as a precautionary measure. 1569 * 1570 * Currently, this is present on a number of the X550 family parts. 1571 */ 1572 (void) ixgbe_setup_eee(hw, B_FALSE); 1573 1574 /* 1575 * Turn on any present SFP Tx laser 1576 */ 1577 ixgbe_enable_tx_laser(hw); 1578 1579 /* 1580 * Power on the PHY 1581 */ 1582 (void) ixgbe_set_phy_power(hw, B_TRUE); 1583 1584 /* 1585 * Save the state of the PHY 1586 */ 1587 ixgbe_get_hw_state(ixgbe); 1588 1589 /* 1590 * Make sure driver has control 1591 */ 1592 ixgbe_get_driver_control(hw); 1593 1594 return (IXGBE_SUCCESS); 1595 } 1596 1597 /* 1598 * ixgbe_chip_stop - Stop the chipset hardware 1599 */ 1600 static void 1601 ixgbe_chip_stop(ixgbe_t *ixgbe) 1602 { 1603 struct ixgbe_hw *hw = &ixgbe->hw; 1604 int rv; 1605 1606 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1607 1608 /* 1609 * Stop interupt generation and disable Tx unit 1610 */ 1611 hw->adapter_stopped = B_FALSE; 1612 (void) ixgbe_stop_adapter(hw); 1613 1614 /* 1615 * Reset the chipset 1616 */ 1617 (void) ixgbe_reset_hw(hw); 1618 1619 /* 1620 * Reset PHY 1621 */ 1622 (void) ixgbe_reset_phy(hw); 1623 1624 /* 1625 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting 1626 * the PHY while doing so. Else, just power down the PHY. 1627 */ 1628 if (hw->phy.ops.enter_lplu != NULL) { 1629 hw->phy.reset_disable = B_TRUE; 1630 rv = hw->phy.ops.enter_lplu(hw); 1631 if (rv != IXGBE_SUCCESS) 1632 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv); 1633 hw->phy.reset_disable = B_FALSE; 1634 } else { 1635 (void) ixgbe_set_phy_power(hw, B_FALSE); 1636 } 1637 1638 /* 1639 * Turn off any present SFP Tx laser 1640 * Expected for health and safety reasons 1641 */ 1642 ixgbe_disable_tx_laser(hw); 1643 1644 /* 1645 * Tell firmware driver is no longer in control 1646 */ 1647 ixgbe_release_driver_control(hw); 1648 1649 } 1650 1651 /* 1652 * ixgbe_reset - Reset the chipset and re-start the driver. 1653 * 1654 * It involves stopping and re-starting the chipset, 1655 * and re-configuring the rx/tx rings. 1656 */ 1657 static int 1658 ixgbe_reset(ixgbe_t *ixgbe) 1659 { 1660 int i; 1661 1662 /* 1663 * Disable and stop the watchdog timer 1664 */ 1665 ixgbe_disable_watchdog_timer(ixgbe); 1666 1667 mutex_enter(&ixgbe->gen_lock); 1668 1669 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1670 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1671 1672 ixgbe_stop(ixgbe, B_FALSE); 1673 1674 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1675 mutex_exit(&ixgbe->gen_lock); 1676 return (IXGBE_FAILURE); 1677 } 1678 1679 /* 1680 * After resetting, need to recheck the link status. 1681 */ 1682 ixgbe->link_check_complete = B_FALSE; 1683 ixgbe->link_check_hrtime = gethrtime() + 1684 (IXGBE_LINK_UP_TIME * 100000000ULL); 1685 1686 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1687 1688 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1689 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1690 mac_tx_ring_update(ixgbe->mac_hdl, 1691 ixgbe->tx_rings[i].ring_handle); 1692 } 1693 } 1694 1695 mutex_exit(&ixgbe->gen_lock); 1696 1697 /* 1698 * Enable and start the watchdog timer 1699 */ 1700 ixgbe_enable_watchdog_timer(ixgbe); 1701 1702 return (IXGBE_SUCCESS); 1703 } 1704 1705 /* 1706 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1707 */ 1708 static void 1709 ixgbe_tx_clean(ixgbe_t *ixgbe) 1710 { 1711 ixgbe_tx_ring_t *tx_ring; 1712 tx_control_block_t *tcb; 1713 link_list_t pending_list; 1714 uint32_t desc_num; 1715 int i, j; 1716 1717 LINK_LIST_INIT(&pending_list); 1718 1719 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1720 tx_ring = &ixgbe->tx_rings[i]; 1721 1722 mutex_enter(&tx_ring->recycle_lock); 1723 1724 /* 1725 * Clean the pending tx data - the pending packets in the 1726 * work_list that have no chances to be transmitted again. 1727 * 1728 * We must ensure the chipset is stopped or the link is down 1729 * before cleaning the transmit packets. 1730 */ 1731 desc_num = 0; 1732 for (j = 0; j < tx_ring->ring_size; j++) { 1733 tcb = tx_ring->work_list[j]; 1734 if (tcb != NULL) { 1735 desc_num += tcb->desc_num; 1736 1737 tx_ring->work_list[j] = NULL; 1738 1739 ixgbe_free_tcb(tcb); 1740 1741 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1742 } 1743 } 1744 1745 if (desc_num > 0) { 1746 atomic_add_32(&tx_ring->tbd_free, desc_num); 1747 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1748 1749 /* 1750 * Reset the head and tail pointers of the tbd ring; 1751 * Reset the writeback head if it's enable. 1752 */ 1753 tx_ring->tbd_head = 0; 1754 tx_ring->tbd_tail = 0; 1755 if (ixgbe->tx_head_wb_enable) 1756 *tx_ring->tbd_head_wb = 0; 1757 1758 IXGBE_WRITE_REG(&ixgbe->hw, 1759 IXGBE_TDH(tx_ring->index), 0); 1760 IXGBE_WRITE_REG(&ixgbe->hw, 1761 IXGBE_TDT(tx_ring->index), 0); 1762 } 1763 1764 mutex_exit(&tx_ring->recycle_lock); 1765 1766 /* 1767 * Add the tx control blocks in the pending list to 1768 * the free list. 1769 */ 1770 ixgbe_put_free_list(tx_ring, &pending_list); 1771 } 1772 } 1773 1774 /* 1775 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1776 * transmitted. 1777 */ 1778 static boolean_t 1779 ixgbe_tx_drain(ixgbe_t *ixgbe) 1780 { 1781 ixgbe_tx_ring_t *tx_ring; 1782 boolean_t done; 1783 int i, j; 1784 1785 /* 1786 * Wait for a specific time to allow pending tx packets 1787 * to be transmitted. 1788 * 1789 * Check the counter tbd_free to see if transmission is done. 1790 * No lock protection is needed here. 1791 * 1792 * Return B_TRUE if all pending packets have been transmitted; 1793 * Otherwise return B_FALSE; 1794 */ 1795 for (i = 0; i < TX_DRAIN_TIME; i++) { 1796 1797 done = B_TRUE; 1798 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1799 tx_ring = &ixgbe->tx_rings[j]; 1800 done = done && 1801 (tx_ring->tbd_free == tx_ring->ring_size); 1802 } 1803 1804 if (done) 1805 break; 1806 1807 msec_delay(1); 1808 } 1809 1810 return (done); 1811 } 1812 1813 /* 1814 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1815 */ 1816 static boolean_t 1817 ixgbe_rx_drain(ixgbe_t *ixgbe) 1818 { 1819 boolean_t done = B_TRUE; 1820 int i; 1821 1822 /* 1823 * Polling the rx free list to check if those rx buffers held by 1824 * the upper layer are released. 1825 * 1826 * Check the counter rcb_free to see if all pending buffers are 1827 * released. No lock protection is needed here. 1828 * 1829 * Return B_TRUE if all pending buffers have been released; 1830 * Otherwise return B_FALSE; 1831 */ 1832 for (i = 0; i < RX_DRAIN_TIME; i++) { 1833 done = (ixgbe->rcb_pending == 0); 1834 1835 if (done) 1836 break; 1837 1838 msec_delay(1); 1839 } 1840 1841 return (done); 1842 } 1843 1844 /* 1845 * ixgbe_start - Start the driver/chipset. 1846 */ 1847 int 1848 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1849 { 1850 struct ixgbe_hw *hw = &ixgbe->hw; 1851 int i; 1852 1853 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1854 1855 if (alloc_buffer) { 1856 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1857 ixgbe_error(ixgbe, 1858 "Failed to allocate software receive rings"); 1859 return (IXGBE_FAILURE); 1860 } 1861 1862 /* Allocate buffers for all the rx/tx rings */ 1863 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1864 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1865 return (IXGBE_FAILURE); 1866 } 1867 1868 ixgbe->tx_ring_init = B_TRUE; 1869 } else { 1870 ixgbe->tx_ring_init = B_FALSE; 1871 } 1872 1873 for (i = 0; i < ixgbe->num_rx_rings; i++) 1874 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1875 for (i = 0; i < ixgbe->num_tx_rings; i++) 1876 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1877 1878 /* 1879 * Start the chipset hardware 1880 */ 1881 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1882 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1883 goto start_failure; 1884 } 1885 1886 /* 1887 * Configure link now for X550 1888 * 1889 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the 1890 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550, 1891 * the resting state of the link would be the maximum speed that 1892 * autonegotiation will allow (usually 10Gb, infrastructure allowing) 1893 * so we never bothered with explicitly setting the link to 10Gb as it 1894 * would already be at that state on driver attach. With X550, we must 1895 * trigger a re-negotiation of the link in order to switch from a LPLU 1896 * 1Gb link to 10Gb (cable and link partner permitting.) 1897 */ 1898 if (hw->mac.type == ixgbe_mac_X550 || 1899 hw->mac.type == ixgbe_mac_X550EM_a || 1900 hw->mac.type == ixgbe_mac_X550EM_x) { 1901 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE); 1902 ixgbe_get_hw_state(ixgbe); 1903 } 1904 1905 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1906 goto start_failure; 1907 } 1908 1909 /* 1910 * Setup the rx/tx rings 1911 */ 1912 ixgbe_setup_rings(ixgbe); 1913 1914 /* 1915 * ixgbe_start() will be called when resetting, however if reset 1916 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1917 * before enabling the interrupts. 1918 */ 1919 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1920 | IXGBE_STALL| IXGBE_OVERTEMP)); 1921 1922 /* 1923 * Enable adapter interrupts 1924 * The interrupts must be enabled after the driver state is START 1925 */ 1926 ixgbe_enable_adapter_interrupts(ixgbe); 1927 1928 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1929 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1930 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1931 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1932 1933 return (IXGBE_SUCCESS); 1934 1935 start_failure: 1936 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1937 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1938 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1939 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1940 1941 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1942 1943 return (IXGBE_FAILURE); 1944 } 1945 1946 /* 1947 * ixgbe_stop - Stop the driver/chipset. 1948 */ 1949 void 1950 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1951 { 1952 int i; 1953 1954 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1955 1956 /* 1957 * Disable the adapter interrupts 1958 */ 1959 ixgbe_disable_adapter_interrupts(ixgbe); 1960 1961 /* 1962 * Drain the pending tx packets 1963 */ 1964 (void) ixgbe_tx_drain(ixgbe); 1965 1966 for (i = 0; i < ixgbe->num_rx_rings; i++) 1967 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1968 for (i = 0; i < ixgbe->num_tx_rings; i++) 1969 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1970 1971 /* 1972 * Stop the chipset hardware 1973 */ 1974 ixgbe_chip_stop(ixgbe); 1975 1976 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1977 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1978 } 1979 1980 /* 1981 * Clean the pending tx data/resources 1982 */ 1983 ixgbe_tx_clean(ixgbe); 1984 1985 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1986 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1987 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1988 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1989 1990 if (ixgbe->link_state == LINK_STATE_UP) { 1991 ixgbe->link_state = LINK_STATE_UNKNOWN; 1992 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1993 } 1994 1995 if (free_buffer) { 1996 /* 1997 * Release the DMA/memory resources of rx/tx rings 1998 */ 1999 ixgbe_free_dma(ixgbe); 2000 ixgbe_free_rx_data(ixgbe); 2001 } 2002 } 2003 2004 /* 2005 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 2006 */ 2007 /* ARGSUSED */ 2008 static int 2009 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 2010 void *arg1, void *arg2) 2011 { 2012 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 2013 2014 switch (cbaction) { 2015 /* IRM callback */ 2016 int count; 2017 case DDI_CB_INTR_ADD: 2018 case DDI_CB_INTR_REMOVE: 2019 count = (int)(uintptr_t)cbarg; 2020 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 2021 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 2022 int, ixgbe->intr_cnt); 2023 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 2024 DDI_SUCCESS) { 2025 ixgbe_error(ixgbe, 2026 "IRM CB: Failed to adjust interrupts"); 2027 goto cb_fail; 2028 } 2029 break; 2030 default: 2031 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 2032 cbaction); 2033 return (DDI_ENOTSUP); 2034 } 2035 return (DDI_SUCCESS); 2036 cb_fail: 2037 return (DDI_FAILURE); 2038 } 2039 2040 /* 2041 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 2042 */ 2043 static int 2044 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 2045 { 2046 int i, rc, actual; 2047 2048 if (count == 0) 2049 return (DDI_SUCCESS); 2050 2051 if ((cbaction == DDI_CB_INTR_ADD && 2052 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 2053 (cbaction == DDI_CB_INTR_REMOVE && 2054 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 2055 return (DDI_FAILURE); 2056 2057 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 2058 return (DDI_FAILURE); 2059 } 2060 2061 for (i = 0; i < ixgbe->num_rx_rings; i++) 2062 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 2063 for (i = 0; i < ixgbe->num_tx_rings; i++) 2064 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 2065 2066 mutex_enter(&ixgbe->gen_lock); 2067 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 2068 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 2069 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 2070 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 2071 2072 ixgbe_stop(ixgbe, B_FALSE); 2073 /* 2074 * Disable interrupts 2075 */ 2076 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 2077 rc = ixgbe_disable_intrs(ixgbe); 2078 ASSERT(rc == IXGBE_SUCCESS); 2079 } 2080 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 2081 2082 /* 2083 * Remove interrupt handlers 2084 */ 2085 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 2086 ixgbe_rem_intr_handlers(ixgbe); 2087 } 2088 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 2089 2090 /* 2091 * Clear vect_map 2092 */ 2093 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 2094 switch (cbaction) { 2095 case DDI_CB_INTR_ADD: 2096 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 2097 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 2098 DDI_INTR_ALLOC_NORMAL); 2099 if (rc != DDI_SUCCESS || actual != count) { 2100 ixgbe_log(ixgbe, "Adjust interrupts failed." 2101 "return: %d, irm cb size: %d, actual: %d", 2102 rc, count, actual); 2103 goto intr_adjust_fail; 2104 } 2105 ixgbe->intr_cnt += count; 2106 break; 2107 2108 case DDI_CB_INTR_REMOVE: 2109 for (i = ixgbe->intr_cnt - count; 2110 i < ixgbe->intr_cnt; i ++) { 2111 rc = ddi_intr_free(ixgbe->htable[i]); 2112 ixgbe->htable[i] = NULL; 2113 if (rc != DDI_SUCCESS) { 2114 ixgbe_log(ixgbe, "Adjust interrupts failed." 2115 "return: %d, irm cb size: %d, actual: %d", 2116 rc, count, actual); 2117 goto intr_adjust_fail; 2118 } 2119 } 2120 ixgbe->intr_cnt -= count; 2121 break; 2122 } 2123 2124 /* 2125 * Get priority for first vector, assume remaining are all the same 2126 */ 2127 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 2128 if (rc != DDI_SUCCESS) { 2129 ixgbe_log(ixgbe, 2130 "Get interrupt priority failed: %d", rc); 2131 goto intr_adjust_fail; 2132 } 2133 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 2134 if (rc != DDI_SUCCESS) { 2135 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 2136 goto intr_adjust_fail; 2137 } 2138 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 2139 2140 /* 2141 * Map rings to interrupt vectors 2142 */ 2143 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 2144 ixgbe_error(ixgbe, 2145 "IRM CB: Failed to map interrupts to vectors"); 2146 goto intr_adjust_fail; 2147 } 2148 2149 /* 2150 * Add interrupt handlers 2151 */ 2152 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 2153 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 2154 goto intr_adjust_fail; 2155 } 2156 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 2157 2158 /* 2159 * Now that mutex locks are initialized, and the chip is also 2160 * initialized, enable interrupts. 2161 */ 2162 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 2163 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 2164 goto intr_adjust_fail; 2165 } 2166 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 2167 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 2168 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 2169 goto intr_adjust_fail; 2170 } 2171 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 2172 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 2173 ixgbe->ixgbe_state |= IXGBE_STARTED; 2174 mutex_exit(&ixgbe->gen_lock); 2175 2176 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2177 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 2178 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 2179 } 2180 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2181 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 2182 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 2183 } 2184 2185 /* Wakeup all Tx rings */ 2186 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2187 mac_tx_ring_update(ixgbe->mac_hdl, 2188 ixgbe->tx_rings[i].ring_handle); 2189 } 2190 2191 IXGBE_DEBUGLOG_3(ixgbe, 2192 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 2193 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 2194 return (DDI_SUCCESS); 2195 2196 intr_adjust_fail: 2197 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 2198 mutex_exit(&ixgbe->gen_lock); 2199 return (DDI_FAILURE); 2200 } 2201 2202 /* 2203 * ixgbe_intr_cb_register - Register interrupt callback function. 2204 */ 2205 static int 2206 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 2207 { 2208 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 2209 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 2210 return (IXGBE_FAILURE); 2211 } 2212 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 2213 return (IXGBE_SUCCESS); 2214 } 2215 2216 /* 2217 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 2218 */ 2219 static int 2220 ixgbe_alloc_rings(ixgbe_t *ixgbe) 2221 { 2222 /* 2223 * Allocate memory space for rx rings 2224 */ 2225 ixgbe->rx_rings = kmem_zalloc( 2226 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 2227 KM_NOSLEEP); 2228 2229 if (ixgbe->rx_rings == NULL) { 2230 return (IXGBE_FAILURE); 2231 } 2232 2233 /* 2234 * Allocate memory space for tx rings 2235 */ 2236 ixgbe->tx_rings = kmem_zalloc( 2237 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 2238 KM_NOSLEEP); 2239 2240 if (ixgbe->tx_rings == NULL) { 2241 kmem_free(ixgbe->rx_rings, 2242 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2243 ixgbe->rx_rings = NULL; 2244 return (IXGBE_FAILURE); 2245 } 2246 2247 /* 2248 * Allocate memory space for rx ring groups 2249 */ 2250 ixgbe->rx_groups = kmem_zalloc( 2251 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 2252 KM_NOSLEEP); 2253 2254 if (ixgbe->rx_groups == NULL) { 2255 kmem_free(ixgbe->rx_rings, 2256 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2257 kmem_free(ixgbe->tx_rings, 2258 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2259 ixgbe->rx_rings = NULL; 2260 ixgbe->tx_rings = NULL; 2261 return (IXGBE_FAILURE); 2262 } 2263 2264 return (IXGBE_SUCCESS); 2265 } 2266 2267 /* 2268 * ixgbe_free_rings - Free the memory space of rx/tx rings. 2269 */ 2270 static void 2271 ixgbe_free_rings(ixgbe_t *ixgbe) 2272 { 2273 if (ixgbe->rx_rings != NULL) { 2274 kmem_free(ixgbe->rx_rings, 2275 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2276 ixgbe->rx_rings = NULL; 2277 } 2278 2279 if (ixgbe->tx_rings != NULL) { 2280 kmem_free(ixgbe->tx_rings, 2281 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2282 ixgbe->tx_rings = NULL; 2283 } 2284 2285 if (ixgbe->rx_groups != NULL) { 2286 kmem_free(ixgbe->rx_groups, 2287 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 2288 ixgbe->rx_groups = NULL; 2289 } 2290 } 2291 2292 static int 2293 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 2294 { 2295 ixgbe_rx_ring_t *rx_ring; 2296 int i; 2297 2298 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2299 rx_ring = &ixgbe->rx_rings[i]; 2300 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 2301 goto alloc_rx_rings_failure; 2302 } 2303 return (IXGBE_SUCCESS); 2304 2305 alloc_rx_rings_failure: 2306 ixgbe_free_rx_data(ixgbe); 2307 return (IXGBE_FAILURE); 2308 } 2309 2310 static void 2311 ixgbe_free_rx_data(ixgbe_t *ixgbe) 2312 { 2313 ixgbe_rx_ring_t *rx_ring; 2314 ixgbe_rx_data_t *rx_data; 2315 int i; 2316 2317 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2318 rx_ring = &ixgbe->rx_rings[i]; 2319 2320 mutex_enter(&ixgbe->rx_pending_lock); 2321 rx_data = rx_ring->rx_data; 2322 2323 if (rx_data != NULL) { 2324 rx_data->flag |= IXGBE_RX_STOPPED; 2325 2326 if (rx_data->rcb_pending == 0) { 2327 ixgbe_free_rx_ring_data(rx_data); 2328 rx_ring->rx_data = NULL; 2329 } 2330 } 2331 2332 mutex_exit(&ixgbe->rx_pending_lock); 2333 } 2334 } 2335 2336 /* 2337 * ixgbe_setup_rings - Setup rx/tx rings. 2338 */ 2339 static void 2340 ixgbe_setup_rings(ixgbe_t *ixgbe) 2341 { 2342 /* 2343 * Setup the rx/tx rings, including the following: 2344 * 2345 * 1. Setup the descriptor ring and the control block buffers; 2346 * 2. Initialize necessary registers for receive/transmit; 2347 * 3. Initialize software pointers/parameters for receive/transmit; 2348 */ 2349 ixgbe_setup_rx(ixgbe); 2350 2351 ixgbe_setup_tx(ixgbe); 2352 } 2353 2354 static void 2355 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2356 { 2357 ixgbe_t *ixgbe = rx_ring->ixgbe; 2358 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2359 struct ixgbe_hw *hw = &ixgbe->hw; 2360 rx_control_block_t *rcb; 2361 union ixgbe_adv_rx_desc *rbd; 2362 uint32_t size; 2363 uint32_t buf_low; 2364 uint32_t buf_high; 2365 uint32_t reg_val; 2366 int i; 2367 2368 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2369 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2370 2371 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2372 rcb = rx_data->work_list[i]; 2373 rbd = &rx_data->rbd_ring[i]; 2374 2375 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2376 rbd->read.hdr_addr = 0; 2377 } 2378 2379 /* 2380 * Initialize the length register 2381 */ 2382 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2383 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2384 2385 /* 2386 * Initialize the base address registers 2387 */ 2388 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2389 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2390 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2391 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2392 2393 /* 2394 * Setup head & tail pointers 2395 */ 2396 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2397 rx_data->ring_size - 1); 2398 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2399 2400 rx_data->rbd_next = 0; 2401 rx_data->lro_first = 0; 2402 2403 /* 2404 * Setup the Receive Descriptor Control Register (RXDCTL) 2405 * PTHRESH=32 descriptors (half the internal cache) 2406 * HTHRESH=0 descriptors (to minimize latency on fetch) 2407 * WTHRESH defaults to 1 (writeback each descriptor) 2408 */ 2409 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2410 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2411 2412 /* Not a valid value for 82599, X540 or X550 */ 2413 if (hw->mac.type == ixgbe_mac_82598EB) { 2414 reg_val |= 0x0020; /* pthresh */ 2415 } 2416 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2417 2418 if (hw->mac.type == ixgbe_mac_82599EB || 2419 hw->mac.type == ixgbe_mac_X540 || 2420 hw->mac.type == ixgbe_mac_X550 || 2421 hw->mac.type == ixgbe_mac_X550EM_x || 2422 hw->mac.type == ixgbe_mac_X550EM_a) { 2423 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2424 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2425 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2426 } 2427 2428 /* 2429 * Setup the Split and Replication Receive Control Register. 2430 * Set the rx buffer size and the advanced descriptor type. 2431 */ 2432 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2433 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2434 reg_val |= IXGBE_SRRCTL_DROP_EN; 2435 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2436 } 2437 2438 static void 2439 ixgbe_setup_rx(ixgbe_t *ixgbe) 2440 { 2441 ixgbe_rx_ring_t *rx_ring; 2442 struct ixgbe_hw *hw = &ixgbe->hw; 2443 uint32_t reg_val; 2444 uint32_t i; 2445 uint32_t psrtype_rss_bit; 2446 2447 /* 2448 * Ensure that Rx is disabled while setting up 2449 * the Rx unit and Rx descriptor ring(s) 2450 */ 2451 ixgbe_disable_rx(hw); 2452 2453 /* PSRTYPE must be configured for 82599 */ 2454 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2455 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2456 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2457 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2458 reg_val |= IXGBE_PSRTYPE_L2HDR; 2459 reg_val |= 0x80000000; 2460 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2461 } else { 2462 if (ixgbe->num_rx_groups > 32) { 2463 psrtype_rss_bit = 0x20000000; 2464 } else { 2465 psrtype_rss_bit = 0x40000000; 2466 } 2467 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2468 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2469 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2470 reg_val |= IXGBE_PSRTYPE_L2HDR; 2471 reg_val |= psrtype_rss_bit; 2472 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2473 } 2474 } 2475 2476 /* 2477 * Set filter control in FCTRL to determine types of packets are passed 2478 * up to the driver. 2479 * - Pass broadcast packets. 2480 * - Do not pass flow control pause frames (82598-specific) 2481 */ 2482 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2483 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */ 2484 if (hw->mac.type == ixgbe_mac_82598EB) { 2485 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */ 2486 } 2487 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2488 2489 /* 2490 * Hardware checksum settings 2491 */ 2492 if (ixgbe->rx_hcksum_enable) { 2493 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2494 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2495 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2496 } 2497 2498 /* 2499 * Setup VMDq and RSS for multiple receive queues 2500 */ 2501 switch (ixgbe->classify_mode) { 2502 case IXGBE_CLASSIFY_RSS: 2503 /* 2504 * One group, only RSS is needed when more than 2505 * one ring enabled. 2506 */ 2507 ixgbe_setup_rss(ixgbe); 2508 break; 2509 2510 case IXGBE_CLASSIFY_VMDQ: 2511 /* 2512 * Multiple groups, each group has one ring, 2513 * only VMDq is needed. 2514 */ 2515 ixgbe_setup_vmdq(ixgbe); 2516 break; 2517 2518 case IXGBE_CLASSIFY_VMDQ_RSS: 2519 /* 2520 * Multiple groups and multiple rings, both 2521 * VMDq and RSS are needed. 2522 */ 2523 ixgbe_setup_vmdq_rss(ixgbe); 2524 break; 2525 2526 default: 2527 break; 2528 } 2529 2530 /* 2531 * Enable the receive unit. This must be done after filter 2532 * control is set in FCTRL. On 82598, we disable the descriptor monitor. 2533 * 82598 is the only adapter which defines this RXCTRL option. 2534 */ 2535 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 2536 if (hw->mac.type == ixgbe_mac_82598EB) 2537 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */ 2538 reg_val |= IXGBE_RXCTRL_RXEN; 2539 (void) ixgbe_enable_rx_dma(hw, reg_val); 2540 2541 /* 2542 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2543 */ 2544 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2545 rx_ring = &ixgbe->rx_rings[i]; 2546 ixgbe_setup_rx_ring(rx_ring); 2547 } 2548 2549 /* 2550 * The 82598 controller gives us the RNBC (Receive No Buffer 2551 * Count) register to determine the number of frames dropped 2552 * due to no available descriptors on the destination queue. 2553 * However, this register was removed starting with 82599 and 2554 * it was replaced with the RQSMR/QPRDC registers. The nice 2555 * thing about the new registers is that they allow you to map 2556 * groups of queues to specific stat registers. The bad thing 2557 * is there are only 16 slots in the stat registers, so this 2558 * won't work when we have 32 Rx groups. Instead, we map all 2559 * queues to the zero slot of the stat registers, giving us a 2560 * global counter at QPRDC[0] (with the equivalent semantics 2561 * of RNBC). Perhaps future controllers will have more slots 2562 * and we can implement per-group counters. 2563 */ 2564 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2565 uint32_t index = ixgbe->rx_rings[i].hw_index; 2566 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), 0); 2567 } 2568 2569 /* 2570 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2571 * by four bytes if the packet has a VLAN field, so includes MTU, 2572 * ethernet header and frame check sequence. 2573 * Register is MAXFRS in 82599. 2574 */ 2575 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD); 2576 reg_val &= ~IXGBE_MHADD_MFS_MASK; 2577 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header) 2578 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2579 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2580 2581 /* 2582 * Setup Jumbo Frame enable bit 2583 */ 2584 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2585 if (ixgbe->default_mtu > ETHERMTU) 2586 reg_val |= IXGBE_HLREG0_JUMBOEN; 2587 else 2588 reg_val &= ~IXGBE_HLREG0_JUMBOEN; 2589 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2590 2591 /* 2592 * Setup RSC for multiple receive queues. 2593 */ 2594 if (ixgbe->lro_enable) { 2595 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2596 /* 2597 * Make sure rx_buf_size * MAXDESC not greater 2598 * than 65535. 2599 * Intel recommends 4 for MAXDESC field value. 2600 */ 2601 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2602 reg_val |= IXGBE_RSCCTL_RSCEN; 2603 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2604 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2605 else 2606 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2607 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2608 } 2609 2610 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2611 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2612 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2613 2614 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2615 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2616 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2617 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2618 2619 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2620 } 2621 } 2622 2623 static void 2624 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2625 { 2626 ixgbe_t *ixgbe = tx_ring->ixgbe; 2627 struct ixgbe_hw *hw = &ixgbe->hw; 2628 uint32_t size; 2629 uint32_t buf_low; 2630 uint32_t buf_high; 2631 uint32_t reg_val; 2632 2633 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2634 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2635 2636 /* 2637 * Initialize the length register 2638 */ 2639 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2640 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2641 2642 /* 2643 * Initialize the base address registers 2644 */ 2645 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2646 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2647 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2648 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2649 2650 /* 2651 * Setup head & tail pointers 2652 */ 2653 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2654 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2655 2656 /* 2657 * Setup head write-back 2658 */ 2659 if (ixgbe->tx_head_wb_enable) { 2660 /* 2661 * The memory of the head write-back is allocated using 2662 * the extra tbd beyond the tail of the tbd ring. 2663 */ 2664 tx_ring->tbd_head_wb = (uint32_t *) 2665 ((uintptr_t)tx_ring->tbd_area.address + size); 2666 *tx_ring->tbd_head_wb = 0; 2667 2668 buf_low = (uint32_t) 2669 (tx_ring->tbd_area.dma_address + size); 2670 buf_high = (uint32_t) 2671 ((tx_ring->tbd_area.dma_address + size) >> 32); 2672 2673 /* Set the head write-back enable bit */ 2674 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2675 2676 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2677 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2678 2679 /* 2680 * Turn off relaxed ordering for head write back or it will 2681 * cause problems with the tx recycling 2682 */ 2683 2684 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ? 2685 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) : 2686 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index)); 2687 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2688 if (hw->mac.type == ixgbe_mac_82598EB) { 2689 IXGBE_WRITE_REG(hw, 2690 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2691 } else { 2692 IXGBE_WRITE_REG(hw, 2693 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val); 2694 } 2695 } else { 2696 tx_ring->tbd_head_wb = NULL; 2697 } 2698 2699 tx_ring->tbd_head = 0; 2700 tx_ring->tbd_tail = 0; 2701 tx_ring->tbd_free = tx_ring->ring_size; 2702 2703 if (ixgbe->tx_ring_init == B_TRUE) { 2704 tx_ring->tcb_head = 0; 2705 tx_ring->tcb_tail = 0; 2706 tx_ring->tcb_free = tx_ring->free_list_size; 2707 } 2708 2709 /* 2710 * Initialize the s/w context structure 2711 */ 2712 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2713 } 2714 2715 static void 2716 ixgbe_setup_tx(ixgbe_t *ixgbe) 2717 { 2718 struct ixgbe_hw *hw = &ixgbe->hw; 2719 ixgbe_tx_ring_t *tx_ring; 2720 uint32_t reg_val; 2721 int i; 2722 2723 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2724 tx_ring = &ixgbe->tx_rings[i]; 2725 ixgbe_setup_tx_ring(tx_ring); 2726 } 2727 2728 /* 2729 * Setup the per-ring statistics mapping. We map all Tx queues 2730 * to slot 0 to stay consistent with Rx. 2731 */ 2732 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2733 switch (hw->mac.type) { 2734 case ixgbe_mac_82598EB: 2735 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 0); 2736 break; 2737 2738 default: 2739 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 0); 2740 break; 2741 } 2742 } 2743 2744 /* 2745 * Enable CRC appending and TX padding (for short tx frames) 2746 */ 2747 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2748 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2749 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2750 2751 /* 2752 * enable DMA for 82599, X540 and X550 parts 2753 */ 2754 if (hw->mac.type == ixgbe_mac_82599EB || 2755 hw->mac.type == ixgbe_mac_X540 || 2756 hw->mac.type == ixgbe_mac_X550 || 2757 hw->mac.type == ixgbe_mac_X550EM_x || 2758 hw->mac.type == ixgbe_mac_X550EM_a) { 2759 /* DMATXCTL.TE must be set after all Tx config is complete */ 2760 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2761 reg_val |= IXGBE_DMATXCTL_TE; 2762 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2763 2764 /* Disable arbiter to set MTQC */ 2765 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2766 reg_val |= IXGBE_RTTDCS_ARBDIS; 2767 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2768 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2769 reg_val &= ~IXGBE_RTTDCS_ARBDIS; 2770 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2771 } 2772 2773 /* 2774 * Enabling tx queues .. 2775 * For 82599 must be done after DMATXCTL.TE is set 2776 */ 2777 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2778 tx_ring = &ixgbe->tx_rings[i]; 2779 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2780 reg_val |= IXGBE_TXDCTL_ENABLE; 2781 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2782 } 2783 } 2784 2785 /* 2786 * ixgbe_setup_rss - Setup receive-side scaling feature. 2787 */ 2788 static void 2789 ixgbe_setup_rss(ixgbe_t *ixgbe) 2790 { 2791 struct ixgbe_hw *hw = &ixgbe->hw; 2792 uint32_t mrqc; 2793 2794 /* 2795 * Initialize RETA/ERETA table 2796 */ 2797 ixgbe_setup_rss_table(ixgbe); 2798 2799 /* 2800 * Enable RSS & perform hash on these packet types 2801 */ 2802 mrqc = IXGBE_MRQC_RSSEN | 2803 IXGBE_MRQC_RSS_FIELD_IPV4 | 2804 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2805 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2806 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2807 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2808 IXGBE_MRQC_RSS_FIELD_IPV6 | 2809 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2810 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2811 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2812 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2813 } 2814 2815 /* 2816 * ixgbe_setup_vmdq - Setup MAC classification feature 2817 */ 2818 static void 2819 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2820 { 2821 struct ixgbe_hw *hw = &ixgbe->hw; 2822 uint32_t vmdctl, i, vtctl; 2823 2824 /* 2825 * Setup the VMDq Control register, enable VMDq based on 2826 * packet destination MAC address: 2827 */ 2828 switch (hw->mac.type) { 2829 case ixgbe_mac_82598EB: 2830 /* 2831 * VMDq Enable = 1; 2832 * VMDq Filter = 0; MAC filtering 2833 * Default VMDq output index = 0; 2834 */ 2835 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2836 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2837 break; 2838 2839 case ixgbe_mac_82599EB: 2840 case ixgbe_mac_X540: 2841 case ixgbe_mac_X550: 2842 case ixgbe_mac_X550EM_x: 2843 case ixgbe_mac_X550EM_a: 2844 /* 2845 * Enable VMDq-only. 2846 */ 2847 vmdctl = IXGBE_MRQC_VMDQEN; 2848 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2849 2850 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2851 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2852 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2853 } 2854 2855 /* 2856 * Enable Virtualization and Replication. 2857 */ 2858 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2859 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2860 2861 /* 2862 * Enable receiving packets to all VFs 2863 */ 2864 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2865 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2866 break; 2867 2868 default: 2869 break; 2870 } 2871 } 2872 2873 /* 2874 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2875 */ 2876 static void 2877 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2878 { 2879 struct ixgbe_hw *hw = &ixgbe->hw; 2880 uint32_t i, mrqc; 2881 uint32_t vtctl, vmdctl; 2882 2883 /* 2884 * Initialize RETA/ERETA table 2885 */ 2886 ixgbe_setup_rss_table(ixgbe); 2887 2888 /* 2889 * Enable and setup RSS and VMDq 2890 */ 2891 switch (hw->mac.type) { 2892 case ixgbe_mac_82598EB: 2893 /* 2894 * Enable RSS & Setup RSS Hash functions 2895 */ 2896 mrqc = IXGBE_MRQC_RSSEN | 2897 IXGBE_MRQC_RSS_FIELD_IPV4 | 2898 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2899 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2900 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2901 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2902 IXGBE_MRQC_RSS_FIELD_IPV6 | 2903 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2904 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2905 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2906 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2907 2908 /* 2909 * Enable and Setup VMDq 2910 * VMDq Filter = 0; MAC filtering 2911 * Default VMDq output index = 0; 2912 */ 2913 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2914 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2915 break; 2916 2917 case ixgbe_mac_82599EB: 2918 case ixgbe_mac_X540: 2919 case ixgbe_mac_X550: 2920 case ixgbe_mac_X550EM_x: 2921 case ixgbe_mac_X550EM_a: 2922 /* 2923 * Enable RSS & Setup RSS Hash functions 2924 */ 2925 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2926 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2927 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2928 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2929 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2930 IXGBE_MRQC_RSS_FIELD_IPV6 | 2931 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2932 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2933 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2934 2935 /* 2936 * Enable VMDq+RSS. 2937 */ 2938 if (ixgbe->num_rx_groups > 32) { 2939 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2940 } else { 2941 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2942 } 2943 2944 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2945 2946 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2947 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2948 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2949 } 2950 break; 2951 2952 default: 2953 break; 2954 2955 } 2956 2957 if (hw->mac.type == ixgbe_mac_82599EB || 2958 hw->mac.type == ixgbe_mac_X540 || 2959 hw->mac.type == ixgbe_mac_X550 || 2960 hw->mac.type == ixgbe_mac_X550EM_x || 2961 hw->mac.type == ixgbe_mac_X550EM_a) { 2962 /* 2963 * Enable Virtualization and Replication. 2964 */ 2965 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2966 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2967 2968 /* 2969 * Enable receiving packets to all VFs 2970 */ 2971 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2972 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2973 } 2974 } 2975 2976 /* 2977 * ixgbe_setup_rss_table - Setup RSS table 2978 */ 2979 static void 2980 ixgbe_setup_rss_table(ixgbe_t *ixgbe) 2981 { 2982 struct ixgbe_hw *hw = &ixgbe->hw; 2983 uint32_t i, j; 2984 uint32_t random; 2985 uint32_t reta; 2986 uint32_t ring_per_group; 2987 uint32_t ring; 2988 uint32_t table_size; 2989 uint32_t index_mult; 2990 uint32_t rxcsum; 2991 2992 /* 2993 * Set multiplier for RETA setup and table size based on MAC type. 2994 * RETA table sizes vary by model: 2995 * 2996 * 82598, 82599, X540: 128 table entries. 2997 * X550: 512 table entries. 2998 */ 2999 index_mult = 0x1; 3000 table_size = 128; 3001 switch (ixgbe->hw.mac.type) { 3002 case ixgbe_mac_82598EB: 3003 index_mult = 0x11; 3004 break; 3005 case ixgbe_mac_X550: 3006 case ixgbe_mac_X550EM_x: 3007 case ixgbe_mac_X550EM_a: 3008 table_size = 512; 3009 break; 3010 default: 3011 break; 3012 } 3013 3014 /* 3015 * Fill out RSS redirection table. The configuation of the indices is 3016 * hardware-dependent. 3017 * 3018 * 82598: 8 bits wide containing two 4 bit RSS indices 3019 * 82599, X540: 8 bits wide containing one 4 bit RSS index 3020 * X550: 8 bits wide containing one 6 bit RSS index 3021 */ 3022 reta = 0; 3023 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3024 3025 for (i = 0, j = 0; i < table_size; i++, j++) { 3026 if (j == ring_per_group) j = 0; 3027 3028 /* 3029 * The low 8 bits are for hash value (n+0); 3030 * The next 8 bits are for hash value (n+1), etc. 3031 */ 3032 ring = (j * index_mult); 3033 reta = reta >> 8; 3034 reta = reta | (((uint32_t)ring) << 24); 3035 3036 if ((i & 3) == 3) { 3037 /* 3038 * The first 128 table entries are programmed into the 3039 * RETA register, with any beyond that (eg; on X550) 3040 * into ERETA. 3041 */ 3042 if (i < 128) 3043 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3044 else 3045 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3046 reta); 3047 reta = 0; 3048 } 3049 } 3050 3051 /* 3052 * Fill out hash function seeds with a random constant 3053 */ 3054 for (i = 0; i < 10; i++) { 3055 (void) random_get_pseudo_bytes((uint8_t *)&random, 3056 sizeof (uint32_t)); 3057 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 3058 } 3059 3060 /* 3061 * Disable Packet Checksum to enable RSS for multiple receive queues. 3062 * It is an adapter hardware limitation that Packet Checksum is 3063 * mutually exclusive with RSS. 3064 */ 3065 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3066 rxcsum |= IXGBE_RXCSUM_PCSD; 3067 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 3068 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3069 } 3070 3071 /* 3072 * ixgbe_init_unicst - Initialize the unicast addresses. 3073 */ 3074 static void 3075 ixgbe_init_unicst(ixgbe_t *ixgbe) 3076 { 3077 struct ixgbe_hw *hw = &ixgbe->hw; 3078 uint8_t *mac_addr; 3079 int slot; 3080 /* 3081 * Here we should consider two situations: 3082 * 3083 * 1. Chipset is initialized at the first time, 3084 * Clear all the multiple unicast addresses. 3085 * 3086 * 2. Chipset is reset 3087 * Recover the multiple unicast addresses from the 3088 * software data structure to the RAR registers. 3089 */ 3090 if (!ixgbe->unicst_init) { 3091 /* 3092 * Initialize the multiple unicast addresses 3093 */ 3094 ixgbe->unicst_total = hw->mac.num_rar_entries; 3095 ixgbe->unicst_avail = ixgbe->unicst_total; 3096 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3097 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 3098 bzero(mac_addr, ETHERADDRL); 3099 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0); 3100 ixgbe->unicst_addr[slot].mac.set = 0; 3101 } 3102 ixgbe->unicst_init = B_TRUE; 3103 } else { 3104 /* Re-configure the RAR registers */ 3105 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3106 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 3107 if (ixgbe->unicst_addr[slot].mac.set == 1) { 3108 (void) ixgbe_set_rar(hw, slot, mac_addr, 3109 ixgbe->unicst_addr[slot].mac.group_index, 3110 IXGBE_RAH_AV); 3111 } else { 3112 bzero(mac_addr, ETHERADDRL); 3113 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0); 3114 } 3115 } 3116 } 3117 } 3118 3119 /* 3120 * ixgbe_unicst_find - Find the slot for the specified unicast address 3121 */ 3122 int 3123 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 3124 { 3125 int slot; 3126 3127 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3128 3129 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3130 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 3131 mac_addr, ETHERADDRL) == 0) 3132 return (slot); 3133 } 3134 3135 return (-1); 3136 } 3137 3138 /* 3139 * ixgbe_multicst_add - Add a multicst address. 3140 */ 3141 int 3142 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 3143 { 3144 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3145 3146 if ((multiaddr[0] & 01) == 0) { 3147 return (EINVAL); 3148 } 3149 3150 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 3151 return (ENOENT); 3152 } 3153 3154 bcopy(multiaddr, 3155 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 3156 ixgbe->mcast_count++; 3157 3158 /* 3159 * Update the multicast table in the hardware 3160 */ 3161 ixgbe_setup_multicst(ixgbe); 3162 3163 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3164 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3165 return (EIO); 3166 } 3167 3168 return (0); 3169 } 3170 3171 /* 3172 * ixgbe_multicst_remove - Remove a multicst address. 3173 */ 3174 int 3175 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 3176 { 3177 int i; 3178 3179 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3180 3181 for (i = 0; i < ixgbe->mcast_count; i++) { 3182 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 3183 ETHERADDRL) == 0) { 3184 for (i++; i < ixgbe->mcast_count; i++) { 3185 ixgbe->mcast_table[i - 1] = 3186 ixgbe->mcast_table[i]; 3187 } 3188 ixgbe->mcast_count--; 3189 break; 3190 } 3191 } 3192 3193 /* 3194 * Update the multicast table in the hardware 3195 */ 3196 ixgbe_setup_multicst(ixgbe); 3197 3198 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3199 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3200 return (EIO); 3201 } 3202 3203 return (0); 3204 } 3205 3206 /* 3207 * ixgbe_setup_multicast - Setup multicast data structures. 3208 * 3209 * This routine initializes all of the multicast related structures 3210 * and save them in the hardware registers. 3211 */ 3212 static void 3213 ixgbe_setup_multicst(ixgbe_t *ixgbe) 3214 { 3215 uint8_t *mc_addr_list; 3216 uint32_t mc_addr_count; 3217 struct ixgbe_hw *hw = &ixgbe->hw; 3218 3219 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3220 3221 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 3222 3223 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 3224 mc_addr_count = ixgbe->mcast_count; 3225 3226 /* 3227 * Update the multicast addresses to the MTA registers 3228 */ 3229 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 3230 ixgbe_mc_table_itr, TRUE); 3231 } 3232 3233 /* 3234 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 3235 * 3236 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 3237 * Different chipsets may have different allowed configuration of vmdq and rss. 3238 */ 3239 static void 3240 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 3241 { 3242 struct ixgbe_hw *hw = &ixgbe->hw; 3243 uint32_t ring_per_group; 3244 3245 switch (hw->mac.type) { 3246 case ixgbe_mac_82598EB: 3247 /* 3248 * 82598 supports the following combination: 3249 * vmdq no. x rss no. 3250 * [5..16] x 1 3251 * [1..4] x [1..16] 3252 * However 8 rss queue per pool (vmdq) is sufficient for 3253 * most cases. 3254 */ 3255 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3256 if (ixgbe->num_rx_groups > 4) { 3257 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 3258 } else { 3259 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3260 min(8, ring_per_group); 3261 } 3262 3263 break; 3264 3265 case ixgbe_mac_82599EB: 3266 case ixgbe_mac_X540: 3267 case ixgbe_mac_X550: 3268 case ixgbe_mac_X550EM_x: 3269 case ixgbe_mac_X550EM_a: 3270 /* 3271 * 82599 supports the following combination: 3272 * vmdq no. x rss no. 3273 * [33..64] x [1..2] 3274 * [2..32] x [1..4] 3275 * 1 x [1..16] 3276 * However 8 rss queue per pool (vmdq) is sufficient for 3277 * most cases. 3278 * 3279 * For now, treat X540 and X550 like the 82599. 3280 */ 3281 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3282 if (ixgbe->num_rx_groups == 1) { 3283 ixgbe->num_rx_rings = min(8, ring_per_group); 3284 } else if (ixgbe->num_rx_groups <= 32) { 3285 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3286 min(4, ring_per_group); 3287 } else if (ixgbe->num_rx_groups <= 64) { 3288 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3289 min(2, ring_per_group); 3290 } 3291 break; 3292 3293 default: 3294 break; 3295 } 3296 3297 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3298 3299 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 3300 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3301 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 3302 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 3303 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 3304 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 3305 } else { 3306 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 3307 } 3308 3309 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 3310 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 3311 } 3312 3313 /* 3314 * ixgbe_get_conf - Get driver configurations set in driver.conf. 3315 * 3316 * This routine gets user-configured values out of the configuration 3317 * file ixgbe.conf. 3318 * 3319 * For each configurable value, there is a minimum, a maximum, and a 3320 * default. 3321 * If user does not configure a value, use the default. 3322 * If user configures below the minimum, use the minumum. 3323 * If user configures above the maximum, use the maxumum. 3324 */ 3325 static void 3326 ixgbe_get_conf(ixgbe_t *ixgbe) 3327 { 3328 struct ixgbe_hw *hw = &ixgbe->hw; 3329 uint32_t flow_control; 3330 3331 /* 3332 * ixgbe driver supports the following user configurations: 3333 * 3334 * Jumbo frame configuration: 3335 * default_mtu 3336 * 3337 * Ethernet flow control configuration: 3338 * flow_control 3339 * 3340 * Multiple rings configurations: 3341 * tx_queue_number 3342 * tx_ring_size 3343 * rx_queue_number 3344 * rx_ring_size 3345 * 3346 * Call ixgbe_get_prop() to get the value for a specific 3347 * configuration parameter. 3348 */ 3349 3350 /* 3351 * Jumbo frame configuration - max_frame_size controls host buffer 3352 * allocation, so includes MTU, ethernet header, vlan tag and 3353 * frame check sequence. 3354 */ 3355 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 3356 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 3357 3358 ixgbe->max_frame_size = ixgbe->default_mtu + 3359 sizeof (struct ether_vlan_header) + ETHERFCSL; 3360 3361 /* 3362 * Ethernet flow control configuration 3363 */ 3364 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 3365 ixgbe_fc_none, 3, ixgbe_fc_none); 3366 if (flow_control == 3) 3367 flow_control = ixgbe_fc_default; 3368 3369 /* 3370 * fc.requested mode is what the user requests. After autoneg, 3371 * fc.current_mode will be the flow_control mode that was negotiated. 3372 */ 3373 hw->fc.requested_mode = flow_control; 3374 3375 /* 3376 * Multiple rings configurations 3377 */ 3378 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 3379 ixgbe->capab->min_tx_que_num, 3380 ixgbe->capab->max_tx_que_num, 3381 ixgbe->capab->def_tx_que_num); 3382 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 3383 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 3384 3385 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 3386 ixgbe->capab->min_rx_que_num, 3387 ixgbe->capab->max_rx_que_num, 3388 ixgbe->capab->def_rx_que_num); 3389 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 3390 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 3391 3392 /* 3393 * Multiple groups configuration 3394 */ 3395 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3396 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3397 ixgbe->capab->def_rx_grp_num); 3398 3399 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3400 0, 1, DEFAULT_MR_ENABLE); 3401 3402 if (ixgbe->mr_enable == B_FALSE) { 3403 ixgbe->num_tx_rings = 1; 3404 ixgbe->num_rx_rings = 1; 3405 ixgbe->num_rx_groups = 1; 3406 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3407 } else { 3408 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3409 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3410 /* 3411 * The combination of num_rx_rings and num_rx_groups 3412 * may be not supported by h/w. We need to adjust 3413 * them to appropriate values. 3414 */ 3415 ixgbe_setup_vmdq_rss_conf(ixgbe); 3416 } 3417 3418 /* 3419 * Tunable used to force an interrupt type. The only use is 3420 * for testing of the lesser interrupt types. 3421 * 0 = don't force interrupt type 3422 * 1 = force interrupt type MSI-X 3423 * 2 = force interrupt type MSI 3424 * 3 = force interrupt type Legacy 3425 */ 3426 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3427 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3428 3429 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3430 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3431 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3432 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3433 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3434 0, 1, DEFAULT_LSO_ENABLE); 3435 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3436 0, 1, DEFAULT_LRO_ENABLE); 3437 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3438 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3439 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3440 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3441 3442 /* Head Write Back not recommended for 82599, X540 and X550 */ 3443 if (hw->mac.type == ixgbe_mac_82599EB || 3444 hw->mac.type == ixgbe_mac_X540 || 3445 hw->mac.type == ixgbe_mac_X550 || 3446 hw->mac.type == ixgbe_mac_X550EM_x || 3447 hw->mac.type == ixgbe_mac_X550EM_a) { 3448 ixgbe->tx_head_wb_enable = B_FALSE; 3449 } 3450 3451 /* 3452 * ixgbe LSO needs the tx h/w checksum support. 3453 * LSO will be disabled if tx h/w checksum is not 3454 * enabled. 3455 */ 3456 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3457 ixgbe->lso_enable = B_FALSE; 3458 } 3459 3460 /* 3461 * ixgbe LRO needs the rx h/w checksum support. 3462 * LRO will be disabled if rx h/w checksum is not 3463 * enabled. 3464 */ 3465 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3466 ixgbe->lro_enable = B_FALSE; 3467 } 3468 3469 /* 3470 * ixgbe LRO only supported by 82599, X540 and X550 3471 */ 3472 if (hw->mac.type == ixgbe_mac_82598EB) { 3473 ixgbe->lro_enable = B_FALSE; 3474 } 3475 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3476 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3477 DEFAULT_TX_COPY_THRESHOLD); 3478 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3479 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3480 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3481 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3482 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3483 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3484 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3485 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3486 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3487 3488 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3489 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3490 DEFAULT_RX_COPY_THRESHOLD); 3491 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3492 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3493 DEFAULT_RX_LIMIT_PER_INTR); 3494 3495 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3496 ixgbe->capab->min_intr_throttle, 3497 ixgbe->capab->max_intr_throttle, 3498 ixgbe->capab->def_intr_throttle); 3499 /* 3500 * 82599, X540 and X550 require the interrupt throttling rate is 3501 * a multiple of 8. This is enforced by the register definiton. 3502 */ 3503 if (hw->mac.type == ixgbe_mac_82599EB || 3504 hw->mac.type == ixgbe_mac_X540 || 3505 hw->mac.type == ixgbe_mac_X550 || 3506 hw->mac.type == ixgbe_mac_X550EM_x || 3507 hw->mac.type == ixgbe_mac_X550EM_a) 3508 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3509 3510 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe, 3511 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP); 3512 } 3513 3514 static void 3515 ixgbe_init_params(ixgbe_t *ixgbe) 3516 { 3517 struct ixgbe_hw *hw = &ixgbe->hw; 3518 ixgbe_link_speed speeds_supported = 0; 3519 boolean_t negotiate; 3520 3521 /* 3522 * Get a list of speeds the adapter supports. If the hw struct hasn't 3523 * been populated with this information yet, retrieve it from the 3524 * adapter and save it to our own variable. 3525 * 3526 * On certain adapters, such as ones which use SFPs, the contents of 3527 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not 3528 * updated, so we must rely on calling ixgbe_get_link_capabilities() 3529 * in order to ascertain the speeds which we are capable of supporting, 3530 * and in the case of SFP-equipped adapters, which speed we are 3531 * advertising. If ixgbe_get_link_capabilities() fails for some reason, 3532 * we'll go with a default list of speeds as a last resort. 3533 */ 3534 speeds_supported = hw->phy.speeds_supported; 3535 3536 if (speeds_supported == 0) { 3537 if (ixgbe_get_link_capabilities(hw, &speeds_supported, 3538 &negotiate) != IXGBE_SUCCESS) { 3539 if (hw->mac.type == ixgbe_mac_82598EB) { 3540 speeds_supported = 3541 IXGBE_LINK_SPEED_82598_AUTONEG; 3542 } else { 3543 speeds_supported = 3544 IXGBE_LINK_SPEED_82599_AUTONEG; 3545 } 3546 } 3547 } 3548 ixgbe->speeds_supported = speeds_supported; 3549 3550 /* 3551 * By default, all supported speeds are enabled and advertised. 3552 */ 3553 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) { 3554 ixgbe->param_en_10000fdx_cap = 1; 3555 ixgbe->param_adv_10000fdx_cap = 1; 3556 } else { 3557 ixgbe->param_en_10000fdx_cap = 0; 3558 ixgbe->param_adv_10000fdx_cap = 0; 3559 } 3560 3561 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) { 3562 ixgbe->param_en_5000fdx_cap = 1; 3563 ixgbe->param_adv_5000fdx_cap = 1; 3564 } else { 3565 ixgbe->param_en_5000fdx_cap = 0; 3566 ixgbe->param_adv_5000fdx_cap = 0; 3567 } 3568 3569 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) { 3570 ixgbe->param_en_2500fdx_cap = 1; 3571 ixgbe->param_adv_2500fdx_cap = 1; 3572 } else { 3573 ixgbe->param_en_2500fdx_cap = 0; 3574 ixgbe->param_adv_2500fdx_cap = 0; 3575 } 3576 3577 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) { 3578 ixgbe->param_en_1000fdx_cap = 1; 3579 ixgbe->param_adv_1000fdx_cap = 1; 3580 } else { 3581 ixgbe->param_en_1000fdx_cap = 0; 3582 ixgbe->param_adv_1000fdx_cap = 0; 3583 } 3584 3585 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) { 3586 ixgbe->param_en_100fdx_cap = 1; 3587 ixgbe->param_adv_100fdx_cap = 1; 3588 } else { 3589 ixgbe->param_en_100fdx_cap = 0; 3590 ixgbe->param_adv_100fdx_cap = 0; 3591 } 3592 3593 ixgbe->param_pause_cap = 1; 3594 ixgbe->param_asym_pause_cap = 1; 3595 ixgbe->param_rem_fault = 0; 3596 3597 ixgbe->param_adv_autoneg_cap = 1; 3598 ixgbe->param_adv_pause_cap = 1; 3599 ixgbe->param_adv_asym_pause_cap = 1; 3600 ixgbe->param_adv_rem_fault = 0; 3601 3602 ixgbe->param_lp_10000fdx_cap = 0; 3603 ixgbe->param_lp_5000fdx_cap = 0; 3604 ixgbe->param_lp_2500fdx_cap = 0; 3605 ixgbe->param_lp_1000fdx_cap = 0; 3606 ixgbe->param_lp_100fdx_cap = 0; 3607 ixgbe->param_lp_autoneg_cap = 0; 3608 ixgbe->param_lp_pause_cap = 0; 3609 ixgbe->param_lp_asym_pause_cap = 0; 3610 ixgbe->param_lp_rem_fault = 0; 3611 } 3612 3613 /* 3614 * ixgbe_get_prop - Get a property value out of the configuration file 3615 * ixgbe.conf. 3616 * 3617 * Caller provides the name of the property, a default value, a minimum 3618 * value, and a maximum value. 3619 * 3620 * Return configured value of the property, with default, minimum and 3621 * maximum properly applied. 3622 */ 3623 static int 3624 ixgbe_get_prop(ixgbe_t *ixgbe, 3625 char *propname, /* name of the property */ 3626 int minval, /* minimum acceptable value */ 3627 int maxval, /* maximim acceptable value */ 3628 int defval) /* default value */ 3629 { 3630 int value; 3631 3632 /* 3633 * Call ddi_prop_get_int() to read the conf settings 3634 */ 3635 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3636 DDI_PROP_DONTPASS, propname, defval); 3637 if (value > maxval) 3638 value = maxval; 3639 3640 if (value < minval) 3641 value = minval; 3642 3643 return (value); 3644 } 3645 3646 /* 3647 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3648 */ 3649 int 3650 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3651 { 3652 struct ixgbe_hw *hw = &ixgbe->hw; 3653 ixgbe_link_speed advertised = 0; 3654 3655 /* 3656 * Assemble a list of enabled speeds to auto-negotiate with. 3657 */ 3658 if (ixgbe->param_en_10000fdx_cap == 1) 3659 advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3660 3661 if (ixgbe->param_en_5000fdx_cap == 1) 3662 advertised |= IXGBE_LINK_SPEED_5GB_FULL; 3663 3664 if (ixgbe->param_en_2500fdx_cap == 1) 3665 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL; 3666 3667 if (ixgbe->param_en_1000fdx_cap == 1) 3668 advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3669 3670 if (ixgbe->param_en_100fdx_cap == 1) 3671 advertised |= IXGBE_LINK_SPEED_100_FULL; 3672 3673 /* 3674 * As a last resort, autoneg with a default list of speeds. 3675 */ 3676 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) { 3677 ixgbe_notice(ixgbe, "Invalid link settings. Setting link " 3678 "to autonegotiate with full capabilities."); 3679 3680 if (hw->mac.type == ixgbe_mac_82598EB) 3681 advertised = IXGBE_LINK_SPEED_82598_AUTONEG; 3682 else 3683 advertised = IXGBE_LINK_SPEED_82599_AUTONEG; 3684 } 3685 3686 if (setup_hw) { 3687 if (ixgbe_setup_link(&ixgbe->hw, advertised, 3688 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) { 3689 ixgbe_notice(ixgbe, "Setup link failed on this " 3690 "device."); 3691 return (IXGBE_FAILURE); 3692 } 3693 } 3694 3695 return (IXGBE_SUCCESS); 3696 } 3697 3698 /* 3699 * ixgbe_driver_link_check - Link status processing. 3700 * 3701 * This function can be called in both kernel context and interrupt context 3702 */ 3703 static void 3704 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3705 { 3706 struct ixgbe_hw *hw = &ixgbe->hw; 3707 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3708 boolean_t link_up = B_FALSE; 3709 boolean_t link_changed = B_FALSE; 3710 3711 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3712 3713 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 3714 if (link_up) { 3715 ixgbe->link_check_complete = B_TRUE; 3716 3717 /* Link is up, enable flow control settings */ 3718 (void) ixgbe_fc_enable(hw); 3719 3720 /* 3721 * The Link is up, check whether it was marked as down earlier 3722 */ 3723 if (ixgbe->link_state != LINK_STATE_UP) { 3724 switch (speed) { 3725 case IXGBE_LINK_SPEED_10GB_FULL: 3726 ixgbe->link_speed = SPEED_10GB; 3727 break; 3728 case IXGBE_LINK_SPEED_5GB_FULL: 3729 ixgbe->link_speed = SPEED_5GB; 3730 break; 3731 case IXGBE_LINK_SPEED_2_5GB_FULL: 3732 ixgbe->link_speed = SPEED_2_5GB; 3733 break; 3734 case IXGBE_LINK_SPEED_1GB_FULL: 3735 ixgbe->link_speed = SPEED_1GB; 3736 break; 3737 case IXGBE_LINK_SPEED_100_FULL: 3738 ixgbe->link_speed = SPEED_100; 3739 } 3740 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3741 ixgbe->link_state = LINK_STATE_UP; 3742 link_changed = B_TRUE; 3743 } 3744 } else { 3745 if (ixgbe->link_check_complete == B_TRUE || 3746 (ixgbe->link_check_complete == B_FALSE && 3747 gethrtime() >= ixgbe->link_check_hrtime)) { 3748 /* 3749 * The link is really down 3750 */ 3751 ixgbe->link_check_complete = B_TRUE; 3752 3753 if (ixgbe->link_state != LINK_STATE_DOWN) { 3754 ixgbe->link_speed = 0; 3755 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3756 ixgbe->link_state = LINK_STATE_DOWN; 3757 link_changed = B_TRUE; 3758 } 3759 } 3760 } 3761 3762 /* 3763 * If we are in an interrupt context, need to re-enable the 3764 * interrupt, which was automasked 3765 */ 3766 if (servicing_interrupt() != 0) { 3767 ixgbe->eims |= IXGBE_EICR_LSC; 3768 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3769 } 3770 3771 if (link_changed) { 3772 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3773 } 3774 } 3775 3776 /* 3777 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3778 */ 3779 static void 3780 ixgbe_sfp_check(void *arg) 3781 { 3782 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3783 uint32_t eicr = ixgbe->eicr; 3784 struct ixgbe_hw *hw = &ixgbe->hw; 3785 3786 mutex_enter(&ixgbe->gen_lock); 3787 (void) hw->phy.ops.identify_sfp(hw); 3788 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) { 3789 /* clear the interrupt */ 3790 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw)); 3791 3792 /* if link up, do multispeed fiber setup */ 3793 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3794 B_TRUE); 3795 ixgbe_driver_link_check(ixgbe); 3796 ixgbe_get_hw_state(ixgbe); 3797 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) { 3798 /* clear the interrupt */ 3799 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw)); 3800 3801 /* if link up, do sfp module setup */ 3802 (void) hw->mac.ops.setup_sfp(hw); 3803 3804 /* do multispeed fiber setup */ 3805 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3806 B_TRUE); 3807 ixgbe_driver_link_check(ixgbe); 3808 ixgbe_get_hw_state(ixgbe); 3809 } 3810 mutex_exit(&ixgbe->gen_lock); 3811 3812 /* 3813 * We need to fully re-check the link later. 3814 */ 3815 ixgbe->link_check_complete = B_FALSE; 3816 ixgbe->link_check_hrtime = gethrtime() + 3817 (IXGBE_LINK_UP_TIME * 100000000ULL); 3818 } 3819 3820 /* 3821 * ixgbe_overtemp_check - overtemp module processing done in taskq 3822 * 3823 * This routine will only be called on adapters with temperature sensor. 3824 * The indication of over-temperature can be either SDP0 interrupt or the link 3825 * status change interrupt. 3826 */ 3827 static void 3828 ixgbe_overtemp_check(void *arg) 3829 { 3830 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3831 struct ixgbe_hw *hw = &ixgbe->hw; 3832 uint32_t eicr = ixgbe->eicr; 3833 ixgbe_link_speed speed; 3834 boolean_t link_up; 3835 3836 mutex_enter(&ixgbe->gen_lock); 3837 3838 /* make sure we know current state of link */ 3839 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 3840 3841 /* check over-temp condition */ 3842 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) || 3843 (eicr & IXGBE_EICR_LSC)) { 3844 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3845 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3846 3847 /* 3848 * Disable the adapter interrupts 3849 */ 3850 ixgbe_disable_adapter_interrupts(ixgbe); 3851 3852 /* 3853 * Disable Rx/Tx units 3854 */ 3855 (void) ixgbe_stop_adapter(hw); 3856 3857 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3858 ixgbe_error(ixgbe, 3859 "Problem: Network adapter has been stopped " 3860 "because it has overheated"); 3861 ixgbe_error(ixgbe, 3862 "Action: Restart the computer. " 3863 "If the problem persists, power off the system " 3864 "and replace the adapter"); 3865 } 3866 } 3867 3868 /* write to clear the interrupt */ 3869 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3870 3871 mutex_exit(&ixgbe->gen_lock); 3872 } 3873 3874 /* 3875 * ixgbe_phy_check - taskq to process interrupts from an external PHY 3876 * 3877 * This routine will only be called on adapters with external PHYs 3878 * (such as X550) that may be trying to raise our attention to some event. 3879 * Currently, this is limited to claiming PHY overtemperature and link status 3880 * change (LSC) events, however this may expand to include other things in 3881 * future adapters. 3882 */ 3883 static void 3884 ixgbe_phy_check(void *arg) 3885 { 3886 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3887 struct ixgbe_hw *hw = &ixgbe->hw; 3888 int rv; 3889 3890 mutex_enter(&ixgbe->gen_lock); 3891 3892 /* 3893 * X550 baseT PHY overtemp and LSC events are handled here. 3894 * 3895 * If an overtemp event occurs, it will be reflected in the 3896 * return value of phy.ops.handle_lasi() and the common code will 3897 * automatically power off the baseT PHY. This is our cue to trigger 3898 * an FMA event. 3899 * 3900 * If a link status change event occurs, phy.ops.handle_lasi() will 3901 * automatically initiate a link setup between the integrated KR PHY 3902 * and the external X557 PHY to ensure that the link speed between 3903 * them matches the link speed of the baseT link. 3904 */ 3905 rv = ixgbe_handle_lasi(hw); 3906 3907 if (rv == IXGBE_ERR_OVERTEMP) { 3908 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3909 3910 /* 3911 * Disable the adapter interrupts 3912 */ 3913 ixgbe_disable_adapter_interrupts(ixgbe); 3914 3915 /* 3916 * Disable Rx/Tx units 3917 */ 3918 (void) ixgbe_stop_adapter(hw); 3919 3920 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3921 ixgbe_error(ixgbe, 3922 "Problem: Network adapter has been stopped due to a " 3923 "overtemperature event being detected."); 3924 ixgbe_error(ixgbe, 3925 "Action: Shut down or restart the computer. If the issue " 3926 "persists, please take action in accordance with the " 3927 "recommendations from your system vendor."); 3928 } 3929 3930 mutex_exit(&ixgbe->gen_lock); 3931 } 3932 3933 /* 3934 * ixgbe_link_timer - timer for link status detection 3935 */ 3936 static void 3937 ixgbe_link_timer(void *arg) 3938 { 3939 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3940 3941 mutex_enter(&ixgbe->gen_lock); 3942 ixgbe_driver_link_check(ixgbe); 3943 mutex_exit(&ixgbe->gen_lock); 3944 } 3945 3946 /* 3947 * ixgbe_local_timer - Driver watchdog function. 3948 * 3949 * This function will handle the transmit stall check and other routines. 3950 */ 3951 static void 3952 ixgbe_local_timer(void *arg) 3953 { 3954 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3955 3956 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3957 goto out; 3958 3959 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3960 ixgbe->reset_count++; 3961 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3962 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3963 goto out; 3964 } 3965 3966 if (ixgbe_stall_check(ixgbe)) { 3967 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3968 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3969 3970 ixgbe->reset_count++; 3971 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3972 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3973 } 3974 3975 out: 3976 ixgbe_restart_watchdog_timer(ixgbe); 3977 } 3978 3979 /* 3980 * ixgbe_stall_check - Check for transmit stall. 3981 * 3982 * This function checks if the adapter is stalled (in transmit). 3983 * 3984 * It is called each time the watchdog timeout is invoked. 3985 * If the transmit descriptor reclaim continuously fails, 3986 * the watchdog value will increment by 1. If the watchdog 3987 * value exceeds the threshold, the ixgbe is assumed to 3988 * have stalled and need to be reset. 3989 */ 3990 static boolean_t 3991 ixgbe_stall_check(ixgbe_t *ixgbe) 3992 { 3993 ixgbe_tx_ring_t *tx_ring; 3994 boolean_t result; 3995 int i; 3996 3997 if (ixgbe->link_state != LINK_STATE_UP) 3998 return (B_FALSE); 3999 4000 /* 4001 * If any tx ring is stalled, we'll reset the chipset 4002 */ 4003 result = B_FALSE; 4004 for (i = 0; i < ixgbe->num_tx_rings; i++) { 4005 tx_ring = &ixgbe->tx_rings[i]; 4006 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 4007 tx_ring->tx_recycle(tx_ring); 4008 } 4009 4010 if (tx_ring->recycle_fail > 0) 4011 tx_ring->stall_watchdog++; 4012 else 4013 tx_ring->stall_watchdog = 0; 4014 4015 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 4016 result = B_TRUE; 4017 break; 4018 } 4019 } 4020 4021 if (result) { 4022 tx_ring->stall_watchdog = 0; 4023 tx_ring->recycle_fail = 0; 4024 } 4025 4026 return (result); 4027 } 4028 4029 4030 /* 4031 * is_valid_mac_addr - Check if the mac address is valid. 4032 */ 4033 static boolean_t 4034 is_valid_mac_addr(uint8_t *mac_addr) 4035 { 4036 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 4037 const uint8_t addr_test2[6] = 4038 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 4039 4040 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 4041 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 4042 return (B_FALSE); 4043 4044 return (B_TRUE); 4045 } 4046 4047 static boolean_t 4048 ixgbe_find_mac_address(ixgbe_t *ixgbe) 4049 { 4050 #ifdef __sparc 4051 struct ixgbe_hw *hw = &ixgbe->hw; 4052 uchar_t *bytes; 4053 struct ether_addr sysaddr; 4054 uint_t nelts; 4055 int err; 4056 boolean_t found = B_FALSE; 4057 4058 /* 4059 * The "vendor's factory-set address" may already have 4060 * been extracted from the chip, but if the property 4061 * "local-mac-address" is set we use that instead. 4062 * 4063 * We check whether it looks like an array of 6 4064 * bytes (which it should, if OBP set it). If we can't 4065 * make sense of it this way, we'll ignore it. 4066 */ 4067 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 4068 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 4069 if (err == DDI_PROP_SUCCESS) { 4070 if (nelts == ETHERADDRL) { 4071 while (nelts--) 4072 hw->mac.addr[nelts] = bytes[nelts]; 4073 found = B_TRUE; 4074 } 4075 ddi_prop_free(bytes); 4076 } 4077 4078 /* 4079 * Look up the OBP property "local-mac-address?". If the user has set 4080 * 'local-mac-address? = false', use "the system address" instead. 4081 */ 4082 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 4083 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 4084 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 4085 if (localetheraddr(NULL, &sysaddr) != 0) { 4086 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 4087 found = B_TRUE; 4088 } 4089 } 4090 ddi_prop_free(bytes); 4091 } 4092 4093 /* 4094 * Finally(!), if there's a valid "mac-address" property (created 4095 * if we netbooted from this interface), we must use this instead 4096 * of any of the above to ensure that the NFS/install server doesn't 4097 * get confused by the address changing as illumos takes over! 4098 */ 4099 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 4100 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 4101 if (err == DDI_PROP_SUCCESS) { 4102 if (nelts == ETHERADDRL) { 4103 while (nelts--) 4104 hw->mac.addr[nelts] = bytes[nelts]; 4105 found = B_TRUE; 4106 } 4107 ddi_prop_free(bytes); 4108 } 4109 4110 if (found) { 4111 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 4112 return (B_TRUE); 4113 } 4114 #else 4115 _NOTE(ARGUNUSED(ixgbe)); 4116 #endif 4117 4118 return (B_TRUE); 4119 } 4120 4121 #pragma inline(ixgbe_arm_watchdog_timer) 4122 static void 4123 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 4124 { 4125 /* 4126 * Fire a watchdog timer 4127 */ 4128 ixgbe->watchdog_tid = 4129 timeout(ixgbe_local_timer, 4130 (void *)ixgbe, 1 * drv_usectohz(1000000)); 4131 4132 } 4133 4134 /* 4135 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 4136 */ 4137 void 4138 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 4139 { 4140 mutex_enter(&ixgbe->watchdog_lock); 4141 4142 if (!ixgbe->watchdog_enable) { 4143 ixgbe->watchdog_enable = B_TRUE; 4144 ixgbe->watchdog_start = B_TRUE; 4145 ixgbe_arm_watchdog_timer(ixgbe); 4146 } 4147 4148 mutex_exit(&ixgbe->watchdog_lock); 4149 } 4150 4151 /* 4152 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 4153 */ 4154 void 4155 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 4156 { 4157 timeout_id_t tid; 4158 4159 mutex_enter(&ixgbe->watchdog_lock); 4160 4161 ixgbe->watchdog_enable = B_FALSE; 4162 ixgbe->watchdog_start = B_FALSE; 4163 tid = ixgbe->watchdog_tid; 4164 ixgbe->watchdog_tid = 0; 4165 4166 mutex_exit(&ixgbe->watchdog_lock); 4167 4168 if (tid != 0) 4169 (void) untimeout(tid); 4170 } 4171 4172 /* 4173 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 4174 */ 4175 void 4176 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 4177 { 4178 mutex_enter(&ixgbe->watchdog_lock); 4179 4180 if (ixgbe->watchdog_enable) { 4181 if (!ixgbe->watchdog_start) { 4182 ixgbe->watchdog_start = B_TRUE; 4183 ixgbe_arm_watchdog_timer(ixgbe); 4184 } 4185 } 4186 4187 mutex_exit(&ixgbe->watchdog_lock); 4188 } 4189 4190 /* 4191 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 4192 */ 4193 static void 4194 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 4195 { 4196 mutex_enter(&ixgbe->watchdog_lock); 4197 4198 if (ixgbe->watchdog_start) 4199 ixgbe_arm_watchdog_timer(ixgbe); 4200 4201 mutex_exit(&ixgbe->watchdog_lock); 4202 } 4203 4204 /* 4205 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 4206 */ 4207 void 4208 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 4209 { 4210 timeout_id_t tid; 4211 4212 mutex_enter(&ixgbe->watchdog_lock); 4213 4214 ixgbe->watchdog_start = B_FALSE; 4215 tid = ixgbe->watchdog_tid; 4216 ixgbe->watchdog_tid = 0; 4217 4218 mutex_exit(&ixgbe->watchdog_lock); 4219 4220 if (tid != 0) 4221 (void) untimeout(tid); 4222 } 4223 4224 /* 4225 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 4226 */ 4227 static void 4228 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 4229 { 4230 struct ixgbe_hw *hw = &ixgbe->hw; 4231 4232 /* 4233 * mask all interrupts off 4234 */ 4235 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 4236 4237 /* 4238 * for MSI-X, also disable autoclear 4239 */ 4240 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 4241 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 4242 } 4243 4244 IXGBE_WRITE_FLUSH(hw); 4245 } 4246 4247 /* 4248 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 4249 */ 4250 static void 4251 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 4252 { 4253 struct ixgbe_hw *hw = &ixgbe->hw; 4254 uint32_t eiac, eiam; 4255 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4256 4257 /* interrupt types to enable */ 4258 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 4259 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 4260 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 4261 4262 /* enable automask on "other" causes that this adapter can generate */ 4263 eiam = ixgbe->capab->other_intr; 4264 4265 /* 4266 * msi-x mode 4267 */ 4268 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 4269 /* enable autoclear but not on bits 29:20 */ 4270 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 4271 4272 /* general purpose interrupt enable */ 4273 gpie |= (IXGBE_GPIE_MSIX_MODE 4274 | IXGBE_GPIE_PBA_SUPPORT 4275 | IXGBE_GPIE_OCD 4276 | IXGBE_GPIE_EIAME); 4277 /* 4278 * non-msi-x mode 4279 */ 4280 } else { 4281 4282 /* disable autoclear, leave gpie at default */ 4283 eiac = 0; 4284 4285 /* 4286 * General purpose interrupt enable. 4287 * For 82599, X540 and X550, extended interrupt 4288 * automask enable only in MSI or MSI-X mode 4289 */ 4290 if ((hw->mac.type == ixgbe_mac_82598EB) || 4291 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 4292 gpie |= IXGBE_GPIE_EIAME; 4293 } 4294 } 4295 4296 /* Enable specific "other" interrupt types */ 4297 switch (hw->mac.type) { 4298 case ixgbe_mac_82598EB: 4299 gpie |= ixgbe->capab->other_gpie; 4300 break; 4301 4302 case ixgbe_mac_82599EB: 4303 case ixgbe_mac_X540: 4304 case ixgbe_mac_X550: 4305 case ixgbe_mac_X550EM_x: 4306 case ixgbe_mac_X550EM_a: 4307 gpie |= ixgbe->capab->other_gpie; 4308 4309 /* Enable RSC Delay 8us when LRO enabled */ 4310 if (ixgbe->lro_enable) { 4311 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 4312 } 4313 break; 4314 4315 default: 4316 break; 4317 } 4318 4319 /* write to interrupt control registers */ 4320 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4321 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 4322 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 4323 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4324 IXGBE_WRITE_FLUSH(hw); 4325 } 4326 4327 /* 4328 * ixgbe_loopback_ioctl - Loopback support. 4329 */ 4330 enum ioc_reply 4331 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 4332 { 4333 lb_info_sz_t *lbsp; 4334 lb_property_t *lbpp; 4335 uint32_t *lbmp; 4336 uint32_t size; 4337 uint32_t value; 4338 4339 if (mp->b_cont == NULL) 4340 return (IOC_INVAL); 4341 4342 switch (iocp->ioc_cmd) { 4343 default: 4344 return (IOC_INVAL); 4345 4346 case LB_GET_INFO_SIZE: 4347 size = sizeof (lb_info_sz_t); 4348 if (iocp->ioc_count != size) 4349 return (IOC_INVAL); 4350 4351 value = sizeof (lb_normal); 4352 value += sizeof (lb_mac); 4353 value += sizeof (lb_external); 4354 4355 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 4356 *lbsp = value; 4357 break; 4358 4359 case LB_GET_INFO: 4360 value = sizeof (lb_normal); 4361 value += sizeof (lb_mac); 4362 value += sizeof (lb_external); 4363 4364 size = value; 4365 if (iocp->ioc_count != size) 4366 return (IOC_INVAL); 4367 4368 value = 0; 4369 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 4370 4371 lbpp[value++] = lb_normal; 4372 lbpp[value++] = lb_mac; 4373 lbpp[value++] = lb_external; 4374 break; 4375 4376 case LB_GET_MODE: 4377 size = sizeof (uint32_t); 4378 if (iocp->ioc_count != size) 4379 return (IOC_INVAL); 4380 4381 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 4382 *lbmp = ixgbe->loopback_mode; 4383 break; 4384 4385 case LB_SET_MODE: 4386 size = 0; 4387 if (iocp->ioc_count != sizeof (uint32_t)) 4388 return (IOC_INVAL); 4389 4390 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 4391 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 4392 return (IOC_INVAL); 4393 break; 4394 } 4395 4396 iocp->ioc_count = size; 4397 iocp->ioc_error = 0; 4398 4399 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4400 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4401 return (IOC_INVAL); 4402 } 4403 4404 return (IOC_REPLY); 4405 } 4406 4407 /* 4408 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 4409 */ 4410 static boolean_t 4411 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 4412 { 4413 if (mode == ixgbe->loopback_mode) 4414 return (B_TRUE); 4415 4416 ixgbe->loopback_mode = mode; 4417 4418 if (mode == IXGBE_LB_NONE) { 4419 /* 4420 * Reset the chip 4421 */ 4422 (void) ixgbe_reset(ixgbe); 4423 return (B_TRUE); 4424 } 4425 4426 mutex_enter(&ixgbe->gen_lock); 4427 4428 switch (mode) { 4429 default: 4430 mutex_exit(&ixgbe->gen_lock); 4431 return (B_FALSE); 4432 4433 case IXGBE_LB_EXTERNAL: 4434 break; 4435 4436 case IXGBE_LB_INTERNAL_MAC: 4437 ixgbe_set_internal_mac_loopback(ixgbe); 4438 break; 4439 } 4440 4441 mutex_exit(&ixgbe->gen_lock); 4442 4443 return (B_TRUE); 4444 } 4445 4446 /* 4447 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 4448 */ 4449 static void 4450 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 4451 { 4452 struct ixgbe_hw *hw; 4453 uint32_t reg; 4454 uint8_t atlas; 4455 4456 hw = &ixgbe->hw; 4457 4458 /* 4459 * Setup MAC loopback 4460 */ 4461 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 4462 reg |= IXGBE_HLREG0_LPBK; 4463 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 4464 4465 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4466 reg &= ~IXGBE_AUTOC_LMS_MASK; 4467 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4468 4469 /* 4470 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 4471 */ 4472 switch (hw->mac.type) { 4473 case ixgbe_mac_82598EB: 4474 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4475 &atlas); 4476 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 4477 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4478 atlas); 4479 4480 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4481 &atlas); 4482 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 4483 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4484 atlas); 4485 4486 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4487 &atlas); 4488 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 4489 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4490 atlas); 4491 4492 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4493 &atlas); 4494 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 4495 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4496 atlas); 4497 break; 4498 4499 case ixgbe_mac_82599EB: 4500 case ixgbe_mac_X540: 4501 case ixgbe_mac_X550: 4502 case ixgbe_mac_X550EM_x: 4503 case ixgbe_mac_X550EM_a: 4504 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4505 reg |= (IXGBE_AUTOC_FLU | 4506 IXGBE_AUTOC_10G_KX4); 4507 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4508 4509 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 4510 B_FALSE); 4511 break; 4512 4513 default: 4514 break; 4515 } 4516 } 4517 4518 #pragma inline(ixgbe_intr_rx_work) 4519 /* 4520 * ixgbe_intr_rx_work - RX processing of ISR. 4521 */ 4522 static void 4523 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 4524 { 4525 mblk_t *mp; 4526 4527 mutex_enter(&rx_ring->rx_lock); 4528 4529 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4530 mutex_exit(&rx_ring->rx_lock); 4531 4532 if (mp != NULL) 4533 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4534 rx_ring->ring_gen_num); 4535 } 4536 4537 #pragma inline(ixgbe_intr_tx_work) 4538 /* 4539 * ixgbe_intr_tx_work - TX processing of ISR. 4540 */ 4541 static void 4542 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 4543 { 4544 ixgbe_t *ixgbe = tx_ring->ixgbe; 4545 4546 /* 4547 * Recycle the tx descriptors 4548 */ 4549 tx_ring->tx_recycle(tx_ring); 4550 4551 /* 4552 * Schedule the re-transmit 4553 */ 4554 if (tx_ring->reschedule && 4555 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 4556 tx_ring->reschedule = B_FALSE; 4557 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 4558 tx_ring->ring_handle); 4559 tx_ring->stat_reschedule++; 4560 } 4561 } 4562 4563 #pragma inline(ixgbe_intr_other_work) 4564 /* 4565 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4566 */ 4567 static void 4568 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4569 { 4570 struct ixgbe_hw *hw = &ixgbe->hw; 4571 4572 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4573 4574 /* 4575 * handle link status change 4576 */ 4577 if (eicr & IXGBE_EICR_LSC) { 4578 ixgbe_driver_link_check(ixgbe); 4579 ixgbe_get_hw_state(ixgbe); 4580 } 4581 4582 /* 4583 * check for fan failure on adapters with fans 4584 */ 4585 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4586 (eicr & IXGBE_EICR_GPI_SDP1)) { 4587 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4588 4589 /* 4590 * Disable the adapter interrupts 4591 */ 4592 ixgbe_disable_adapter_interrupts(ixgbe); 4593 4594 /* 4595 * Disable Rx/Tx units 4596 */ 4597 (void) ixgbe_stop_adapter(&ixgbe->hw); 4598 4599 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4600 ixgbe_error(ixgbe, 4601 "Problem: Network adapter has been stopped " 4602 "because the fan has stopped.\n"); 4603 ixgbe_error(ixgbe, 4604 "Action: Replace the adapter.\n"); 4605 4606 /* re-enable the interrupt, which was automasked */ 4607 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4608 } 4609 4610 /* 4611 * Do SFP check for adapters with hot-plug capability 4612 */ 4613 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4614 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) || 4615 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) { 4616 ixgbe->eicr = eicr; 4617 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4618 ixgbe_sfp_check, (void *)ixgbe, 4619 DDI_NOSLEEP)) != DDI_SUCCESS) { 4620 ixgbe_log(ixgbe, "No memory available to dispatch " 4621 "taskq for SFP check"); 4622 } 4623 } 4624 4625 /* 4626 * Do over-temperature check for adapters with temp sensor 4627 */ 4628 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4629 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) || 4630 (eicr & IXGBE_EICR_LSC))) { 4631 ixgbe->eicr = eicr; 4632 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4633 ixgbe_overtemp_check, (void *)ixgbe, 4634 DDI_NOSLEEP)) != DDI_SUCCESS) { 4635 ixgbe_log(ixgbe, "No memory available to dispatch " 4636 "taskq for overtemp check"); 4637 } 4638 } 4639 4640 /* 4641 * Process an external PHY interrupt 4642 */ 4643 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 4644 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 4645 ixgbe->eicr = eicr; 4646 if ((ddi_taskq_dispatch(ixgbe->phy_taskq, 4647 ixgbe_phy_check, (void *)ixgbe, 4648 DDI_NOSLEEP)) != DDI_SUCCESS) { 4649 ixgbe_log(ixgbe, "No memory available to dispatch " 4650 "taskq for PHY check"); 4651 } 4652 } 4653 } 4654 4655 /* 4656 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4657 */ 4658 static uint_t 4659 ixgbe_intr_legacy(void *arg1, void *arg2) 4660 { 4661 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4662 struct ixgbe_hw *hw = &ixgbe->hw; 4663 ixgbe_tx_ring_t *tx_ring; 4664 ixgbe_rx_ring_t *rx_ring; 4665 uint32_t eicr; 4666 mblk_t *mp; 4667 boolean_t tx_reschedule; 4668 uint_t result; 4669 4670 _NOTE(ARGUNUSED(arg2)); 4671 4672 mutex_enter(&ixgbe->gen_lock); 4673 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4674 mutex_exit(&ixgbe->gen_lock); 4675 return (DDI_INTR_UNCLAIMED); 4676 } 4677 4678 mp = NULL; 4679 tx_reschedule = B_FALSE; 4680 4681 /* 4682 * Any bit set in eicr: claim this interrupt 4683 */ 4684 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4685 4686 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4687 mutex_exit(&ixgbe->gen_lock); 4688 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4689 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4690 return (DDI_INTR_CLAIMED); 4691 } 4692 4693 if (eicr) { 4694 /* 4695 * For legacy interrupt, we have only one interrupt, 4696 * so we have only one rx ring and one tx ring enabled. 4697 */ 4698 ASSERT(ixgbe->num_rx_rings == 1); 4699 ASSERT(ixgbe->num_tx_rings == 1); 4700 4701 /* 4702 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4703 */ 4704 if (eicr & 0x1) { 4705 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4706 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4707 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4708 /* 4709 * Clean the rx descriptors 4710 */ 4711 rx_ring = &ixgbe->rx_rings[0]; 4712 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4713 } 4714 4715 /* 4716 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4717 */ 4718 if (eicr & 0x2) { 4719 /* 4720 * Recycle the tx descriptors 4721 */ 4722 tx_ring = &ixgbe->tx_rings[0]; 4723 tx_ring->tx_recycle(tx_ring); 4724 4725 /* 4726 * Schedule the re-transmit 4727 */ 4728 tx_reschedule = (tx_ring->reschedule && 4729 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4730 } 4731 4732 /* any interrupt type other than tx/rx */ 4733 if (eicr & ixgbe->capab->other_intr) { 4734 switch (hw->mac.type) { 4735 case ixgbe_mac_82598EB: 4736 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4737 break; 4738 4739 case ixgbe_mac_82599EB: 4740 case ixgbe_mac_X540: 4741 case ixgbe_mac_X550: 4742 case ixgbe_mac_X550EM_x: 4743 case ixgbe_mac_X550EM_a: 4744 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4745 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4746 break; 4747 4748 default: 4749 break; 4750 } 4751 ixgbe_intr_other_work(ixgbe, eicr); 4752 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4753 } 4754 4755 mutex_exit(&ixgbe->gen_lock); 4756 4757 result = DDI_INTR_CLAIMED; 4758 } else { 4759 mutex_exit(&ixgbe->gen_lock); 4760 4761 /* 4762 * No interrupt cause bits set: don't claim this interrupt. 4763 */ 4764 result = DDI_INTR_UNCLAIMED; 4765 } 4766 4767 /* re-enable the interrupts which were automasked */ 4768 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4769 4770 /* 4771 * Do the following work outside of the gen_lock 4772 */ 4773 if (mp != NULL) { 4774 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4775 rx_ring->ring_gen_num); 4776 } 4777 4778 if (tx_reschedule) { 4779 tx_ring->reschedule = B_FALSE; 4780 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4781 tx_ring->stat_reschedule++; 4782 } 4783 4784 return (result); 4785 } 4786 4787 /* 4788 * ixgbe_intr_msi - Interrupt handler for MSI. 4789 */ 4790 static uint_t 4791 ixgbe_intr_msi(void *arg1, void *arg2) 4792 { 4793 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4794 struct ixgbe_hw *hw = &ixgbe->hw; 4795 uint32_t eicr; 4796 4797 _NOTE(ARGUNUSED(arg2)); 4798 4799 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4800 4801 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4802 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4803 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4804 return (DDI_INTR_CLAIMED); 4805 } 4806 4807 /* 4808 * For MSI interrupt, we have only one vector, 4809 * so we have only one rx ring and one tx ring enabled. 4810 */ 4811 ASSERT(ixgbe->num_rx_rings == 1); 4812 ASSERT(ixgbe->num_tx_rings == 1); 4813 4814 /* 4815 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4816 */ 4817 if (eicr & 0x1) { 4818 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4819 } 4820 4821 /* 4822 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4823 */ 4824 if (eicr & 0x2) { 4825 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4826 } 4827 4828 /* any interrupt type other than tx/rx */ 4829 if (eicr & ixgbe->capab->other_intr) { 4830 mutex_enter(&ixgbe->gen_lock); 4831 switch (hw->mac.type) { 4832 case ixgbe_mac_82598EB: 4833 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4834 break; 4835 4836 case ixgbe_mac_82599EB: 4837 case ixgbe_mac_X540: 4838 case ixgbe_mac_X550: 4839 case ixgbe_mac_X550EM_x: 4840 case ixgbe_mac_X550EM_a: 4841 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4842 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4843 break; 4844 4845 default: 4846 break; 4847 } 4848 ixgbe_intr_other_work(ixgbe, eicr); 4849 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4850 mutex_exit(&ixgbe->gen_lock); 4851 } 4852 4853 /* re-enable the interrupts which were automasked */ 4854 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4855 4856 return (DDI_INTR_CLAIMED); 4857 } 4858 4859 /* 4860 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4861 */ 4862 static uint_t 4863 ixgbe_intr_msix(void *arg1, void *arg2) 4864 { 4865 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4866 ixgbe_t *ixgbe = vect->ixgbe; 4867 struct ixgbe_hw *hw = &ixgbe->hw; 4868 uint32_t eicr; 4869 int r_idx = 0; 4870 4871 _NOTE(ARGUNUSED(arg2)); 4872 4873 /* 4874 * Clean each rx ring that has its bit set in the map 4875 */ 4876 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4877 while (r_idx >= 0) { 4878 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4879 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4880 (ixgbe->num_rx_rings - 1)); 4881 } 4882 4883 /* 4884 * Clean each tx ring that has its bit set in the map 4885 */ 4886 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4887 while (r_idx >= 0) { 4888 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4889 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4890 (ixgbe->num_tx_rings - 1)); 4891 } 4892 4893 4894 /* 4895 * Clean other interrupt (link change) that has its bit set in the map 4896 */ 4897 if (BT_TEST(vect->other_map, 0) == 1) { 4898 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4899 4900 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4901 DDI_FM_OK) { 4902 ddi_fm_service_impact(ixgbe->dip, 4903 DDI_SERVICE_DEGRADED); 4904 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4905 return (DDI_INTR_CLAIMED); 4906 } 4907 4908 /* 4909 * Check "other" cause bits: any interrupt type other than tx/rx 4910 */ 4911 if (eicr & ixgbe->capab->other_intr) { 4912 mutex_enter(&ixgbe->gen_lock); 4913 switch (hw->mac.type) { 4914 case ixgbe_mac_82598EB: 4915 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4916 ixgbe_intr_other_work(ixgbe, eicr); 4917 break; 4918 4919 case ixgbe_mac_82599EB: 4920 case ixgbe_mac_X540: 4921 case ixgbe_mac_X550: 4922 case ixgbe_mac_X550EM_x: 4923 case ixgbe_mac_X550EM_a: 4924 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4925 ixgbe_intr_other_work(ixgbe, eicr); 4926 break; 4927 4928 default: 4929 break; 4930 } 4931 mutex_exit(&ixgbe->gen_lock); 4932 } 4933 4934 /* re-enable the interrupts which were automasked */ 4935 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4936 } 4937 4938 return (DDI_INTR_CLAIMED); 4939 } 4940 4941 /* 4942 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4943 * 4944 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4945 * if not successful, try Legacy. 4946 * ixgbe->intr_force can be used to force sequence to start with 4947 * any of the 3 types. 4948 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4949 */ 4950 static int 4951 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4952 { 4953 dev_info_t *devinfo; 4954 int intr_types; 4955 int rc; 4956 4957 devinfo = ixgbe->dip; 4958 4959 /* 4960 * Get supported interrupt types 4961 */ 4962 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4963 4964 if (rc != DDI_SUCCESS) { 4965 ixgbe_log(ixgbe, 4966 "Get supported interrupt types failed: %d", rc); 4967 return (IXGBE_FAILURE); 4968 } 4969 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4970 4971 ixgbe->intr_type = 0; 4972 4973 /* 4974 * Install MSI-X interrupts 4975 */ 4976 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4977 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4978 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4979 if (rc == IXGBE_SUCCESS) 4980 return (IXGBE_SUCCESS); 4981 4982 ixgbe_log(ixgbe, 4983 "Allocate MSI-X failed, trying MSI interrupts..."); 4984 } 4985 4986 /* 4987 * MSI-X not used, force rings and groups to 1 4988 */ 4989 ixgbe->num_rx_rings = 1; 4990 ixgbe->num_rx_groups = 1; 4991 ixgbe->num_tx_rings = 1; 4992 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4993 ixgbe_log(ixgbe, 4994 "MSI-X not used, force rings and groups number to 1"); 4995 4996 /* 4997 * Install MSI interrupts 4998 */ 4999 if ((intr_types & DDI_INTR_TYPE_MSI) && 5000 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 5001 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 5002 if (rc == IXGBE_SUCCESS) 5003 return (IXGBE_SUCCESS); 5004 5005 ixgbe_log(ixgbe, 5006 "Allocate MSI failed, trying Legacy interrupts..."); 5007 } 5008 5009 /* 5010 * Install legacy interrupts 5011 */ 5012 if (intr_types & DDI_INTR_TYPE_FIXED) { 5013 /* 5014 * Disallow legacy interrupts for X550. X550 has a silicon 5015 * bug which prevents Shared Legacy interrupts from working. 5016 * For details, please reference: 5017 * 5018 * Intel Ethernet Controller X550 Specification Update rev. 2.1 5019 * May 2016, erratum 22: PCIe Interrupt Status Bit 5020 */ 5021 if (ixgbe->hw.mac.type == ixgbe_mac_X550 || 5022 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x || 5023 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a || 5024 ixgbe->hw.mac.type == ixgbe_mac_X550_vf || 5025 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf || 5026 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) { 5027 ixgbe_log(ixgbe, 5028 "Legacy interrupts are not supported on this " 5029 "adapter. Please use MSI or MSI-X instead."); 5030 return (IXGBE_FAILURE); 5031 } 5032 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 5033 if (rc == IXGBE_SUCCESS) 5034 return (IXGBE_SUCCESS); 5035 5036 ixgbe_log(ixgbe, 5037 "Allocate Legacy interrupts failed"); 5038 } 5039 5040 /* 5041 * If none of the 3 types succeeded, return failure 5042 */ 5043 return (IXGBE_FAILURE); 5044 } 5045 5046 /* 5047 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 5048 * 5049 * For legacy and MSI, only 1 handle is needed. For MSI-X, 5050 * if fewer than 2 handles are available, return failure. 5051 * Upon success, this maps the vectors to rx and tx rings for 5052 * interrupts. 5053 */ 5054 static int 5055 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 5056 { 5057 dev_info_t *devinfo; 5058 int request, count, actual; 5059 int minimum; 5060 int rc; 5061 uint32_t ring_per_group; 5062 5063 devinfo = ixgbe->dip; 5064 5065 switch (intr_type) { 5066 case DDI_INTR_TYPE_FIXED: 5067 request = 1; /* Request 1 legacy interrupt handle */ 5068 minimum = 1; 5069 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 5070 break; 5071 5072 case DDI_INTR_TYPE_MSI: 5073 request = 1; /* Request 1 MSI interrupt handle */ 5074 minimum = 1; 5075 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 5076 break; 5077 5078 case DDI_INTR_TYPE_MSIX: 5079 /* 5080 * Best number of vectors for the adapter is 5081 * (# rx rings + # tx rings), however we will 5082 * limit the request number. 5083 */ 5084 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 5085 if (request > ixgbe->capab->max_ring_vect) 5086 request = ixgbe->capab->max_ring_vect; 5087 minimum = 1; 5088 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 5089 break; 5090 5091 default: 5092 ixgbe_log(ixgbe, 5093 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 5094 intr_type); 5095 return (IXGBE_FAILURE); 5096 } 5097 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 5098 request, minimum); 5099 5100 /* 5101 * Get number of supported interrupts 5102 */ 5103 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 5104 if ((rc != DDI_SUCCESS) || (count < minimum)) { 5105 ixgbe_log(ixgbe, 5106 "Get interrupt number failed. Return: %d, count: %d", 5107 rc, count); 5108 return (IXGBE_FAILURE); 5109 } 5110 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 5111 5112 actual = 0; 5113 ixgbe->intr_cnt = 0; 5114 ixgbe->intr_cnt_max = 0; 5115 ixgbe->intr_cnt_min = 0; 5116 5117 /* 5118 * Allocate an array of interrupt handles 5119 */ 5120 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 5121 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 5122 5123 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 5124 request, &actual, DDI_INTR_ALLOC_NORMAL); 5125 if (rc != DDI_SUCCESS) { 5126 ixgbe_log(ixgbe, "Allocate interrupts failed. " 5127 "return: %d, request: %d, actual: %d", 5128 rc, request, actual); 5129 goto alloc_handle_fail; 5130 } 5131 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 5132 5133 /* 5134 * upper/lower limit of interrupts 5135 */ 5136 ixgbe->intr_cnt = actual; 5137 ixgbe->intr_cnt_max = request; 5138 ixgbe->intr_cnt_min = minimum; 5139 5140 /* 5141 * rss number per group should not exceed the rx interrupt number, 5142 * else need to adjust rx ring number. 5143 */ 5144 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5145 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 5146 if (actual < ring_per_group) { 5147 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 5148 ixgbe_setup_vmdq_rss_conf(ixgbe); 5149 } 5150 5151 /* 5152 * Now we know the actual number of vectors. Here we map the vector 5153 * to other, rx rings and tx ring. 5154 */ 5155 if (actual < minimum) { 5156 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 5157 actual); 5158 goto alloc_handle_fail; 5159 } 5160 5161 /* 5162 * Get priority for first vector, assume remaining are all the same 5163 */ 5164 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 5165 if (rc != DDI_SUCCESS) { 5166 ixgbe_log(ixgbe, 5167 "Get interrupt priority failed: %d", rc); 5168 goto alloc_handle_fail; 5169 } 5170 5171 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 5172 if (rc != DDI_SUCCESS) { 5173 ixgbe_log(ixgbe, 5174 "Get interrupt cap failed: %d", rc); 5175 goto alloc_handle_fail; 5176 } 5177 5178 ixgbe->intr_type = intr_type; 5179 5180 return (IXGBE_SUCCESS); 5181 5182 alloc_handle_fail: 5183 ixgbe_rem_intrs(ixgbe); 5184 5185 return (IXGBE_FAILURE); 5186 } 5187 5188 /* 5189 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 5190 * 5191 * Before adding the interrupt handlers, the interrupt vectors have 5192 * been allocated, and the rx/tx rings have also been allocated. 5193 */ 5194 static int 5195 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 5196 { 5197 int vector = 0; 5198 int rc; 5199 5200 switch (ixgbe->intr_type) { 5201 case DDI_INTR_TYPE_MSIX: 5202 /* 5203 * Add interrupt handler for all vectors 5204 */ 5205 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 5206 /* 5207 * install pointer to vect_map[vector] 5208 */ 5209 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5210 (ddi_intr_handler_t *)ixgbe_intr_msix, 5211 (void *)&ixgbe->vect_map[vector], NULL); 5212 5213 if (rc != DDI_SUCCESS) { 5214 ixgbe_log(ixgbe, 5215 "Add interrupt handler failed. " 5216 "return: %d, vector: %d", rc, vector); 5217 for (vector--; vector >= 0; vector--) { 5218 (void) ddi_intr_remove_handler( 5219 ixgbe->htable[vector]); 5220 } 5221 return (IXGBE_FAILURE); 5222 } 5223 } 5224 5225 break; 5226 5227 case DDI_INTR_TYPE_MSI: 5228 /* 5229 * Add interrupt handlers for the only vector 5230 */ 5231 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5232 (ddi_intr_handler_t *)ixgbe_intr_msi, 5233 (void *)ixgbe, NULL); 5234 5235 if (rc != DDI_SUCCESS) { 5236 ixgbe_log(ixgbe, 5237 "Add MSI interrupt handler failed: %d", rc); 5238 return (IXGBE_FAILURE); 5239 } 5240 5241 break; 5242 5243 case DDI_INTR_TYPE_FIXED: 5244 /* 5245 * Add interrupt handlers for the only vector 5246 */ 5247 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5248 (ddi_intr_handler_t *)ixgbe_intr_legacy, 5249 (void *)ixgbe, NULL); 5250 5251 if (rc != DDI_SUCCESS) { 5252 ixgbe_log(ixgbe, 5253 "Add legacy interrupt handler failed: %d", rc); 5254 return (IXGBE_FAILURE); 5255 } 5256 5257 break; 5258 5259 default: 5260 return (IXGBE_FAILURE); 5261 } 5262 5263 return (IXGBE_SUCCESS); 5264 } 5265 5266 #pragma inline(ixgbe_map_rxring_to_vector) 5267 /* 5268 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 5269 */ 5270 static void 5271 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 5272 { 5273 /* 5274 * Set bit in map 5275 */ 5276 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5277 5278 /* 5279 * Count bits set 5280 */ 5281 ixgbe->vect_map[v_idx].rxr_cnt++; 5282 5283 /* 5284 * Remember bit position 5285 */ 5286 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 5287 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 5288 } 5289 5290 #pragma inline(ixgbe_map_txring_to_vector) 5291 /* 5292 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 5293 */ 5294 static void 5295 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 5296 { 5297 /* 5298 * Set bit in map 5299 */ 5300 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 5301 5302 /* 5303 * Count bits set 5304 */ 5305 ixgbe->vect_map[v_idx].txr_cnt++; 5306 5307 /* 5308 * Remember bit position 5309 */ 5310 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 5311 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 5312 } 5313 5314 /* 5315 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 5316 * allocation register (IVAR). 5317 * cause: 5318 * -1 : other cause 5319 * 0 : rx 5320 * 1 : tx 5321 */ 5322 static void 5323 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 5324 int8_t cause) 5325 { 5326 struct ixgbe_hw *hw = &ixgbe->hw; 5327 u32 ivar, index; 5328 5329 switch (hw->mac.type) { 5330 case ixgbe_mac_82598EB: 5331 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5332 if (cause == -1) { 5333 cause = 0; 5334 } 5335 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5336 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5337 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 5338 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 5339 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5340 break; 5341 5342 case ixgbe_mac_82599EB: 5343 case ixgbe_mac_X540: 5344 case ixgbe_mac_X550: 5345 case ixgbe_mac_X550EM_x: 5346 case ixgbe_mac_X550EM_a: 5347 if (cause == -1) { 5348 /* other causes */ 5349 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5350 index = (intr_alloc_entry & 1) * 8; 5351 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5352 ivar &= ~(0xFF << index); 5353 ivar |= (msix_vector << index); 5354 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5355 } else { 5356 /* tx or rx causes */ 5357 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5358 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5359 ivar = IXGBE_READ_REG(hw, 5360 IXGBE_IVAR(intr_alloc_entry >> 1)); 5361 ivar &= ~(0xFF << index); 5362 ivar |= (msix_vector << index); 5363 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5364 ivar); 5365 } 5366 break; 5367 5368 default: 5369 break; 5370 } 5371 } 5372 5373 /* 5374 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 5375 * given interrupt vector allocation register (IVAR). 5376 * cause: 5377 * -1 : other cause 5378 * 0 : rx 5379 * 1 : tx 5380 */ 5381 static void 5382 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 5383 { 5384 struct ixgbe_hw *hw = &ixgbe->hw; 5385 u32 ivar, index; 5386 5387 switch (hw->mac.type) { 5388 case ixgbe_mac_82598EB: 5389 if (cause == -1) { 5390 cause = 0; 5391 } 5392 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5393 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5394 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 5395 (intr_alloc_entry & 0x3))); 5396 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5397 break; 5398 5399 case ixgbe_mac_82599EB: 5400 case ixgbe_mac_X540: 5401 case ixgbe_mac_X550: 5402 case ixgbe_mac_X550EM_x: 5403 case ixgbe_mac_X550EM_a: 5404 if (cause == -1) { 5405 /* other causes */ 5406 index = (intr_alloc_entry & 1) * 8; 5407 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5408 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 5409 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5410 } else { 5411 /* tx or rx causes */ 5412 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5413 ivar = IXGBE_READ_REG(hw, 5414 IXGBE_IVAR(intr_alloc_entry >> 1)); 5415 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 5416 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5417 ivar); 5418 } 5419 break; 5420 5421 default: 5422 break; 5423 } 5424 } 5425 5426 /* 5427 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 5428 * given interrupt vector allocation register (IVAR). 5429 * cause: 5430 * -1 : other cause 5431 * 0 : rx 5432 * 1 : tx 5433 */ 5434 static void 5435 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 5436 { 5437 struct ixgbe_hw *hw = &ixgbe->hw; 5438 u32 ivar, index; 5439 5440 switch (hw->mac.type) { 5441 case ixgbe_mac_82598EB: 5442 if (cause == -1) { 5443 cause = 0; 5444 } 5445 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5446 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5447 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 5448 (intr_alloc_entry & 0x3))); 5449 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5450 break; 5451 5452 case ixgbe_mac_82599EB: 5453 case ixgbe_mac_X540: 5454 case ixgbe_mac_X550: 5455 case ixgbe_mac_X550EM_x: 5456 case ixgbe_mac_X550EM_a: 5457 if (cause == -1) { 5458 /* other causes */ 5459 index = (intr_alloc_entry & 1) * 8; 5460 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5461 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 5462 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5463 } else { 5464 /* tx or rx causes */ 5465 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5466 ivar = IXGBE_READ_REG(hw, 5467 IXGBE_IVAR(intr_alloc_entry >> 1)); 5468 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 5469 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5470 ivar); 5471 } 5472 break; 5473 5474 default: 5475 break; 5476 } 5477 } 5478 5479 /* 5480 * Convert the rx ring index driver maintained to the rx ring index 5481 * in h/w. 5482 */ 5483 static uint32_t 5484 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 5485 { 5486 5487 struct ixgbe_hw *hw = &ixgbe->hw; 5488 uint32_t rx_ring_per_group, hw_rx_index; 5489 5490 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 5491 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 5492 return (sw_rx_index); 5493 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 5494 switch (hw->mac.type) { 5495 case ixgbe_mac_82598EB: 5496 return (sw_rx_index); 5497 5498 case ixgbe_mac_82599EB: 5499 case ixgbe_mac_X540: 5500 case ixgbe_mac_X550: 5501 case ixgbe_mac_X550EM_x: 5502 case ixgbe_mac_X550EM_a: 5503 return (sw_rx_index * 2); 5504 5505 default: 5506 break; 5507 } 5508 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 5509 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5510 5511 switch (hw->mac.type) { 5512 case ixgbe_mac_82598EB: 5513 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 5514 16 + (sw_rx_index % rx_ring_per_group); 5515 return (hw_rx_index); 5516 5517 case ixgbe_mac_82599EB: 5518 case ixgbe_mac_X540: 5519 case ixgbe_mac_X550: 5520 case ixgbe_mac_X550EM_x: 5521 case ixgbe_mac_X550EM_a: 5522 if (ixgbe->num_rx_groups > 32) { 5523 hw_rx_index = (sw_rx_index / 5524 rx_ring_per_group) * 2 + 5525 (sw_rx_index % rx_ring_per_group); 5526 } else { 5527 hw_rx_index = (sw_rx_index / 5528 rx_ring_per_group) * 4 + 5529 (sw_rx_index % rx_ring_per_group); 5530 } 5531 return (hw_rx_index); 5532 5533 default: 5534 break; 5535 } 5536 } 5537 5538 /* 5539 * Should never reach. Just to make compiler happy. 5540 */ 5541 return (sw_rx_index); 5542 } 5543 5544 /* 5545 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 5546 * 5547 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 5548 * to vector[0 - (intr_cnt -1)]. 5549 */ 5550 static int 5551 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 5552 { 5553 int i, vector = 0; 5554 5555 /* initialize vector map */ 5556 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 5557 for (i = 0; i < ixgbe->intr_cnt; i++) { 5558 ixgbe->vect_map[i].ixgbe = ixgbe; 5559 } 5560 5561 /* 5562 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 5563 * tx rings[0] on RTxQ[1]. 5564 */ 5565 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5566 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 5567 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 5568 return (IXGBE_SUCCESS); 5569 } 5570 5571 /* 5572 * Interrupts/vectors mapping for MSI-X 5573 */ 5574 5575 /* 5576 * Map other interrupt to vector 0, 5577 * Set bit in map and count the bits set. 5578 */ 5579 BT_SET(ixgbe->vect_map[vector].other_map, 0); 5580 ixgbe->vect_map[vector].other_cnt++; 5581 5582 /* 5583 * Map rx ring interrupts to vectors 5584 */ 5585 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5586 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 5587 vector = (vector +1) % ixgbe->intr_cnt; 5588 } 5589 5590 /* 5591 * Map tx ring interrupts to vectors 5592 */ 5593 for (i = 0; i < ixgbe->num_tx_rings; i++) { 5594 ixgbe_map_txring_to_vector(ixgbe, i, vector); 5595 vector = (vector +1) % ixgbe->intr_cnt; 5596 } 5597 5598 return (IXGBE_SUCCESS); 5599 } 5600 5601 /* 5602 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 5603 * 5604 * This relies on ring/vector mapping already set up in the 5605 * vect_map[] structures 5606 */ 5607 static void 5608 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 5609 { 5610 struct ixgbe_hw *hw = &ixgbe->hw; 5611 ixgbe_intr_vector_t *vect; /* vector bitmap */ 5612 int r_idx; /* ring index */ 5613 int v_idx; /* vector index */ 5614 uint32_t hw_index; 5615 5616 /* 5617 * Clear any previous entries 5618 */ 5619 switch (hw->mac.type) { 5620 case ixgbe_mac_82598EB: 5621 for (v_idx = 0; v_idx < 25; v_idx++) 5622 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5623 break; 5624 5625 case ixgbe_mac_82599EB: 5626 case ixgbe_mac_X540: 5627 case ixgbe_mac_X550: 5628 case ixgbe_mac_X550EM_x: 5629 case ixgbe_mac_X550EM_a: 5630 for (v_idx = 0; v_idx < 64; v_idx++) 5631 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5632 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5633 break; 5634 5635 default: 5636 break; 5637 } 5638 5639 /* 5640 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5641 * tx rings[0] will use RTxQ[1]. 5642 */ 5643 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5644 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5645 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5646 return; 5647 } 5648 5649 /* 5650 * For MSI-X interrupt, "Other" is always on vector[0]. 5651 */ 5652 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5653 5654 /* 5655 * For each interrupt vector, populate the IVAR table 5656 */ 5657 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5658 vect = &ixgbe->vect_map[v_idx]; 5659 5660 /* 5661 * For each rx ring bit set 5662 */ 5663 r_idx = bt_getlowbit(vect->rx_map, 0, 5664 (ixgbe->num_rx_rings - 1)); 5665 5666 while (r_idx >= 0) { 5667 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5668 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5669 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5670 (ixgbe->num_rx_rings - 1)); 5671 } 5672 5673 /* 5674 * For each tx ring bit set 5675 */ 5676 r_idx = bt_getlowbit(vect->tx_map, 0, 5677 (ixgbe->num_tx_rings - 1)); 5678 5679 while (r_idx >= 0) { 5680 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5681 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5682 (ixgbe->num_tx_rings - 1)); 5683 } 5684 } 5685 } 5686 5687 /* 5688 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5689 */ 5690 static void 5691 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5692 { 5693 int i; 5694 int rc; 5695 5696 for (i = 0; i < ixgbe->intr_cnt; i++) { 5697 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5698 if (rc != DDI_SUCCESS) { 5699 IXGBE_DEBUGLOG_1(ixgbe, 5700 "Remove intr handler failed: %d", rc); 5701 } 5702 } 5703 } 5704 5705 /* 5706 * ixgbe_rem_intrs - Remove the allocated interrupts. 5707 */ 5708 static void 5709 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5710 { 5711 int i; 5712 int rc; 5713 5714 for (i = 0; i < ixgbe->intr_cnt; i++) { 5715 rc = ddi_intr_free(ixgbe->htable[i]); 5716 if (rc != DDI_SUCCESS) { 5717 IXGBE_DEBUGLOG_1(ixgbe, 5718 "Free intr failed: %d", rc); 5719 } 5720 } 5721 5722 kmem_free(ixgbe->htable, ixgbe->intr_size); 5723 ixgbe->htable = NULL; 5724 } 5725 5726 /* 5727 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5728 */ 5729 static int 5730 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5731 { 5732 int i; 5733 int rc; 5734 5735 /* 5736 * Enable interrupts 5737 */ 5738 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5739 /* 5740 * Call ddi_intr_block_enable() for MSI 5741 */ 5742 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5743 if (rc != DDI_SUCCESS) { 5744 ixgbe_log(ixgbe, 5745 "Enable block intr failed: %d", rc); 5746 return (IXGBE_FAILURE); 5747 } 5748 } else { 5749 /* 5750 * Call ddi_intr_enable() for Legacy/MSI non block enable 5751 */ 5752 for (i = 0; i < ixgbe->intr_cnt; i++) { 5753 rc = ddi_intr_enable(ixgbe->htable[i]); 5754 if (rc != DDI_SUCCESS) { 5755 ixgbe_log(ixgbe, 5756 "Enable intr failed: %d", rc); 5757 return (IXGBE_FAILURE); 5758 } 5759 } 5760 } 5761 5762 return (IXGBE_SUCCESS); 5763 } 5764 5765 /* 5766 * ixgbe_disable_intrs - Disable all the interrupts. 5767 */ 5768 static int 5769 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5770 { 5771 int i; 5772 int rc; 5773 5774 /* 5775 * Disable all interrupts 5776 */ 5777 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5778 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5779 if (rc != DDI_SUCCESS) { 5780 ixgbe_log(ixgbe, 5781 "Disable block intr failed: %d", rc); 5782 return (IXGBE_FAILURE); 5783 } 5784 } else { 5785 for (i = 0; i < ixgbe->intr_cnt; i++) { 5786 rc = ddi_intr_disable(ixgbe->htable[i]); 5787 if (rc != DDI_SUCCESS) { 5788 ixgbe_log(ixgbe, 5789 "Disable intr failed: %d", rc); 5790 return (IXGBE_FAILURE); 5791 } 5792 } 5793 } 5794 5795 return (IXGBE_SUCCESS); 5796 } 5797 5798 /* 5799 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5800 */ 5801 static void 5802 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5803 { 5804 struct ixgbe_hw *hw = &ixgbe->hw; 5805 ixgbe_link_speed speed = 0; 5806 boolean_t link_up = B_FALSE; 5807 uint32_t pcs1g_anlp = 0; 5808 5809 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5810 ixgbe->param_lp_1000fdx_cap = 0; 5811 ixgbe->param_lp_100fdx_cap = 0; 5812 5813 /* check for link, don't wait */ 5814 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 5815 5816 /* 5817 * Update the observed Link Partner's capabilities. Not all adapters 5818 * can provide full information on the LP's capable speeds, so we 5819 * provide what we can. 5820 */ 5821 if (link_up) { 5822 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5823 5824 ixgbe->param_lp_1000fdx_cap = 5825 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5826 ixgbe->param_lp_100fdx_cap = 5827 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5828 } 5829 5830 /* 5831 * Update GLD's notion of the adapter's currently advertised speeds. 5832 * Since the common code doesn't always record the current autonegotiate 5833 * settings in the phy struct for all parts (specifically, adapters with 5834 * SFPs) we first test to see if it is 0, and if so, we fall back to 5835 * using the adapter's speed capabilities which we saved during instance 5836 * init in ixgbe_init_params(). 5837 * 5838 * Adapters with SFPs will always be shown as advertising all of their 5839 * supported speeds, and adapters with baseT PHYs (where the phy struct 5840 * is maintained by the common code) will always have a factual view of 5841 * their currently-advertised speeds. In the case of SFPs, this is 5842 * acceptable as we default to advertising all speeds that the adapter 5843 * claims to support, and those properties are immutable; unlike on 5844 * baseT (copper) PHYs, where speeds can be enabled or disabled at will. 5845 */ 5846 speed = hw->phy.autoneg_advertised; 5847 if (speed == 0) 5848 speed = ixgbe->speeds_supported; 5849 5850 ixgbe->param_adv_10000fdx_cap = 5851 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0; 5852 ixgbe->param_adv_5000fdx_cap = 5853 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0; 5854 ixgbe->param_adv_2500fdx_cap = 5855 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0; 5856 ixgbe->param_adv_1000fdx_cap = 5857 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0; 5858 ixgbe->param_adv_100fdx_cap = 5859 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0; 5860 } 5861 5862 /* 5863 * ixgbe_get_driver_control - Notify that driver is in control of device. 5864 */ 5865 static void 5866 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5867 { 5868 uint32_t ctrl_ext; 5869 5870 /* 5871 * Notify firmware that driver is in control of device 5872 */ 5873 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5874 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5875 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5876 } 5877 5878 /* 5879 * ixgbe_release_driver_control - Notify that driver is no longer in control 5880 * of device. 5881 */ 5882 static void 5883 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5884 { 5885 uint32_t ctrl_ext; 5886 5887 /* 5888 * Notify firmware that driver is no longer in control of device 5889 */ 5890 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5891 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5892 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5893 } 5894 5895 /* 5896 * ixgbe_atomic_reserve - Atomic decrease operation. 5897 */ 5898 int 5899 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5900 { 5901 uint32_t oldval; 5902 uint32_t newval; 5903 5904 /* 5905 * ATOMICALLY 5906 */ 5907 do { 5908 oldval = *count_p; 5909 if (oldval < n) 5910 return (-1); 5911 newval = oldval - n; 5912 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5913 5914 return (newval); 5915 } 5916 5917 /* 5918 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5919 */ 5920 static uint8_t * 5921 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5922 { 5923 uint8_t *addr = *upd_ptr; 5924 uint8_t *new_ptr; 5925 5926 _NOTE(ARGUNUSED(hw)); 5927 _NOTE(ARGUNUSED(vmdq)); 5928 5929 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5930 *upd_ptr = new_ptr; 5931 return (addr); 5932 } 5933 5934 /* 5935 * FMA support 5936 */ 5937 int 5938 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5939 { 5940 ddi_fm_error_t de; 5941 5942 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5943 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5944 return (de.fme_status); 5945 } 5946 5947 int 5948 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5949 { 5950 ddi_fm_error_t de; 5951 5952 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5953 return (de.fme_status); 5954 } 5955 5956 /* 5957 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5958 */ 5959 static int 5960 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5961 { 5962 _NOTE(ARGUNUSED(impl_data)); 5963 /* 5964 * as the driver can always deal with an error in any dma or 5965 * access handle, we can just return the fme_status value. 5966 */ 5967 pci_ereport_post(dip, err, NULL); 5968 return (err->fme_status); 5969 } 5970 5971 static void 5972 ixgbe_fm_init(ixgbe_t *ixgbe) 5973 { 5974 ddi_iblock_cookie_t iblk; 5975 int fma_dma_flag; 5976 5977 /* 5978 * Only register with IO Fault Services if we have some capability 5979 */ 5980 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5981 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5982 } else { 5983 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5984 } 5985 5986 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5987 fma_dma_flag = 1; 5988 } else { 5989 fma_dma_flag = 0; 5990 } 5991 5992 ixgbe_set_fma_flags(fma_dma_flag); 5993 5994 if (ixgbe->fm_capabilities) { 5995 5996 /* 5997 * Register capabilities with IO Fault Services 5998 */ 5999 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 6000 6001 /* 6002 * Initialize pci ereport capabilities if ereport capable 6003 */ 6004 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 6005 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6006 pci_ereport_setup(ixgbe->dip); 6007 6008 /* 6009 * Register error callback if error callback capable 6010 */ 6011 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6012 ddi_fm_handler_register(ixgbe->dip, 6013 ixgbe_fm_error_cb, (void*) ixgbe); 6014 } 6015 } 6016 6017 static void 6018 ixgbe_fm_fini(ixgbe_t *ixgbe) 6019 { 6020 /* 6021 * Only unregister FMA capabilities if they are registered 6022 */ 6023 if (ixgbe->fm_capabilities) { 6024 6025 /* 6026 * Release any resources allocated by pci_ereport_setup() 6027 */ 6028 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 6029 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6030 pci_ereport_teardown(ixgbe->dip); 6031 6032 /* 6033 * Un-register error callback if error callback capable 6034 */ 6035 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6036 ddi_fm_handler_unregister(ixgbe->dip); 6037 6038 /* 6039 * Unregister from IO Fault Service 6040 */ 6041 ddi_fm_fini(ixgbe->dip); 6042 } 6043 } 6044 6045 void 6046 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 6047 { 6048 uint64_t ena; 6049 char buf[FM_MAX_CLASS]; 6050 6051 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 6052 ena = fm_ena_generate(0, FM_ENA_FMT1); 6053 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 6054 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 6055 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 6056 } 6057 } 6058 6059 static int 6060 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 6061 { 6062 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 6063 6064 mutex_enter(&rx_ring->rx_lock); 6065 rx_ring->ring_gen_num = mr_gen_num; 6066 mutex_exit(&rx_ring->rx_lock); 6067 return (0); 6068 } 6069 6070 /* 6071 * Get the global ring index by a ring index within a group. 6072 */ 6073 static int 6074 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 6075 { 6076 ixgbe_rx_ring_t *rx_ring; 6077 int i; 6078 6079 for (i = 0; i < ixgbe->num_rx_rings; i++) { 6080 rx_ring = &ixgbe->rx_rings[i]; 6081 if (rx_ring->group_index == gindex) 6082 rindex--; 6083 if (rindex < 0) 6084 return (i); 6085 } 6086 6087 return (-1); 6088 } 6089 6090 /* 6091 * Callback funtion for MAC layer to register all rings. 6092 */ 6093 /* ARGSUSED */ 6094 void 6095 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 6096 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 6097 { 6098 ixgbe_t *ixgbe = (ixgbe_t *)arg; 6099 mac_intr_t *mintr = &infop->mri_intr; 6100 6101 switch (rtype) { 6102 case MAC_RING_TYPE_RX: { 6103 /* 6104 * 'index' is the ring index within the group. 6105 * Need to get the global ring index by searching in groups. 6106 */ 6107 int global_ring_index = ixgbe_get_rx_ring_index( 6108 ixgbe, group_index, ring_index); 6109 6110 ASSERT(global_ring_index >= 0); 6111 6112 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 6113 rx_ring->ring_handle = rh; 6114 6115 infop->mri_driver = (mac_ring_driver_t)rx_ring; 6116 infop->mri_start = ixgbe_ring_start; 6117 infop->mri_stop = NULL; 6118 infop->mri_poll = ixgbe_ring_rx_poll; 6119 infop->mri_stat = ixgbe_rx_ring_stat; 6120 6121 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 6122 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 6123 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 6124 if (ixgbe->intr_type & 6125 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 6126 mintr->mi_ddi_handle = 6127 ixgbe->htable[rx_ring->intr_vector]; 6128 } 6129 6130 break; 6131 } 6132 case MAC_RING_TYPE_TX: { 6133 ASSERT(group_index == -1); 6134 ASSERT(ring_index < ixgbe->num_tx_rings); 6135 6136 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 6137 tx_ring->ring_handle = rh; 6138 6139 infop->mri_driver = (mac_ring_driver_t)tx_ring; 6140 infop->mri_start = NULL; 6141 infop->mri_stop = NULL; 6142 infop->mri_tx = ixgbe_ring_tx; 6143 infop->mri_stat = ixgbe_tx_ring_stat; 6144 if (ixgbe->intr_type & 6145 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 6146 mintr->mi_ddi_handle = 6147 ixgbe->htable[tx_ring->intr_vector]; 6148 } 6149 break; 6150 } 6151 default: 6152 break; 6153 } 6154 } 6155 6156 /* 6157 * Callback funtion for MAC layer to register all groups. 6158 */ 6159 void 6160 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 6161 mac_group_info_t *infop, mac_group_handle_t gh) 6162 { 6163 ixgbe_t *ixgbe = (ixgbe_t *)arg; 6164 6165 switch (rtype) { 6166 case MAC_RING_TYPE_RX: { 6167 ixgbe_rx_group_t *rx_group; 6168 6169 rx_group = &ixgbe->rx_groups[index]; 6170 rx_group->group_handle = gh; 6171 6172 infop->mgi_driver = (mac_group_driver_t)rx_group; 6173 infop->mgi_start = NULL; 6174 infop->mgi_stop = NULL; 6175 infop->mgi_addmac = ixgbe_addmac; 6176 infop->mgi_remmac = ixgbe_remmac; 6177 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 6178 6179 break; 6180 } 6181 case MAC_RING_TYPE_TX: 6182 break; 6183 default: 6184 break; 6185 } 6186 } 6187 6188 /* 6189 * Enable interrupt on the specificed rx ring. 6190 */ 6191 int 6192 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 6193 { 6194 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 6195 ixgbe_t *ixgbe = rx_ring->ixgbe; 6196 int r_idx = rx_ring->index; 6197 int hw_r_idx = rx_ring->hw_index; 6198 int v_idx = rx_ring->intr_vector; 6199 6200 mutex_enter(&ixgbe->gen_lock); 6201 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 6202 mutex_exit(&ixgbe->gen_lock); 6203 /* 6204 * Simply return 0. 6205 * Interrupts are being adjusted. ixgbe_intr_adjust() 6206 * will eventually re-enable the interrupt when it's 6207 * done with the adjustment. 6208 */ 6209 return (0); 6210 } 6211 6212 /* 6213 * To enable interrupt by setting the VAL bit of given interrupt 6214 * vector allocation register (IVAR). 6215 */ 6216 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 6217 6218 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 6219 6220 /* 6221 * Trigger a Rx interrupt on this ring 6222 */ 6223 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 6224 IXGBE_WRITE_FLUSH(&ixgbe->hw); 6225 6226 mutex_exit(&ixgbe->gen_lock); 6227 6228 return (0); 6229 } 6230 6231 /* 6232 * Disable interrupt on the specificed rx ring. 6233 */ 6234 int 6235 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 6236 { 6237 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 6238 ixgbe_t *ixgbe = rx_ring->ixgbe; 6239 int r_idx = rx_ring->index; 6240 int hw_r_idx = rx_ring->hw_index; 6241 int v_idx = rx_ring->intr_vector; 6242 6243 mutex_enter(&ixgbe->gen_lock); 6244 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 6245 mutex_exit(&ixgbe->gen_lock); 6246 /* 6247 * Simply return 0. 6248 * In the rare case where an interrupt is being 6249 * disabled while interrupts are being adjusted, 6250 * we don't fail the operation. No interrupts will 6251 * be generated while they are adjusted, and 6252 * ixgbe_intr_adjust() will cause the interrupts 6253 * to be re-enabled once it completes. Note that 6254 * in this case, packets may be delivered to the 6255 * stack via interrupts before xgbe_rx_ring_intr_enable() 6256 * is called again. This is acceptable since interrupt 6257 * adjustment is infrequent, and the stack will be 6258 * able to handle these packets. 6259 */ 6260 return (0); 6261 } 6262 6263 /* 6264 * To disable interrupt by clearing the VAL bit of given interrupt 6265 * vector allocation register (IVAR). 6266 */ 6267 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 6268 6269 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 6270 6271 mutex_exit(&ixgbe->gen_lock); 6272 6273 return (0); 6274 } 6275 6276 /* 6277 * Add a mac address. 6278 */ 6279 static int 6280 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 6281 { 6282 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 6283 ixgbe_t *ixgbe = rx_group->ixgbe; 6284 struct ixgbe_hw *hw = &ixgbe->hw; 6285 int slot, i; 6286 6287 mutex_enter(&ixgbe->gen_lock); 6288 6289 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 6290 mutex_exit(&ixgbe->gen_lock); 6291 return (ECANCELED); 6292 } 6293 6294 if (ixgbe->unicst_avail == 0) { 6295 /* no slots available */ 6296 mutex_exit(&ixgbe->gen_lock); 6297 return (ENOSPC); 6298 } 6299 6300 /* 6301 * The first ixgbe->num_rx_groups slots are reserved for each respective 6302 * group. The rest slots are shared by all groups. While adding a 6303 * MAC address, reserved slots are firstly checked then the shared 6304 * slots are searched. 6305 */ 6306 slot = -1; 6307 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 6308 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 6309 if (ixgbe->unicst_addr[i].mac.set == 0) { 6310 slot = i; 6311 break; 6312 } 6313 } 6314 } else { 6315 slot = rx_group->index; 6316 } 6317 6318 if (slot == -1) { 6319 /* no slots available */ 6320 mutex_exit(&ixgbe->gen_lock); 6321 return (ENOSPC); 6322 } 6323 6324 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 6325 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 6326 rx_group->index, IXGBE_RAH_AV); 6327 ixgbe->unicst_addr[slot].mac.set = 1; 6328 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 6329 ixgbe->unicst_avail--; 6330 6331 mutex_exit(&ixgbe->gen_lock); 6332 6333 return (0); 6334 } 6335 6336 /* 6337 * Remove a mac address. 6338 */ 6339 static int 6340 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 6341 { 6342 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 6343 ixgbe_t *ixgbe = rx_group->ixgbe; 6344 struct ixgbe_hw *hw = &ixgbe->hw; 6345 int slot; 6346 6347 mutex_enter(&ixgbe->gen_lock); 6348 6349 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 6350 mutex_exit(&ixgbe->gen_lock); 6351 return (ECANCELED); 6352 } 6353 6354 slot = ixgbe_unicst_find(ixgbe, mac_addr); 6355 if (slot == -1) { 6356 mutex_exit(&ixgbe->gen_lock); 6357 return (EINVAL); 6358 } 6359 6360 if (ixgbe->unicst_addr[slot].mac.set == 0) { 6361 mutex_exit(&ixgbe->gen_lock); 6362 return (EINVAL); 6363 } 6364 6365 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 6366 (void) ixgbe_clear_rar(hw, slot); 6367 ixgbe->unicst_addr[slot].mac.set = 0; 6368 ixgbe->unicst_avail++; 6369 6370 mutex_exit(&ixgbe->gen_lock); 6371 6372 return (0); 6373 } 6374