1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright (c) 2012, Joyent, Inc. All rights reserved. 29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 30 */ 31 32 #include "ixgbe_sw.h" 33 34 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 35 static char ixgbe_version[] = "ixgbe 1.1.7"; 36 37 /* 38 * Local function protoypes 39 */ 40 static int ixgbe_register_mac(ixgbe_t *); 41 static int ixgbe_identify_hardware(ixgbe_t *); 42 static int ixgbe_regs_map(ixgbe_t *); 43 static void ixgbe_init_properties(ixgbe_t *); 44 static int ixgbe_init_driver_settings(ixgbe_t *); 45 static void ixgbe_init_locks(ixgbe_t *); 46 static void ixgbe_destroy_locks(ixgbe_t *); 47 static int ixgbe_init(ixgbe_t *); 48 static int ixgbe_chip_start(ixgbe_t *); 49 static void ixgbe_chip_stop(ixgbe_t *); 50 static int ixgbe_reset(ixgbe_t *); 51 static void ixgbe_tx_clean(ixgbe_t *); 52 static boolean_t ixgbe_tx_drain(ixgbe_t *); 53 static boolean_t ixgbe_rx_drain(ixgbe_t *); 54 static int ixgbe_alloc_rings(ixgbe_t *); 55 static void ixgbe_free_rings(ixgbe_t *); 56 static int ixgbe_alloc_rx_data(ixgbe_t *); 57 static void ixgbe_free_rx_data(ixgbe_t *); 58 static void ixgbe_setup_rings(ixgbe_t *); 59 static void ixgbe_setup_rx(ixgbe_t *); 60 static void ixgbe_setup_tx(ixgbe_t *); 61 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 62 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 63 static void ixgbe_setup_rss(ixgbe_t *); 64 static void ixgbe_setup_vmdq(ixgbe_t *); 65 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 66 static void ixgbe_init_unicst(ixgbe_t *); 67 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 68 static void ixgbe_setup_multicst(ixgbe_t *); 69 static void ixgbe_get_hw_state(ixgbe_t *); 70 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 71 static void ixgbe_get_conf(ixgbe_t *); 72 static void ixgbe_init_params(ixgbe_t *); 73 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 74 static void ixgbe_driver_link_check(ixgbe_t *); 75 static void ixgbe_sfp_check(void *); 76 static void ixgbe_overtemp_check(void *); 77 static void ixgbe_link_timer(void *); 78 static void ixgbe_local_timer(void *); 79 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 80 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 81 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 82 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 83 static boolean_t is_valid_mac_addr(uint8_t *); 84 static boolean_t ixgbe_stall_check(ixgbe_t *); 85 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 86 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 87 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 88 static int ixgbe_alloc_intrs(ixgbe_t *); 89 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 90 static int ixgbe_add_intr_handlers(ixgbe_t *); 91 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 92 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 93 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 94 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 95 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 96 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 97 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 98 static void ixgbe_setup_adapter_vector(ixgbe_t *); 99 static void ixgbe_rem_intr_handlers(ixgbe_t *); 100 static void ixgbe_rem_intrs(ixgbe_t *); 101 static int ixgbe_enable_intrs(ixgbe_t *); 102 static int ixgbe_disable_intrs(ixgbe_t *); 103 static uint_t ixgbe_intr_legacy(void *, void *); 104 static uint_t ixgbe_intr_msi(void *, void *); 105 static uint_t ixgbe_intr_msix(void *, void *); 106 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 107 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 108 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 109 static void ixgbe_get_driver_control(struct ixgbe_hw *); 110 static int ixgbe_addmac(void *, const uint8_t *); 111 static int ixgbe_remmac(void *, const uint8_t *); 112 static void ixgbe_release_driver_control(struct ixgbe_hw *); 113 114 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 115 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 116 static int ixgbe_resume(dev_info_t *); 117 static int ixgbe_suspend(dev_info_t *); 118 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 119 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 120 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 121 static int ixgbe_intr_cb_register(ixgbe_t *); 122 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 123 124 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 125 const void *impl_data); 126 static void ixgbe_fm_init(ixgbe_t *); 127 static void ixgbe_fm_fini(ixgbe_t *); 128 129 char *ixgbe_priv_props[] = { 130 "_tx_copy_thresh", 131 "_tx_recycle_thresh", 132 "_tx_overload_thresh", 133 "_tx_resched_thresh", 134 "_rx_copy_thresh", 135 "_rx_limit_per_intr", 136 "_intr_throttling", 137 "_adv_pause_cap", 138 "_adv_asym_pause_cap", 139 NULL 140 }; 141 142 #define IXGBE_MAX_PRIV_PROPS \ 143 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 144 145 static struct cb_ops ixgbe_cb_ops = { 146 nulldev, /* cb_open */ 147 nulldev, /* cb_close */ 148 nodev, /* cb_strategy */ 149 nodev, /* cb_print */ 150 nodev, /* cb_dump */ 151 nodev, /* cb_read */ 152 nodev, /* cb_write */ 153 nodev, /* cb_ioctl */ 154 nodev, /* cb_devmap */ 155 nodev, /* cb_mmap */ 156 nodev, /* cb_segmap */ 157 nochpoll, /* cb_chpoll */ 158 ddi_prop_op, /* cb_prop_op */ 159 NULL, /* cb_stream */ 160 D_MP | D_HOTPLUG, /* cb_flag */ 161 CB_REV, /* cb_rev */ 162 nodev, /* cb_aread */ 163 nodev /* cb_awrite */ 164 }; 165 166 static struct dev_ops ixgbe_dev_ops = { 167 DEVO_REV, /* devo_rev */ 168 0, /* devo_refcnt */ 169 NULL, /* devo_getinfo */ 170 nulldev, /* devo_identify */ 171 nulldev, /* devo_probe */ 172 ixgbe_attach, /* devo_attach */ 173 ixgbe_detach, /* devo_detach */ 174 nodev, /* devo_reset */ 175 &ixgbe_cb_ops, /* devo_cb_ops */ 176 NULL, /* devo_bus_ops */ 177 ddi_power, /* devo_power */ 178 ddi_quiesce_not_supported, /* devo_quiesce */ 179 }; 180 181 static struct modldrv ixgbe_modldrv = { 182 &mod_driverops, /* Type of module. This one is a driver */ 183 ixgbe_ident, /* Discription string */ 184 &ixgbe_dev_ops /* driver ops */ 185 }; 186 187 static struct modlinkage ixgbe_modlinkage = { 188 MODREV_1, &ixgbe_modldrv, NULL 189 }; 190 191 /* 192 * Access attributes for register mapping 193 */ 194 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 195 DDI_DEVICE_ATTR_V1, 196 DDI_STRUCTURE_LE_ACC, 197 DDI_STRICTORDER_ACC, 198 DDI_FLAGERR_ACC 199 }; 200 201 /* 202 * Loopback property 203 */ 204 static lb_property_t lb_normal = { 205 normal, "normal", IXGBE_LB_NONE 206 }; 207 208 static lb_property_t lb_mac = { 209 internal, "MAC", IXGBE_LB_INTERNAL_MAC 210 }; 211 212 static lb_property_t lb_external = { 213 external, "External", IXGBE_LB_EXTERNAL 214 }; 215 216 #define IXGBE_M_CALLBACK_FLAGS \ 217 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 218 219 static mac_callbacks_t ixgbe_m_callbacks = { 220 IXGBE_M_CALLBACK_FLAGS, 221 ixgbe_m_stat, 222 ixgbe_m_start, 223 ixgbe_m_stop, 224 ixgbe_m_promisc, 225 ixgbe_m_multicst, 226 NULL, 227 NULL, 228 NULL, 229 ixgbe_m_ioctl, 230 ixgbe_m_getcapab, 231 NULL, 232 NULL, 233 ixgbe_m_setprop, 234 ixgbe_m_getprop, 235 ixgbe_m_propinfo 236 }; 237 238 /* 239 * Initialize capabilities of each supported adapter type 240 */ 241 static adapter_info_t ixgbe_82598eb_cap = { 242 64, /* maximum number of rx queues */ 243 1, /* minimum number of rx queues */ 244 64, /* default number of rx queues */ 245 16, /* maximum number of rx groups */ 246 1, /* minimum number of rx groups */ 247 1, /* default number of rx groups */ 248 32, /* maximum number of tx queues */ 249 1, /* minimum number of tx queues */ 250 8, /* default number of tx queues */ 251 16366, /* maximum MTU size */ 252 0xFFFF, /* maximum interrupt throttle rate */ 253 0, /* minimum interrupt throttle rate */ 254 200, /* default interrupt throttle rate */ 255 18, /* maximum total msix vectors */ 256 16, /* maximum number of ring vectors */ 257 2, /* maximum number of other vectors */ 258 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 259 0, /* "other" interrupt types enable mask */ 260 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 261 | IXGBE_FLAG_RSS_CAPABLE 262 | IXGBE_FLAG_VMDQ_CAPABLE) 263 }; 264 265 static adapter_info_t ixgbe_82599eb_cap = { 266 128, /* maximum number of rx queues */ 267 1, /* minimum number of rx queues */ 268 128, /* default number of rx queues */ 269 64, /* maximum number of rx groups */ 270 1, /* minimum number of rx groups */ 271 1, /* default number of rx groups */ 272 128, /* maximum number of tx queues */ 273 1, /* minimum number of tx queues */ 274 8, /* default number of tx queues */ 275 15500, /* maximum MTU size */ 276 0xFF8, /* maximum interrupt throttle rate */ 277 0, /* minimum interrupt throttle rate */ 278 200, /* default interrupt throttle rate */ 279 64, /* maximum total msix vectors */ 280 16, /* maximum number of ring vectors */ 281 2, /* maximum number of other vectors */ 282 (IXGBE_EICR_LSC 283 | IXGBE_EICR_GPI_SDP1 284 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 285 286 (IXGBE_SDP1_GPIEN 287 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 288 289 (IXGBE_FLAG_DCA_CAPABLE 290 | IXGBE_FLAG_RSS_CAPABLE 291 | IXGBE_FLAG_VMDQ_CAPABLE 292 | IXGBE_FLAG_RSC_CAPABLE 293 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 294 }; 295 296 static adapter_info_t ixgbe_X540_cap = { 297 128, /* maximum number of rx queues */ 298 1, /* minimum number of rx queues */ 299 128, /* default number of rx queues */ 300 64, /* maximum number of rx groups */ 301 1, /* minimum number of rx groups */ 302 1, /* default number of rx groups */ 303 128, /* maximum number of tx queues */ 304 1, /* minimum number of tx queues */ 305 8, /* default number of tx queues */ 306 15500, /* maximum MTU size */ 307 0xFF8, /* maximum interrupt throttle rate */ 308 0, /* minimum interrupt throttle rate */ 309 200, /* default interrupt throttle rate */ 310 64, /* maximum total msix vectors */ 311 16, /* maximum number of ring vectors */ 312 2, /* maximum number of other vectors */ 313 (IXGBE_EICR_LSC 314 | IXGBE_EICR_GPI_SDP1 315 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 316 317 (IXGBE_SDP1_GPIEN 318 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 319 320 (IXGBE_FLAG_DCA_CAPABLE 321 | IXGBE_FLAG_RSS_CAPABLE 322 | IXGBE_FLAG_VMDQ_CAPABLE 323 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 324 }; 325 326 /* 327 * Module Initialization Functions. 328 */ 329 330 int 331 _init(void) 332 { 333 int status; 334 335 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 336 337 status = mod_install(&ixgbe_modlinkage); 338 339 if (status != DDI_SUCCESS) { 340 mac_fini_ops(&ixgbe_dev_ops); 341 } 342 343 return (status); 344 } 345 346 int 347 _fini(void) 348 { 349 int status; 350 351 status = mod_remove(&ixgbe_modlinkage); 352 353 if (status == DDI_SUCCESS) { 354 mac_fini_ops(&ixgbe_dev_ops); 355 } 356 357 return (status); 358 } 359 360 int 361 _info(struct modinfo *modinfop) 362 { 363 int status; 364 365 status = mod_info(&ixgbe_modlinkage, modinfop); 366 367 return (status); 368 } 369 370 /* 371 * ixgbe_attach - Driver attach. 372 * 373 * This function is the device specific initialization entry 374 * point. This entry point is required and must be written. 375 * The DDI_ATTACH command must be provided in the attach entry 376 * point. When attach() is called with cmd set to DDI_ATTACH, 377 * all normal kernel services (such as kmem_alloc(9F)) are 378 * available for use by the driver. 379 * 380 * The attach() function will be called once for each instance 381 * of the device on the system with cmd set to DDI_ATTACH. 382 * Until attach() succeeds, the only driver entry points which 383 * may be called are open(9E) and getinfo(9E). 384 */ 385 static int 386 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 387 { 388 ixgbe_t *ixgbe; 389 struct ixgbe_osdep *osdep; 390 struct ixgbe_hw *hw; 391 int instance; 392 char taskqname[32]; 393 394 /* 395 * Check the command and perform corresponding operations 396 */ 397 switch (cmd) { 398 default: 399 return (DDI_FAILURE); 400 401 case DDI_RESUME: 402 return (ixgbe_resume(devinfo)); 403 404 case DDI_ATTACH: 405 break; 406 } 407 408 /* Get the device instance */ 409 instance = ddi_get_instance(devinfo); 410 411 /* Allocate memory for the instance data structure */ 412 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 413 414 ixgbe->dip = devinfo; 415 ixgbe->instance = instance; 416 417 hw = &ixgbe->hw; 418 osdep = &ixgbe->osdep; 419 hw->back = osdep; 420 osdep->ixgbe = ixgbe; 421 422 /* Attach the instance pointer to the dev_info data structure */ 423 ddi_set_driver_private(devinfo, ixgbe); 424 425 /* 426 * Initialize for fma support 427 */ 428 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 429 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 430 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 431 ixgbe_fm_init(ixgbe); 432 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 433 434 /* 435 * Map PCI config space registers 436 */ 437 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 438 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 439 goto attach_fail; 440 } 441 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 442 443 /* 444 * Identify the chipset family 445 */ 446 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 447 ixgbe_error(ixgbe, "Failed to identify hardware"); 448 goto attach_fail; 449 } 450 451 /* 452 * Map device registers 453 */ 454 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 455 ixgbe_error(ixgbe, "Failed to map device registers"); 456 goto attach_fail; 457 } 458 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 459 460 /* 461 * Initialize driver parameters 462 */ 463 ixgbe_init_properties(ixgbe); 464 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 465 466 /* 467 * Register interrupt callback 468 */ 469 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 470 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 471 goto attach_fail; 472 } 473 474 /* 475 * Allocate interrupts 476 */ 477 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 478 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 479 goto attach_fail; 480 } 481 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 482 483 /* 484 * Allocate rx/tx rings based on the ring numbers. 485 * The actual numbers of rx/tx rings are decided by the number of 486 * allocated interrupt vectors, so we should allocate the rings after 487 * interrupts are allocated. 488 */ 489 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 490 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 491 goto attach_fail; 492 } 493 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 494 495 /* 496 * Map rings to interrupt vectors 497 */ 498 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 499 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 500 goto attach_fail; 501 } 502 503 /* 504 * Add interrupt handlers 505 */ 506 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 507 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 508 goto attach_fail; 509 } 510 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 511 512 /* 513 * Create a taskq for sfp-change 514 */ 515 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 516 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 517 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 518 ixgbe_error(ixgbe, "sfp_taskq create failed"); 519 goto attach_fail; 520 } 521 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 522 523 /* 524 * Create a taskq for over-temp 525 */ 526 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 527 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 528 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 529 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 530 goto attach_fail; 531 } 532 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 533 534 /* 535 * Initialize driver parameters 536 */ 537 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 538 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 539 goto attach_fail; 540 } 541 542 /* 543 * Initialize mutexes for this device. 544 * Do this before enabling the interrupt handler and 545 * register the softint to avoid the condition where 546 * interrupt handler can try using uninitialized mutex. 547 */ 548 ixgbe_init_locks(ixgbe); 549 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 550 551 /* 552 * Initialize chipset hardware 553 */ 554 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 555 ixgbe_error(ixgbe, "Failed to initialize adapter"); 556 goto attach_fail; 557 } 558 ixgbe->link_check_complete = B_FALSE; 559 ixgbe->link_check_hrtime = gethrtime() + 560 (IXGBE_LINK_UP_TIME * 100000000ULL); 561 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 562 563 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 564 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 565 goto attach_fail; 566 } 567 568 /* 569 * Initialize statistics 570 */ 571 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 572 ixgbe_error(ixgbe, "Failed to initialize statistics"); 573 goto attach_fail; 574 } 575 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 576 577 /* 578 * Register the driver to the MAC 579 */ 580 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 581 ixgbe_error(ixgbe, "Failed to register MAC"); 582 goto attach_fail; 583 } 584 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 585 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 586 587 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 588 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 589 if (ixgbe->periodic_id == 0) { 590 ixgbe_error(ixgbe, "Failed to add the link check timer"); 591 goto attach_fail; 592 } 593 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 594 595 /* 596 * Now that mutex locks are initialized, and the chip is also 597 * initialized, enable interrupts. 598 */ 599 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 600 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 601 goto attach_fail; 602 } 603 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 604 605 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version); 606 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 607 608 return (DDI_SUCCESS); 609 610 attach_fail: 611 ixgbe_unconfigure(devinfo, ixgbe); 612 return (DDI_FAILURE); 613 } 614 615 /* 616 * ixgbe_detach - Driver detach. 617 * 618 * The detach() function is the complement of the attach routine. 619 * If cmd is set to DDI_DETACH, detach() is used to remove the 620 * state associated with a given instance of a device node 621 * prior to the removal of that instance from the system. 622 * 623 * The detach() function will be called once for each instance 624 * of the device for which there has been a successful attach() 625 * once there are no longer any opens on the device. 626 * 627 * Interrupts routine are disabled, All memory allocated by this 628 * driver are freed. 629 */ 630 static int 631 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 632 { 633 ixgbe_t *ixgbe; 634 635 /* 636 * Check detach command 637 */ 638 switch (cmd) { 639 default: 640 return (DDI_FAILURE); 641 642 case DDI_SUSPEND: 643 return (ixgbe_suspend(devinfo)); 644 645 case DDI_DETACH: 646 break; 647 } 648 649 /* 650 * Get the pointer to the driver private data structure 651 */ 652 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 653 if (ixgbe == NULL) 654 return (DDI_FAILURE); 655 656 /* 657 * If the device is still running, it needs to be stopped first. 658 * This check is necessary because under some specific circumstances, 659 * the detach routine can be called without stopping the interface 660 * first. 661 */ 662 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 663 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 664 mutex_enter(&ixgbe->gen_lock); 665 ixgbe_stop(ixgbe, B_TRUE); 666 mutex_exit(&ixgbe->gen_lock); 667 /* Disable and stop the watchdog timer */ 668 ixgbe_disable_watchdog_timer(ixgbe); 669 } 670 671 /* 672 * Check if there are still rx buffers held by the upper layer. 673 * If so, fail the detach. 674 */ 675 if (!ixgbe_rx_drain(ixgbe)) 676 return (DDI_FAILURE); 677 678 /* 679 * Do the remaining unconfigure routines 680 */ 681 ixgbe_unconfigure(devinfo, ixgbe); 682 683 return (DDI_SUCCESS); 684 } 685 686 static void 687 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 688 { 689 /* 690 * Disable interrupt 691 */ 692 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 693 (void) ixgbe_disable_intrs(ixgbe); 694 } 695 696 /* 697 * remove the link check timer 698 */ 699 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 700 if (ixgbe->periodic_id != NULL) { 701 ddi_periodic_delete(ixgbe->periodic_id); 702 ixgbe->periodic_id = NULL; 703 } 704 } 705 706 /* 707 * Unregister MAC 708 */ 709 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 710 (void) mac_unregister(ixgbe->mac_hdl); 711 } 712 713 /* 714 * Free statistics 715 */ 716 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 717 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 718 } 719 720 /* 721 * Remove interrupt handlers 722 */ 723 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 724 ixgbe_rem_intr_handlers(ixgbe); 725 } 726 727 /* 728 * Remove taskq for sfp-status-change 729 */ 730 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 731 ddi_taskq_destroy(ixgbe->sfp_taskq); 732 } 733 734 /* 735 * Remove taskq for over-temp 736 */ 737 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 738 ddi_taskq_destroy(ixgbe->overtemp_taskq); 739 } 740 741 /* 742 * Remove interrupts 743 */ 744 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 745 ixgbe_rem_intrs(ixgbe); 746 } 747 748 /* 749 * Unregister interrupt callback handler 750 */ 751 (void) ddi_cb_unregister(ixgbe->cb_hdl); 752 753 /* 754 * Remove driver properties 755 */ 756 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 757 (void) ddi_prop_remove_all(devinfo); 758 } 759 760 /* 761 * Stop the chipset 762 */ 763 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 764 mutex_enter(&ixgbe->gen_lock); 765 ixgbe_chip_stop(ixgbe); 766 mutex_exit(&ixgbe->gen_lock); 767 } 768 769 /* 770 * Free register handle 771 */ 772 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 773 if (ixgbe->osdep.reg_handle != NULL) 774 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 775 } 776 777 /* 778 * Free PCI config handle 779 */ 780 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 781 if (ixgbe->osdep.cfg_handle != NULL) 782 pci_config_teardown(&ixgbe->osdep.cfg_handle); 783 } 784 785 /* 786 * Free locks 787 */ 788 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 789 ixgbe_destroy_locks(ixgbe); 790 } 791 792 /* 793 * Free the rx/tx rings 794 */ 795 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 796 ixgbe_free_rings(ixgbe); 797 } 798 799 /* 800 * Unregister FMA capabilities 801 */ 802 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 803 ixgbe_fm_fini(ixgbe); 804 } 805 806 /* 807 * Free the driver data structure 808 */ 809 kmem_free(ixgbe, sizeof (ixgbe_t)); 810 811 ddi_set_driver_private(devinfo, NULL); 812 } 813 814 /* 815 * ixgbe_register_mac - Register the driver and its function pointers with 816 * the GLD interface. 817 */ 818 static int 819 ixgbe_register_mac(ixgbe_t *ixgbe) 820 { 821 struct ixgbe_hw *hw = &ixgbe->hw; 822 mac_register_t *mac; 823 int status; 824 825 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 826 return (IXGBE_FAILURE); 827 828 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 829 mac->m_driver = ixgbe; 830 mac->m_dip = ixgbe->dip; 831 mac->m_src_addr = hw->mac.addr; 832 mac->m_callbacks = &ixgbe_m_callbacks; 833 mac->m_min_sdu = 0; 834 mac->m_max_sdu = ixgbe->default_mtu; 835 mac->m_margin = VLAN_TAGSZ; 836 mac->m_priv_props = ixgbe_priv_props; 837 mac->m_v12n = MAC_VIRT_LEVEL1; 838 839 status = mac_register(mac, &ixgbe->mac_hdl); 840 841 mac_free(mac); 842 843 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 844 } 845 846 /* 847 * ixgbe_identify_hardware - Identify the type of the chipset. 848 */ 849 static int 850 ixgbe_identify_hardware(ixgbe_t *ixgbe) 851 { 852 struct ixgbe_hw *hw = &ixgbe->hw; 853 struct ixgbe_osdep *osdep = &ixgbe->osdep; 854 855 /* 856 * Get the device id 857 */ 858 hw->vendor_id = 859 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 860 hw->device_id = 861 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 862 hw->revision_id = 863 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 864 hw->subsystem_device_id = 865 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 866 hw->subsystem_vendor_id = 867 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 868 869 /* 870 * Set the mac type of the adapter based on the device id 871 */ 872 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 873 return (IXGBE_FAILURE); 874 } 875 876 /* 877 * Install adapter capabilities 878 */ 879 switch (hw->mac.type) { 880 case ixgbe_mac_82598EB: 881 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 882 ixgbe->capab = &ixgbe_82598eb_cap; 883 884 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 885 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 886 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 887 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 888 } 889 break; 890 891 case ixgbe_mac_82599EB: 892 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 893 ixgbe->capab = &ixgbe_82599eb_cap; 894 895 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 896 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 897 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 898 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 899 } 900 break; 901 902 case ixgbe_mac_X540: 903 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n"); 904 ixgbe->capab = &ixgbe_X540_cap; 905 /* 906 * For now, X540 is all set in its capab structure. 907 * As other X540 variants show up, things can change here. 908 */ 909 break; 910 911 default: 912 IXGBE_DEBUGLOG_1(ixgbe, 913 "adapter not supported in ixgbe_identify_hardware(): %d\n", 914 hw->mac.type); 915 return (IXGBE_FAILURE); 916 } 917 918 return (IXGBE_SUCCESS); 919 } 920 921 /* 922 * ixgbe_regs_map - Map the device registers. 923 * 924 */ 925 static int 926 ixgbe_regs_map(ixgbe_t *ixgbe) 927 { 928 dev_info_t *devinfo = ixgbe->dip; 929 struct ixgbe_hw *hw = &ixgbe->hw; 930 struct ixgbe_osdep *osdep = &ixgbe->osdep; 931 off_t mem_size; 932 933 /* 934 * First get the size of device registers to be mapped. 935 */ 936 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 937 != DDI_SUCCESS) { 938 return (IXGBE_FAILURE); 939 } 940 941 /* 942 * Call ddi_regs_map_setup() to map registers 943 */ 944 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 945 (caddr_t *)&hw->hw_addr, 0, 946 mem_size, &ixgbe_regs_acc_attr, 947 &osdep->reg_handle)) != DDI_SUCCESS) { 948 return (IXGBE_FAILURE); 949 } 950 951 return (IXGBE_SUCCESS); 952 } 953 954 /* 955 * ixgbe_init_properties - Initialize driver properties. 956 */ 957 static void 958 ixgbe_init_properties(ixgbe_t *ixgbe) 959 { 960 /* 961 * Get conf file properties, including link settings 962 * jumbo frames, ring number, descriptor number, etc. 963 */ 964 ixgbe_get_conf(ixgbe); 965 966 ixgbe_init_params(ixgbe); 967 } 968 969 /* 970 * ixgbe_init_driver_settings - Initialize driver settings. 971 * 972 * The settings include hardware function pointers, bus information, 973 * rx/tx rings settings, link state, and any other parameters that 974 * need to be setup during driver initialization. 975 */ 976 static int 977 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 978 { 979 struct ixgbe_hw *hw = &ixgbe->hw; 980 dev_info_t *devinfo = ixgbe->dip; 981 ixgbe_rx_ring_t *rx_ring; 982 ixgbe_rx_group_t *rx_group; 983 ixgbe_tx_ring_t *tx_ring; 984 uint32_t rx_size; 985 uint32_t tx_size; 986 uint32_t ring_per_group; 987 int i; 988 989 /* 990 * Initialize chipset specific hardware function pointers 991 */ 992 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 993 return (IXGBE_FAILURE); 994 } 995 996 /* 997 * Get the system page size 998 */ 999 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 1000 1001 /* 1002 * Set rx buffer size 1003 * 1004 * The IP header alignment room is counted in the calculation. 1005 * The rx buffer size is in unit of 1K that is required by the 1006 * chipset hardware. 1007 */ 1008 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 1009 ixgbe->rx_buf_size = ((rx_size >> 10) + 1010 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1011 1012 /* 1013 * Set tx buffer size 1014 */ 1015 tx_size = ixgbe->max_frame_size; 1016 ixgbe->tx_buf_size = ((tx_size >> 10) + 1017 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1018 1019 /* 1020 * Initialize rx/tx rings/groups parameters 1021 */ 1022 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 1023 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1024 rx_ring = &ixgbe->rx_rings[i]; 1025 rx_ring->index = i; 1026 rx_ring->ixgbe = ixgbe; 1027 rx_ring->group_index = i / ring_per_group; 1028 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 1029 } 1030 1031 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1032 rx_group = &ixgbe->rx_groups[i]; 1033 rx_group->index = i; 1034 rx_group->ixgbe = ixgbe; 1035 } 1036 1037 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1038 tx_ring = &ixgbe->tx_rings[i]; 1039 tx_ring->index = i; 1040 tx_ring->ixgbe = ixgbe; 1041 if (ixgbe->tx_head_wb_enable) 1042 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1043 else 1044 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1045 1046 tx_ring->ring_size = ixgbe->tx_ring_size; 1047 tx_ring->free_list_size = ixgbe->tx_ring_size + 1048 (ixgbe->tx_ring_size >> 1); 1049 } 1050 1051 /* 1052 * Initialize values of interrupt throttling rate 1053 */ 1054 for (i = 1; i < MAX_INTR_VECTOR; i++) 1055 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1056 1057 /* 1058 * The initial link state should be "unknown" 1059 */ 1060 ixgbe->link_state = LINK_STATE_UNKNOWN; 1061 1062 return (IXGBE_SUCCESS); 1063 } 1064 1065 /* 1066 * ixgbe_init_locks - Initialize locks. 1067 */ 1068 static void 1069 ixgbe_init_locks(ixgbe_t *ixgbe) 1070 { 1071 ixgbe_rx_ring_t *rx_ring; 1072 ixgbe_tx_ring_t *tx_ring; 1073 int i; 1074 1075 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1076 rx_ring = &ixgbe->rx_rings[i]; 1077 mutex_init(&rx_ring->rx_lock, NULL, 1078 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1079 } 1080 1081 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1082 tx_ring = &ixgbe->tx_rings[i]; 1083 mutex_init(&tx_ring->tx_lock, NULL, 1084 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1085 mutex_init(&tx_ring->recycle_lock, NULL, 1086 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1087 mutex_init(&tx_ring->tcb_head_lock, NULL, 1088 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1089 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1090 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1091 } 1092 1093 mutex_init(&ixgbe->gen_lock, NULL, 1094 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1095 1096 mutex_init(&ixgbe->watchdog_lock, NULL, 1097 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1098 } 1099 1100 /* 1101 * ixgbe_destroy_locks - Destroy locks. 1102 */ 1103 static void 1104 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1105 { 1106 ixgbe_rx_ring_t *rx_ring; 1107 ixgbe_tx_ring_t *tx_ring; 1108 int i; 1109 1110 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1111 rx_ring = &ixgbe->rx_rings[i]; 1112 mutex_destroy(&rx_ring->rx_lock); 1113 } 1114 1115 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1116 tx_ring = &ixgbe->tx_rings[i]; 1117 mutex_destroy(&tx_ring->tx_lock); 1118 mutex_destroy(&tx_ring->recycle_lock); 1119 mutex_destroy(&tx_ring->tcb_head_lock); 1120 mutex_destroy(&tx_ring->tcb_tail_lock); 1121 } 1122 1123 mutex_destroy(&ixgbe->gen_lock); 1124 mutex_destroy(&ixgbe->watchdog_lock); 1125 } 1126 1127 static int 1128 ixgbe_resume(dev_info_t *devinfo) 1129 { 1130 ixgbe_t *ixgbe; 1131 int i; 1132 1133 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1134 if (ixgbe == NULL) 1135 return (DDI_FAILURE); 1136 1137 mutex_enter(&ixgbe->gen_lock); 1138 1139 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1140 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1141 mutex_exit(&ixgbe->gen_lock); 1142 return (DDI_FAILURE); 1143 } 1144 1145 /* 1146 * Enable and start the watchdog timer 1147 */ 1148 ixgbe_enable_watchdog_timer(ixgbe); 1149 } 1150 1151 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1152 1153 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1154 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1155 mac_tx_ring_update(ixgbe->mac_hdl, 1156 ixgbe->tx_rings[i].ring_handle); 1157 } 1158 } 1159 1160 mutex_exit(&ixgbe->gen_lock); 1161 1162 return (DDI_SUCCESS); 1163 } 1164 1165 static int 1166 ixgbe_suspend(dev_info_t *devinfo) 1167 { 1168 ixgbe_t *ixgbe; 1169 1170 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1171 if (ixgbe == NULL) 1172 return (DDI_FAILURE); 1173 1174 mutex_enter(&ixgbe->gen_lock); 1175 1176 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1177 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1178 mutex_exit(&ixgbe->gen_lock); 1179 return (DDI_SUCCESS); 1180 } 1181 ixgbe_stop(ixgbe, B_FALSE); 1182 1183 mutex_exit(&ixgbe->gen_lock); 1184 1185 /* 1186 * Disable and stop the watchdog timer 1187 */ 1188 ixgbe_disable_watchdog_timer(ixgbe); 1189 1190 return (DDI_SUCCESS); 1191 } 1192 1193 /* 1194 * ixgbe_init - Initialize the device. 1195 */ 1196 static int 1197 ixgbe_init(ixgbe_t *ixgbe) 1198 { 1199 struct ixgbe_hw *hw = &ixgbe->hw; 1200 1201 mutex_enter(&ixgbe->gen_lock); 1202 1203 /* 1204 * Reset chipset to put the hardware in a known state 1205 * before we try to do anything with the eeprom. 1206 */ 1207 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1208 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1209 goto init_fail; 1210 } 1211 1212 /* 1213 * Need to init eeprom before validating the checksum. 1214 */ 1215 if (ixgbe_init_eeprom_params(hw) < 0) { 1216 ixgbe_error(ixgbe, 1217 "Unable to intitialize the eeprom interface."); 1218 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1219 goto init_fail; 1220 } 1221 1222 /* 1223 * NVM validation 1224 */ 1225 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1226 /* 1227 * Some PCI-E parts fail the first check due to 1228 * the link being in sleep state. Call it again, 1229 * if it fails a second time it's a real issue. 1230 */ 1231 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1232 ixgbe_error(ixgbe, 1233 "Invalid NVM checksum. Please contact " 1234 "the vendor to update the NVM."); 1235 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1236 goto init_fail; 1237 } 1238 } 1239 1240 /* 1241 * Setup default flow control thresholds - enable/disable 1242 * & flow control type is controlled by ixgbe.conf 1243 */ 1244 hw->fc.high_water[0] = DEFAULT_FCRTH; 1245 hw->fc.low_water[0] = DEFAULT_FCRTL; 1246 hw->fc.pause_time = DEFAULT_FCPAUSE; 1247 hw->fc.send_xon = B_TRUE; 1248 1249 /* 1250 * Initialize link settings 1251 */ 1252 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1253 1254 /* 1255 * Initialize the chipset hardware 1256 */ 1257 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1258 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1259 goto init_fail; 1260 } 1261 1262 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1263 goto init_fail; 1264 } 1265 1266 mutex_exit(&ixgbe->gen_lock); 1267 return (IXGBE_SUCCESS); 1268 1269 init_fail: 1270 /* 1271 * Reset PHY 1272 */ 1273 (void) ixgbe_reset_phy(hw); 1274 1275 mutex_exit(&ixgbe->gen_lock); 1276 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1277 return (IXGBE_FAILURE); 1278 } 1279 1280 /* 1281 * ixgbe_chip_start - Initialize and start the chipset hardware. 1282 */ 1283 static int 1284 ixgbe_chip_start(ixgbe_t *ixgbe) 1285 { 1286 struct ixgbe_hw *hw = &ixgbe->hw; 1287 int ret_val, i; 1288 1289 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1290 1291 /* 1292 * Get the mac address 1293 * This function should handle SPARC case correctly. 1294 */ 1295 if (!ixgbe_find_mac_address(ixgbe)) { 1296 ixgbe_error(ixgbe, "Failed to get the mac address"); 1297 return (IXGBE_FAILURE); 1298 } 1299 1300 /* 1301 * Validate the mac address 1302 */ 1303 (void) ixgbe_init_rx_addrs(hw); 1304 if (!is_valid_mac_addr(hw->mac.addr)) { 1305 ixgbe_error(ixgbe, "Invalid mac address"); 1306 return (IXGBE_FAILURE); 1307 } 1308 1309 /* 1310 * Configure/Initialize hardware 1311 */ 1312 ret_val = ixgbe_init_hw(hw); 1313 if (ret_val != IXGBE_SUCCESS) { 1314 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1315 ixgbe_error(ixgbe, 1316 "This 82599 device is pre-release and contains" 1317 " outdated firmware, please contact your hardware" 1318 " vendor for a replacement."); 1319 } else { 1320 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1321 return (IXGBE_FAILURE); 1322 } 1323 } 1324 1325 /* 1326 * Re-enable relaxed ordering for performance. It is disabled 1327 * by default in the hardware init. 1328 */ 1329 if (ixgbe->relax_order_enable == B_TRUE) 1330 ixgbe_enable_relaxed_ordering(hw); 1331 1332 /* 1333 * Setup adapter interrupt vectors 1334 */ 1335 ixgbe_setup_adapter_vector(ixgbe); 1336 1337 /* 1338 * Initialize unicast addresses. 1339 */ 1340 ixgbe_init_unicst(ixgbe); 1341 1342 /* 1343 * Setup and initialize the mctable structures. 1344 */ 1345 ixgbe_setup_multicst(ixgbe); 1346 1347 /* 1348 * Set interrupt throttling rate 1349 */ 1350 for (i = 0; i < ixgbe->intr_cnt; i++) { 1351 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1352 } 1353 1354 /* 1355 * Save the state of the phy 1356 */ 1357 ixgbe_get_hw_state(ixgbe); 1358 1359 /* 1360 * Make sure driver has control 1361 */ 1362 ixgbe_get_driver_control(hw); 1363 1364 return (IXGBE_SUCCESS); 1365 } 1366 1367 /* 1368 * ixgbe_chip_stop - Stop the chipset hardware 1369 */ 1370 static void 1371 ixgbe_chip_stop(ixgbe_t *ixgbe) 1372 { 1373 struct ixgbe_hw *hw = &ixgbe->hw; 1374 1375 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1376 1377 /* 1378 * Tell firmware driver is no longer in control 1379 */ 1380 ixgbe_release_driver_control(hw); 1381 1382 /* 1383 * Reset the chipset 1384 */ 1385 (void) ixgbe_reset_hw(hw); 1386 1387 /* 1388 * Reset PHY 1389 */ 1390 (void) ixgbe_reset_phy(hw); 1391 } 1392 1393 /* 1394 * ixgbe_reset - Reset the chipset and re-start the driver. 1395 * 1396 * It involves stopping and re-starting the chipset, 1397 * and re-configuring the rx/tx rings. 1398 */ 1399 static int 1400 ixgbe_reset(ixgbe_t *ixgbe) 1401 { 1402 int i; 1403 1404 /* 1405 * Disable and stop the watchdog timer 1406 */ 1407 ixgbe_disable_watchdog_timer(ixgbe); 1408 1409 mutex_enter(&ixgbe->gen_lock); 1410 1411 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1412 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1413 1414 ixgbe_stop(ixgbe, B_FALSE); 1415 1416 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1417 mutex_exit(&ixgbe->gen_lock); 1418 return (IXGBE_FAILURE); 1419 } 1420 1421 /* 1422 * After resetting, need to recheck the link status. 1423 */ 1424 ixgbe->link_check_complete = B_FALSE; 1425 ixgbe->link_check_hrtime = gethrtime() + 1426 (IXGBE_LINK_UP_TIME * 100000000ULL); 1427 1428 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1429 1430 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1431 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1432 mac_tx_ring_update(ixgbe->mac_hdl, 1433 ixgbe->tx_rings[i].ring_handle); 1434 } 1435 } 1436 1437 mutex_exit(&ixgbe->gen_lock); 1438 1439 /* 1440 * Enable and start the watchdog timer 1441 */ 1442 ixgbe_enable_watchdog_timer(ixgbe); 1443 1444 return (IXGBE_SUCCESS); 1445 } 1446 1447 /* 1448 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1449 */ 1450 static void 1451 ixgbe_tx_clean(ixgbe_t *ixgbe) 1452 { 1453 ixgbe_tx_ring_t *tx_ring; 1454 tx_control_block_t *tcb; 1455 link_list_t pending_list; 1456 uint32_t desc_num; 1457 int i, j; 1458 1459 LINK_LIST_INIT(&pending_list); 1460 1461 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1462 tx_ring = &ixgbe->tx_rings[i]; 1463 1464 mutex_enter(&tx_ring->recycle_lock); 1465 1466 /* 1467 * Clean the pending tx data - the pending packets in the 1468 * work_list that have no chances to be transmitted again. 1469 * 1470 * We must ensure the chipset is stopped or the link is down 1471 * before cleaning the transmit packets. 1472 */ 1473 desc_num = 0; 1474 for (j = 0; j < tx_ring->ring_size; j++) { 1475 tcb = tx_ring->work_list[j]; 1476 if (tcb != NULL) { 1477 desc_num += tcb->desc_num; 1478 1479 tx_ring->work_list[j] = NULL; 1480 1481 ixgbe_free_tcb(tcb); 1482 1483 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1484 } 1485 } 1486 1487 if (desc_num > 0) { 1488 atomic_add_32(&tx_ring->tbd_free, desc_num); 1489 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1490 1491 /* 1492 * Reset the head and tail pointers of the tbd ring; 1493 * Reset the writeback head if it's enable. 1494 */ 1495 tx_ring->tbd_head = 0; 1496 tx_ring->tbd_tail = 0; 1497 if (ixgbe->tx_head_wb_enable) 1498 *tx_ring->tbd_head_wb = 0; 1499 1500 IXGBE_WRITE_REG(&ixgbe->hw, 1501 IXGBE_TDH(tx_ring->index), 0); 1502 IXGBE_WRITE_REG(&ixgbe->hw, 1503 IXGBE_TDT(tx_ring->index), 0); 1504 } 1505 1506 mutex_exit(&tx_ring->recycle_lock); 1507 1508 /* 1509 * Add the tx control blocks in the pending list to 1510 * the free list. 1511 */ 1512 ixgbe_put_free_list(tx_ring, &pending_list); 1513 } 1514 } 1515 1516 /* 1517 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1518 * transmitted. 1519 */ 1520 static boolean_t 1521 ixgbe_tx_drain(ixgbe_t *ixgbe) 1522 { 1523 ixgbe_tx_ring_t *tx_ring; 1524 boolean_t done; 1525 int i, j; 1526 1527 /* 1528 * Wait for a specific time to allow pending tx packets 1529 * to be transmitted. 1530 * 1531 * Check the counter tbd_free to see if transmission is done. 1532 * No lock protection is needed here. 1533 * 1534 * Return B_TRUE if all pending packets have been transmitted; 1535 * Otherwise return B_FALSE; 1536 */ 1537 for (i = 0; i < TX_DRAIN_TIME; i++) { 1538 1539 done = B_TRUE; 1540 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1541 tx_ring = &ixgbe->tx_rings[j]; 1542 done = done && 1543 (tx_ring->tbd_free == tx_ring->ring_size); 1544 } 1545 1546 if (done) 1547 break; 1548 1549 msec_delay(1); 1550 } 1551 1552 return (done); 1553 } 1554 1555 /* 1556 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1557 */ 1558 static boolean_t 1559 ixgbe_rx_drain(ixgbe_t *ixgbe) 1560 { 1561 boolean_t done = B_TRUE; 1562 int i; 1563 1564 /* 1565 * Polling the rx free list to check if those rx buffers held by 1566 * the upper layer are released. 1567 * 1568 * Check the counter rcb_free to see if all pending buffers are 1569 * released. No lock protection is needed here. 1570 * 1571 * Return B_TRUE if all pending buffers have been released; 1572 * Otherwise return B_FALSE; 1573 */ 1574 for (i = 0; i < RX_DRAIN_TIME; i++) { 1575 done = (ixgbe->rcb_pending == 0); 1576 1577 if (done) 1578 break; 1579 1580 msec_delay(1); 1581 } 1582 1583 return (done); 1584 } 1585 1586 /* 1587 * ixgbe_start - Start the driver/chipset. 1588 */ 1589 int 1590 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1591 { 1592 int i; 1593 1594 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1595 1596 if (alloc_buffer) { 1597 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1598 ixgbe_error(ixgbe, 1599 "Failed to allocate software receive rings"); 1600 return (IXGBE_FAILURE); 1601 } 1602 1603 /* Allocate buffers for all the rx/tx rings */ 1604 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1605 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1606 return (IXGBE_FAILURE); 1607 } 1608 1609 ixgbe->tx_ring_init = B_TRUE; 1610 } else { 1611 ixgbe->tx_ring_init = B_FALSE; 1612 } 1613 1614 for (i = 0; i < ixgbe->num_rx_rings; i++) 1615 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1616 for (i = 0; i < ixgbe->num_tx_rings; i++) 1617 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1618 1619 /* 1620 * Start the chipset hardware 1621 */ 1622 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1623 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1624 goto start_failure; 1625 } 1626 1627 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1628 goto start_failure; 1629 } 1630 1631 /* 1632 * Setup the rx/tx rings 1633 */ 1634 ixgbe_setup_rings(ixgbe); 1635 1636 /* 1637 * ixgbe_start() will be called when resetting, however if reset 1638 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1639 * before enabling the interrupts. 1640 */ 1641 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1642 | IXGBE_STALL| IXGBE_OVERTEMP)); 1643 1644 /* 1645 * Enable adapter interrupts 1646 * The interrupts must be enabled after the driver state is START 1647 */ 1648 ixgbe_enable_adapter_interrupts(ixgbe); 1649 1650 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1651 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1652 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1653 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1654 1655 return (IXGBE_SUCCESS); 1656 1657 start_failure: 1658 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1659 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1660 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1661 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1662 1663 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1664 1665 return (IXGBE_FAILURE); 1666 } 1667 1668 /* 1669 * ixgbe_stop - Stop the driver/chipset. 1670 */ 1671 void 1672 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1673 { 1674 int i; 1675 1676 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1677 1678 /* 1679 * Disable the adapter interrupts 1680 */ 1681 ixgbe_disable_adapter_interrupts(ixgbe); 1682 1683 /* 1684 * Drain the pending tx packets 1685 */ 1686 (void) ixgbe_tx_drain(ixgbe); 1687 1688 for (i = 0; i < ixgbe->num_rx_rings; i++) 1689 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1690 for (i = 0; i < ixgbe->num_tx_rings; i++) 1691 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1692 1693 /* 1694 * Stop the chipset hardware 1695 */ 1696 ixgbe_chip_stop(ixgbe); 1697 1698 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1699 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1700 } 1701 1702 /* 1703 * Clean the pending tx data/resources 1704 */ 1705 ixgbe_tx_clean(ixgbe); 1706 1707 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1708 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1709 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1710 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1711 1712 if (ixgbe->link_state == LINK_STATE_UP) { 1713 ixgbe->link_state = LINK_STATE_UNKNOWN; 1714 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1715 } 1716 1717 if (free_buffer) { 1718 /* 1719 * Release the DMA/memory resources of rx/tx rings 1720 */ 1721 ixgbe_free_dma(ixgbe); 1722 ixgbe_free_rx_data(ixgbe); 1723 } 1724 } 1725 1726 /* 1727 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 1728 */ 1729 /* ARGSUSED */ 1730 static int 1731 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 1732 void *arg1, void *arg2) 1733 { 1734 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 1735 1736 switch (cbaction) { 1737 /* IRM callback */ 1738 int count; 1739 case DDI_CB_INTR_ADD: 1740 case DDI_CB_INTR_REMOVE: 1741 count = (int)(uintptr_t)cbarg; 1742 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 1743 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 1744 int, ixgbe->intr_cnt); 1745 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 1746 DDI_SUCCESS) { 1747 ixgbe_error(ixgbe, 1748 "IRM CB: Failed to adjust interrupts"); 1749 goto cb_fail; 1750 } 1751 break; 1752 default: 1753 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 1754 cbaction); 1755 return (DDI_ENOTSUP); 1756 } 1757 return (DDI_SUCCESS); 1758 cb_fail: 1759 return (DDI_FAILURE); 1760 } 1761 1762 /* 1763 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 1764 */ 1765 static int 1766 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 1767 { 1768 int i, rc, actual; 1769 1770 if (count == 0) 1771 return (DDI_SUCCESS); 1772 1773 if ((cbaction == DDI_CB_INTR_ADD && 1774 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 1775 (cbaction == DDI_CB_INTR_REMOVE && 1776 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 1777 return (DDI_FAILURE); 1778 1779 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1780 return (DDI_FAILURE); 1781 } 1782 1783 for (i = 0; i < ixgbe->num_rx_rings; i++) 1784 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 1785 for (i = 0; i < ixgbe->num_tx_rings; i++) 1786 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 1787 1788 mutex_enter(&ixgbe->gen_lock); 1789 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1790 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 1791 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1792 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 1793 1794 ixgbe_stop(ixgbe, B_FALSE); 1795 /* 1796 * Disable interrupts 1797 */ 1798 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1799 rc = ixgbe_disable_intrs(ixgbe); 1800 ASSERT(rc == IXGBE_SUCCESS); 1801 } 1802 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 1803 1804 /* 1805 * Remove interrupt handlers 1806 */ 1807 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 1808 ixgbe_rem_intr_handlers(ixgbe); 1809 } 1810 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 1811 1812 /* 1813 * Clear vect_map 1814 */ 1815 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 1816 switch (cbaction) { 1817 case DDI_CB_INTR_ADD: 1818 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 1819 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 1820 DDI_INTR_ALLOC_NORMAL); 1821 if (rc != DDI_SUCCESS || actual != count) { 1822 ixgbe_log(ixgbe, "Adjust interrupts failed." 1823 "return: %d, irm cb size: %d, actual: %d", 1824 rc, count, actual); 1825 goto intr_adjust_fail; 1826 } 1827 ixgbe->intr_cnt += count; 1828 break; 1829 1830 case DDI_CB_INTR_REMOVE: 1831 for (i = ixgbe->intr_cnt - count; 1832 i < ixgbe->intr_cnt; i ++) { 1833 rc = ddi_intr_free(ixgbe->htable[i]); 1834 ixgbe->htable[i] = NULL; 1835 if (rc != DDI_SUCCESS) { 1836 ixgbe_log(ixgbe, "Adjust interrupts failed." 1837 "return: %d, irm cb size: %d, actual: %d", 1838 rc, count, actual); 1839 goto intr_adjust_fail; 1840 } 1841 } 1842 ixgbe->intr_cnt -= count; 1843 break; 1844 } 1845 1846 /* 1847 * Get priority for first vector, assume remaining are all the same 1848 */ 1849 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 1850 if (rc != DDI_SUCCESS) { 1851 ixgbe_log(ixgbe, 1852 "Get interrupt priority failed: %d", rc); 1853 goto intr_adjust_fail; 1854 } 1855 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 1856 if (rc != DDI_SUCCESS) { 1857 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 1858 goto intr_adjust_fail; 1859 } 1860 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 1861 1862 /* 1863 * Map rings to interrupt vectors 1864 */ 1865 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 1866 ixgbe_error(ixgbe, 1867 "IRM CB: Failed to map interrupts to vectors"); 1868 goto intr_adjust_fail; 1869 } 1870 1871 /* 1872 * Add interrupt handlers 1873 */ 1874 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 1875 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 1876 goto intr_adjust_fail; 1877 } 1878 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 1879 1880 /* 1881 * Now that mutex locks are initialized, and the chip is also 1882 * initialized, enable interrupts. 1883 */ 1884 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 1885 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 1886 goto intr_adjust_fail; 1887 } 1888 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 1889 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1890 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 1891 goto intr_adjust_fail; 1892 } 1893 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 1894 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1895 ixgbe->ixgbe_state |= IXGBE_STARTED; 1896 mutex_exit(&ixgbe->gen_lock); 1897 1898 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1899 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 1900 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 1901 } 1902 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1903 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 1904 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 1905 } 1906 1907 /* Wakeup all Tx rings */ 1908 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1909 mac_tx_ring_update(ixgbe->mac_hdl, 1910 ixgbe->tx_rings[i].ring_handle); 1911 } 1912 1913 IXGBE_DEBUGLOG_3(ixgbe, 1914 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 1915 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 1916 return (DDI_SUCCESS); 1917 1918 intr_adjust_fail: 1919 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1920 mutex_exit(&ixgbe->gen_lock); 1921 return (DDI_FAILURE); 1922 } 1923 1924 /* 1925 * ixgbe_intr_cb_register - Register interrupt callback function. 1926 */ 1927 static int 1928 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 1929 { 1930 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 1931 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 1932 return (IXGBE_FAILURE); 1933 } 1934 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 1935 return (IXGBE_SUCCESS); 1936 } 1937 1938 /* 1939 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 1940 */ 1941 static int 1942 ixgbe_alloc_rings(ixgbe_t *ixgbe) 1943 { 1944 /* 1945 * Allocate memory space for rx rings 1946 */ 1947 ixgbe->rx_rings = kmem_zalloc( 1948 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 1949 KM_NOSLEEP); 1950 1951 if (ixgbe->rx_rings == NULL) { 1952 return (IXGBE_FAILURE); 1953 } 1954 1955 /* 1956 * Allocate memory space for tx rings 1957 */ 1958 ixgbe->tx_rings = kmem_zalloc( 1959 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 1960 KM_NOSLEEP); 1961 1962 if (ixgbe->tx_rings == NULL) { 1963 kmem_free(ixgbe->rx_rings, 1964 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1965 ixgbe->rx_rings = NULL; 1966 return (IXGBE_FAILURE); 1967 } 1968 1969 /* 1970 * Allocate memory space for rx ring groups 1971 */ 1972 ixgbe->rx_groups = kmem_zalloc( 1973 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 1974 KM_NOSLEEP); 1975 1976 if (ixgbe->rx_groups == NULL) { 1977 kmem_free(ixgbe->rx_rings, 1978 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1979 kmem_free(ixgbe->tx_rings, 1980 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 1981 ixgbe->rx_rings = NULL; 1982 ixgbe->tx_rings = NULL; 1983 return (IXGBE_FAILURE); 1984 } 1985 1986 return (IXGBE_SUCCESS); 1987 } 1988 1989 /* 1990 * ixgbe_free_rings - Free the memory space of rx/tx rings. 1991 */ 1992 static void 1993 ixgbe_free_rings(ixgbe_t *ixgbe) 1994 { 1995 if (ixgbe->rx_rings != NULL) { 1996 kmem_free(ixgbe->rx_rings, 1997 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 1998 ixgbe->rx_rings = NULL; 1999 } 2000 2001 if (ixgbe->tx_rings != NULL) { 2002 kmem_free(ixgbe->tx_rings, 2003 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2004 ixgbe->tx_rings = NULL; 2005 } 2006 2007 if (ixgbe->rx_groups != NULL) { 2008 kmem_free(ixgbe->rx_groups, 2009 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 2010 ixgbe->rx_groups = NULL; 2011 } 2012 } 2013 2014 static int 2015 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 2016 { 2017 ixgbe_rx_ring_t *rx_ring; 2018 int i; 2019 2020 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2021 rx_ring = &ixgbe->rx_rings[i]; 2022 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 2023 goto alloc_rx_rings_failure; 2024 } 2025 return (IXGBE_SUCCESS); 2026 2027 alloc_rx_rings_failure: 2028 ixgbe_free_rx_data(ixgbe); 2029 return (IXGBE_FAILURE); 2030 } 2031 2032 static void 2033 ixgbe_free_rx_data(ixgbe_t *ixgbe) 2034 { 2035 ixgbe_rx_ring_t *rx_ring; 2036 ixgbe_rx_data_t *rx_data; 2037 int i; 2038 2039 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2040 rx_ring = &ixgbe->rx_rings[i]; 2041 2042 mutex_enter(&ixgbe->rx_pending_lock); 2043 rx_data = rx_ring->rx_data; 2044 2045 if (rx_data != NULL) { 2046 rx_data->flag |= IXGBE_RX_STOPPED; 2047 2048 if (rx_data->rcb_pending == 0) { 2049 ixgbe_free_rx_ring_data(rx_data); 2050 rx_ring->rx_data = NULL; 2051 } 2052 } 2053 2054 mutex_exit(&ixgbe->rx_pending_lock); 2055 } 2056 } 2057 2058 /* 2059 * ixgbe_setup_rings - Setup rx/tx rings. 2060 */ 2061 static void 2062 ixgbe_setup_rings(ixgbe_t *ixgbe) 2063 { 2064 /* 2065 * Setup the rx/tx rings, including the following: 2066 * 2067 * 1. Setup the descriptor ring and the control block buffers; 2068 * 2. Initialize necessary registers for receive/transmit; 2069 * 3. Initialize software pointers/parameters for receive/transmit; 2070 */ 2071 ixgbe_setup_rx(ixgbe); 2072 2073 ixgbe_setup_tx(ixgbe); 2074 } 2075 2076 static void 2077 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2078 { 2079 ixgbe_t *ixgbe = rx_ring->ixgbe; 2080 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2081 struct ixgbe_hw *hw = &ixgbe->hw; 2082 rx_control_block_t *rcb; 2083 union ixgbe_adv_rx_desc *rbd; 2084 uint32_t size; 2085 uint32_t buf_low; 2086 uint32_t buf_high; 2087 uint32_t reg_val; 2088 int i; 2089 2090 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2091 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2092 2093 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2094 rcb = rx_data->work_list[i]; 2095 rbd = &rx_data->rbd_ring[i]; 2096 2097 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2098 rbd->read.hdr_addr = NULL; 2099 } 2100 2101 /* 2102 * Initialize the length register 2103 */ 2104 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2105 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2106 2107 /* 2108 * Initialize the base address registers 2109 */ 2110 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2111 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2112 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2113 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2114 2115 /* 2116 * Setup head & tail pointers 2117 */ 2118 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2119 rx_data->ring_size - 1); 2120 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2121 2122 rx_data->rbd_next = 0; 2123 rx_data->lro_first = 0; 2124 2125 /* 2126 * Setup the Receive Descriptor Control Register (RXDCTL) 2127 * PTHRESH=32 descriptors (half the internal cache) 2128 * HTHRESH=0 descriptors (to minimize latency on fetch) 2129 * WTHRESH defaults to 1 (writeback each descriptor) 2130 */ 2131 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2132 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2133 2134 /* Not a valid value for 82599 or X540 */ 2135 if (hw->mac.type == ixgbe_mac_82598EB) { 2136 reg_val |= 0x0020; /* pthresh */ 2137 } 2138 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2139 2140 if (hw->mac.type == ixgbe_mac_82599EB || 2141 hw->mac.type == ixgbe_mac_X540) { 2142 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2143 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2144 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2145 } 2146 2147 /* 2148 * Setup the Split and Replication Receive Control Register. 2149 * Set the rx buffer size and the advanced descriptor type. 2150 */ 2151 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2152 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2153 reg_val |= IXGBE_SRRCTL_DROP_EN; 2154 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2155 } 2156 2157 static void 2158 ixgbe_setup_rx(ixgbe_t *ixgbe) 2159 { 2160 ixgbe_rx_ring_t *rx_ring; 2161 struct ixgbe_hw *hw = &ixgbe->hw; 2162 uint32_t reg_val; 2163 uint32_t ring_mapping; 2164 uint32_t i, index; 2165 uint32_t psrtype_rss_bit; 2166 2167 /* PSRTYPE must be configured for 82599 */ 2168 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2169 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2170 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2171 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2172 reg_val |= IXGBE_PSRTYPE_L2HDR; 2173 reg_val |= 0x80000000; 2174 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2175 } else { 2176 if (ixgbe->num_rx_groups > 32) { 2177 psrtype_rss_bit = 0x20000000; 2178 } else { 2179 psrtype_rss_bit = 0x40000000; 2180 } 2181 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2182 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2183 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2184 reg_val |= IXGBE_PSRTYPE_L2HDR; 2185 reg_val |= psrtype_rss_bit; 2186 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2187 } 2188 } 2189 2190 /* 2191 * Set filter control in FCTRL to accept broadcast packets and do 2192 * not pass pause frames to host. Flow control settings are already 2193 * in this register, so preserve them. 2194 */ 2195 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2196 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 2197 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 2198 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2199 2200 /* 2201 * Hardware checksum settings 2202 */ 2203 if (ixgbe->rx_hcksum_enable) { 2204 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2205 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2206 } 2207 2208 /* 2209 * Setup VMDq and RSS for multiple receive queues 2210 */ 2211 switch (ixgbe->classify_mode) { 2212 case IXGBE_CLASSIFY_RSS: 2213 /* 2214 * One group, only RSS is needed when more than 2215 * one ring enabled. 2216 */ 2217 ixgbe_setup_rss(ixgbe); 2218 break; 2219 2220 case IXGBE_CLASSIFY_VMDQ: 2221 /* 2222 * Multiple groups, each group has one ring, 2223 * only VMDq is needed. 2224 */ 2225 ixgbe_setup_vmdq(ixgbe); 2226 break; 2227 2228 case IXGBE_CLASSIFY_VMDQ_RSS: 2229 /* 2230 * Multiple groups and multiple rings, both 2231 * VMDq and RSS are needed. 2232 */ 2233 ixgbe_setup_vmdq_rss(ixgbe); 2234 break; 2235 2236 default: 2237 break; 2238 } 2239 2240 /* 2241 * Enable the receive unit. This must be done after filter 2242 * control is set in FCTRL. 2243 */ 2244 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 2245 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 2246 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 2247 2248 /* 2249 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2250 */ 2251 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2252 rx_ring = &ixgbe->rx_rings[i]; 2253 ixgbe_setup_rx_ring(rx_ring); 2254 } 2255 2256 /* 2257 * Setup the per-ring statistics mapping. 2258 */ 2259 ring_mapping = 0; 2260 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2261 index = ixgbe->rx_rings[i].hw_index; 2262 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2263 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2264 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2265 } 2266 2267 /* 2268 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2269 * by four bytes if the packet has a VLAN field, so includes MTU, 2270 * ethernet header and frame check sequence. 2271 * Register is MAXFRS in 82599. 2272 */ 2273 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 2274 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2275 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2276 2277 /* 2278 * Setup Jumbo Frame enable bit 2279 */ 2280 if (ixgbe->default_mtu > ETHERMTU) { 2281 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2282 reg_val |= IXGBE_HLREG0_JUMBOEN; 2283 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2284 } 2285 2286 /* 2287 * Setup RSC for multiple receive queues. 2288 */ 2289 if (ixgbe->lro_enable) { 2290 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2291 /* 2292 * Make sure rx_buf_size * MAXDESC not greater 2293 * than 65535. 2294 * Intel recommends 4 for MAXDESC field value. 2295 */ 2296 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2297 reg_val |= IXGBE_RSCCTL_RSCEN; 2298 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2299 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2300 else 2301 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2302 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2303 } 2304 2305 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2306 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2307 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2308 2309 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2310 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2311 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2312 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2313 2314 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2315 } 2316 } 2317 2318 static void 2319 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2320 { 2321 ixgbe_t *ixgbe = tx_ring->ixgbe; 2322 struct ixgbe_hw *hw = &ixgbe->hw; 2323 uint32_t size; 2324 uint32_t buf_low; 2325 uint32_t buf_high; 2326 uint32_t reg_val; 2327 2328 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2329 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2330 2331 /* 2332 * Initialize the length register 2333 */ 2334 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2335 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2336 2337 /* 2338 * Initialize the base address registers 2339 */ 2340 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2341 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2342 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2343 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2344 2345 /* 2346 * Setup head & tail pointers 2347 */ 2348 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2349 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2350 2351 /* 2352 * Setup head write-back 2353 */ 2354 if (ixgbe->tx_head_wb_enable) { 2355 /* 2356 * The memory of the head write-back is allocated using 2357 * the extra tbd beyond the tail of the tbd ring. 2358 */ 2359 tx_ring->tbd_head_wb = (uint32_t *) 2360 ((uintptr_t)tx_ring->tbd_area.address + size); 2361 *tx_ring->tbd_head_wb = 0; 2362 2363 buf_low = (uint32_t) 2364 (tx_ring->tbd_area.dma_address + size); 2365 buf_high = (uint32_t) 2366 ((tx_ring->tbd_area.dma_address + size) >> 32); 2367 2368 /* Set the head write-back enable bit */ 2369 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2370 2371 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2372 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2373 2374 /* 2375 * Turn off relaxed ordering for head write back or it will 2376 * cause problems with the tx recycling 2377 */ 2378 2379 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ? 2380 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) : 2381 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index)); 2382 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2383 if (hw->mac.type == ixgbe_mac_82598EB) { 2384 IXGBE_WRITE_REG(hw, 2385 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2386 } else { 2387 IXGBE_WRITE_REG(hw, 2388 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val); 2389 } 2390 } else { 2391 tx_ring->tbd_head_wb = NULL; 2392 } 2393 2394 tx_ring->tbd_head = 0; 2395 tx_ring->tbd_tail = 0; 2396 tx_ring->tbd_free = tx_ring->ring_size; 2397 2398 if (ixgbe->tx_ring_init == B_TRUE) { 2399 tx_ring->tcb_head = 0; 2400 tx_ring->tcb_tail = 0; 2401 tx_ring->tcb_free = tx_ring->free_list_size; 2402 } 2403 2404 /* 2405 * Initialize the s/w context structure 2406 */ 2407 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2408 } 2409 2410 static void 2411 ixgbe_setup_tx(ixgbe_t *ixgbe) 2412 { 2413 struct ixgbe_hw *hw = &ixgbe->hw; 2414 ixgbe_tx_ring_t *tx_ring; 2415 uint32_t reg_val; 2416 uint32_t ring_mapping; 2417 int i; 2418 2419 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2420 tx_ring = &ixgbe->tx_rings[i]; 2421 ixgbe_setup_tx_ring(tx_ring); 2422 } 2423 2424 /* 2425 * Setup the per-ring statistics mapping. 2426 */ 2427 ring_mapping = 0; 2428 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2429 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2430 if ((i & 0x3) == 0x3) { 2431 switch (hw->mac.type) { 2432 case ixgbe_mac_82598EB: 2433 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2434 ring_mapping); 2435 break; 2436 2437 case ixgbe_mac_82599EB: 2438 case ixgbe_mac_X540: 2439 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2440 ring_mapping); 2441 break; 2442 2443 default: 2444 break; 2445 } 2446 2447 ring_mapping = 0; 2448 } 2449 } 2450 if (i & 0x3) { 2451 switch (hw->mac.type) { 2452 case ixgbe_mac_82598EB: 2453 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2454 break; 2455 2456 case ixgbe_mac_82599EB: 2457 case ixgbe_mac_X540: 2458 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2459 break; 2460 2461 default: 2462 break; 2463 } 2464 } 2465 2466 /* 2467 * Enable CRC appending and TX padding (for short tx frames) 2468 */ 2469 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2470 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2471 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2472 2473 /* 2474 * enable DMA for 82599 and X540 parts 2475 */ 2476 if (hw->mac.type == ixgbe_mac_82599EB || 2477 hw->mac.type == ixgbe_mac_X540) { 2478 /* DMATXCTL.TE must be set after all Tx config is complete */ 2479 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2480 reg_val |= IXGBE_DMATXCTL_TE; 2481 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2482 2483 /* Disable arbiter to set MTQC */ 2484 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2485 reg_val |= IXGBE_RTTDCS_ARBDIS; 2486 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2487 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2488 reg_val &= ~IXGBE_RTTDCS_ARBDIS; 2489 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2490 } 2491 2492 /* 2493 * Enabling tx queues .. 2494 * For 82599 must be done after DMATXCTL.TE is set 2495 */ 2496 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2497 tx_ring = &ixgbe->tx_rings[i]; 2498 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2499 reg_val |= IXGBE_TXDCTL_ENABLE; 2500 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2501 } 2502 } 2503 2504 /* 2505 * ixgbe_setup_rss - Setup receive-side scaling feature. 2506 */ 2507 static void 2508 ixgbe_setup_rss(ixgbe_t *ixgbe) 2509 { 2510 struct ixgbe_hw *hw = &ixgbe->hw; 2511 uint32_t i, mrqc, rxcsum; 2512 uint32_t random; 2513 uint32_t reta; 2514 uint32_t ring_per_group; 2515 2516 /* 2517 * Fill out redirection table 2518 */ 2519 reta = 0; 2520 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2521 2522 for (i = 0; i < 128; i++) { 2523 reta = (reta << 8) | (i % ring_per_group) | 2524 ((i % ring_per_group) << 4); 2525 if ((i & 3) == 3) 2526 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2527 } 2528 2529 /* 2530 * Fill out hash function seeds with a random constant 2531 */ 2532 for (i = 0; i < 10; i++) { 2533 (void) random_get_pseudo_bytes((uint8_t *)&random, 2534 sizeof (uint32_t)); 2535 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2536 } 2537 2538 /* 2539 * Enable RSS & perform hash on these packet types 2540 */ 2541 mrqc = IXGBE_MRQC_RSSEN | 2542 IXGBE_MRQC_RSS_FIELD_IPV4 | 2543 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2544 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2545 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2546 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2547 IXGBE_MRQC_RSS_FIELD_IPV6 | 2548 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2549 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2550 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2551 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2552 2553 /* 2554 * Disable Packet Checksum to enable RSS for multiple receive queues. 2555 * It is an adapter hardware limitation that Packet Checksum is 2556 * mutually exclusive with RSS. 2557 */ 2558 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2559 rxcsum |= IXGBE_RXCSUM_PCSD; 2560 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2561 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2562 } 2563 2564 /* 2565 * ixgbe_setup_vmdq - Setup MAC classification feature 2566 */ 2567 static void 2568 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2569 { 2570 struct ixgbe_hw *hw = &ixgbe->hw; 2571 uint32_t vmdctl, i, vtctl; 2572 2573 /* 2574 * Setup the VMDq Control register, enable VMDq based on 2575 * packet destination MAC address: 2576 */ 2577 switch (hw->mac.type) { 2578 case ixgbe_mac_82598EB: 2579 /* 2580 * VMDq Enable = 1; 2581 * VMDq Filter = 0; MAC filtering 2582 * Default VMDq output index = 0; 2583 */ 2584 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2585 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2586 break; 2587 2588 case ixgbe_mac_82599EB: 2589 case ixgbe_mac_X540: 2590 /* 2591 * Enable VMDq-only. 2592 */ 2593 vmdctl = IXGBE_MRQC_VMDQEN; 2594 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2595 2596 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2597 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2598 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2599 } 2600 2601 /* 2602 * Enable Virtualization and Replication. 2603 */ 2604 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2605 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2606 2607 /* 2608 * Enable receiving packets to all VFs 2609 */ 2610 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2611 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2612 break; 2613 2614 default: 2615 break; 2616 } 2617 } 2618 2619 /* 2620 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2621 */ 2622 static void 2623 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2624 { 2625 struct ixgbe_hw *hw = &ixgbe->hw; 2626 uint32_t i, mrqc, rxcsum; 2627 uint32_t random; 2628 uint32_t reta; 2629 uint32_t ring_per_group; 2630 uint32_t vmdctl, vtctl; 2631 2632 /* 2633 * Fill out redirection table 2634 */ 2635 reta = 0; 2636 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2637 for (i = 0; i < 128; i++) { 2638 reta = (reta << 8) | (i % ring_per_group) | 2639 ((i % ring_per_group) << 4); 2640 if ((i & 3) == 3) 2641 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2642 } 2643 2644 /* 2645 * Fill out hash function seeds with a random constant 2646 */ 2647 for (i = 0; i < 10; i++) { 2648 (void) random_get_pseudo_bytes((uint8_t *)&random, 2649 sizeof (uint32_t)); 2650 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2651 } 2652 2653 /* 2654 * Enable and setup RSS and VMDq 2655 */ 2656 switch (hw->mac.type) { 2657 case ixgbe_mac_82598EB: 2658 /* 2659 * Enable RSS & Setup RSS Hash functions 2660 */ 2661 mrqc = IXGBE_MRQC_RSSEN | 2662 IXGBE_MRQC_RSS_FIELD_IPV4 | 2663 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2664 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2665 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2666 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2667 IXGBE_MRQC_RSS_FIELD_IPV6 | 2668 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2669 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2670 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2671 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2672 2673 /* 2674 * Enable and Setup VMDq 2675 * VMDq Filter = 0; MAC filtering 2676 * Default VMDq output index = 0; 2677 */ 2678 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2679 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2680 break; 2681 2682 case ixgbe_mac_82599EB: 2683 case ixgbe_mac_X540: 2684 /* 2685 * Enable RSS & Setup RSS Hash functions 2686 */ 2687 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2688 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2689 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2690 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2691 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2692 IXGBE_MRQC_RSS_FIELD_IPV6 | 2693 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2694 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2695 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2696 2697 /* 2698 * Enable VMDq+RSS. 2699 */ 2700 if (ixgbe->num_rx_groups > 32) { 2701 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2702 } else { 2703 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2704 } 2705 2706 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2707 2708 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2709 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2710 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2711 } 2712 break; 2713 2714 default: 2715 break; 2716 2717 } 2718 2719 /* 2720 * Disable Packet Checksum to enable RSS for multiple receive queues. 2721 * It is an adapter hardware limitation that Packet Checksum is 2722 * mutually exclusive with RSS. 2723 */ 2724 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2725 rxcsum |= IXGBE_RXCSUM_PCSD; 2726 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2727 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2728 2729 if (hw->mac.type == ixgbe_mac_82599EB || 2730 hw->mac.type == ixgbe_mac_X540) { 2731 /* 2732 * Enable Virtualization and Replication. 2733 */ 2734 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2735 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2736 2737 /* 2738 * Enable receiving packets to all VFs 2739 */ 2740 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2741 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2742 } 2743 } 2744 2745 /* 2746 * ixgbe_init_unicst - Initialize the unicast addresses. 2747 */ 2748 static void 2749 ixgbe_init_unicst(ixgbe_t *ixgbe) 2750 { 2751 struct ixgbe_hw *hw = &ixgbe->hw; 2752 uint8_t *mac_addr; 2753 int slot; 2754 /* 2755 * Here we should consider two situations: 2756 * 2757 * 1. Chipset is initialized at the first time, 2758 * Clear all the multiple unicast addresses. 2759 * 2760 * 2. Chipset is reset 2761 * Recover the multiple unicast addresses from the 2762 * software data structure to the RAR registers. 2763 */ 2764 if (!ixgbe->unicst_init) { 2765 /* 2766 * Initialize the multiple unicast addresses 2767 */ 2768 ixgbe->unicst_total = hw->mac.num_rar_entries; 2769 ixgbe->unicst_avail = ixgbe->unicst_total; 2770 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2771 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2772 bzero(mac_addr, ETHERADDRL); 2773 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2774 ixgbe->unicst_addr[slot].mac.set = 0; 2775 } 2776 ixgbe->unicst_init = B_TRUE; 2777 } else { 2778 /* Re-configure the RAR registers */ 2779 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2780 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2781 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2782 (void) ixgbe_set_rar(hw, slot, mac_addr, 2783 ixgbe->unicst_addr[slot].mac.group_index, 2784 IXGBE_RAH_AV); 2785 } else { 2786 bzero(mac_addr, ETHERADDRL); 2787 (void) ixgbe_set_rar(hw, slot, mac_addr, 2788 NULL, NULL); 2789 } 2790 } 2791 } 2792 } 2793 2794 /* 2795 * ixgbe_unicst_find - Find the slot for the specified unicast address 2796 */ 2797 int 2798 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2799 { 2800 int slot; 2801 2802 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2803 2804 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2805 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2806 mac_addr, ETHERADDRL) == 0) 2807 return (slot); 2808 } 2809 2810 return (-1); 2811 } 2812 2813 /* 2814 * ixgbe_multicst_add - Add a multicst address. 2815 */ 2816 int 2817 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2818 { 2819 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2820 2821 if ((multiaddr[0] & 01) == 0) { 2822 return (EINVAL); 2823 } 2824 2825 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2826 return (ENOENT); 2827 } 2828 2829 bcopy(multiaddr, 2830 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2831 ixgbe->mcast_count++; 2832 2833 /* 2834 * Update the multicast table in the hardware 2835 */ 2836 ixgbe_setup_multicst(ixgbe); 2837 2838 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2839 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2840 return (EIO); 2841 } 2842 2843 return (0); 2844 } 2845 2846 /* 2847 * ixgbe_multicst_remove - Remove a multicst address. 2848 */ 2849 int 2850 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2851 { 2852 int i; 2853 2854 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2855 2856 for (i = 0; i < ixgbe->mcast_count; i++) { 2857 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2858 ETHERADDRL) == 0) { 2859 for (i++; i < ixgbe->mcast_count; i++) { 2860 ixgbe->mcast_table[i - 1] = 2861 ixgbe->mcast_table[i]; 2862 } 2863 ixgbe->mcast_count--; 2864 break; 2865 } 2866 } 2867 2868 /* 2869 * Update the multicast table in the hardware 2870 */ 2871 ixgbe_setup_multicst(ixgbe); 2872 2873 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2874 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2875 return (EIO); 2876 } 2877 2878 return (0); 2879 } 2880 2881 /* 2882 * ixgbe_setup_multicast - Setup multicast data structures. 2883 * 2884 * This routine initializes all of the multicast related structures 2885 * and save them in the hardware registers. 2886 */ 2887 static void 2888 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2889 { 2890 uint8_t *mc_addr_list; 2891 uint32_t mc_addr_count; 2892 struct ixgbe_hw *hw = &ixgbe->hw; 2893 2894 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2895 2896 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2897 2898 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2899 mc_addr_count = ixgbe->mcast_count; 2900 2901 /* 2902 * Update the multicast addresses to the MTA registers 2903 */ 2904 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2905 ixgbe_mc_table_itr, TRUE); 2906 } 2907 2908 /* 2909 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 2910 * 2911 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 2912 * Different chipsets may have different allowed configuration of vmdq and rss. 2913 */ 2914 static void 2915 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 2916 { 2917 struct ixgbe_hw *hw = &ixgbe->hw; 2918 uint32_t ring_per_group; 2919 2920 switch (hw->mac.type) { 2921 case ixgbe_mac_82598EB: 2922 /* 2923 * 82598 supports the following combination: 2924 * vmdq no. x rss no. 2925 * [5..16] x 1 2926 * [1..4] x [1..16] 2927 * However 8 rss queue per pool (vmdq) is sufficient for 2928 * most cases. 2929 */ 2930 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2931 if (ixgbe->num_rx_groups > 4) { 2932 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 2933 } else { 2934 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2935 min(8, ring_per_group); 2936 } 2937 2938 break; 2939 2940 case ixgbe_mac_82599EB: 2941 case ixgbe_mac_X540: 2942 /* 2943 * 82599 supports the following combination: 2944 * vmdq no. x rss no. 2945 * [33..64] x [1..2] 2946 * [2..32] x [1..4] 2947 * 1 x [1..16] 2948 * However 8 rss queue per pool (vmdq) is sufficient for 2949 * most cases. 2950 * 2951 * For now, treat X540 like the 82599. 2952 */ 2953 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2954 if (ixgbe->num_rx_groups == 1) { 2955 ixgbe->num_rx_rings = min(8, ring_per_group); 2956 } else if (ixgbe->num_rx_groups <= 32) { 2957 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2958 min(4, ring_per_group); 2959 } else if (ixgbe->num_rx_groups <= 64) { 2960 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2961 min(2, ring_per_group); 2962 } 2963 break; 2964 2965 default: 2966 break; 2967 } 2968 2969 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2970 2971 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 2972 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 2973 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 2974 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 2975 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 2976 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 2977 } else { 2978 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 2979 } 2980 2981 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 2982 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 2983 } 2984 2985 /* 2986 * ixgbe_get_conf - Get driver configurations set in driver.conf. 2987 * 2988 * This routine gets user-configured values out of the configuration 2989 * file ixgbe.conf. 2990 * 2991 * For each configurable value, there is a minimum, a maximum, and a 2992 * default. 2993 * If user does not configure a value, use the default. 2994 * If user configures below the minimum, use the minumum. 2995 * If user configures above the maximum, use the maxumum. 2996 */ 2997 static void 2998 ixgbe_get_conf(ixgbe_t *ixgbe) 2999 { 3000 struct ixgbe_hw *hw = &ixgbe->hw; 3001 uint32_t flow_control; 3002 3003 /* 3004 * ixgbe driver supports the following user configurations: 3005 * 3006 * Jumbo frame configuration: 3007 * default_mtu 3008 * 3009 * Ethernet flow control configuration: 3010 * flow_control 3011 * 3012 * Multiple rings configurations: 3013 * tx_queue_number 3014 * tx_ring_size 3015 * rx_queue_number 3016 * rx_ring_size 3017 * 3018 * Call ixgbe_get_prop() to get the value for a specific 3019 * configuration parameter. 3020 */ 3021 3022 /* 3023 * Jumbo frame configuration - max_frame_size controls host buffer 3024 * allocation, so includes MTU, ethernet header, vlan tag and 3025 * frame check sequence. 3026 */ 3027 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 3028 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 3029 3030 ixgbe->max_frame_size = ixgbe->default_mtu + 3031 sizeof (struct ether_vlan_header) + ETHERFCSL; 3032 3033 /* 3034 * Ethernet flow control configuration 3035 */ 3036 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 3037 ixgbe_fc_none, 3, ixgbe_fc_none); 3038 if (flow_control == 3) 3039 flow_control = ixgbe_fc_default; 3040 3041 /* 3042 * fc.requested mode is what the user requests. After autoneg, 3043 * fc.current_mode will be the flow_control mode that was negotiated. 3044 */ 3045 hw->fc.requested_mode = flow_control; 3046 3047 /* 3048 * Multiple rings configurations 3049 */ 3050 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 3051 ixgbe->capab->min_tx_que_num, 3052 ixgbe->capab->max_tx_que_num, 3053 ixgbe->capab->def_tx_que_num); 3054 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 3055 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 3056 3057 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 3058 ixgbe->capab->min_rx_que_num, 3059 ixgbe->capab->max_rx_que_num, 3060 ixgbe->capab->def_rx_que_num); 3061 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 3062 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 3063 3064 /* 3065 * Multiple groups configuration 3066 */ 3067 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3068 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3069 ixgbe->capab->def_rx_grp_num); 3070 3071 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3072 0, 1, DEFAULT_MR_ENABLE); 3073 3074 if (ixgbe->mr_enable == B_FALSE) { 3075 ixgbe->num_tx_rings = 1; 3076 ixgbe->num_rx_rings = 1; 3077 ixgbe->num_rx_groups = 1; 3078 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3079 } else { 3080 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3081 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3082 /* 3083 * The combination of num_rx_rings and num_rx_groups 3084 * may be not supported by h/w. We need to adjust 3085 * them to appropriate values. 3086 */ 3087 ixgbe_setup_vmdq_rss_conf(ixgbe); 3088 } 3089 3090 /* 3091 * Tunable used to force an interrupt type. The only use is 3092 * for testing of the lesser interrupt types. 3093 * 0 = don't force interrupt type 3094 * 1 = force interrupt type MSI-X 3095 * 2 = force interrupt type MSI 3096 * 3 = force interrupt type Legacy 3097 */ 3098 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3099 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3100 3101 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3102 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3103 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3104 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3105 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3106 0, 1, DEFAULT_LSO_ENABLE); 3107 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3108 0, 1, DEFAULT_LRO_ENABLE); 3109 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3110 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3111 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3112 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3113 3114 /* Head Write Back not recommended for 82599 and X540 */ 3115 if (hw->mac.type == ixgbe_mac_82599EB || 3116 hw->mac.type == ixgbe_mac_X540) { 3117 ixgbe->tx_head_wb_enable = B_FALSE; 3118 } 3119 3120 /* 3121 * ixgbe LSO needs the tx h/w checksum support. 3122 * LSO will be disabled if tx h/w checksum is not 3123 * enabled. 3124 */ 3125 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3126 ixgbe->lso_enable = B_FALSE; 3127 } 3128 3129 /* 3130 * ixgbe LRO needs the rx h/w checksum support. 3131 * LRO will be disabled if rx h/w checksum is not 3132 * enabled. 3133 */ 3134 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3135 ixgbe->lro_enable = B_FALSE; 3136 } 3137 3138 /* 3139 * ixgbe LRO only been supported by 82599 and X540 now 3140 */ 3141 if (hw->mac.type == ixgbe_mac_82598EB) { 3142 ixgbe->lro_enable = B_FALSE; 3143 } 3144 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3145 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3146 DEFAULT_TX_COPY_THRESHOLD); 3147 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3148 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3149 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3150 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3151 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3152 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3153 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3154 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3155 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3156 3157 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3158 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3159 DEFAULT_RX_COPY_THRESHOLD); 3160 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3161 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3162 DEFAULT_RX_LIMIT_PER_INTR); 3163 3164 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3165 ixgbe->capab->min_intr_throttle, 3166 ixgbe->capab->max_intr_throttle, 3167 ixgbe->capab->def_intr_throttle); 3168 /* 3169 * 82599 and X540 require the interrupt throttling rate is 3170 * a multiple of 8. This is enforced by the register 3171 * definiton. 3172 */ 3173 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540) 3174 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3175 } 3176 3177 static void 3178 ixgbe_init_params(ixgbe_t *ixgbe) 3179 { 3180 ixgbe->param_en_10000fdx_cap = 1; 3181 ixgbe->param_en_1000fdx_cap = 1; 3182 ixgbe->param_en_100fdx_cap = 1; 3183 ixgbe->param_adv_10000fdx_cap = 1; 3184 ixgbe->param_adv_1000fdx_cap = 1; 3185 ixgbe->param_adv_100fdx_cap = 1; 3186 3187 ixgbe->param_pause_cap = 1; 3188 ixgbe->param_asym_pause_cap = 1; 3189 ixgbe->param_rem_fault = 0; 3190 3191 ixgbe->param_adv_autoneg_cap = 1; 3192 ixgbe->param_adv_pause_cap = 1; 3193 ixgbe->param_adv_asym_pause_cap = 1; 3194 ixgbe->param_adv_rem_fault = 0; 3195 3196 ixgbe->param_lp_10000fdx_cap = 0; 3197 ixgbe->param_lp_1000fdx_cap = 0; 3198 ixgbe->param_lp_100fdx_cap = 0; 3199 ixgbe->param_lp_autoneg_cap = 0; 3200 ixgbe->param_lp_pause_cap = 0; 3201 ixgbe->param_lp_asym_pause_cap = 0; 3202 ixgbe->param_lp_rem_fault = 0; 3203 } 3204 3205 /* 3206 * ixgbe_get_prop - Get a property value out of the configuration file 3207 * ixgbe.conf. 3208 * 3209 * Caller provides the name of the property, a default value, a minimum 3210 * value, and a maximum value. 3211 * 3212 * Return configured value of the property, with default, minimum and 3213 * maximum properly applied. 3214 */ 3215 static int 3216 ixgbe_get_prop(ixgbe_t *ixgbe, 3217 char *propname, /* name of the property */ 3218 int minval, /* minimum acceptable value */ 3219 int maxval, /* maximim acceptable value */ 3220 int defval) /* default value */ 3221 { 3222 int value; 3223 3224 /* 3225 * Call ddi_prop_get_int() to read the conf settings 3226 */ 3227 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3228 DDI_PROP_DONTPASS, propname, defval); 3229 if (value > maxval) 3230 value = maxval; 3231 3232 if (value < minval) 3233 value = minval; 3234 3235 return (value); 3236 } 3237 3238 /* 3239 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3240 */ 3241 int 3242 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3243 { 3244 u32 autoneg_advertised = 0; 3245 3246 /* 3247 * No half duplex support with 10Gb parts 3248 */ 3249 if (ixgbe->param_adv_10000fdx_cap == 1) 3250 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3251 3252 if (ixgbe->param_adv_1000fdx_cap == 1) 3253 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3254 3255 if (ixgbe->param_adv_100fdx_cap == 1) 3256 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 3257 3258 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) { 3259 ixgbe_notice(ixgbe, "Invalid link settings. Setup link " 3260 "to autonegotiation with full link capabilities."); 3261 3262 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL | 3263 IXGBE_LINK_SPEED_1GB_FULL | 3264 IXGBE_LINK_SPEED_100_FULL; 3265 } 3266 3267 if (setup_hw) { 3268 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised, 3269 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) { 3270 ixgbe_notice(ixgbe, "Setup link failed on this " 3271 "device."); 3272 return (IXGBE_FAILURE); 3273 } 3274 } 3275 3276 return (IXGBE_SUCCESS); 3277 } 3278 3279 /* 3280 * ixgbe_driver_link_check - Link status processing. 3281 * 3282 * This function can be called in both kernel context and interrupt context 3283 */ 3284 static void 3285 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3286 { 3287 struct ixgbe_hw *hw = &ixgbe->hw; 3288 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3289 boolean_t link_up = B_FALSE; 3290 boolean_t link_changed = B_FALSE; 3291 3292 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3293 3294 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3295 if (link_up) { 3296 ixgbe->link_check_complete = B_TRUE; 3297 3298 /* Link is up, enable flow control settings */ 3299 (void) ixgbe_fc_enable(hw); 3300 3301 /* 3302 * The Link is up, check whether it was marked as down earlier 3303 */ 3304 if (ixgbe->link_state != LINK_STATE_UP) { 3305 switch (speed) { 3306 case IXGBE_LINK_SPEED_10GB_FULL: 3307 ixgbe->link_speed = SPEED_10GB; 3308 break; 3309 case IXGBE_LINK_SPEED_1GB_FULL: 3310 ixgbe->link_speed = SPEED_1GB; 3311 break; 3312 case IXGBE_LINK_SPEED_100_FULL: 3313 ixgbe->link_speed = SPEED_100; 3314 } 3315 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3316 ixgbe->link_state = LINK_STATE_UP; 3317 link_changed = B_TRUE; 3318 } 3319 } else { 3320 if (ixgbe->link_check_complete == B_TRUE || 3321 (ixgbe->link_check_complete == B_FALSE && 3322 gethrtime() >= ixgbe->link_check_hrtime)) { 3323 /* 3324 * The link is really down 3325 */ 3326 ixgbe->link_check_complete = B_TRUE; 3327 3328 if (ixgbe->link_state != LINK_STATE_DOWN) { 3329 ixgbe->link_speed = 0; 3330 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3331 ixgbe->link_state = LINK_STATE_DOWN; 3332 link_changed = B_TRUE; 3333 } 3334 } 3335 } 3336 3337 /* 3338 * If we are in an interrupt context, need to re-enable the 3339 * interrupt, which was automasked 3340 */ 3341 if (servicing_interrupt() != 0) { 3342 ixgbe->eims |= IXGBE_EICR_LSC; 3343 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3344 } 3345 3346 if (link_changed) { 3347 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3348 } 3349 } 3350 3351 /* 3352 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3353 */ 3354 static void 3355 ixgbe_sfp_check(void *arg) 3356 { 3357 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3358 uint32_t eicr = ixgbe->eicr; 3359 struct ixgbe_hw *hw = &ixgbe->hw; 3360 3361 mutex_enter(&ixgbe->gen_lock); 3362 if (eicr & IXGBE_EICR_GPI_SDP1) { 3363 /* clear the interrupt */ 3364 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 3365 3366 /* if link up, do multispeed fiber setup */ 3367 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3368 B_TRUE, B_TRUE); 3369 ixgbe_driver_link_check(ixgbe); 3370 ixgbe_get_hw_state(ixgbe); 3371 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 3372 /* clear the interrupt */ 3373 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 3374 3375 /* if link up, do sfp module setup */ 3376 (void) hw->mac.ops.setup_sfp(hw); 3377 3378 /* do multispeed fiber setup */ 3379 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3380 B_TRUE, B_TRUE); 3381 ixgbe_driver_link_check(ixgbe); 3382 ixgbe_get_hw_state(ixgbe); 3383 } 3384 mutex_exit(&ixgbe->gen_lock); 3385 3386 /* 3387 * We need to fully re-check the link later. 3388 */ 3389 ixgbe->link_check_complete = B_FALSE; 3390 ixgbe->link_check_hrtime = gethrtime() + 3391 (IXGBE_LINK_UP_TIME * 100000000ULL); 3392 } 3393 3394 /* 3395 * ixgbe_overtemp_check - overtemp module processing done in taskq 3396 * 3397 * This routine will only be called on adapters with temperature sensor. 3398 * The indication of over-temperature can be either SDP0 interrupt or the link 3399 * status change interrupt. 3400 */ 3401 static void 3402 ixgbe_overtemp_check(void *arg) 3403 { 3404 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3405 struct ixgbe_hw *hw = &ixgbe->hw; 3406 uint32_t eicr = ixgbe->eicr; 3407 ixgbe_link_speed speed; 3408 boolean_t link_up; 3409 3410 mutex_enter(&ixgbe->gen_lock); 3411 3412 /* make sure we know current state of link */ 3413 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3414 3415 /* check over-temp condition */ 3416 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) || 3417 (eicr & IXGBE_EICR_LSC)) { 3418 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3419 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3420 3421 /* 3422 * Disable the adapter interrupts 3423 */ 3424 ixgbe_disable_adapter_interrupts(ixgbe); 3425 3426 /* 3427 * Disable Rx/Tx units 3428 */ 3429 (void) ixgbe_stop_adapter(hw); 3430 3431 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3432 ixgbe_error(ixgbe, 3433 "Problem: Network adapter has been stopped " 3434 "because it has overheated"); 3435 ixgbe_error(ixgbe, 3436 "Action: Restart the computer. " 3437 "If the problem persists, power off the system " 3438 "and replace the adapter"); 3439 } 3440 } 3441 3442 /* write to clear the interrupt */ 3443 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3444 3445 mutex_exit(&ixgbe->gen_lock); 3446 } 3447 3448 /* 3449 * ixgbe_link_timer - timer for link status detection 3450 */ 3451 static void 3452 ixgbe_link_timer(void *arg) 3453 { 3454 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3455 3456 mutex_enter(&ixgbe->gen_lock); 3457 ixgbe_driver_link_check(ixgbe); 3458 mutex_exit(&ixgbe->gen_lock); 3459 } 3460 3461 /* 3462 * ixgbe_local_timer - Driver watchdog function. 3463 * 3464 * This function will handle the transmit stall check and other routines. 3465 */ 3466 static void 3467 ixgbe_local_timer(void *arg) 3468 { 3469 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3470 3471 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3472 goto out; 3473 3474 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3475 ixgbe->reset_count++; 3476 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3477 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3478 goto out; 3479 } 3480 3481 if (ixgbe_stall_check(ixgbe)) { 3482 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3483 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3484 3485 ixgbe->reset_count++; 3486 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3487 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3488 } 3489 3490 out: 3491 ixgbe_restart_watchdog_timer(ixgbe); 3492 } 3493 3494 /* 3495 * ixgbe_stall_check - Check for transmit stall. 3496 * 3497 * This function checks if the adapter is stalled (in transmit). 3498 * 3499 * It is called each time the watchdog timeout is invoked. 3500 * If the transmit descriptor reclaim continuously fails, 3501 * the watchdog value will increment by 1. If the watchdog 3502 * value exceeds the threshold, the ixgbe is assumed to 3503 * have stalled and need to be reset. 3504 */ 3505 static boolean_t 3506 ixgbe_stall_check(ixgbe_t *ixgbe) 3507 { 3508 ixgbe_tx_ring_t *tx_ring; 3509 boolean_t result; 3510 int i; 3511 3512 if (ixgbe->link_state != LINK_STATE_UP) 3513 return (B_FALSE); 3514 3515 /* 3516 * If any tx ring is stalled, we'll reset the chipset 3517 */ 3518 result = B_FALSE; 3519 for (i = 0; i < ixgbe->num_tx_rings; i++) { 3520 tx_ring = &ixgbe->tx_rings[i]; 3521 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 3522 tx_ring->tx_recycle(tx_ring); 3523 } 3524 3525 if (tx_ring->recycle_fail > 0) 3526 tx_ring->stall_watchdog++; 3527 else 3528 tx_ring->stall_watchdog = 0; 3529 3530 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3531 result = B_TRUE; 3532 break; 3533 } 3534 } 3535 3536 if (result) { 3537 tx_ring->stall_watchdog = 0; 3538 tx_ring->recycle_fail = 0; 3539 } 3540 3541 return (result); 3542 } 3543 3544 3545 /* 3546 * is_valid_mac_addr - Check if the mac address is valid. 3547 */ 3548 static boolean_t 3549 is_valid_mac_addr(uint8_t *mac_addr) 3550 { 3551 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3552 const uint8_t addr_test2[6] = 3553 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3554 3555 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3556 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3557 return (B_FALSE); 3558 3559 return (B_TRUE); 3560 } 3561 3562 static boolean_t 3563 ixgbe_find_mac_address(ixgbe_t *ixgbe) 3564 { 3565 #ifdef __sparc 3566 struct ixgbe_hw *hw = &ixgbe->hw; 3567 uchar_t *bytes; 3568 struct ether_addr sysaddr; 3569 uint_t nelts; 3570 int err; 3571 boolean_t found = B_FALSE; 3572 3573 /* 3574 * The "vendor's factory-set address" may already have 3575 * been extracted from the chip, but if the property 3576 * "local-mac-address" is set we use that instead. 3577 * 3578 * We check whether it looks like an array of 6 3579 * bytes (which it should, if OBP set it). If we can't 3580 * make sense of it this way, we'll ignore it. 3581 */ 3582 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3583 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3584 if (err == DDI_PROP_SUCCESS) { 3585 if (nelts == ETHERADDRL) { 3586 while (nelts--) 3587 hw->mac.addr[nelts] = bytes[nelts]; 3588 found = B_TRUE; 3589 } 3590 ddi_prop_free(bytes); 3591 } 3592 3593 /* 3594 * Look up the OBP property "local-mac-address?". If the user has set 3595 * 'local-mac-address? = false', use "the system address" instead. 3596 */ 3597 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 3598 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3599 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3600 if (localetheraddr(NULL, &sysaddr) != 0) { 3601 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3602 found = B_TRUE; 3603 } 3604 } 3605 ddi_prop_free(bytes); 3606 } 3607 3608 /* 3609 * Finally(!), if there's a valid "mac-address" property (created 3610 * if we netbooted from this interface), we must use this instead 3611 * of any of the above to ensure that the NFS/install server doesn't 3612 * get confused by the address changing as Solaris takes over! 3613 */ 3614 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3615 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3616 if (err == DDI_PROP_SUCCESS) { 3617 if (nelts == ETHERADDRL) { 3618 while (nelts--) 3619 hw->mac.addr[nelts] = bytes[nelts]; 3620 found = B_TRUE; 3621 } 3622 ddi_prop_free(bytes); 3623 } 3624 3625 if (found) { 3626 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3627 return (B_TRUE); 3628 } 3629 #else 3630 _NOTE(ARGUNUSED(ixgbe)); 3631 #endif 3632 3633 return (B_TRUE); 3634 } 3635 3636 #pragma inline(ixgbe_arm_watchdog_timer) 3637 static void 3638 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 3639 { 3640 /* 3641 * Fire a watchdog timer 3642 */ 3643 ixgbe->watchdog_tid = 3644 timeout(ixgbe_local_timer, 3645 (void *)ixgbe, 1 * drv_usectohz(1000000)); 3646 3647 } 3648 3649 /* 3650 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 3651 */ 3652 void 3653 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 3654 { 3655 mutex_enter(&ixgbe->watchdog_lock); 3656 3657 if (!ixgbe->watchdog_enable) { 3658 ixgbe->watchdog_enable = B_TRUE; 3659 ixgbe->watchdog_start = B_TRUE; 3660 ixgbe_arm_watchdog_timer(ixgbe); 3661 } 3662 3663 mutex_exit(&ixgbe->watchdog_lock); 3664 } 3665 3666 /* 3667 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 3668 */ 3669 void 3670 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 3671 { 3672 timeout_id_t tid; 3673 3674 mutex_enter(&ixgbe->watchdog_lock); 3675 3676 ixgbe->watchdog_enable = B_FALSE; 3677 ixgbe->watchdog_start = B_FALSE; 3678 tid = ixgbe->watchdog_tid; 3679 ixgbe->watchdog_tid = 0; 3680 3681 mutex_exit(&ixgbe->watchdog_lock); 3682 3683 if (tid != 0) 3684 (void) untimeout(tid); 3685 } 3686 3687 /* 3688 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 3689 */ 3690 void 3691 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 3692 { 3693 mutex_enter(&ixgbe->watchdog_lock); 3694 3695 if (ixgbe->watchdog_enable) { 3696 if (!ixgbe->watchdog_start) { 3697 ixgbe->watchdog_start = B_TRUE; 3698 ixgbe_arm_watchdog_timer(ixgbe); 3699 } 3700 } 3701 3702 mutex_exit(&ixgbe->watchdog_lock); 3703 } 3704 3705 /* 3706 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 3707 */ 3708 static void 3709 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 3710 { 3711 mutex_enter(&ixgbe->watchdog_lock); 3712 3713 if (ixgbe->watchdog_start) 3714 ixgbe_arm_watchdog_timer(ixgbe); 3715 3716 mutex_exit(&ixgbe->watchdog_lock); 3717 } 3718 3719 /* 3720 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 3721 */ 3722 void 3723 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 3724 { 3725 timeout_id_t tid; 3726 3727 mutex_enter(&ixgbe->watchdog_lock); 3728 3729 ixgbe->watchdog_start = B_FALSE; 3730 tid = ixgbe->watchdog_tid; 3731 ixgbe->watchdog_tid = 0; 3732 3733 mutex_exit(&ixgbe->watchdog_lock); 3734 3735 if (tid != 0) 3736 (void) untimeout(tid); 3737 } 3738 3739 /* 3740 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 3741 */ 3742 static void 3743 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 3744 { 3745 struct ixgbe_hw *hw = &ixgbe->hw; 3746 3747 /* 3748 * mask all interrupts off 3749 */ 3750 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 3751 3752 /* 3753 * for MSI-X, also disable autoclear 3754 */ 3755 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3756 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 3757 } 3758 3759 IXGBE_WRITE_FLUSH(hw); 3760 } 3761 3762 /* 3763 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 3764 */ 3765 static void 3766 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 3767 { 3768 struct ixgbe_hw *hw = &ixgbe->hw; 3769 uint32_t eiac, eiam; 3770 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 3771 3772 /* interrupt types to enable */ 3773 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 3774 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 3775 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 3776 3777 /* enable automask on "other" causes that this adapter can generate */ 3778 eiam = ixgbe->capab->other_intr; 3779 3780 /* 3781 * msi-x mode 3782 */ 3783 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3784 /* enable autoclear but not on bits 29:20 */ 3785 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 3786 3787 /* general purpose interrupt enable */ 3788 gpie |= (IXGBE_GPIE_MSIX_MODE 3789 | IXGBE_GPIE_PBA_SUPPORT 3790 | IXGBE_GPIE_OCD 3791 | IXGBE_GPIE_EIAME); 3792 /* 3793 * non-msi-x mode 3794 */ 3795 } else { 3796 3797 /* disable autoclear, leave gpie at default */ 3798 eiac = 0; 3799 3800 /* 3801 * General purpose interrupt enable. 3802 * For 82599 or X540, extended interrupt automask enable 3803 * only in MSI or MSI-X mode 3804 */ 3805 if ((hw->mac.type == ixgbe_mac_82598EB) || 3806 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 3807 gpie |= IXGBE_GPIE_EIAME; 3808 } 3809 } 3810 3811 /* Enable specific "other" interrupt types */ 3812 switch (hw->mac.type) { 3813 case ixgbe_mac_82598EB: 3814 gpie |= ixgbe->capab->other_gpie; 3815 break; 3816 3817 case ixgbe_mac_82599EB: 3818 case ixgbe_mac_X540: 3819 gpie |= ixgbe->capab->other_gpie; 3820 3821 /* Enable RSC Delay 8us when LRO enabled */ 3822 if (ixgbe->lro_enable) { 3823 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 3824 } 3825 break; 3826 3827 default: 3828 break; 3829 } 3830 3831 /* write to interrupt control registers */ 3832 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3833 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3834 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3835 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3836 IXGBE_WRITE_FLUSH(hw); 3837 } 3838 3839 /* 3840 * ixgbe_loopback_ioctl - Loopback support. 3841 */ 3842 enum ioc_reply 3843 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3844 { 3845 lb_info_sz_t *lbsp; 3846 lb_property_t *lbpp; 3847 uint32_t *lbmp; 3848 uint32_t size; 3849 uint32_t value; 3850 3851 if (mp->b_cont == NULL) 3852 return (IOC_INVAL); 3853 3854 switch (iocp->ioc_cmd) { 3855 default: 3856 return (IOC_INVAL); 3857 3858 case LB_GET_INFO_SIZE: 3859 size = sizeof (lb_info_sz_t); 3860 if (iocp->ioc_count != size) 3861 return (IOC_INVAL); 3862 3863 value = sizeof (lb_normal); 3864 value += sizeof (lb_mac); 3865 value += sizeof (lb_external); 3866 3867 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3868 *lbsp = value; 3869 break; 3870 3871 case LB_GET_INFO: 3872 value = sizeof (lb_normal); 3873 value += sizeof (lb_mac); 3874 value += sizeof (lb_external); 3875 3876 size = value; 3877 if (iocp->ioc_count != size) 3878 return (IOC_INVAL); 3879 3880 value = 0; 3881 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3882 3883 lbpp[value++] = lb_normal; 3884 lbpp[value++] = lb_mac; 3885 lbpp[value++] = lb_external; 3886 break; 3887 3888 case LB_GET_MODE: 3889 size = sizeof (uint32_t); 3890 if (iocp->ioc_count != size) 3891 return (IOC_INVAL); 3892 3893 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3894 *lbmp = ixgbe->loopback_mode; 3895 break; 3896 3897 case LB_SET_MODE: 3898 size = 0; 3899 if (iocp->ioc_count != sizeof (uint32_t)) 3900 return (IOC_INVAL); 3901 3902 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3903 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3904 return (IOC_INVAL); 3905 break; 3906 } 3907 3908 iocp->ioc_count = size; 3909 iocp->ioc_error = 0; 3910 3911 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3912 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3913 return (IOC_INVAL); 3914 } 3915 3916 return (IOC_REPLY); 3917 } 3918 3919 /* 3920 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3921 */ 3922 static boolean_t 3923 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3924 { 3925 if (mode == ixgbe->loopback_mode) 3926 return (B_TRUE); 3927 3928 ixgbe->loopback_mode = mode; 3929 3930 if (mode == IXGBE_LB_NONE) { 3931 /* 3932 * Reset the chip 3933 */ 3934 (void) ixgbe_reset(ixgbe); 3935 return (B_TRUE); 3936 } 3937 3938 mutex_enter(&ixgbe->gen_lock); 3939 3940 switch (mode) { 3941 default: 3942 mutex_exit(&ixgbe->gen_lock); 3943 return (B_FALSE); 3944 3945 case IXGBE_LB_EXTERNAL: 3946 break; 3947 3948 case IXGBE_LB_INTERNAL_MAC: 3949 ixgbe_set_internal_mac_loopback(ixgbe); 3950 break; 3951 } 3952 3953 mutex_exit(&ixgbe->gen_lock); 3954 3955 return (B_TRUE); 3956 } 3957 3958 /* 3959 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 3960 */ 3961 static void 3962 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 3963 { 3964 struct ixgbe_hw *hw; 3965 uint32_t reg; 3966 uint8_t atlas; 3967 3968 hw = &ixgbe->hw; 3969 3970 /* 3971 * Setup MAC loopback 3972 */ 3973 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 3974 reg |= IXGBE_HLREG0_LPBK; 3975 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 3976 3977 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 3978 reg &= ~IXGBE_AUTOC_LMS_MASK; 3979 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 3980 3981 /* 3982 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 3983 */ 3984 switch (hw->mac.type) { 3985 case ixgbe_mac_82598EB: 3986 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3987 &atlas); 3988 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 3989 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 3990 atlas); 3991 3992 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3993 &atlas); 3994 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 3995 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 3996 atlas); 3997 3998 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 3999 &atlas); 4000 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 4001 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4002 atlas); 4003 4004 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4005 &atlas); 4006 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 4007 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4008 atlas); 4009 break; 4010 4011 case ixgbe_mac_82599EB: 4012 case ixgbe_mac_X540: 4013 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4014 reg |= (IXGBE_AUTOC_FLU | 4015 IXGBE_AUTOC_10G_KX4); 4016 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4017 4018 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 4019 B_FALSE, B_TRUE); 4020 break; 4021 4022 default: 4023 break; 4024 } 4025 } 4026 4027 #pragma inline(ixgbe_intr_rx_work) 4028 /* 4029 * ixgbe_intr_rx_work - RX processing of ISR. 4030 */ 4031 static void 4032 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 4033 { 4034 mblk_t *mp; 4035 4036 mutex_enter(&rx_ring->rx_lock); 4037 4038 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4039 mutex_exit(&rx_ring->rx_lock); 4040 4041 if (mp != NULL) 4042 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4043 rx_ring->ring_gen_num); 4044 } 4045 4046 #pragma inline(ixgbe_intr_tx_work) 4047 /* 4048 * ixgbe_intr_tx_work - TX processing of ISR. 4049 */ 4050 static void 4051 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 4052 { 4053 ixgbe_t *ixgbe = tx_ring->ixgbe; 4054 4055 /* 4056 * Recycle the tx descriptors 4057 */ 4058 tx_ring->tx_recycle(tx_ring); 4059 4060 /* 4061 * Schedule the re-transmit 4062 */ 4063 if (tx_ring->reschedule && 4064 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 4065 tx_ring->reschedule = B_FALSE; 4066 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 4067 tx_ring->ring_handle); 4068 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4069 } 4070 } 4071 4072 #pragma inline(ixgbe_intr_other_work) 4073 /* 4074 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4075 */ 4076 static void 4077 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4078 { 4079 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4080 4081 /* 4082 * handle link status change 4083 */ 4084 if (eicr & IXGBE_EICR_LSC) { 4085 ixgbe_driver_link_check(ixgbe); 4086 ixgbe_get_hw_state(ixgbe); 4087 } 4088 4089 /* 4090 * check for fan failure on adapters with fans 4091 */ 4092 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4093 (eicr & IXGBE_EICR_GPI_SDP1)) { 4094 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4095 4096 /* 4097 * Disable the adapter interrupts 4098 */ 4099 ixgbe_disable_adapter_interrupts(ixgbe); 4100 4101 /* 4102 * Disable Rx/Tx units 4103 */ 4104 (void) ixgbe_stop_adapter(&ixgbe->hw); 4105 4106 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4107 ixgbe_error(ixgbe, 4108 "Problem: Network adapter has been stopped " 4109 "because the fan has stopped.\n"); 4110 ixgbe_error(ixgbe, 4111 "Action: Replace the adapter.\n"); 4112 4113 /* re-enable the interrupt, which was automasked */ 4114 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4115 } 4116 4117 /* 4118 * Do SFP check for adapters with hot-plug capability 4119 */ 4120 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4121 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) { 4122 ixgbe->eicr = eicr; 4123 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4124 ixgbe_sfp_check, (void *)ixgbe, 4125 DDI_NOSLEEP)) != DDI_SUCCESS) { 4126 ixgbe_log(ixgbe, "No memory available to dispatch " 4127 "taskq for SFP check"); 4128 } 4129 } 4130 4131 /* 4132 * Do over-temperature check for adapters with temp sensor 4133 */ 4134 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4135 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) { 4136 ixgbe->eicr = eicr; 4137 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4138 ixgbe_overtemp_check, (void *)ixgbe, 4139 DDI_NOSLEEP)) != DDI_SUCCESS) { 4140 ixgbe_log(ixgbe, "No memory available to dispatch " 4141 "taskq for overtemp check"); 4142 } 4143 } 4144 } 4145 4146 /* 4147 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4148 */ 4149 static uint_t 4150 ixgbe_intr_legacy(void *arg1, void *arg2) 4151 { 4152 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4153 struct ixgbe_hw *hw = &ixgbe->hw; 4154 ixgbe_tx_ring_t *tx_ring; 4155 ixgbe_rx_ring_t *rx_ring; 4156 uint32_t eicr; 4157 mblk_t *mp; 4158 boolean_t tx_reschedule; 4159 uint_t result; 4160 4161 _NOTE(ARGUNUSED(arg2)); 4162 4163 mutex_enter(&ixgbe->gen_lock); 4164 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4165 mutex_exit(&ixgbe->gen_lock); 4166 return (DDI_INTR_UNCLAIMED); 4167 } 4168 4169 mp = NULL; 4170 tx_reschedule = B_FALSE; 4171 4172 /* 4173 * Any bit set in eicr: claim this interrupt 4174 */ 4175 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4176 4177 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4178 mutex_exit(&ixgbe->gen_lock); 4179 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4180 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4181 return (DDI_INTR_CLAIMED); 4182 } 4183 4184 if (eicr) { 4185 /* 4186 * For legacy interrupt, we have only one interrupt, 4187 * so we have only one rx ring and one tx ring enabled. 4188 */ 4189 ASSERT(ixgbe->num_rx_rings == 1); 4190 ASSERT(ixgbe->num_tx_rings == 1); 4191 4192 /* 4193 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4194 */ 4195 if (eicr & 0x1) { 4196 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4197 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4198 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4199 /* 4200 * Clean the rx descriptors 4201 */ 4202 rx_ring = &ixgbe->rx_rings[0]; 4203 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4204 } 4205 4206 /* 4207 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4208 */ 4209 if (eicr & 0x2) { 4210 /* 4211 * Recycle the tx descriptors 4212 */ 4213 tx_ring = &ixgbe->tx_rings[0]; 4214 tx_ring->tx_recycle(tx_ring); 4215 4216 /* 4217 * Schedule the re-transmit 4218 */ 4219 tx_reschedule = (tx_ring->reschedule && 4220 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4221 } 4222 4223 /* any interrupt type other than tx/rx */ 4224 if (eicr & ixgbe->capab->other_intr) { 4225 switch (hw->mac.type) { 4226 case ixgbe_mac_82598EB: 4227 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4228 break; 4229 4230 case ixgbe_mac_82599EB: 4231 case ixgbe_mac_X540: 4232 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4233 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4234 break; 4235 4236 default: 4237 break; 4238 } 4239 ixgbe_intr_other_work(ixgbe, eicr); 4240 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4241 } 4242 4243 mutex_exit(&ixgbe->gen_lock); 4244 4245 result = DDI_INTR_CLAIMED; 4246 } else { 4247 mutex_exit(&ixgbe->gen_lock); 4248 4249 /* 4250 * No interrupt cause bits set: don't claim this interrupt. 4251 */ 4252 result = DDI_INTR_UNCLAIMED; 4253 } 4254 4255 /* re-enable the interrupts which were automasked */ 4256 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4257 4258 /* 4259 * Do the following work outside of the gen_lock 4260 */ 4261 if (mp != NULL) { 4262 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4263 rx_ring->ring_gen_num); 4264 } 4265 4266 if (tx_reschedule) { 4267 tx_ring->reschedule = B_FALSE; 4268 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4269 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4270 } 4271 4272 return (result); 4273 } 4274 4275 /* 4276 * ixgbe_intr_msi - Interrupt handler for MSI. 4277 */ 4278 static uint_t 4279 ixgbe_intr_msi(void *arg1, void *arg2) 4280 { 4281 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4282 struct ixgbe_hw *hw = &ixgbe->hw; 4283 uint32_t eicr; 4284 4285 _NOTE(ARGUNUSED(arg2)); 4286 4287 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4288 4289 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4290 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4291 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4292 return (DDI_INTR_CLAIMED); 4293 } 4294 4295 /* 4296 * For MSI interrupt, we have only one vector, 4297 * so we have only one rx ring and one tx ring enabled. 4298 */ 4299 ASSERT(ixgbe->num_rx_rings == 1); 4300 ASSERT(ixgbe->num_tx_rings == 1); 4301 4302 /* 4303 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4304 */ 4305 if (eicr & 0x1) { 4306 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4307 } 4308 4309 /* 4310 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4311 */ 4312 if (eicr & 0x2) { 4313 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4314 } 4315 4316 /* any interrupt type other than tx/rx */ 4317 if (eicr & ixgbe->capab->other_intr) { 4318 mutex_enter(&ixgbe->gen_lock); 4319 switch (hw->mac.type) { 4320 case ixgbe_mac_82598EB: 4321 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4322 break; 4323 4324 case ixgbe_mac_82599EB: 4325 case ixgbe_mac_X540: 4326 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4327 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4328 break; 4329 4330 default: 4331 break; 4332 } 4333 ixgbe_intr_other_work(ixgbe, eicr); 4334 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4335 mutex_exit(&ixgbe->gen_lock); 4336 } 4337 4338 /* re-enable the interrupts which were automasked */ 4339 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4340 4341 return (DDI_INTR_CLAIMED); 4342 } 4343 4344 /* 4345 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4346 */ 4347 static uint_t 4348 ixgbe_intr_msix(void *arg1, void *arg2) 4349 { 4350 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4351 ixgbe_t *ixgbe = vect->ixgbe; 4352 struct ixgbe_hw *hw = &ixgbe->hw; 4353 uint32_t eicr; 4354 int r_idx = 0; 4355 4356 _NOTE(ARGUNUSED(arg2)); 4357 4358 /* 4359 * Clean each rx ring that has its bit set in the map 4360 */ 4361 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4362 while (r_idx >= 0) { 4363 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4364 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4365 (ixgbe->num_rx_rings - 1)); 4366 } 4367 4368 /* 4369 * Clean each tx ring that has its bit set in the map 4370 */ 4371 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4372 while (r_idx >= 0) { 4373 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4374 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4375 (ixgbe->num_tx_rings - 1)); 4376 } 4377 4378 4379 /* 4380 * Clean other interrupt (link change) that has its bit set in the map 4381 */ 4382 if (BT_TEST(vect->other_map, 0) == 1) { 4383 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4384 4385 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4386 DDI_FM_OK) { 4387 ddi_fm_service_impact(ixgbe->dip, 4388 DDI_SERVICE_DEGRADED); 4389 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4390 return (DDI_INTR_CLAIMED); 4391 } 4392 4393 /* 4394 * Check "other" cause bits: any interrupt type other than tx/rx 4395 */ 4396 if (eicr & ixgbe->capab->other_intr) { 4397 mutex_enter(&ixgbe->gen_lock); 4398 switch (hw->mac.type) { 4399 case ixgbe_mac_82598EB: 4400 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4401 ixgbe_intr_other_work(ixgbe, eicr); 4402 break; 4403 4404 case ixgbe_mac_82599EB: 4405 case ixgbe_mac_X540: 4406 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4407 ixgbe_intr_other_work(ixgbe, eicr); 4408 break; 4409 4410 default: 4411 break; 4412 } 4413 mutex_exit(&ixgbe->gen_lock); 4414 } 4415 4416 /* re-enable the interrupts which were automasked */ 4417 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4418 } 4419 4420 return (DDI_INTR_CLAIMED); 4421 } 4422 4423 /* 4424 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4425 * 4426 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4427 * if not successful, try Legacy. 4428 * ixgbe->intr_force can be used to force sequence to start with 4429 * any of the 3 types. 4430 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4431 */ 4432 static int 4433 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4434 { 4435 dev_info_t *devinfo; 4436 int intr_types; 4437 int rc; 4438 4439 devinfo = ixgbe->dip; 4440 4441 /* 4442 * Get supported interrupt types 4443 */ 4444 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4445 4446 if (rc != DDI_SUCCESS) { 4447 ixgbe_log(ixgbe, 4448 "Get supported interrupt types failed: %d", rc); 4449 return (IXGBE_FAILURE); 4450 } 4451 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4452 4453 ixgbe->intr_type = 0; 4454 4455 /* 4456 * Install MSI-X interrupts 4457 */ 4458 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4459 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4460 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4461 if (rc == IXGBE_SUCCESS) 4462 return (IXGBE_SUCCESS); 4463 4464 ixgbe_log(ixgbe, 4465 "Allocate MSI-X failed, trying MSI interrupts..."); 4466 } 4467 4468 /* 4469 * MSI-X not used, force rings and groups to 1 4470 */ 4471 ixgbe->num_rx_rings = 1; 4472 ixgbe->num_rx_groups = 1; 4473 ixgbe->num_tx_rings = 1; 4474 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4475 ixgbe_log(ixgbe, 4476 "MSI-X not used, force rings and groups number to 1"); 4477 4478 /* 4479 * Install MSI interrupts 4480 */ 4481 if ((intr_types & DDI_INTR_TYPE_MSI) && 4482 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 4483 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 4484 if (rc == IXGBE_SUCCESS) 4485 return (IXGBE_SUCCESS); 4486 4487 ixgbe_log(ixgbe, 4488 "Allocate MSI failed, trying Legacy interrupts..."); 4489 } 4490 4491 /* 4492 * Install legacy interrupts 4493 */ 4494 if (intr_types & DDI_INTR_TYPE_FIXED) { 4495 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 4496 if (rc == IXGBE_SUCCESS) 4497 return (IXGBE_SUCCESS); 4498 4499 ixgbe_log(ixgbe, 4500 "Allocate Legacy interrupts failed"); 4501 } 4502 4503 /* 4504 * If none of the 3 types succeeded, return failure 4505 */ 4506 return (IXGBE_FAILURE); 4507 } 4508 4509 /* 4510 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 4511 * 4512 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4513 * if fewer than 2 handles are available, return failure. 4514 * Upon success, this maps the vectors to rx and tx rings for 4515 * interrupts. 4516 */ 4517 static int 4518 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 4519 { 4520 dev_info_t *devinfo; 4521 int request, count, actual; 4522 int minimum; 4523 int rc; 4524 uint32_t ring_per_group; 4525 4526 devinfo = ixgbe->dip; 4527 4528 switch (intr_type) { 4529 case DDI_INTR_TYPE_FIXED: 4530 request = 1; /* Request 1 legacy interrupt handle */ 4531 minimum = 1; 4532 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 4533 break; 4534 4535 case DDI_INTR_TYPE_MSI: 4536 request = 1; /* Request 1 MSI interrupt handle */ 4537 minimum = 1; 4538 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 4539 break; 4540 4541 case DDI_INTR_TYPE_MSIX: 4542 /* 4543 * Best number of vectors for the adapter is 4544 * (# rx rings + # tx rings), however we will 4545 * limit the request number. 4546 */ 4547 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 4548 if (request > ixgbe->capab->max_ring_vect) 4549 request = ixgbe->capab->max_ring_vect; 4550 minimum = 1; 4551 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 4552 break; 4553 4554 default: 4555 ixgbe_log(ixgbe, 4556 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 4557 intr_type); 4558 return (IXGBE_FAILURE); 4559 } 4560 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 4561 request, minimum); 4562 4563 /* 4564 * Get number of supported interrupts 4565 */ 4566 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4567 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4568 ixgbe_log(ixgbe, 4569 "Get interrupt number failed. Return: %d, count: %d", 4570 rc, count); 4571 return (IXGBE_FAILURE); 4572 } 4573 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 4574 4575 actual = 0; 4576 ixgbe->intr_cnt = 0; 4577 ixgbe->intr_cnt_max = 0; 4578 ixgbe->intr_cnt_min = 0; 4579 4580 /* 4581 * Allocate an array of interrupt handles 4582 */ 4583 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 4584 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 4585 4586 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 4587 request, &actual, DDI_INTR_ALLOC_NORMAL); 4588 if (rc != DDI_SUCCESS) { 4589 ixgbe_log(ixgbe, "Allocate interrupts failed. " 4590 "return: %d, request: %d, actual: %d", 4591 rc, request, actual); 4592 goto alloc_handle_fail; 4593 } 4594 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 4595 4596 /* 4597 * upper/lower limit of interrupts 4598 */ 4599 ixgbe->intr_cnt = actual; 4600 ixgbe->intr_cnt_max = request; 4601 ixgbe->intr_cnt_min = minimum; 4602 4603 /* 4604 * rss number per group should not exceed the rx interrupt number, 4605 * else need to adjust rx ring number. 4606 */ 4607 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4608 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 4609 if (actual < ring_per_group) { 4610 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 4611 ixgbe_setup_vmdq_rss_conf(ixgbe); 4612 } 4613 4614 /* 4615 * Now we know the actual number of vectors. Here we map the vector 4616 * to other, rx rings and tx ring. 4617 */ 4618 if (actual < minimum) { 4619 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 4620 actual); 4621 goto alloc_handle_fail; 4622 } 4623 4624 /* 4625 * Get priority for first vector, assume remaining are all the same 4626 */ 4627 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 4628 if (rc != DDI_SUCCESS) { 4629 ixgbe_log(ixgbe, 4630 "Get interrupt priority failed: %d", rc); 4631 goto alloc_handle_fail; 4632 } 4633 4634 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 4635 if (rc != DDI_SUCCESS) { 4636 ixgbe_log(ixgbe, 4637 "Get interrupt cap failed: %d", rc); 4638 goto alloc_handle_fail; 4639 } 4640 4641 ixgbe->intr_type = intr_type; 4642 4643 return (IXGBE_SUCCESS); 4644 4645 alloc_handle_fail: 4646 ixgbe_rem_intrs(ixgbe); 4647 4648 return (IXGBE_FAILURE); 4649 } 4650 4651 /* 4652 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 4653 * 4654 * Before adding the interrupt handlers, the interrupt vectors have 4655 * been allocated, and the rx/tx rings have also been allocated. 4656 */ 4657 static int 4658 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 4659 { 4660 int vector = 0; 4661 int rc; 4662 4663 switch (ixgbe->intr_type) { 4664 case DDI_INTR_TYPE_MSIX: 4665 /* 4666 * Add interrupt handler for all vectors 4667 */ 4668 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 4669 /* 4670 * install pointer to vect_map[vector] 4671 */ 4672 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4673 (ddi_intr_handler_t *)ixgbe_intr_msix, 4674 (void *)&ixgbe->vect_map[vector], NULL); 4675 4676 if (rc != DDI_SUCCESS) { 4677 ixgbe_log(ixgbe, 4678 "Add interrupt handler failed. " 4679 "return: %d, vector: %d", rc, vector); 4680 for (vector--; vector >= 0; vector--) { 4681 (void) ddi_intr_remove_handler( 4682 ixgbe->htable[vector]); 4683 } 4684 return (IXGBE_FAILURE); 4685 } 4686 } 4687 4688 break; 4689 4690 case DDI_INTR_TYPE_MSI: 4691 /* 4692 * Add interrupt handlers for the only vector 4693 */ 4694 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4695 (ddi_intr_handler_t *)ixgbe_intr_msi, 4696 (void *)ixgbe, NULL); 4697 4698 if (rc != DDI_SUCCESS) { 4699 ixgbe_log(ixgbe, 4700 "Add MSI interrupt handler failed: %d", rc); 4701 return (IXGBE_FAILURE); 4702 } 4703 4704 break; 4705 4706 case DDI_INTR_TYPE_FIXED: 4707 /* 4708 * Add interrupt handlers for the only vector 4709 */ 4710 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4711 (ddi_intr_handler_t *)ixgbe_intr_legacy, 4712 (void *)ixgbe, NULL); 4713 4714 if (rc != DDI_SUCCESS) { 4715 ixgbe_log(ixgbe, 4716 "Add legacy interrupt handler failed: %d", rc); 4717 return (IXGBE_FAILURE); 4718 } 4719 4720 break; 4721 4722 default: 4723 return (IXGBE_FAILURE); 4724 } 4725 4726 return (IXGBE_SUCCESS); 4727 } 4728 4729 #pragma inline(ixgbe_map_rxring_to_vector) 4730 /* 4731 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 4732 */ 4733 static void 4734 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 4735 { 4736 /* 4737 * Set bit in map 4738 */ 4739 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4740 4741 /* 4742 * Count bits set 4743 */ 4744 ixgbe->vect_map[v_idx].rxr_cnt++; 4745 4746 /* 4747 * Remember bit position 4748 */ 4749 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 4750 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 4751 } 4752 4753 #pragma inline(ixgbe_map_txring_to_vector) 4754 /* 4755 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 4756 */ 4757 static void 4758 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 4759 { 4760 /* 4761 * Set bit in map 4762 */ 4763 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 4764 4765 /* 4766 * Count bits set 4767 */ 4768 ixgbe->vect_map[v_idx].txr_cnt++; 4769 4770 /* 4771 * Remember bit position 4772 */ 4773 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 4774 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 4775 } 4776 4777 /* 4778 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 4779 * allocation register (IVAR). 4780 * cause: 4781 * -1 : other cause 4782 * 0 : rx 4783 * 1 : tx 4784 */ 4785 static void 4786 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 4787 int8_t cause) 4788 { 4789 struct ixgbe_hw *hw = &ixgbe->hw; 4790 u32 ivar, index; 4791 4792 switch (hw->mac.type) { 4793 case ixgbe_mac_82598EB: 4794 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4795 if (cause == -1) { 4796 cause = 0; 4797 } 4798 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4799 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4800 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 4801 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 4802 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4803 break; 4804 4805 case ixgbe_mac_82599EB: 4806 case ixgbe_mac_X540: 4807 if (cause == -1) { 4808 /* other causes */ 4809 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4810 index = (intr_alloc_entry & 1) * 8; 4811 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4812 ivar &= ~(0xFF << index); 4813 ivar |= (msix_vector << index); 4814 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4815 } else { 4816 /* tx or rx causes */ 4817 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4818 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4819 ivar = IXGBE_READ_REG(hw, 4820 IXGBE_IVAR(intr_alloc_entry >> 1)); 4821 ivar &= ~(0xFF << index); 4822 ivar |= (msix_vector << index); 4823 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4824 ivar); 4825 } 4826 break; 4827 4828 default: 4829 break; 4830 } 4831 } 4832 4833 /* 4834 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 4835 * given interrupt vector allocation register (IVAR). 4836 * cause: 4837 * -1 : other cause 4838 * 0 : rx 4839 * 1 : tx 4840 */ 4841 static void 4842 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4843 { 4844 struct ixgbe_hw *hw = &ixgbe->hw; 4845 u32 ivar, index; 4846 4847 switch (hw->mac.type) { 4848 case ixgbe_mac_82598EB: 4849 if (cause == -1) { 4850 cause = 0; 4851 } 4852 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4853 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4854 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 4855 (intr_alloc_entry & 0x3))); 4856 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4857 break; 4858 4859 case ixgbe_mac_82599EB: 4860 case ixgbe_mac_X540: 4861 if (cause == -1) { 4862 /* other causes */ 4863 index = (intr_alloc_entry & 1) * 8; 4864 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4865 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4866 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4867 } else { 4868 /* tx or rx causes */ 4869 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4870 ivar = IXGBE_READ_REG(hw, 4871 IXGBE_IVAR(intr_alloc_entry >> 1)); 4872 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4873 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4874 ivar); 4875 } 4876 break; 4877 4878 default: 4879 break; 4880 } 4881 } 4882 4883 /* 4884 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 4885 * given interrupt vector allocation register (IVAR). 4886 * cause: 4887 * -1 : other cause 4888 * 0 : rx 4889 * 1 : tx 4890 */ 4891 static void 4892 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4893 { 4894 struct ixgbe_hw *hw = &ixgbe->hw; 4895 u32 ivar, index; 4896 4897 switch (hw->mac.type) { 4898 case ixgbe_mac_82598EB: 4899 if (cause == -1) { 4900 cause = 0; 4901 } 4902 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4903 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4904 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 4905 (intr_alloc_entry & 0x3))); 4906 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4907 break; 4908 4909 case ixgbe_mac_82599EB: 4910 case ixgbe_mac_X540: 4911 if (cause == -1) { 4912 /* other causes */ 4913 index = (intr_alloc_entry & 1) * 8; 4914 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4915 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4916 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4917 } else { 4918 /* tx or rx causes */ 4919 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4920 ivar = IXGBE_READ_REG(hw, 4921 IXGBE_IVAR(intr_alloc_entry >> 1)); 4922 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4923 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4924 ivar); 4925 } 4926 break; 4927 4928 default: 4929 break; 4930 } 4931 } 4932 4933 /* 4934 * Convert the rx ring index driver maintained to the rx ring index 4935 * in h/w. 4936 */ 4937 static uint32_t 4938 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 4939 { 4940 4941 struct ixgbe_hw *hw = &ixgbe->hw; 4942 uint32_t rx_ring_per_group, hw_rx_index; 4943 4944 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 4945 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 4946 return (sw_rx_index); 4947 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 4948 switch (hw->mac.type) { 4949 case ixgbe_mac_82598EB: 4950 return (sw_rx_index); 4951 4952 case ixgbe_mac_82599EB: 4953 case ixgbe_mac_X540: 4954 return (sw_rx_index * 2); 4955 4956 default: 4957 break; 4958 } 4959 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 4960 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4961 4962 switch (hw->mac.type) { 4963 case ixgbe_mac_82598EB: 4964 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 4965 16 + (sw_rx_index % rx_ring_per_group); 4966 return (hw_rx_index); 4967 4968 case ixgbe_mac_82599EB: 4969 case ixgbe_mac_X540: 4970 if (ixgbe->num_rx_groups > 32) { 4971 hw_rx_index = (sw_rx_index / 4972 rx_ring_per_group) * 2 + 4973 (sw_rx_index % rx_ring_per_group); 4974 } else { 4975 hw_rx_index = (sw_rx_index / 4976 rx_ring_per_group) * 4 + 4977 (sw_rx_index % rx_ring_per_group); 4978 } 4979 return (hw_rx_index); 4980 4981 default: 4982 break; 4983 } 4984 } 4985 4986 /* 4987 * Should never reach. Just to make compiler happy. 4988 */ 4989 return (sw_rx_index); 4990 } 4991 4992 /* 4993 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 4994 * 4995 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 4996 * to vector[0 - (intr_cnt -1)]. 4997 */ 4998 static int 4999 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 5000 { 5001 int i, vector = 0; 5002 5003 /* initialize vector map */ 5004 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 5005 for (i = 0; i < ixgbe->intr_cnt; i++) { 5006 ixgbe->vect_map[i].ixgbe = ixgbe; 5007 } 5008 5009 /* 5010 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 5011 * tx rings[0] on RTxQ[1]. 5012 */ 5013 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5014 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 5015 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 5016 return (IXGBE_SUCCESS); 5017 } 5018 5019 /* 5020 * Interrupts/vectors mapping for MSI-X 5021 */ 5022 5023 /* 5024 * Map other interrupt to vector 0, 5025 * Set bit in map and count the bits set. 5026 */ 5027 BT_SET(ixgbe->vect_map[vector].other_map, 0); 5028 ixgbe->vect_map[vector].other_cnt++; 5029 5030 /* 5031 * Map rx ring interrupts to vectors 5032 */ 5033 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5034 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 5035 vector = (vector +1) % ixgbe->intr_cnt; 5036 } 5037 5038 /* 5039 * Map tx ring interrupts to vectors 5040 */ 5041 for (i = 0; i < ixgbe->num_tx_rings; i++) { 5042 ixgbe_map_txring_to_vector(ixgbe, i, vector); 5043 vector = (vector +1) % ixgbe->intr_cnt; 5044 } 5045 5046 return (IXGBE_SUCCESS); 5047 } 5048 5049 /* 5050 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 5051 * 5052 * This relies on ring/vector mapping already set up in the 5053 * vect_map[] structures 5054 */ 5055 static void 5056 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 5057 { 5058 struct ixgbe_hw *hw = &ixgbe->hw; 5059 ixgbe_intr_vector_t *vect; /* vector bitmap */ 5060 int r_idx; /* ring index */ 5061 int v_idx; /* vector index */ 5062 uint32_t hw_index; 5063 5064 /* 5065 * Clear any previous entries 5066 */ 5067 switch (hw->mac.type) { 5068 case ixgbe_mac_82598EB: 5069 for (v_idx = 0; v_idx < 25; v_idx++) 5070 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5071 break; 5072 5073 case ixgbe_mac_82599EB: 5074 case ixgbe_mac_X540: 5075 for (v_idx = 0; v_idx < 64; v_idx++) 5076 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5077 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5078 break; 5079 5080 default: 5081 break; 5082 } 5083 5084 /* 5085 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5086 * tx rings[0] will use RTxQ[1]. 5087 */ 5088 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5089 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5090 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5091 return; 5092 } 5093 5094 /* 5095 * For MSI-X interrupt, "Other" is always on vector[0]. 5096 */ 5097 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5098 5099 /* 5100 * For each interrupt vector, populate the IVAR table 5101 */ 5102 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5103 vect = &ixgbe->vect_map[v_idx]; 5104 5105 /* 5106 * For each rx ring bit set 5107 */ 5108 r_idx = bt_getlowbit(vect->rx_map, 0, 5109 (ixgbe->num_rx_rings - 1)); 5110 5111 while (r_idx >= 0) { 5112 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5113 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5114 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5115 (ixgbe->num_rx_rings - 1)); 5116 } 5117 5118 /* 5119 * For each tx ring bit set 5120 */ 5121 r_idx = bt_getlowbit(vect->tx_map, 0, 5122 (ixgbe->num_tx_rings - 1)); 5123 5124 while (r_idx >= 0) { 5125 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5126 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5127 (ixgbe->num_tx_rings - 1)); 5128 } 5129 } 5130 } 5131 5132 /* 5133 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5134 */ 5135 static void 5136 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5137 { 5138 int i; 5139 int rc; 5140 5141 for (i = 0; i < ixgbe->intr_cnt; i++) { 5142 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5143 if (rc != DDI_SUCCESS) { 5144 IXGBE_DEBUGLOG_1(ixgbe, 5145 "Remove intr handler failed: %d", rc); 5146 } 5147 } 5148 } 5149 5150 /* 5151 * ixgbe_rem_intrs - Remove the allocated interrupts. 5152 */ 5153 static void 5154 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5155 { 5156 int i; 5157 int rc; 5158 5159 for (i = 0; i < ixgbe->intr_cnt; i++) { 5160 rc = ddi_intr_free(ixgbe->htable[i]); 5161 if (rc != DDI_SUCCESS) { 5162 IXGBE_DEBUGLOG_1(ixgbe, 5163 "Free intr failed: %d", rc); 5164 } 5165 } 5166 5167 kmem_free(ixgbe->htable, ixgbe->intr_size); 5168 ixgbe->htable = NULL; 5169 } 5170 5171 /* 5172 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5173 */ 5174 static int 5175 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5176 { 5177 int i; 5178 int rc; 5179 5180 /* 5181 * Enable interrupts 5182 */ 5183 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5184 /* 5185 * Call ddi_intr_block_enable() for MSI 5186 */ 5187 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5188 if (rc != DDI_SUCCESS) { 5189 ixgbe_log(ixgbe, 5190 "Enable block intr failed: %d", rc); 5191 return (IXGBE_FAILURE); 5192 } 5193 } else { 5194 /* 5195 * Call ddi_intr_enable() for Legacy/MSI non block enable 5196 */ 5197 for (i = 0; i < ixgbe->intr_cnt; i++) { 5198 rc = ddi_intr_enable(ixgbe->htable[i]); 5199 if (rc != DDI_SUCCESS) { 5200 ixgbe_log(ixgbe, 5201 "Enable intr failed: %d", rc); 5202 return (IXGBE_FAILURE); 5203 } 5204 } 5205 } 5206 5207 return (IXGBE_SUCCESS); 5208 } 5209 5210 /* 5211 * ixgbe_disable_intrs - Disable all the interrupts. 5212 */ 5213 static int 5214 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5215 { 5216 int i; 5217 int rc; 5218 5219 /* 5220 * Disable all interrupts 5221 */ 5222 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5223 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5224 if (rc != DDI_SUCCESS) { 5225 ixgbe_log(ixgbe, 5226 "Disable block intr failed: %d", rc); 5227 return (IXGBE_FAILURE); 5228 } 5229 } else { 5230 for (i = 0; i < ixgbe->intr_cnt; i++) { 5231 rc = ddi_intr_disable(ixgbe->htable[i]); 5232 if (rc != DDI_SUCCESS) { 5233 ixgbe_log(ixgbe, 5234 "Disable intr failed: %d", rc); 5235 return (IXGBE_FAILURE); 5236 } 5237 } 5238 } 5239 5240 return (IXGBE_SUCCESS); 5241 } 5242 5243 /* 5244 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5245 */ 5246 static void 5247 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5248 { 5249 struct ixgbe_hw *hw = &ixgbe->hw; 5250 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 5251 boolean_t link_up = B_FALSE; 5252 uint32_t pcs1g_anlp = 0; 5253 uint32_t pcs1g_ana = 0; 5254 boolean_t autoneg = B_FALSE; 5255 5256 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5257 ixgbe->param_lp_1000fdx_cap = 0; 5258 ixgbe->param_lp_100fdx_cap = 0; 5259 5260 /* check for link, don't wait */ 5261 (void) ixgbe_check_link(hw, &speed, &link_up, false); 5262 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 5263 5264 if (link_up) { 5265 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5266 5267 ixgbe->param_lp_1000fdx_cap = 5268 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5269 ixgbe->param_lp_100fdx_cap = 5270 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5271 } 5272 5273 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg); 5274 5275 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5276 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0; 5277 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5278 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0; 5279 } 5280 5281 /* 5282 * ixgbe_get_driver_control - Notify that driver is in control of device. 5283 */ 5284 static void 5285 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5286 { 5287 uint32_t ctrl_ext; 5288 5289 /* 5290 * Notify firmware that driver is in control of device 5291 */ 5292 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5293 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5294 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5295 } 5296 5297 /* 5298 * ixgbe_release_driver_control - Notify that driver is no longer in control 5299 * of device. 5300 */ 5301 static void 5302 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5303 { 5304 uint32_t ctrl_ext; 5305 5306 /* 5307 * Notify firmware that driver is no longer in control of device 5308 */ 5309 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5310 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5311 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5312 } 5313 5314 /* 5315 * ixgbe_atomic_reserve - Atomic decrease operation. 5316 */ 5317 int 5318 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5319 { 5320 uint32_t oldval; 5321 uint32_t newval; 5322 5323 /* 5324 * ATOMICALLY 5325 */ 5326 do { 5327 oldval = *count_p; 5328 if (oldval < n) 5329 return (-1); 5330 newval = oldval - n; 5331 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5332 5333 return (newval); 5334 } 5335 5336 /* 5337 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5338 */ 5339 static uint8_t * 5340 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5341 { 5342 uint8_t *addr = *upd_ptr; 5343 uint8_t *new_ptr; 5344 5345 _NOTE(ARGUNUSED(hw)); 5346 _NOTE(ARGUNUSED(vmdq)); 5347 5348 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5349 *upd_ptr = new_ptr; 5350 return (addr); 5351 } 5352 5353 /* 5354 * FMA support 5355 */ 5356 int 5357 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5358 { 5359 ddi_fm_error_t de; 5360 5361 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5362 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5363 return (de.fme_status); 5364 } 5365 5366 int 5367 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5368 { 5369 ddi_fm_error_t de; 5370 5371 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5372 return (de.fme_status); 5373 } 5374 5375 /* 5376 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5377 */ 5378 static int 5379 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5380 { 5381 _NOTE(ARGUNUSED(impl_data)); 5382 /* 5383 * as the driver can always deal with an error in any dma or 5384 * access handle, we can just return the fme_status value. 5385 */ 5386 pci_ereport_post(dip, err, NULL); 5387 return (err->fme_status); 5388 } 5389 5390 static void 5391 ixgbe_fm_init(ixgbe_t *ixgbe) 5392 { 5393 ddi_iblock_cookie_t iblk; 5394 int fma_dma_flag; 5395 5396 /* 5397 * Only register with IO Fault Services if we have some capability 5398 */ 5399 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5400 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5401 } else { 5402 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5403 } 5404 5405 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5406 fma_dma_flag = 1; 5407 } else { 5408 fma_dma_flag = 0; 5409 } 5410 5411 ixgbe_set_fma_flags(fma_dma_flag); 5412 5413 if (ixgbe->fm_capabilities) { 5414 5415 /* 5416 * Register capabilities with IO Fault Services 5417 */ 5418 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 5419 5420 /* 5421 * Initialize pci ereport capabilities if ereport capable 5422 */ 5423 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5424 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5425 pci_ereport_setup(ixgbe->dip); 5426 5427 /* 5428 * Register error callback if error callback capable 5429 */ 5430 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5431 ddi_fm_handler_register(ixgbe->dip, 5432 ixgbe_fm_error_cb, (void*) ixgbe); 5433 } 5434 } 5435 5436 static void 5437 ixgbe_fm_fini(ixgbe_t *ixgbe) 5438 { 5439 /* 5440 * Only unregister FMA capabilities if they are registered 5441 */ 5442 if (ixgbe->fm_capabilities) { 5443 5444 /* 5445 * Release any resources allocated by pci_ereport_setup() 5446 */ 5447 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5448 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5449 pci_ereport_teardown(ixgbe->dip); 5450 5451 /* 5452 * Un-register error callback if error callback capable 5453 */ 5454 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5455 ddi_fm_handler_unregister(ixgbe->dip); 5456 5457 /* 5458 * Unregister from IO Fault Service 5459 */ 5460 ddi_fm_fini(ixgbe->dip); 5461 } 5462 } 5463 5464 void 5465 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 5466 { 5467 uint64_t ena; 5468 char buf[FM_MAX_CLASS]; 5469 5470 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5471 ena = fm_ena_generate(0, FM_ENA_FMT1); 5472 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 5473 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 5474 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5475 } 5476 } 5477 5478 static int 5479 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 5480 { 5481 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 5482 5483 mutex_enter(&rx_ring->rx_lock); 5484 rx_ring->ring_gen_num = mr_gen_num; 5485 mutex_exit(&rx_ring->rx_lock); 5486 return (0); 5487 } 5488 5489 /* 5490 * Get the global ring index by a ring index within a group. 5491 */ 5492 static int 5493 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 5494 { 5495 ixgbe_rx_ring_t *rx_ring; 5496 int i; 5497 5498 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5499 rx_ring = &ixgbe->rx_rings[i]; 5500 if (rx_ring->group_index == gindex) 5501 rindex--; 5502 if (rindex < 0) 5503 return (i); 5504 } 5505 5506 return (-1); 5507 } 5508 5509 /* 5510 * Callback funtion for MAC layer to register all rings. 5511 */ 5512 /* ARGSUSED */ 5513 void 5514 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 5515 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 5516 { 5517 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5518 mac_intr_t *mintr = &infop->mri_intr; 5519 5520 switch (rtype) { 5521 case MAC_RING_TYPE_RX: { 5522 /* 5523 * 'index' is the ring index within the group. 5524 * Need to get the global ring index by searching in groups. 5525 */ 5526 int global_ring_index = ixgbe_get_rx_ring_index( 5527 ixgbe, group_index, ring_index); 5528 5529 ASSERT(global_ring_index >= 0); 5530 5531 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 5532 rx_ring->ring_handle = rh; 5533 5534 infop->mri_driver = (mac_ring_driver_t)rx_ring; 5535 infop->mri_start = ixgbe_ring_start; 5536 infop->mri_stop = NULL; 5537 infop->mri_poll = ixgbe_ring_rx_poll; 5538 infop->mri_stat = ixgbe_rx_ring_stat; 5539 5540 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 5541 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 5542 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 5543 if (ixgbe->intr_type & 5544 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5545 mintr->mi_ddi_handle = 5546 ixgbe->htable[rx_ring->intr_vector]; 5547 } 5548 5549 break; 5550 } 5551 case MAC_RING_TYPE_TX: { 5552 ASSERT(group_index == -1); 5553 ASSERT(ring_index < ixgbe->num_tx_rings); 5554 5555 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 5556 tx_ring->ring_handle = rh; 5557 5558 infop->mri_driver = (mac_ring_driver_t)tx_ring; 5559 infop->mri_start = NULL; 5560 infop->mri_stop = NULL; 5561 infop->mri_tx = ixgbe_ring_tx; 5562 infop->mri_stat = ixgbe_tx_ring_stat; 5563 if (ixgbe->intr_type & 5564 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5565 mintr->mi_ddi_handle = 5566 ixgbe->htable[tx_ring->intr_vector]; 5567 } 5568 break; 5569 } 5570 default: 5571 break; 5572 } 5573 } 5574 5575 /* 5576 * Callback funtion for MAC layer to register all groups. 5577 */ 5578 void 5579 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 5580 mac_group_info_t *infop, mac_group_handle_t gh) 5581 { 5582 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5583 5584 switch (rtype) { 5585 case MAC_RING_TYPE_RX: { 5586 ixgbe_rx_group_t *rx_group; 5587 5588 rx_group = &ixgbe->rx_groups[index]; 5589 rx_group->group_handle = gh; 5590 5591 infop->mgi_driver = (mac_group_driver_t)rx_group; 5592 infop->mgi_start = NULL; 5593 infop->mgi_stop = NULL; 5594 infop->mgi_addmac = ixgbe_addmac; 5595 infop->mgi_remmac = ixgbe_remmac; 5596 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 5597 5598 break; 5599 } 5600 case MAC_RING_TYPE_TX: 5601 break; 5602 default: 5603 break; 5604 } 5605 } 5606 5607 /* 5608 * Enable interrupt on the specificed rx ring. 5609 */ 5610 int 5611 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 5612 { 5613 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5614 ixgbe_t *ixgbe = rx_ring->ixgbe; 5615 int r_idx = rx_ring->index; 5616 int hw_r_idx = rx_ring->hw_index; 5617 int v_idx = rx_ring->intr_vector; 5618 5619 mutex_enter(&ixgbe->gen_lock); 5620 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5621 mutex_exit(&ixgbe->gen_lock); 5622 /* 5623 * Simply return 0. 5624 * Interrupts are being adjusted. ixgbe_intr_adjust() 5625 * will eventually re-enable the interrupt when it's 5626 * done with the adjustment. 5627 */ 5628 return (0); 5629 } 5630 5631 /* 5632 * To enable interrupt by setting the VAL bit of given interrupt 5633 * vector allocation register (IVAR). 5634 */ 5635 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 5636 5637 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5638 5639 /* 5640 * Trigger a Rx interrupt on this ring 5641 */ 5642 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 5643 IXGBE_WRITE_FLUSH(&ixgbe->hw); 5644 5645 mutex_exit(&ixgbe->gen_lock); 5646 5647 return (0); 5648 } 5649 5650 /* 5651 * Disable interrupt on the specificed rx ring. 5652 */ 5653 int 5654 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 5655 { 5656 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5657 ixgbe_t *ixgbe = rx_ring->ixgbe; 5658 int r_idx = rx_ring->index; 5659 int hw_r_idx = rx_ring->hw_index; 5660 int v_idx = rx_ring->intr_vector; 5661 5662 mutex_enter(&ixgbe->gen_lock); 5663 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5664 mutex_exit(&ixgbe->gen_lock); 5665 /* 5666 * Simply return 0. 5667 * In the rare case where an interrupt is being 5668 * disabled while interrupts are being adjusted, 5669 * we don't fail the operation. No interrupts will 5670 * be generated while they are adjusted, and 5671 * ixgbe_intr_adjust() will cause the interrupts 5672 * to be re-enabled once it completes. Note that 5673 * in this case, packets may be delivered to the 5674 * stack via interrupts before xgbe_rx_ring_intr_enable() 5675 * is called again. This is acceptable since interrupt 5676 * adjustment is infrequent, and the stack will be 5677 * able to handle these packets. 5678 */ 5679 return (0); 5680 } 5681 5682 /* 5683 * To disable interrupt by clearing the VAL bit of given interrupt 5684 * vector allocation register (IVAR). 5685 */ 5686 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 5687 5688 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 5689 5690 mutex_exit(&ixgbe->gen_lock); 5691 5692 return (0); 5693 } 5694 5695 /* 5696 * Add a mac address. 5697 */ 5698 static int 5699 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 5700 { 5701 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5702 ixgbe_t *ixgbe = rx_group->ixgbe; 5703 struct ixgbe_hw *hw = &ixgbe->hw; 5704 int slot, i; 5705 5706 mutex_enter(&ixgbe->gen_lock); 5707 5708 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5709 mutex_exit(&ixgbe->gen_lock); 5710 return (ECANCELED); 5711 } 5712 5713 if (ixgbe->unicst_avail == 0) { 5714 /* no slots available */ 5715 mutex_exit(&ixgbe->gen_lock); 5716 return (ENOSPC); 5717 } 5718 5719 /* 5720 * The first ixgbe->num_rx_groups slots are reserved for each respective 5721 * group. The rest slots are shared by all groups. While adding a 5722 * MAC address, reserved slots are firstly checked then the shared 5723 * slots are searched. 5724 */ 5725 slot = -1; 5726 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 5727 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 5728 if (ixgbe->unicst_addr[i].mac.set == 0) { 5729 slot = i; 5730 break; 5731 } 5732 } 5733 } else { 5734 slot = rx_group->index; 5735 } 5736 5737 if (slot == -1) { 5738 /* no slots available */ 5739 mutex_exit(&ixgbe->gen_lock); 5740 return (ENOSPC); 5741 } 5742 5743 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5744 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 5745 rx_group->index, IXGBE_RAH_AV); 5746 ixgbe->unicst_addr[slot].mac.set = 1; 5747 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 5748 ixgbe->unicst_avail--; 5749 5750 mutex_exit(&ixgbe->gen_lock); 5751 5752 return (0); 5753 } 5754 5755 /* 5756 * Remove a mac address. 5757 */ 5758 static int 5759 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 5760 { 5761 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5762 ixgbe_t *ixgbe = rx_group->ixgbe; 5763 struct ixgbe_hw *hw = &ixgbe->hw; 5764 int slot; 5765 5766 mutex_enter(&ixgbe->gen_lock); 5767 5768 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5769 mutex_exit(&ixgbe->gen_lock); 5770 return (ECANCELED); 5771 } 5772 5773 slot = ixgbe_unicst_find(ixgbe, mac_addr); 5774 if (slot == -1) { 5775 mutex_exit(&ixgbe->gen_lock); 5776 return (EINVAL); 5777 } 5778 5779 if (ixgbe->unicst_addr[slot].mac.set == 0) { 5780 mutex_exit(&ixgbe->gen_lock); 5781 return (EINVAL); 5782 } 5783 5784 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5785 (void) ixgbe_clear_rar(hw, slot); 5786 ixgbe->unicst_addr[slot].mac.set = 0; 5787 ixgbe->unicst_avail++; 5788 5789 mutex_exit(&ixgbe->gen_lock); 5790 5791 return (0); 5792 } 5793