1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright (c) 2012, Joyent, Inc. All rights reserved. 29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved. 31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved. 32 */ 33 34 #include "ixgbe_sw.h" 35 36 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 37 static char ixgbe_version[] = "ixgbe 1.1.7"; 38 39 /* 40 * Local function protoypes 41 */ 42 static int ixgbe_register_mac(ixgbe_t *); 43 static int ixgbe_identify_hardware(ixgbe_t *); 44 static int ixgbe_regs_map(ixgbe_t *); 45 static void ixgbe_init_properties(ixgbe_t *); 46 static int ixgbe_init_driver_settings(ixgbe_t *); 47 static void ixgbe_init_locks(ixgbe_t *); 48 static void ixgbe_destroy_locks(ixgbe_t *); 49 static int ixgbe_init(ixgbe_t *); 50 static int ixgbe_chip_start(ixgbe_t *); 51 static void ixgbe_chip_stop(ixgbe_t *); 52 static int ixgbe_reset(ixgbe_t *); 53 static void ixgbe_tx_clean(ixgbe_t *); 54 static boolean_t ixgbe_tx_drain(ixgbe_t *); 55 static boolean_t ixgbe_rx_drain(ixgbe_t *); 56 static int ixgbe_alloc_rings(ixgbe_t *); 57 static void ixgbe_free_rings(ixgbe_t *); 58 static int ixgbe_alloc_rx_data(ixgbe_t *); 59 static void ixgbe_free_rx_data(ixgbe_t *); 60 static void ixgbe_setup_rings(ixgbe_t *); 61 static void ixgbe_setup_rx(ixgbe_t *); 62 static void ixgbe_setup_tx(ixgbe_t *); 63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 65 static void ixgbe_setup_rss(ixgbe_t *); 66 static void ixgbe_setup_vmdq(ixgbe_t *); 67 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 68 static void ixgbe_init_unicst(ixgbe_t *); 69 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 70 static void ixgbe_setup_multicst(ixgbe_t *); 71 static void ixgbe_get_hw_state(ixgbe_t *); 72 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 73 static void ixgbe_get_conf(ixgbe_t *); 74 static void ixgbe_init_params(ixgbe_t *); 75 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 76 static void ixgbe_driver_link_check(ixgbe_t *); 77 static void ixgbe_sfp_check(void *); 78 static void ixgbe_overtemp_check(void *); 79 static void ixgbe_link_timer(void *); 80 static void ixgbe_local_timer(void *); 81 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 82 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 83 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 84 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 85 static boolean_t is_valid_mac_addr(uint8_t *); 86 static boolean_t ixgbe_stall_check(ixgbe_t *); 87 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 88 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 89 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 90 static int ixgbe_alloc_intrs(ixgbe_t *); 91 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 92 static int ixgbe_add_intr_handlers(ixgbe_t *); 93 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 94 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 95 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 96 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 97 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 98 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 99 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 100 static void ixgbe_setup_adapter_vector(ixgbe_t *); 101 static void ixgbe_rem_intr_handlers(ixgbe_t *); 102 static void ixgbe_rem_intrs(ixgbe_t *); 103 static int ixgbe_enable_intrs(ixgbe_t *); 104 static int ixgbe_disable_intrs(ixgbe_t *); 105 static uint_t ixgbe_intr_legacy(void *, void *); 106 static uint_t ixgbe_intr_msi(void *, void *); 107 static uint_t ixgbe_intr_msix(void *, void *); 108 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 109 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 110 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 111 static void ixgbe_get_driver_control(struct ixgbe_hw *); 112 static int ixgbe_addmac(void *, const uint8_t *); 113 static int ixgbe_remmac(void *, const uint8_t *); 114 static void ixgbe_release_driver_control(struct ixgbe_hw *); 115 116 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 117 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 118 static int ixgbe_resume(dev_info_t *); 119 static int ixgbe_suspend(dev_info_t *); 120 static int ixgbe_quiesce(dev_info_t *); 121 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 122 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 123 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 124 static int ixgbe_intr_cb_register(ixgbe_t *); 125 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 126 127 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 128 const void *impl_data); 129 static void ixgbe_fm_init(ixgbe_t *); 130 static void ixgbe_fm_fini(ixgbe_t *); 131 132 char *ixgbe_priv_props[] = { 133 "_tx_copy_thresh", 134 "_tx_recycle_thresh", 135 "_tx_overload_thresh", 136 "_tx_resched_thresh", 137 "_rx_copy_thresh", 138 "_rx_limit_per_intr", 139 "_intr_throttling", 140 "_adv_pause_cap", 141 "_adv_asym_pause_cap", 142 NULL 143 }; 144 145 #define IXGBE_MAX_PRIV_PROPS \ 146 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 147 148 static struct cb_ops ixgbe_cb_ops = { 149 nulldev, /* cb_open */ 150 nulldev, /* cb_close */ 151 nodev, /* cb_strategy */ 152 nodev, /* cb_print */ 153 nodev, /* cb_dump */ 154 nodev, /* cb_read */ 155 nodev, /* cb_write */ 156 nodev, /* cb_ioctl */ 157 nodev, /* cb_devmap */ 158 nodev, /* cb_mmap */ 159 nodev, /* cb_segmap */ 160 nochpoll, /* cb_chpoll */ 161 ddi_prop_op, /* cb_prop_op */ 162 NULL, /* cb_stream */ 163 D_MP | D_HOTPLUG, /* cb_flag */ 164 CB_REV, /* cb_rev */ 165 nodev, /* cb_aread */ 166 nodev /* cb_awrite */ 167 }; 168 169 static struct dev_ops ixgbe_dev_ops = { 170 DEVO_REV, /* devo_rev */ 171 0, /* devo_refcnt */ 172 NULL, /* devo_getinfo */ 173 nulldev, /* devo_identify */ 174 nulldev, /* devo_probe */ 175 ixgbe_attach, /* devo_attach */ 176 ixgbe_detach, /* devo_detach */ 177 nodev, /* devo_reset */ 178 &ixgbe_cb_ops, /* devo_cb_ops */ 179 NULL, /* devo_bus_ops */ 180 ddi_power, /* devo_power */ 181 ixgbe_quiesce, /* devo_quiesce */ 182 }; 183 184 static struct modldrv ixgbe_modldrv = { 185 &mod_driverops, /* Type of module. This one is a driver */ 186 ixgbe_ident, /* Discription string */ 187 &ixgbe_dev_ops /* driver ops */ 188 }; 189 190 static struct modlinkage ixgbe_modlinkage = { 191 MODREV_1, &ixgbe_modldrv, NULL 192 }; 193 194 /* 195 * Access attributes for register mapping 196 */ 197 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 198 DDI_DEVICE_ATTR_V1, 199 DDI_STRUCTURE_LE_ACC, 200 DDI_STRICTORDER_ACC, 201 DDI_FLAGERR_ACC 202 }; 203 204 /* 205 * Loopback property 206 */ 207 static lb_property_t lb_normal = { 208 normal, "normal", IXGBE_LB_NONE 209 }; 210 211 static lb_property_t lb_mac = { 212 internal, "MAC", IXGBE_LB_INTERNAL_MAC 213 }; 214 215 static lb_property_t lb_external = { 216 external, "External", IXGBE_LB_EXTERNAL 217 }; 218 219 #define IXGBE_M_CALLBACK_FLAGS \ 220 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 221 222 static mac_callbacks_t ixgbe_m_callbacks = { 223 IXGBE_M_CALLBACK_FLAGS, 224 ixgbe_m_stat, 225 ixgbe_m_start, 226 ixgbe_m_stop, 227 ixgbe_m_promisc, 228 ixgbe_m_multicst, 229 NULL, 230 NULL, 231 NULL, 232 ixgbe_m_ioctl, 233 ixgbe_m_getcapab, 234 NULL, 235 NULL, 236 ixgbe_m_setprop, 237 ixgbe_m_getprop, 238 ixgbe_m_propinfo 239 }; 240 241 /* 242 * Initialize capabilities of each supported adapter type 243 */ 244 static adapter_info_t ixgbe_82598eb_cap = { 245 64, /* maximum number of rx queues */ 246 1, /* minimum number of rx queues */ 247 64, /* default number of rx queues */ 248 16, /* maximum number of rx groups */ 249 1, /* minimum number of rx groups */ 250 1, /* default number of rx groups */ 251 32, /* maximum number of tx queues */ 252 1, /* minimum number of tx queues */ 253 8, /* default number of tx queues */ 254 16366, /* maximum MTU size */ 255 0xFFFF, /* maximum interrupt throttle rate */ 256 0, /* minimum interrupt throttle rate */ 257 200, /* default interrupt throttle rate */ 258 18, /* maximum total msix vectors */ 259 16, /* maximum number of ring vectors */ 260 2, /* maximum number of other vectors */ 261 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 262 0, /* "other" interrupt types enable mask */ 263 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 264 | IXGBE_FLAG_RSS_CAPABLE 265 | IXGBE_FLAG_VMDQ_CAPABLE) 266 }; 267 268 static adapter_info_t ixgbe_82599eb_cap = { 269 128, /* maximum number of rx queues */ 270 1, /* minimum number of rx queues */ 271 128, /* default number of rx queues */ 272 64, /* maximum number of rx groups */ 273 1, /* minimum number of rx groups */ 274 1, /* default number of rx groups */ 275 128, /* maximum number of tx queues */ 276 1, /* minimum number of tx queues */ 277 8, /* default number of tx queues */ 278 15500, /* maximum MTU size */ 279 0xFF8, /* maximum interrupt throttle rate */ 280 0, /* minimum interrupt throttle rate */ 281 200, /* default interrupt throttle rate */ 282 64, /* maximum total msix vectors */ 283 16, /* maximum number of ring vectors */ 284 2, /* maximum number of other vectors */ 285 (IXGBE_EICR_LSC 286 | IXGBE_EICR_GPI_SDP1 287 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 288 289 (IXGBE_SDP1_GPIEN 290 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 291 292 (IXGBE_FLAG_DCA_CAPABLE 293 | IXGBE_FLAG_RSS_CAPABLE 294 | IXGBE_FLAG_VMDQ_CAPABLE 295 | IXGBE_FLAG_RSC_CAPABLE 296 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 297 }; 298 299 static adapter_info_t ixgbe_X540_cap = { 300 128, /* maximum number of rx queues */ 301 1, /* minimum number of rx queues */ 302 128, /* default number of rx queues */ 303 64, /* maximum number of rx groups */ 304 1, /* minimum number of rx groups */ 305 1, /* default number of rx groups */ 306 128, /* maximum number of tx queues */ 307 1, /* minimum number of tx queues */ 308 8, /* default number of tx queues */ 309 15500, /* maximum MTU size */ 310 0xFF8, /* maximum interrupt throttle rate */ 311 0, /* minimum interrupt throttle rate */ 312 200, /* default interrupt throttle rate */ 313 64, /* maximum total msix vectors */ 314 16, /* maximum number of ring vectors */ 315 2, /* maximum number of other vectors */ 316 (IXGBE_EICR_LSC 317 | IXGBE_EICR_GPI_SDP1 318 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 319 320 (IXGBE_SDP1_GPIEN 321 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 322 323 (IXGBE_FLAG_DCA_CAPABLE 324 | IXGBE_FLAG_RSS_CAPABLE 325 | IXGBE_FLAG_VMDQ_CAPABLE 326 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 327 }; 328 329 /* 330 * Module Initialization Functions. 331 */ 332 333 int 334 _init(void) 335 { 336 int status; 337 338 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 339 340 status = mod_install(&ixgbe_modlinkage); 341 342 if (status != DDI_SUCCESS) { 343 mac_fini_ops(&ixgbe_dev_ops); 344 } 345 346 return (status); 347 } 348 349 int 350 _fini(void) 351 { 352 int status; 353 354 status = mod_remove(&ixgbe_modlinkage); 355 356 if (status == DDI_SUCCESS) { 357 mac_fini_ops(&ixgbe_dev_ops); 358 } 359 360 return (status); 361 } 362 363 int 364 _info(struct modinfo *modinfop) 365 { 366 int status; 367 368 status = mod_info(&ixgbe_modlinkage, modinfop); 369 370 return (status); 371 } 372 373 /* 374 * ixgbe_attach - Driver attach. 375 * 376 * This function is the device specific initialization entry 377 * point. This entry point is required and must be written. 378 * The DDI_ATTACH command must be provided in the attach entry 379 * point. When attach() is called with cmd set to DDI_ATTACH, 380 * all normal kernel services (such as kmem_alloc(9F)) are 381 * available for use by the driver. 382 * 383 * The attach() function will be called once for each instance 384 * of the device on the system with cmd set to DDI_ATTACH. 385 * Until attach() succeeds, the only driver entry points which 386 * may be called are open(9E) and getinfo(9E). 387 */ 388 static int 389 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 390 { 391 ixgbe_t *ixgbe; 392 struct ixgbe_osdep *osdep; 393 struct ixgbe_hw *hw; 394 int instance; 395 char taskqname[32]; 396 397 /* 398 * Check the command and perform corresponding operations 399 */ 400 switch (cmd) { 401 default: 402 return (DDI_FAILURE); 403 404 case DDI_RESUME: 405 return (ixgbe_resume(devinfo)); 406 407 case DDI_ATTACH: 408 break; 409 } 410 411 /* Get the device instance */ 412 instance = ddi_get_instance(devinfo); 413 414 /* Allocate memory for the instance data structure */ 415 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 416 417 ixgbe->dip = devinfo; 418 ixgbe->instance = instance; 419 420 hw = &ixgbe->hw; 421 osdep = &ixgbe->osdep; 422 hw->back = osdep; 423 osdep->ixgbe = ixgbe; 424 425 /* Attach the instance pointer to the dev_info data structure */ 426 ddi_set_driver_private(devinfo, ixgbe); 427 428 /* 429 * Initialize for fma support 430 */ 431 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 432 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 433 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 434 ixgbe_fm_init(ixgbe); 435 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 436 437 /* 438 * Map PCI config space registers 439 */ 440 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 441 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 442 goto attach_fail; 443 } 444 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 445 446 /* 447 * Identify the chipset family 448 */ 449 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 450 ixgbe_error(ixgbe, "Failed to identify hardware"); 451 goto attach_fail; 452 } 453 454 /* 455 * Map device registers 456 */ 457 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 458 ixgbe_error(ixgbe, "Failed to map device registers"); 459 goto attach_fail; 460 } 461 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 462 463 /* 464 * Initialize driver parameters 465 */ 466 ixgbe_init_properties(ixgbe); 467 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 468 469 /* 470 * Register interrupt callback 471 */ 472 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 473 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 474 goto attach_fail; 475 } 476 477 /* 478 * Allocate interrupts 479 */ 480 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 481 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 482 goto attach_fail; 483 } 484 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 485 486 /* 487 * Allocate rx/tx rings based on the ring numbers. 488 * The actual numbers of rx/tx rings are decided by the number of 489 * allocated interrupt vectors, so we should allocate the rings after 490 * interrupts are allocated. 491 */ 492 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 493 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 494 goto attach_fail; 495 } 496 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 497 498 /* 499 * Map rings to interrupt vectors 500 */ 501 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 502 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 503 goto attach_fail; 504 } 505 506 /* 507 * Add interrupt handlers 508 */ 509 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 510 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 511 goto attach_fail; 512 } 513 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 514 515 /* 516 * Create a taskq for sfp-change 517 */ 518 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 519 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 520 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 521 ixgbe_error(ixgbe, "sfp_taskq create failed"); 522 goto attach_fail; 523 } 524 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 525 526 /* 527 * Create a taskq for over-temp 528 */ 529 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 530 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 531 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 532 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 533 goto attach_fail; 534 } 535 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 536 537 /* 538 * Initialize driver parameters 539 */ 540 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 541 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 542 goto attach_fail; 543 } 544 545 /* 546 * Initialize mutexes for this device. 547 * Do this before enabling the interrupt handler and 548 * register the softint to avoid the condition where 549 * interrupt handler can try using uninitialized mutex. 550 */ 551 ixgbe_init_locks(ixgbe); 552 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 553 554 /* 555 * Initialize chipset hardware 556 */ 557 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 558 ixgbe_error(ixgbe, "Failed to initialize adapter"); 559 goto attach_fail; 560 } 561 ixgbe->link_check_complete = B_FALSE; 562 ixgbe->link_check_hrtime = gethrtime() + 563 (IXGBE_LINK_UP_TIME * 100000000ULL); 564 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 565 566 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 567 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 568 goto attach_fail; 569 } 570 571 /* 572 * Initialize statistics 573 */ 574 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 575 ixgbe_error(ixgbe, "Failed to initialize statistics"); 576 goto attach_fail; 577 } 578 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 579 580 /* 581 * Register the driver to the MAC 582 */ 583 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 584 ixgbe_error(ixgbe, "Failed to register MAC"); 585 goto attach_fail; 586 } 587 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 588 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 589 590 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 591 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 592 if (ixgbe->periodic_id == 0) { 593 ixgbe_error(ixgbe, "Failed to add the link check timer"); 594 goto attach_fail; 595 } 596 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 597 598 /* 599 * Now that mutex locks are initialized, and the chip is also 600 * initialized, enable interrupts. 601 */ 602 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 603 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 604 goto attach_fail; 605 } 606 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 607 608 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version); 609 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 610 611 return (DDI_SUCCESS); 612 613 attach_fail: 614 ixgbe_unconfigure(devinfo, ixgbe); 615 return (DDI_FAILURE); 616 } 617 618 /* 619 * ixgbe_detach - Driver detach. 620 * 621 * The detach() function is the complement of the attach routine. 622 * If cmd is set to DDI_DETACH, detach() is used to remove the 623 * state associated with a given instance of a device node 624 * prior to the removal of that instance from the system. 625 * 626 * The detach() function will be called once for each instance 627 * of the device for which there has been a successful attach() 628 * once there are no longer any opens on the device. 629 * 630 * Interrupts routine are disabled, All memory allocated by this 631 * driver are freed. 632 */ 633 static int 634 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 635 { 636 ixgbe_t *ixgbe; 637 638 /* 639 * Check detach command 640 */ 641 switch (cmd) { 642 default: 643 return (DDI_FAILURE); 644 645 case DDI_SUSPEND: 646 return (ixgbe_suspend(devinfo)); 647 648 case DDI_DETACH: 649 break; 650 } 651 652 /* 653 * Get the pointer to the driver private data structure 654 */ 655 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 656 if (ixgbe == NULL) 657 return (DDI_FAILURE); 658 659 /* 660 * If the device is still running, it needs to be stopped first. 661 * This check is necessary because under some specific circumstances, 662 * the detach routine can be called without stopping the interface 663 * first. 664 */ 665 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 666 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 667 mutex_enter(&ixgbe->gen_lock); 668 ixgbe_stop(ixgbe, B_TRUE); 669 mutex_exit(&ixgbe->gen_lock); 670 /* Disable and stop the watchdog timer */ 671 ixgbe_disable_watchdog_timer(ixgbe); 672 } 673 674 /* 675 * Check if there are still rx buffers held by the upper layer. 676 * If so, fail the detach. 677 */ 678 if (!ixgbe_rx_drain(ixgbe)) 679 return (DDI_FAILURE); 680 681 /* 682 * Do the remaining unconfigure routines 683 */ 684 ixgbe_unconfigure(devinfo, ixgbe); 685 686 return (DDI_SUCCESS); 687 } 688 689 /* 690 * quiesce(9E) entry point. 691 * 692 * This function is called when the system is single-threaded at high 693 * PIL with preemption disabled. Therefore, this function must not be 694 * blocked. 695 * 696 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 697 * DDI_FAILURE indicates an error condition and should almost never happen. 698 */ 699 static int 700 ixgbe_quiesce(dev_info_t *devinfo) 701 { 702 ixgbe_t *ixgbe; 703 struct ixgbe_hw *hw; 704 705 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 706 707 if (ixgbe == NULL) 708 return (DDI_FAILURE); 709 710 hw = &ixgbe->hw; 711 712 /* 713 * Disable the adapter interrupts 714 */ 715 ixgbe_disable_adapter_interrupts(ixgbe); 716 717 /* 718 * Tell firmware driver is no longer in control 719 */ 720 ixgbe_release_driver_control(hw); 721 722 /* 723 * Reset the chipset 724 */ 725 (void) ixgbe_reset_hw(hw); 726 727 /* 728 * Reset PHY 729 */ 730 (void) ixgbe_reset_phy(hw); 731 732 return (DDI_SUCCESS); 733 } 734 735 static void 736 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 737 { 738 /* 739 * Disable interrupt 740 */ 741 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 742 (void) ixgbe_disable_intrs(ixgbe); 743 } 744 745 /* 746 * remove the link check timer 747 */ 748 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 749 if (ixgbe->periodic_id != NULL) { 750 ddi_periodic_delete(ixgbe->periodic_id); 751 ixgbe->periodic_id = NULL; 752 } 753 } 754 755 /* 756 * Unregister MAC 757 */ 758 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 759 (void) mac_unregister(ixgbe->mac_hdl); 760 } 761 762 /* 763 * Free statistics 764 */ 765 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 766 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 767 } 768 769 /* 770 * Remove interrupt handlers 771 */ 772 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 773 ixgbe_rem_intr_handlers(ixgbe); 774 } 775 776 /* 777 * Remove taskq for sfp-status-change 778 */ 779 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 780 ddi_taskq_destroy(ixgbe->sfp_taskq); 781 } 782 783 /* 784 * Remove taskq for over-temp 785 */ 786 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 787 ddi_taskq_destroy(ixgbe->overtemp_taskq); 788 } 789 790 /* 791 * Remove interrupts 792 */ 793 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 794 ixgbe_rem_intrs(ixgbe); 795 } 796 797 /* 798 * Unregister interrupt callback handler 799 */ 800 (void) ddi_cb_unregister(ixgbe->cb_hdl); 801 802 /* 803 * Remove driver properties 804 */ 805 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 806 (void) ddi_prop_remove_all(devinfo); 807 } 808 809 /* 810 * Stop the chipset 811 */ 812 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 813 mutex_enter(&ixgbe->gen_lock); 814 ixgbe_chip_stop(ixgbe); 815 mutex_exit(&ixgbe->gen_lock); 816 } 817 818 /* 819 * Free register handle 820 */ 821 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 822 if (ixgbe->osdep.reg_handle != NULL) 823 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 824 } 825 826 /* 827 * Free PCI config handle 828 */ 829 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 830 if (ixgbe->osdep.cfg_handle != NULL) 831 pci_config_teardown(&ixgbe->osdep.cfg_handle); 832 } 833 834 /* 835 * Free locks 836 */ 837 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 838 ixgbe_destroy_locks(ixgbe); 839 } 840 841 /* 842 * Free the rx/tx rings 843 */ 844 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 845 ixgbe_free_rings(ixgbe); 846 } 847 848 /* 849 * Unregister FMA capabilities 850 */ 851 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 852 ixgbe_fm_fini(ixgbe); 853 } 854 855 /* 856 * Free the driver data structure 857 */ 858 kmem_free(ixgbe, sizeof (ixgbe_t)); 859 860 ddi_set_driver_private(devinfo, NULL); 861 } 862 863 /* 864 * ixgbe_register_mac - Register the driver and its function pointers with 865 * the GLD interface. 866 */ 867 static int 868 ixgbe_register_mac(ixgbe_t *ixgbe) 869 { 870 struct ixgbe_hw *hw = &ixgbe->hw; 871 mac_register_t *mac; 872 int status; 873 874 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 875 return (IXGBE_FAILURE); 876 877 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 878 mac->m_driver = ixgbe; 879 mac->m_dip = ixgbe->dip; 880 mac->m_src_addr = hw->mac.addr; 881 mac->m_callbacks = &ixgbe_m_callbacks; 882 mac->m_min_sdu = 0; 883 mac->m_max_sdu = ixgbe->default_mtu; 884 mac->m_margin = VLAN_TAGSZ; 885 mac->m_priv_props = ixgbe_priv_props; 886 mac->m_v12n = MAC_VIRT_LEVEL1; 887 888 status = mac_register(mac, &ixgbe->mac_hdl); 889 890 mac_free(mac); 891 892 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 893 } 894 895 /* 896 * ixgbe_identify_hardware - Identify the type of the chipset. 897 */ 898 static int 899 ixgbe_identify_hardware(ixgbe_t *ixgbe) 900 { 901 struct ixgbe_hw *hw = &ixgbe->hw; 902 struct ixgbe_osdep *osdep = &ixgbe->osdep; 903 904 /* 905 * Get the device id 906 */ 907 hw->vendor_id = 908 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 909 hw->device_id = 910 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 911 hw->revision_id = 912 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 913 hw->subsystem_device_id = 914 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 915 hw->subsystem_vendor_id = 916 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 917 918 /* 919 * Set the mac type of the adapter based on the device id 920 */ 921 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 922 return (IXGBE_FAILURE); 923 } 924 925 /* 926 * Install adapter capabilities 927 */ 928 switch (hw->mac.type) { 929 case ixgbe_mac_82598EB: 930 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 931 ixgbe->capab = &ixgbe_82598eb_cap; 932 933 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 934 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 935 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 936 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 937 } 938 break; 939 940 case ixgbe_mac_82599EB: 941 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 942 ixgbe->capab = &ixgbe_82599eb_cap; 943 944 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 945 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 946 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 947 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 948 } 949 break; 950 951 case ixgbe_mac_X540: 952 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n"); 953 ixgbe->capab = &ixgbe_X540_cap; 954 /* 955 * For now, X540 is all set in its capab structure. 956 * As other X540 variants show up, things can change here. 957 */ 958 break; 959 960 default: 961 IXGBE_DEBUGLOG_1(ixgbe, 962 "adapter not supported in ixgbe_identify_hardware(): %d\n", 963 hw->mac.type); 964 return (IXGBE_FAILURE); 965 } 966 967 return (IXGBE_SUCCESS); 968 } 969 970 /* 971 * ixgbe_regs_map - Map the device registers. 972 * 973 */ 974 static int 975 ixgbe_regs_map(ixgbe_t *ixgbe) 976 { 977 dev_info_t *devinfo = ixgbe->dip; 978 struct ixgbe_hw *hw = &ixgbe->hw; 979 struct ixgbe_osdep *osdep = &ixgbe->osdep; 980 off_t mem_size; 981 982 /* 983 * First get the size of device registers to be mapped. 984 */ 985 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 986 != DDI_SUCCESS) { 987 return (IXGBE_FAILURE); 988 } 989 990 /* 991 * Call ddi_regs_map_setup() to map registers 992 */ 993 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 994 (caddr_t *)&hw->hw_addr, 0, 995 mem_size, &ixgbe_regs_acc_attr, 996 &osdep->reg_handle)) != DDI_SUCCESS) { 997 return (IXGBE_FAILURE); 998 } 999 1000 return (IXGBE_SUCCESS); 1001 } 1002 1003 /* 1004 * ixgbe_init_properties - Initialize driver properties. 1005 */ 1006 static void 1007 ixgbe_init_properties(ixgbe_t *ixgbe) 1008 { 1009 /* 1010 * Get conf file properties, including link settings 1011 * jumbo frames, ring number, descriptor number, etc. 1012 */ 1013 ixgbe_get_conf(ixgbe); 1014 1015 ixgbe_init_params(ixgbe); 1016 } 1017 1018 /* 1019 * ixgbe_init_driver_settings - Initialize driver settings. 1020 * 1021 * The settings include hardware function pointers, bus information, 1022 * rx/tx rings settings, link state, and any other parameters that 1023 * need to be setup during driver initialization. 1024 */ 1025 static int 1026 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 1027 { 1028 struct ixgbe_hw *hw = &ixgbe->hw; 1029 dev_info_t *devinfo = ixgbe->dip; 1030 ixgbe_rx_ring_t *rx_ring; 1031 ixgbe_rx_group_t *rx_group; 1032 ixgbe_tx_ring_t *tx_ring; 1033 uint32_t rx_size; 1034 uint32_t tx_size; 1035 uint32_t ring_per_group; 1036 int i; 1037 1038 /* 1039 * Initialize chipset specific hardware function pointers 1040 */ 1041 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 1042 return (IXGBE_FAILURE); 1043 } 1044 1045 /* 1046 * Get the system page size 1047 */ 1048 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 1049 1050 /* 1051 * Set rx buffer size 1052 * 1053 * The IP header alignment room is counted in the calculation. 1054 * The rx buffer size is in unit of 1K that is required by the 1055 * chipset hardware. 1056 */ 1057 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 1058 ixgbe->rx_buf_size = ((rx_size >> 10) + 1059 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1060 1061 /* 1062 * Set tx buffer size 1063 */ 1064 tx_size = ixgbe->max_frame_size; 1065 ixgbe->tx_buf_size = ((tx_size >> 10) + 1066 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1067 1068 /* 1069 * Initialize rx/tx rings/groups parameters 1070 */ 1071 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 1072 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1073 rx_ring = &ixgbe->rx_rings[i]; 1074 rx_ring->index = i; 1075 rx_ring->ixgbe = ixgbe; 1076 rx_ring->group_index = i / ring_per_group; 1077 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 1078 } 1079 1080 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1081 rx_group = &ixgbe->rx_groups[i]; 1082 rx_group->index = i; 1083 rx_group->ixgbe = ixgbe; 1084 } 1085 1086 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1087 tx_ring = &ixgbe->tx_rings[i]; 1088 tx_ring->index = i; 1089 tx_ring->ixgbe = ixgbe; 1090 if (ixgbe->tx_head_wb_enable) 1091 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1092 else 1093 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1094 1095 tx_ring->ring_size = ixgbe->tx_ring_size; 1096 tx_ring->free_list_size = ixgbe->tx_ring_size + 1097 (ixgbe->tx_ring_size >> 1); 1098 } 1099 1100 /* 1101 * Initialize values of interrupt throttling rate 1102 */ 1103 for (i = 1; i < MAX_INTR_VECTOR; i++) 1104 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1105 1106 /* 1107 * The initial link state should be "unknown" 1108 */ 1109 ixgbe->link_state = LINK_STATE_UNKNOWN; 1110 1111 return (IXGBE_SUCCESS); 1112 } 1113 1114 /* 1115 * ixgbe_init_locks - Initialize locks. 1116 */ 1117 static void 1118 ixgbe_init_locks(ixgbe_t *ixgbe) 1119 { 1120 ixgbe_rx_ring_t *rx_ring; 1121 ixgbe_tx_ring_t *tx_ring; 1122 int i; 1123 1124 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1125 rx_ring = &ixgbe->rx_rings[i]; 1126 mutex_init(&rx_ring->rx_lock, NULL, 1127 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1128 } 1129 1130 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1131 tx_ring = &ixgbe->tx_rings[i]; 1132 mutex_init(&tx_ring->tx_lock, NULL, 1133 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1134 mutex_init(&tx_ring->recycle_lock, NULL, 1135 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1136 mutex_init(&tx_ring->tcb_head_lock, NULL, 1137 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1138 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1139 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1140 } 1141 1142 mutex_init(&ixgbe->gen_lock, NULL, 1143 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1144 1145 mutex_init(&ixgbe->watchdog_lock, NULL, 1146 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1147 } 1148 1149 /* 1150 * ixgbe_destroy_locks - Destroy locks. 1151 */ 1152 static void 1153 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1154 { 1155 ixgbe_rx_ring_t *rx_ring; 1156 ixgbe_tx_ring_t *tx_ring; 1157 int i; 1158 1159 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1160 rx_ring = &ixgbe->rx_rings[i]; 1161 mutex_destroy(&rx_ring->rx_lock); 1162 } 1163 1164 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1165 tx_ring = &ixgbe->tx_rings[i]; 1166 mutex_destroy(&tx_ring->tx_lock); 1167 mutex_destroy(&tx_ring->recycle_lock); 1168 mutex_destroy(&tx_ring->tcb_head_lock); 1169 mutex_destroy(&tx_ring->tcb_tail_lock); 1170 } 1171 1172 mutex_destroy(&ixgbe->gen_lock); 1173 mutex_destroy(&ixgbe->watchdog_lock); 1174 } 1175 1176 static int 1177 ixgbe_resume(dev_info_t *devinfo) 1178 { 1179 ixgbe_t *ixgbe; 1180 int i; 1181 1182 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1183 if (ixgbe == NULL) 1184 return (DDI_FAILURE); 1185 1186 mutex_enter(&ixgbe->gen_lock); 1187 1188 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1189 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1190 mutex_exit(&ixgbe->gen_lock); 1191 return (DDI_FAILURE); 1192 } 1193 1194 /* 1195 * Enable and start the watchdog timer 1196 */ 1197 ixgbe_enable_watchdog_timer(ixgbe); 1198 } 1199 1200 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1201 1202 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1203 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1204 mac_tx_ring_update(ixgbe->mac_hdl, 1205 ixgbe->tx_rings[i].ring_handle); 1206 } 1207 } 1208 1209 mutex_exit(&ixgbe->gen_lock); 1210 1211 return (DDI_SUCCESS); 1212 } 1213 1214 static int 1215 ixgbe_suspend(dev_info_t *devinfo) 1216 { 1217 ixgbe_t *ixgbe; 1218 1219 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1220 if (ixgbe == NULL) 1221 return (DDI_FAILURE); 1222 1223 mutex_enter(&ixgbe->gen_lock); 1224 1225 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1226 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1227 mutex_exit(&ixgbe->gen_lock); 1228 return (DDI_SUCCESS); 1229 } 1230 ixgbe_stop(ixgbe, B_FALSE); 1231 1232 mutex_exit(&ixgbe->gen_lock); 1233 1234 /* 1235 * Disable and stop the watchdog timer 1236 */ 1237 ixgbe_disable_watchdog_timer(ixgbe); 1238 1239 return (DDI_SUCCESS); 1240 } 1241 1242 /* 1243 * ixgbe_init - Initialize the device. 1244 */ 1245 static int 1246 ixgbe_init(ixgbe_t *ixgbe) 1247 { 1248 struct ixgbe_hw *hw = &ixgbe->hw; 1249 u8 pbanum[IXGBE_PBANUM_LENGTH]; 1250 1251 mutex_enter(&ixgbe->gen_lock); 1252 1253 /* 1254 * Reset chipset to put the hardware in a known state 1255 * before we try to do anything with the eeprom. 1256 */ 1257 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1258 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1259 goto init_fail; 1260 } 1261 1262 /* 1263 * Need to init eeprom before validating the checksum. 1264 */ 1265 if (ixgbe_init_eeprom_params(hw) < 0) { 1266 ixgbe_error(ixgbe, 1267 "Unable to intitialize the eeprom interface."); 1268 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1269 goto init_fail; 1270 } 1271 1272 /* 1273 * NVM validation 1274 */ 1275 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1276 /* 1277 * Some PCI-E parts fail the first check due to 1278 * the link being in sleep state. Call it again, 1279 * if it fails a second time it's a real issue. 1280 */ 1281 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1282 ixgbe_error(ixgbe, 1283 "Invalid NVM checksum. Please contact " 1284 "the vendor to update the NVM."); 1285 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1286 goto init_fail; 1287 } 1288 } 1289 1290 /* 1291 * Setup default flow control thresholds - enable/disable 1292 * & flow control type is controlled by ixgbe.conf 1293 */ 1294 hw->fc.high_water[0] = DEFAULT_FCRTH; 1295 hw->fc.low_water[0] = DEFAULT_FCRTL; 1296 hw->fc.pause_time = DEFAULT_FCPAUSE; 1297 hw->fc.send_xon = B_TRUE; 1298 1299 /* 1300 * Initialize link settings 1301 */ 1302 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1303 1304 /* 1305 * Initialize the chipset hardware 1306 */ 1307 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1308 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1309 goto init_fail; 1310 } 1311 1312 /* 1313 * Read identifying information and place in devinfo. 1314 */ 1315 pbanum[0] = '\0'; 1316 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum)); 1317 if (*pbanum != '\0') { 1318 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip, 1319 "printed-board-assembly", (char *)pbanum); 1320 } 1321 1322 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1323 goto init_fail; 1324 } 1325 1326 mutex_exit(&ixgbe->gen_lock); 1327 return (IXGBE_SUCCESS); 1328 1329 init_fail: 1330 /* 1331 * Reset PHY 1332 */ 1333 (void) ixgbe_reset_phy(hw); 1334 1335 mutex_exit(&ixgbe->gen_lock); 1336 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1337 return (IXGBE_FAILURE); 1338 } 1339 1340 /* 1341 * ixgbe_chip_start - Initialize and start the chipset hardware. 1342 */ 1343 static int 1344 ixgbe_chip_start(ixgbe_t *ixgbe) 1345 { 1346 struct ixgbe_hw *hw = &ixgbe->hw; 1347 int ret_val, i; 1348 1349 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1350 1351 /* 1352 * Get the mac address 1353 * This function should handle SPARC case correctly. 1354 */ 1355 if (!ixgbe_find_mac_address(ixgbe)) { 1356 ixgbe_error(ixgbe, "Failed to get the mac address"); 1357 return (IXGBE_FAILURE); 1358 } 1359 1360 /* 1361 * Validate the mac address 1362 */ 1363 (void) ixgbe_init_rx_addrs(hw); 1364 if (!is_valid_mac_addr(hw->mac.addr)) { 1365 ixgbe_error(ixgbe, "Invalid mac address"); 1366 return (IXGBE_FAILURE); 1367 } 1368 1369 /* 1370 * Configure/Initialize hardware 1371 */ 1372 ret_val = ixgbe_init_hw(hw); 1373 if (ret_val != IXGBE_SUCCESS) { 1374 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1375 ixgbe_error(ixgbe, 1376 "This 82599 device is pre-release and contains" 1377 " outdated firmware, please contact your hardware" 1378 " vendor for a replacement."); 1379 } else { 1380 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1381 return (IXGBE_FAILURE); 1382 } 1383 } 1384 1385 /* 1386 * Re-enable relaxed ordering for performance. It is disabled 1387 * by default in the hardware init. 1388 */ 1389 if (ixgbe->relax_order_enable == B_TRUE) 1390 ixgbe_enable_relaxed_ordering(hw); 1391 1392 /* 1393 * Setup adapter interrupt vectors 1394 */ 1395 ixgbe_setup_adapter_vector(ixgbe); 1396 1397 /* 1398 * Initialize unicast addresses. 1399 */ 1400 ixgbe_init_unicst(ixgbe); 1401 1402 /* 1403 * Setup and initialize the mctable structures. 1404 */ 1405 ixgbe_setup_multicst(ixgbe); 1406 1407 /* 1408 * Set interrupt throttling rate 1409 */ 1410 for (i = 0; i < ixgbe->intr_cnt; i++) { 1411 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1412 } 1413 1414 /* 1415 * Save the state of the phy 1416 */ 1417 ixgbe_get_hw_state(ixgbe); 1418 1419 /* 1420 * Make sure driver has control 1421 */ 1422 ixgbe_get_driver_control(hw); 1423 1424 return (IXGBE_SUCCESS); 1425 } 1426 1427 /* 1428 * ixgbe_chip_stop - Stop the chipset hardware 1429 */ 1430 static void 1431 ixgbe_chip_stop(ixgbe_t *ixgbe) 1432 { 1433 struct ixgbe_hw *hw = &ixgbe->hw; 1434 1435 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1436 1437 /* 1438 * Tell firmware driver is no longer in control 1439 */ 1440 ixgbe_release_driver_control(hw); 1441 1442 /* 1443 * Reset the chipset 1444 */ 1445 (void) ixgbe_reset_hw(hw); 1446 1447 /* 1448 * Reset PHY 1449 */ 1450 (void) ixgbe_reset_phy(hw); 1451 } 1452 1453 /* 1454 * ixgbe_reset - Reset the chipset and re-start the driver. 1455 * 1456 * It involves stopping and re-starting the chipset, 1457 * and re-configuring the rx/tx rings. 1458 */ 1459 static int 1460 ixgbe_reset(ixgbe_t *ixgbe) 1461 { 1462 int i; 1463 1464 /* 1465 * Disable and stop the watchdog timer 1466 */ 1467 ixgbe_disable_watchdog_timer(ixgbe); 1468 1469 mutex_enter(&ixgbe->gen_lock); 1470 1471 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1472 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1473 1474 ixgbe_stop(ixgbe, B_FALSE); 1475 1476 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1477 mutex_exit(&ixgbe->gen_lock); 1478 return (IXGBE_FAILURE); 1479 } 1480 1481 /* 1482 * After resetting, need to recheck the link status. 1483 */ 1484 ixgbe->link_check_complete = B_FALSE; 1485 ixgbe->link_check_hrtime = gethrtime() + 1486 (IXGBE_LINK_UP_TIME * 100000000ULL); 1487 1488 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1489 1490 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1491 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1492 mac_tx_ring_update(ixgbe->mac_hdl, 1493 ixgbe->tx_rings[i].ring_handle); 1494 } 1495 } 1496 1497 mutex_exit(&ixgbe->gen_lock); 1498 1499 /* 1500 * Enable and start the watchdog timer 1501 */ 1502 ixgbe_enable_watchdog_timer(ixgbe); 1503 1504 return (IXGBE_SUCCESS); 1505 } 1506 1507 /* 1508 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1509 */ 1510 static void 1511 ixgbe_tx_clean(ixgbe_t *ixgbe) 1512 { 1513 ixgbe_tx_ring_t *tx_ring; 1514 tx_control_block_t *tcb; 1515 link_list_t pending_list; 1516 uint32_t desc_num; 1517 int i, j; 1518 1519 LINK_LIST_INIT(&pending_list); 1520 1521 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1522 tx_ring = &ixgbe->tx_rings[i]; 1523 1524 mutex_enter(&tx_ring->recycle_lock); 1525 1526 /* 1527 * Clean the pending tx data - the pending packets in the 1528 * work_list that have no chances to be transmitted again. 1529 * 1530 * We must ensure the chipset is stopped or the link is down 1531 * before cleaning the transmit packets. 1532 */ 1533 desc_num = 0; 1534 for (j = 0; j < tx_ring->ring_size; j++) { 1535 tcb = tx_ring->work_list[j]; 1536 if (tcb != NULL) { 1537 desc_num += tcb->desc_num; 1538 1539 tx_ring->work_list[j] = NULL; 1540 1541 ixgbe_free_tcb(tcb); 1542 1543 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1544 } 1545 } 1546 1547 if (desc_num > 0) { 1548 atomic_add_32(&tx_ring->tbd_free, desc_num); 1549 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1550 1551 /* 1552 * Reset the head and tail pointers of the tbd ring; 1553 * Reset the writeback head if it's enable. 1554 */ 1555 tx_ring->tbd_head = 0; 1556 tx_ring->tbd_tail = 0; 1557 if (ixgbe->tx_head_wb_enable) 1558 *tx_ring->tbd_head_wb = 0; 1559 1560 IXGBE_WRITE_REG(&ixgbe->hw, 1561 IXGBE_TDH(tx_ring->index), 0); 1562 IXGBE_WRITE_REG(&ixgbe->hw, 1563 IXGBE_TDT(tx_ring->index), 0); 1564 } 1565 1566 mutex_exit(&tx_ring->recycle_lock); 1567 1568 /* 1569 * Add the tx control blocks in the pending list to 1570 * the free list. 1571 */ 1572 ixgbe_put_free_list(tx_ring, &pending_list); 1573 } 1574 } 1575 1576 /* 1577 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1578 * transmitted. 1579 */ 1580 static boolean_t 1581 ixgbe_tx_drain(ixgbe_t *ixgbe) 1582 { 1583 ixgbe_tx_ring_t *tx_ring; 1584 boolean_t done; 1585 int i, j; 1586 1587 /* 1588 * Wait for a specific time to allow pending tx packets 1589 * to be transmitted. 1590 * 1591 * Check the counter tbd_free to see if transmission is done. 1592 * No lock protection is needed here. 1593 * 1594 * Return B_TRUE if all pending packets have been transmitted; 1595 * Otherwise return B_FALSE; 1596 */ 1597 for (i = 0; i < TX_DRAIN_TIME; i++) { 1598 1599 done = B_TRUE; 1600 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1601 tx_ring = &ixgbe->tx_rings[j]; 1602 done = done && 1603 (tx_ring->tbd_free == tx_ring->ring_size); 1604 } 1605 1606 if (done) 1607 break; 1608 1609 msec_delay(1); 1610 } 1611 1612 return (done); 1613 } 1614 1615 /* 1616 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1617 */ 1618 static boolean_t 1619 ixgbe_rx_drain(ixgbe_t *ixgbe) 1620 { 1621 boolean_t done = B_TRUE; 1622 int i; 1623 1624 /* 1625 * Polling the rx free list to check if those rx buffers held by 1626 * the upper layer are released. 1627 * 1628 * Check the counter rcb_free to see if all pending buffers are 1629 * released. No lock protection is needed here. 1630 * 1631 * Return B_TRUE if all pending buffers have been released; 1632 * Otherwise return B_FALSE; 1633 */ 1634 for (i = 0; i < RX_DRAIN_TIME; i++) { 1635 done = (ixgbe->rcb_pending == 0); 1636 1637 if (done) 1638 break; 1639 1640 msec_delay(1); 1641 } 1642 1643 return (done); 1644 } 1645 1646 /* 1647 * ixgbe_start - Start the driver/chipset. 1648 */ 1649 int 1650 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1651 { 1652 int i; 1653 1654 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1655 1656 if (alloc_buffer) { 1657 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1658 ixgbe_error(ixgbe, 1659 "Failed to allocate software receive rings"); 1660 return (IXGBE_FAILURE); 1661 } 1662 1663 /* Allocate buffers for all the rx/tx rings */ 1664 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1665 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1666 return (IXGBE_FAILURE); 1667 } 1668 1669 ixgbe->tx_ring_init = B_TRUE; 1670 } else { 1671 ixgbe->tx_ring_init = B_FALSE; 1672 } 1673 1674 for (i = 0; i < ixgbe->num_rx_rings; i++) 1675 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1676 for (i = 0; i < ixgbe->num_tx_rings; i++) 1677 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1678 1679 /* 1680 * Start the chipset hardware 1681 */ 1682 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1683 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1684 goto start_failure; 1685 } 1686 1687 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1688 goto start_failure; 1689 } 1690 1691 /* 1692 * Setup the rx/tx rings 1693 */ 1694 ixgbe_setup_rings(ixgbe); 1695 1696 /* 1697 * ixgbe_start() will be called when resetting, however if reset 1698 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1699 * before enabling the interrupts. 1700 */ 1701 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1702 | IXGBE_STALL| IXGBE_OVERTEMP)); 1703 1704 /* 1705 * Enable adapter interrupts 1706 * The interrupts must be enabled after the driver state is START 1707 */ 1708 ixgbe_enable_adapter_interrupts(ixgbe); 1709 1710 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1711 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1712 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1713 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1714 1715 return (IXGBE_SUCCESS); 1716 1717 start_failure: 1718 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1719 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1720 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1721 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1722 1723 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1724 1725 return (IXGBE_FAILURE); 1726 } 1727 1728 /* 1729 * ixgbe_stop - Stop the driver/chipset. 1730 */ 1731 void 1732 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1733 { 1734 int i; 1735 1736 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1737 1738 /* 1739 * Disable the adapter interrupts 1740 */ 1741 ixgbe_disable_adapter_interrupts(ixgbe); 1742 1743 /* 1744 * Drain the pending tx packets 1745 */ 1746 (void) ixgbe_tx_drain(ixgbe); 1747 1748 for (i = 0; i < ixgbe->num_rx_rings; i++) 1749 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1750 for (i = 0; i < ixgbe->num_tx_rings; i++) 1751 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1752 1753 /* 1754 * Stop the chipset hardware 1755 */ 1756 ixgbe_chip_stop(ixgbe); 1757 1758 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1759 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1760 } 1761 1762 /* 1763 * Clean the pending tx data/resources 1764 */ 1765 ixgbe_tx_clean(ixgbe); 1766 1767 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1768 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1769 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1770 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1771 1772 if (ixgbe->link_state == LINK_STATE_UP) { 1773 ixgbe->link_state = LINK_STATE_UNKNOWN; 1774 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1775 } 1776 1777 if (free_buffer) { 1778 /* 1779 * Release the DMA/memory resources of rx/tx rings 1780 */ 1781 ixgbe_free_dma(ixgbe); 1782 ixgbe_free_rx_data(ixgbe); 1783 } 1784 } 1785 1786 /* 1787 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 1788 */ 1789 /* ARGSUSED */ 1790 static int 1791 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 1792 void *arg1, void *arg2) 1793 { 1794 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 1795 1796 switch (cbaction) { 1797 /* IRM callback */ 1798 int count; 1799 case DDI_CB_INTR_ADD: 1800 case DDI_CB_INTR_REMOVE: 1801 count = (int)(uintptr_t)cbarg; 1802 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 1803 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 1804 int, ixgbe->intr_cnt); 1805 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 1806 DDI_SUCCESS) { 1807 ixgbe_error(ixgbe, 1808 "IRM CB: Failed to adjust interrupts"); 1809 goto cb_fail; 1810 } 1811 break; 1812 default: 1813 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 1814 cbaction); 1815 return (DDI_ENOTSUP); 1816 } 1817 return (DDI_SUCCESS); 1818 cb_fail: 1819 return (DDI_FAILURE); 1820 } 1821 1822 /* 1823 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 1824 */ 1825 static int 1826 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 1827 { 1828 int i, rc, actual; 1829 1830 if (count == 0) 1831 return (DDI_SUCCESS); 1832 1833 if ((cbaction == DDI_CB_INTR_ADD && 1834 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 1835 (cbaction == DDI_CB_INTR_REMOVE && 1836 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 1837 return (DDI_FAILURE); 1838 1839 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1840 return (DDI_FAILURE); 1841 } 1842 1843 for (i = 0; i < ixgbe->num_rx_rings; i++) 1844 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 1845 for (i = 0; i < ixgbe->num_tx_rings; i++) 1846 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 1847 1848 mutex_enter(&ixgbe->gen_lock); 1849 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1850 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 1851 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1852 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 1853 1854 ixgbe_stop(ixgbe, B_FALSE); 1855 /* 1856 * Disable interrupts 1857 */ 1858 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1859 rc = ixgbe_disable_intrs(ixgbe); 1860 ASSERT(rc == IXGBE_SUCCESS); 1861 } 1862 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 1863 1864 /* 1865 * Remove interrupt handlers 1866 */ 1867 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 1868 ixgbe_rem_intr_handlers(ixgbe); 1869 } 1870 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 1871 1872 /* 1873 * Clear vect_map 1874 */ 1875 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 1876 switch (cbaction) { 1877 case DDI_CB_INTR_ADD: 1878 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 1879 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 1880 DDI_INTR_ALLOC_NORMAL); 1881 if (rc != DDI_SUCCESS || actual != count) { 1882 ixgbe_log(ixgbe, "Adjust interrupts failed." 1883 "return: %d, irm cb size: %d, actual: %d", 1884 rc, count, actual); 1885 goto intr_adjust_fail; 1886 } 1887 ixgbe->intr_cnt += count; 1888 break; 1889 1890 case DDI_CB_INTR_REMOVE: 1891 for (i = ixgbe->intr_cnt - count; 1892 i < ixgbe->intr_cnt; i ++) { 1893 rc = ddi_intr_free(ixgbe->htable[i]); 1894 ixgbe->htable[i] = NULL; 1895 if (rc != DDI_SUCCESS) { 1896 ixgbe_log(ixgbe, "Adjust interrupts failed." 1897 "return: %d, irm cb size: %d, actual: %d", 1898 rc, count, actual); 1899 goto intr_adjust_fail; 1900 } 1901 } 1902 ixgbe->intr_cnt -= count; 1903 break; 1904 } 1905 1906 /* 1907 * Get priority for first vector, assume remaining are all the same 1908 */ 1909 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 1910 if (rc != DDI_SUCCESS) { 1911 ixgbe_log(ixgbe, 1912 "Get interrupt priority failed: %d", rc); 1913 goto intr_adjust_fail; 1914 } 1915 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 1916 if (rc != DDI_SUCCESS) { 1917 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 1918 goto intr_adjust_fail; 1919 } 1920 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 1921 1922 /* 1923 * Map rings to interrupt vectors 1924 */ 1925 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 1926 ixgbe_error(ixgbe, 1927 "IRM CB: Failed to map interrupts to vectors"); 1928 goto intr_adjust_fail; 1929 } 1930 1931 /* 1932 * Add interrupt handlers 1933 */ 1934 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 1935 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 1936 goto intr_adjust_fail; 1937 } 1938 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 1939 1940 /* 1941 * Now that mutex locks are initialized, and the chip is also 1942 * initialized, enable interrupts. 1943 */ 1944 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 1945 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 1946 goto intr_adjust_fail; 1947 } 1948 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 1949 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1950 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 1951 goto intr_adjust_fail; 1952 } 1953 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 1954 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1955 ixgbe->ixgbe_state |= IXGBE_STARTED; 1956 mutex_exit(&ixgbe->gen_lock); 1957 1958 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1959 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 1960 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 1961 } 1962 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1963 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 1964 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 1965 } 1966 1967 /* Wakeup all Tx rings */ 1968 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1969 mac_tx_ring_update(ixgbe->mac_hdl, 1970 ixgbe->tx_rings[i].ring_handle); 1971 } 1972 1973 IXGBE_DEBUGLOG_3(ixgbe, 1974 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 1975 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 1976 return (DDI_SUCCESS); 1977 1978 intr_adjust_fail: 1979 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1980 mutex_exit(&ixgbe->gen_lock); 1981 return (DDI_FAILURE); 1982 } 1983 1984 /* 1985 * ixgbe_intr_cb_register - Register interrupt callback function. 1986 */ 1987 static int 1988 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 1989 { 1990 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 1991 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 1992 return (IXGBE_FAILURE); 1993 } 1994 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 1995 return (IXGBE_SUCCESS); 1996 } 1997 1998 /* 1999 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 2000 */ 2001 static int 2002 ixgbe_alloc_rings(ixgbe_t *ixgbe) 2003 { 2004 /* 2005 * Allocate memory space for rx rings 2006 */ 2007 ixgbe->rx_rings = kmem_zalloc( 2008 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 2009 KM_NOSLEEP); 2010 2011 if (ixgbe->rx_rings == NULL) { 2012 return (IXGBE_FAILURE); 2013 } 2014 2015 /* 2016 * Allocate memory space for tx rings 2017 */ 2018 ixgbe->tx_rings = kmem_zalloc( 2019 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 2020 KM_NOSLEEP); 2021 2022 if (ixgbe->tx_rings == NULL) { 2023 kmem_free(ixgbe->rx_rings, 2024 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2025 ixgbe->rx_rings = NULL; 2026 return (IXGBE_FAILURE); 2027 } 2028 2029 /* 2030 * Allocate memory space for rx ring groups 2031 */ 2032 ixgbe->rx_groups = kmem_zalloc( 2033 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 2034 KM_NOSLEEP); 2035 2036 if (ixgbe->rx_groups == NULL) { 2037 kmem_free(ixgbe->rx_rings, 2038 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2039 kmem_free(ixgbe->tx_rings, 2040 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2041 ixgbe->rx_rings = NULL; 2042 ixgbe->tx_rings = NULL; 2043 return (IXGBE_FAILURE); 2044 } 2045 2046 return (IXGBE_SUCCESS); 2047 } 2048 2049 /* 2050 * ixgbe_free_rings - Free the memory space of rx/tx rings. 2051 */ 2052 static void 2053 ixgbe_free_rings(ixgbe_t *ixgbe) 2054 { 2055 if (ixgbe->rx_rings != NULL) { 2056 kmem_free(ixgbe->rx_rings, 2057 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2058 ixgbe->rx_rings = NULL; 2059 } 2060 2061 if (ixgbe->tx_rings != NULL) { 2062 kmem_free(ixgbe->tx_rings, 2063 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2064 ixgbe->tx_rings = NULL; 2065 } 2066 2067 if (ixgbe->rx_groups != NULL) { 2068 kmem_free(ixgbe->rx_groups, 2069 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 2070 ixgbe->rx_groups = NULL; 2071 } 2072 } 2073 2074 static int 2075 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 2076 { 2077 ixgbe_rx_ring_t *rx_ring; 2078 int i; 2079 2080 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2081 rx_ring = &ixgbe->rx_rings[i]; 2082 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 2083 goto alloc_rx_rings_failure; 2084 } 2085 return (IXGBE_SUCCESS); 2086 2087 alloc_rx_rings_failure: 2088 ixgbe_free_rx_data(ixgbe); 2089 return (IXGBE_FAILURE); 2090 } 2091 2092 static void 2093 ixgbe_free_rx_data(ixgbe_t *ixgbe) 2094 { 2095 ixgbe_rx_ring_t *rx_ring; 2096 ixgbe_rx_data_t *rx_data; 2097 int i; 2098 2099 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2100 rx_ring = &ixgbe->rx_rings[i]; 2101 2102 mutex_enter(&ixgbe->rx_pending_lock); 2103 rx_data = rx_ring->rx_data; 2104 2105 if (rx_data != NULL) { 2106 rx_data->flag |= IXGBE_RX_STOPPED; 2107 2108 if (rx_data->rcb_pending == 0) { 2109 ixgbe_free_rx_ring_data(rx_data); 2110 rx_ring->rx_data = NULL; 2111 } 2112 } 2113 2114 mutex_exit(&ixgbe->rx_pending_lock); 2115 } 2116 } 2117 2118 /* 2119 * ixgbe_setup_rings - Setup rx/tx rings. 2120 */ 2121 static void 2122 ixgbe_setup_rings(ixgbe_t *ixgbe) 2123 { 2124 /* 2125 * Setup the rx/tx rings, including the following: 2126 * 2127 * 1. Setup the descriptor ring and the control block buffers; 2128 * 2. Initialize necessary registers for receive/transmit; 2129 * 3. Initialize software pointers/parameters for receive/transmit; 2130 */ 2131 ixgbe_setup_rx(ixgbe); 2132 2133 ixgbe_setup_tx(ixgbe); 2134 } 2135 2136 static void 2137 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2138 { 2139 ixgbe_t *ixgbe = rx_ring->ixgbe; 2140 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2141 struct ixgbe_hw *hw = &ixgbe->hw; 2142 rx_control_block_t *rcb; 2143 union ixgbe_adv_rx_desc *rbd; 2144 uint32_t size; 2145 uint32_t buf_low; 2146 uint32_t buf_high; 2147 uint32_t reg_val; 2148 int i; 2149 2150 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2151 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2152 2153 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2154 rcb = rx_data->work_list[i]; 2155 rbd = &rx_data->rbd_ring[i]; 2156 2157 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2158 rbd->read.hdr_addr = NULL; 2159 } 2160 2161 /* 2162 * Initialize the length register 2163 */ 2164 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2165 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2166 2167 /* 2168 * Initialize the base address registers 2169 */ 2170 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2171 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2172 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2173 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2174 2175 /* 2176 * Setup head & tail pointers 2177 */ 2178 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2179 rx_data->ring_size - 1); 2180 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2181 2182 rx_data->rbd_next = 0; 2183 rx_data->lro_first = 0; 2184 2185 /* 2186 * Setup the Receive Descriptor Control Register (RXDCTL) 2187 * PTHRESH=32 descriptors (half the internal cache) 2188 * HTHRESH=0 descriptors (to minimize latency on fetch) 2189 * WTHRESH defaults to 1 (writeback each descriptor) 2190 */ 2191 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2192 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2193 2194 /* Not a valid value for 82599 or X540 */ 2195 if (hw->mac.type == ixgbe_mac_82598EB) { 2196 reg_val |= 0x0020; /* pthresh */ 2197 } 2198 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2199 2200 if (hw->mac.type == ixgbe_mac_82599EB || 2201 hw->mac.type == ixgbe_mac_X540) { 2202 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2203 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2204 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2205 } 2206 2207 /* 2208 * Setup the Split and Replication Receive Control Register. 2209 * Set the rx buffer size and the advanced descriptor type. 2210 */ 2211 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2212 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2213 reg_val |= IXGBE_SRRCTL_DROP_EN; 2214 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2215 } 2216 2217 static void 2218 ixgbe_setup_rx(ixgbe_t *ixgbe) 2219 { 2220 ixgbe_rx_ring_t *rx_ring; 2221 struct ixgbe_hw *hw = &ixgbe->hw; 2222 uint32_t reg_val; 2223 uint32_t ring_mapping; 2224 uint32_t i, index; 2225 uint32_t psrtype_rss_bit; 2226 2227 /* PSRTYPE must be configured for 82599 */ 2228 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2229 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2230 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2231 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2232 reg_val |= IXGBE_PSRTYPE_L2HDR; 2233 reg_val |= 0x80000000; 2234 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2235 } else { 2236 if (ixgbe->num_rx_groups > 32) { 2237 psrtype_rss_bit = 0x20000000; 2238 } else { 2239 psrtype_rss_bit = 0x40000000; 2240 } 2241 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2242 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2243 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2244 reg_val |= IXGBE_PSRTYPE_L2HDR; 2245 reg_val |= psrtype_rss_bit; 2246 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2247 } 2248 } 2249 2250 /* 2251 * Set filter control in FCTRL to accept broadcast packets and do 2252 * not pass pause frames to host. Flow control settings are already 2253 * in this register, so preserve them. 2254 */ 2255 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2256 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 2257 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 2258 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2259 2260 /* 2261 * Hardware checksum settings 2262 */ 2263 if (ixgbe->rx_hcksum_enable) { 2264 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2265 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2266 } 2267 2268 /* 2269 * Setup VMDq and RSS for multiple receive queues 2270 */ 2271 switch (ixgbe->classify_mode) { 2272 case IXGBE_CLASSIFY_RSS: 2273 /* 2274 * One group, only RSS is needed when more than 2275 * one ring enabled. 2276 */ 2277 ixgbe_setup_rss(ixgbe); 2278 break; 2279 2280 case IXGBE_CLASSIFY_VMDQ: 2281 /* 2282 * Multiple groups, each group has one ring, 2283 * only VMDq is needed. 2284 */ 2285 ixgbe_setup_vmdq(ixgbe); 2286 break; 2287 2288 case IXGBE_CLASSIFY_VMDQ_RSS: 2289 /* 2290 * Multiple groups and multiple rings, both 2291 * VMDq and RSS are needed. 2292 */ 2293 ixgbe_setup_vmdq_rss(ixgbe); 2294 break; 2295 2296 default: 2297 break; 2298 } 2299 2300 /* 2301 * Enable the receive unit. This must be done after filter 2302 * control is set in FCTRL. 2303 */ 2304 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 2305 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 2306 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 2307 2308 /* 2309 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2310 */ 2311 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2312 rx_ring = &ixgbe->rx_rings[i]; 2313 ixgbe_setup_rx_ring(rx_ring); 2314 } 2315 2316 /* 2317 * Setup the per-ring statistics mapping. 2318 */ 2319 ring_mapping = 0; 2320 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2321 index = ixgbe->rx_rings[i].hw_index; 2322 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2323 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2324 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2325 } 2326 2327 /* 2328 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2329 * by four bytes if the packet has a VLAN field, so includes MTU, 2330 * ethernet header and frame check sequence. 2331 * Register is MAXFRS in 82599. 2332 */ 2333 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 2334 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2335 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2336 2337 /* 2338 * Setup Jumbo Frame enable bit 2339 */ 2340 if (ixgbe->default_mtu > ETHERMTU) { 2341 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2342 reg_val |= IXGBE_HLREG0_JUMBOEN; 2343 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2344 } 2345 2346 /* 2347 * Setup RSC for multiple receive queues. 2348 */ 2349 if (ixgbe->lro_enable) { 2350 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2351 /* 2352 * Make sure rx_buf_size * MAXDESC not greater 2353 * than 65535. 2354 * Intel recommends 4 for MAXDESC field value. 2355 */ 2356 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2357 reg_val |= IXGBE_RSCCTL_RSCEN; 2358 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2359 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2360 else 2361 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2362 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2363 } 2364 2365 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2366 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2367 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2368 2369 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2370 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2371 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2372 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2373 2374 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2375 } 2376 } 2377 2378 static void 2379 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2380 { 2381 ixgbe_t *ixgbe = tx_ring->ixgbe; 2382 struct ixgbe_hw *hw = &ixgbe->hw; 2383 uint32_t size; 2384 uint32_t buf_low; 2385 uint32_t buf_high; 2386 uint32_t reg_val; 2387 2388 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2389 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2390 2391 /* 2392 * Initialize the length register 2393 */ 2394 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2395 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2396 2397 /* 2398 * Initialize the base address registers 2399 */ 2400 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2401 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2402 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2403 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2404 2405 /* 2406 * Setup head & tail pointers 2407 */ 2408 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2409 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2410 2411 /* 2412 * Setup head write-back 2413 */ 2414 if (ixgbe->tx_head_wb_enable) { 2415 /* 2416 * The memory of the head write-back is allocated using 2417 * the extra tbd beyond the tail of the tbd ring. 2418 */ 2419 tx_ring->tbd_head_wb = (uint32_t *) 2420 ((uintptr_t)tx_ring->tbd_area.address + size); 2421 *tx_ring->tbd_head_wb = 0; 2422 2423 buf_low = (uint32_t) 2424 (tx_ring->tbd_area.dma_address + size); 2425 buf_high = (uint32_t) 2426 ((tx_ring->tbd_area.dma_address + size) >> 32); 2427 2428 /* Set the head write-back enable bit */ 2429 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2430 2431 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2432 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2433 2434 /* 2435 * Turn off relaxed ordering for head write back or it will 2436 * cause problems with the tx recycling 2437 */ 2438 2439 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ? 2440 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) : 2441 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index)); 2442 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2443 if (hw->mac.type == ixgbe_mac_82598EB) { 2444 IXGBE_WRITE_REG(hw, 2445 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2446 } else { 2447 IXGBE_WRITE_REG(hw, 2448 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val); 2449 } 2450 } else { 2451 tx_ring->tbd_head_wb = NULL; 2452 } 2453 2454 tx_ring->tbd_head = 0; 2455 tx_ring->tbd_tail = 0; 2456 tx_ring->tbd_free = tx_ring->ring_size; 2457 2458 if (ixgbe->tx_ring_init == B_TRUE) { 2459 tx_ring->tcb_head = 0; 2460 tx_ring->tcb_tail = 0; 2461 tx_ring->tcb_free = tx_ring->free_list_size; 2462 } 2463 2464 /* 2465 * Initialize the s/w context structure 2466 */ 2467 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2468 } 2469 2470 static void 2471 ixgbe_setup_tx(ixgbe_t *ixgbe) 2472 { 2473 struct ixgbe_hw *hw = &ixgbe->hw; 2474 ixgbe_tx_ring_t *tx_ring; 2475 uint32_t reg_val; 2476 uint32_t ring_mapping; 2477 int i; 2478 2479 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2480 tx_ring = &ixgbe->tx_rings[i]; 2481 ixgbe_setup_tx_ring(tx_ring); 2482 } 2483 2484 /* 2485 * Setup the per-ring statistics mapping. 2486 */ 2487 ring_mapping = 0; 2488 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2489 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2490 if ((i & 0x3) == 0x3) { 2491 switch (hw->mac.type) { 2492 case ixgbe_mac_82598EB: 2493 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2494 ring_mapping); 2495 break; 2496 2497 case ixgbe_mac_82599EB: 2498 case ixgbe_mac_X540: 2499 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2500 ring_mapping); 2501 break; 2502 2503 default: 2504 break; 2505 } 2506 2507 ring_mapping = 0; 2508 } 2509 } 2510 if (i & 0x3) { 2511 switch (hw->mac.type) { 2512 case ixgbe_mac_82598EB: 2513 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2514 break; 2515 2516 case ixgbe_mac_82599EB: 2517 case ixgbe_mac_X540: 2518 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2519 break; 2520 2521 default: 2522 break; 2523 } 2524 } 2525 2526 /* 2527 * Enable CRC appending and TX padding (for short tx frames) 2528 */ 2529 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2530 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2531 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2532 2533 /* 2534 * enable DMA for 82599 and X540 parts 2535 */ 2536 if (hw->mac.type == ixgbe_mac_82599EB || 2537 hw->mac.type == ixgbe_mac_X540) { 2538 /* DMATXCTL.TE must be set after all Tx config is complete */ 2539 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2540 reg_val |= IXGBE_DMATXCTL_TE; 2541 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2542 2543 /* Disable arbiter to set MTQC */ 2544 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2545 reg_val |= IXGBE_RTTDCS_ARBDIS; 2546 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2547 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2548 reg_val &= ~IXGBE_RTTDCS_ARBDIS; 2549 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2550 } 2551 2552 /* 2553 * Enabling tx queues .. 2554 * For 82599 must be done after DMATXCTL.TE is set 2555 */ 2556 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2557 tx_ring = &ixgbe->tx_rings[i]; 2558 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2559 reg_val |= IXGBE_TXDCTL_ENABLE; 2560 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2561 } 2562 } 2563 2564 /* 2565 * ixgbe_setup_rss - Setup receive-side scaling feature. 2566 */ 2567 static void 2568 ixgbe_setup_rss(ixgbe_t *ixgbe) 2569 { 2570 struct ixgbe_hw *hw = &ixgbe->hw; 2571 uint32_t i, mrqc, rxcsum; 2572 uint32_t random; 2573 uint32_t reta; 2574 uint32_t ring_per_group; 2575 2576 /* 2577 * Fill out redirection table 2578 */ 2579 reta = 0; 2580 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2581 2582 for (i = 0; i < 128; i++) { 2583 reta = (reta << 8) | (i % ring_per_group) | 2584 ((i % ring_per_group) << 4); 2585 if ((i & 3) == 3) 2586 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2587 } 2588 2589 /* 2590 * Fill out hash function seeds with a random constant 2591 */ 2592 for (i = 0; i < 10; i++) { 2593 (void) random_get_pseudo_bytes((uint8_t *)&random, 2594 sizeof (uint32_t)); 2595 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2596 } 2597 2598 /* 2599 * Enable RSS & perform hash on these packet types 2600 */ 2601 mrqc = IXGBE_MRQC_RSSEN | 2602 IXGBE_MRQC_RSS_FIELD_IPV4 | 2603 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2604 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2605 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2606 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2607 IXGBE_MRQC_RSS_FIELD_IPV6 | 2608 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2609 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2610 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2611 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2612 2613 /* 2614 * Disable Packet Checksum to enable RSS for multiple receive queues. 2615 * It is an adapter hardware limitation that Packet Checksum is 2616 * mutually exclusive with RSS. 2617 */ 2618 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2619 rxcsum |= IXGBE_RXCSUM_PCSD; 2620 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2621 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2622 } 2623 2624 /* 2625 * ixgbe_setup_vmdq - Setup MAC classification feature 2626 */ 2627 static void 2628 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2629 { 2630 struct ixgbe_hw *hw = &ixgbe->hw; 2631 uint32_t vmdctl, i, vtctl; 2632 2633 /* 2634 * Setup the VMDq Control register, enable VMDq based on 2635 * packet destination MAC address: 2636 */ 2637 switch (hw->mac.type) { 2638 case ixgbe_mac_82598EB: 2639 /* 2640 * VMDq Enable = 1; 2641 * VMDq Filter = 0; MAC filtering 2642 * Default VMDq output index = 0; 2643 */ 2644 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2645 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2646 break; 2647 2648 case ixgbe_mac_82599EB: 2649 case ixgbe_mac_X540: 2650 /* 2651 * Enable VMDq-only. 2652 */ 2653 vmdctl = IXGBE_MRQC_VMDQEN; 2654 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2655 2656 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2657 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2658 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2659 } 2660 2661 /* 2662 * Enable Virtualization and Replication. 2663 */ 2664 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2665 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2666 2667 /* 2668 * Enable receiving packets to all VFs 2669 */ 2670 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2671 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2672 break; 2673 2674 default: 2675 break; 2676 } 2677 } 2678 2679 /* 2680 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2681 */ 2682 static void 2683 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2684 { 2685 struct ixgbe_hw *hw = &ixgbe->hw; 2686 uint32_t i, mrqc, rxcsum; 2687 uint32_t random; 2688 uint32_t reta; 2689 uint32_t ring_per_group; 2690 uint32_t vmdctl, vtctl; 2691 2692 /* 2693 * Fill out redirection table 2694 */ 2695 reta = 0; 2696 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2697 for (i = 0; i < 128; i++) { 2698 reta = (reta << 8) | (i % ring_per_group) | 2699 ((i % ring_per_group) << 4); 2700 if ((i & 3) == 3) 2701 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2702 } 2703 2704 /* 2705 * Fill out hash function seeds with a random constant 2706 */ 2707 for (i = 0; i < 10; i++) { 2708 (void) random_get_pseudo_bytes((uint8_t *)&random, 2709 sizeof (uint32_t)); 2710 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2711 } 2712 2713 /* 2714 * Enable and setup RSS and VMDq 2715 */ 2716 switch (hw->mac.type) { 2717 case ixgbe_mac_82598EB: 2718 /* 2719 * Enable RSS & Setup RSS Hash functions 2720 */ 2721 mrqc = IXGBE_MRQC_RSSEN | 2722 IXGBE_MRQC_RSS_FIELD_IPV4 | 2723 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2724 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2725 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2726 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2727 IXGBE_MRQC_RSS_FIELD_IPV6 | 2728 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2729 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2730 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2731 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2732 2733 /* 2734 * Enable and Setup VMDq 2735 * VMDq Filter = 0; MAC filtering 2736 * Default VMDq output index = 0; 2737 */ 2738 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2739 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2740 break; 2741 2742 case ixgbe_mac_82599EB: 2743 case ixgbe_mac_X540: 2744 /* 2745 * Enable RSS & Setup RSS Hash functions 2746 */ 2747 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2748 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2749 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2750 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2751 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2752 IXGBE_MRQC_RSS_FIELD_IPV6 | 2753 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2754 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2755 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2756 2757 /* 2758 * Enable VMDq+RSS. 2759 */ 2760 if (ixgbe->num_rx_groups > 32) { 2761 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2762 } else { 2763 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2764 } 2765 2766 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2767 2768 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2769 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2770 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2771 } 2772 break; 2773 2774 default: 2775 break; 2776 2777 } 2778 2779 /* 2780 * Disable Packet Checksum to enable RSS for multiple receive queues. 2781 * It is an adapter hardware limitation that Packet Checksum is 2782 * mutually exclusive with RSS. 2783 */ 2784 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2785 rxcsum |= IXGBE_RXCSUM_PCSD; 2786 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2787 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2788 2789 if (hw->mac.type == ixgbe_mac_82599EB || 2790 hw->mac.type == ixgbe_mac_X540) { 2791 /* 2792 * Enable Virtualization and Replication. 2793 */ 2794 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2795 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2796 2797 /* 2798 * Enable receiving packets to all VFs 2799 */ 2800 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2801 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2802 } 2803 } 2804 2805 /* 2806 * ixgbe_init_unicst - Initialize the unicast addresses. 2807 */ 2808 static void 2809 ixgbe_init_unicst(ixgbe_t *ixgbe) 2810 { 2811 struct ixgbe_hw *hw = &ixgbe->hw; 2812 uint8_t *mac_addr; 2813 int slot; 2814 /* 2815 * Here we should consider two situations: 2816 * 2817 * 1. Chipset is initialized at the first time, 2818 * Clear all the multiple unicast addresses. 2819 * 2820 * 2. Chipset is reset 2821 * Recover the multiple unicast addresses from the 2822 * software data structure to the RAR registers. 2823 */ 2824 if (!ixgbe->unicst_init) { 2825 /* 2826 * Initialize the multiple unicast addresses 2827 */ 2828 ixgbe->unicst_total = hw->mac.num_rar_entries; 2829 ixgbe->unicst_avail = ixgbe->unicst_total; 2830 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2831 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2832 bzero(mac_addr, ETHERADDRL); 2833 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2834 ixgbe->unicst_addr[slot].mac.set = 0; 2835 } 2836 ixgbe->unicst_init = B_TRUE; 2837 } else { 2838 /* Re-configure the RAR registers */ 2839 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2840 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2841 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2842 (void) ixgbe_set_rar(hw, slot, mac_addr, 2843 ixgbe->unicst_addr[slot].mac.group_index, 2844 IXGBE_RAH_AV); 2845 } else { 2846 bzero(mac_addr, ETHERADDRL); 2847 (void) ixgbe_set_rar(hw, slot, mac_addr, 2848 NULL, NULL); 2849 } 2850 } 2851 } 2852 } 2853 2854 /* 2855 * ixgbe_unicst_find - Find the slot for the specified unicast address 2856 */ 2857 int 2858 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2859 { 2860 int slot; 2861 2862 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2863 2864 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2865 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2866 mac_addr, ETHERADDRL) == 0) 2867 return (slot); 2868 } 2869 2870 return (-1); 2871 } 2872 2873 /* 2874 * ixgbe_multicst_add - Add a multicst address. 2875 */ 2876 int 2877 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2878 { 2879 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2880 2881 if ((multiaddr[0] & 01) == 0) { 2882 return (EINVAL); 2883 } 2884 2885 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2886 return (ENOENT); 2887 } 2888 2889 bcopy(multiaddr, 2890 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2891 ixgbe->mcast_count++; 2892 2893 /* 2894 * Update the multicast table in the hardware 2895 */ 2896 ixgbe_setup_multicst(ixgbe); 2897 2898 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2899 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2900 return (EIO); 2901 } 2902 2903 return (0); 2904 } 2905 2906 /* 2907 * ixgbe_multicst_remove - Remove a multicst address. 2908 */ 2909 int 2910 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2911 { 2912 int i; 2913 2914 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2915 2916 for (i = 0; i < ixgbe->mcast_count; i++) { 2917 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2918 ETHERADDRL) == 0) { 2919 for (i++; i < ixgbe->mcast_count; i++) { 2920 ixgbe->mcast_table[i - 1] = 2921 ixgbe->mcast_table[i]; 2922 } 2923 ixgbe->mcast_count--; 2924 break; 2925 } 2926 } 2927 2928 /* 2929 * Update the multicast table in the hardware 2930 */ 2931 ixgbe_setup_multicst(ixgbe); 2932 2933 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2934 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2935 return (EIO); 2936 } 2937 2938 return (0); 2939 } 2940 2941 /* 2942 * ixgbe_setup_multicast - Setup multicast data structures. 2943 * 2944 * This routine initializes all of the multicast related structures 2945 * and save them in the hardware registers. 2946 */ 2947 static void 2948 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2949 { 2950 uint8_t *mc_addr_list; 2951 uint32_t mc_addr_count; 2952 struct ixgbe_hw *hw = &ixgbe->hw; 2953 2954 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2955 2956 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2957 2958 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2959 mc_addr_count = ixgbe->mcast_count; 2960 2961 /* 2962 * Update the multicast addresses to the MTA registers 2963 */ 2964 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2965 ixgbe_mc_table_itr, TRUE); 2966 } 2967 2968 /* 2969 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 2970 * 2971 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 2972 * Different chipsets may have different allowed configuration of vmdq and rss. 2973 */ 2974 static void 2975 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 2976 { 2977 struct ixgbe_hw *hw = &ixgbe->hw; 2978 uint32_t ring_per_group; 2979 2980 switch (hw->mac.type) { 2981 case ixgbe_mac_82598EB: 2982 /* 2983 * 82598 supports the following combination: 2984 * vmdq no. x rss no. 2985 * [5..16] x 1 2986 * [1..4] x [1..16] 2987 * However 8 rss queue per pool (vmdq) is sufficient for 2988 * most cases. 2989 */ 2990 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2991 if (ixgbe->num_rx_groups > 4) { 2992 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 2993 } else { 2994 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2995 min(8, ring_per_group); 2996 } 2997 2998 break; 2999 3000 case ixgbe_mac_82599EB: 3001 case ixgbe_mac_X540: 3002 /* 3003 * 82599 supports the following combination: 3004 * vmdq no. x rss no. 3005 * [33..64] x [1..2] 3006 * [2..32] x [1..4] 3007 * 1 x [1..16] 3008 * However 8 rss queue per pool (vmdq) is sufficient for 3009 * most cases. 3010 * 3011 * For now, treat X540 like the 82599. 3012 */ 3013 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3014 if (ixgbe->num_rx_groups == 1) { 3015 ixgbe->num_rx_rings = min(8, ring_per_group); 3016 } else if (ixgbe->num_rx_groups <= 32) { 3017 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3018 min(4, ring_per_group); 3019 } else if (ixgbe->num_rx_groups <= 64) { 3020 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3021 min(2, ring_per_group); 3022 } 3023 break; 3024 3025 default: 3026 break; 3027 } 3028 3029 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3030 3031 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 3032 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3033 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 3034 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 3035 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 3036 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 3037 } else { 3038 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 3039 } 3040 3041 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 3042 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 3043 } 3044 3045 /* 3046 * ixgbe_get_conf - Get driver configurations set in driver.conf. 3047 * 3048 * This routine gets user-configured values out of the configuration 3049 * file ixgbe.conf. 3050 * 3051 * For each configurable value, there is a minimum, a maximum, and a 3052 * default. 3053 * If user does not configure a value, use the default. 3054 * If user configures below the minimum, use the minumum. 3055 * If user configures above the maximum, use the maxumum. 3056 */ 3057 static void 3058 ixgbe_get_conf(ixgbe_t *ixgbe) 3059 { 3060 struct ixgbe_hw *hw = &ixgbe->hw; 3061 uint32_t flow_control; 3062 3063 /* 3064 * ixgbe driver supports the following user configurations: 3065 * 3066 * Jumbo frame configuration: 3067 * default_mtu 3068 * 3069 * Ethernet flow control configuration: 3070 * flow_control 3071 * 3072 * Multiple rings configurations: 3073 * tx_queue_number 3074 * tx_ring_size 3075 * rx_queue_number 3076 * rx_ring_size 3077 * 3078 * Call ixgbe_get_prop() to get the value for a specific 3079 * configuration parameter. 3080 */ 3081 3082 /* 3083 * Jumbo frame configuration - max_frame_size controls host buffer 3084 * allocation, so includes MTU, ethernet header, vlan tag and 3085 * frame check sequence. 3086 */ 3087 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 3088 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 3089 3090 ixgbe->max_frame_size = ixgbe->default_mtu + 3091 sizeof (struct ether_vlan_header) + ETHERFCSL; 3092 3093 /* 3094 * Ethernet flow control configuration 3095 */ 3096 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 3097 ixgbe_fc_none, 3, ixgbe_fc_none); 3098 if (flow_control == 3) 3099 flow_control = ixgbe_fc_default; 3100 3101 /* 3102 * fc.requested mode is what the user requests. After autoneg, 3103 * fc.current_mode will be the flow_control mode that was negotiated. 3104 */ 3105 hw->fc.requested_mode = flow_control; 3106 3107 /* 3108 * Multiple rings configurations 3109 */ 3110 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 3111 ixgbe->capab->min_tx_que_num, 3112 ixgbe->capab->max_tx_que_num, 3113 ixgbe->capab->def_tx_que_num); 3114 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 3115 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 3116 3117 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 3118 ixgbe->capab->min_rx_que_num, 3119 ixgbe->capab->max_rx_que_num, 3120 ixgbe->capab->def_rx_que_num); 3121 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 3122 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 3123 3124 /* 3125 * Multiple groups configuration 3126 */ 3127 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3128 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3129 ixgbe->capab->def_rx_grp_num); 3130 3131 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3132 0, 1, DEFAULT_MR_ENABLE); 3133 3134 if (ixgbe->mr_enable == B_FALSE) { 3135 ixgbe->num_tx_rings = 1; 3136 ixgbe->num_rx_rings = 1; 3137 ixgbe->num_rx_groups = 1; 3138 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3139 } else { 3140 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3141 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3142 /* 3143 * The combination of num_rx_rings and num_rx_groups 3144 * may be not supported by h/w. We need to adjust 3145 * them to appropriate values. 3146 */ 3147 ixgbe_setup_vmdq_rss_conf(ixgbe); 3148 } 3149 3150 /* 3151 * Tunable used to force an interrupt type. The only use is 3152 * for testing of the lesser interrupt types. 3153 * 0 = don't force interrupt type 3154 * 1 = force interrupt type MSI-X 3155 * 2 = force interrupt type MSI 3156 * 3 = force interrupt type Legacy 3157 */ 3158 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3159 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3160 3161 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3162 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3163 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3164 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3165 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3166 0, 1, DEFAULT_LSO_ENABLE); 3167 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3168 0, 1, DEFAULT_LRO_ENABLE); 3169 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3170 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3171 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3172 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3173 3174 /* Head Write Back not recommended for 82599 and X540 */ 3175 if (hw->mac.type == ixgbe_mac_82599EB || 3176 hw->mac.type == ixgbe_mac_X540) { 3177 ixgbe->tx_head_wb_enable = B_FALSE; 3178 } 3179 3180 /* 3181 * ixgbe LSO needs the tx h/w checksum support. 3182 * LSO will be disabled if tx h/w checksum is not 3183 * enabled. 3184 */ 3185 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3186 ixgbe->lso_enable = B_FALSE; 3187 } 3188 3189 /* 3190 * ixgbe LRO needs the rx h/w checksum support. 3191 * LRO will be disabled if rx h/w checksum is not 3192 * enabled. 3193 */ 3194 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3195 ixgbe->lro_enable = B_FALSE; 3196 } 3197 3198 /* 3199 * ixgbe LRO only been supported by 82599 and X540 now 3200 */ 3201 if (hw->mac.type == ixgbe_mac_82598EB) { 3202 ixgbe->lro_enable = B_FALSE; 3203 } 3204 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3205 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3206 DEFAULT_TX_COPY_THRESHOLD); 3207 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3208 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3209 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3210 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3211 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3212 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3213 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3214 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3215 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3216 3217 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3218 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3219 DEFAULT_RX_COPY_THRESHOLD); 3220 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3221 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3222 DEFAULT_RX_LIMIT_PER_INTR); 3223 3224 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3225 ixgbe->capab->min_intr_throttle, 3226 ixgbe->capab->max_intr_throttle, 3227 ixgbe->capab->def_intr_throttle); 3228 /* 3229 * 82599 and X540 require the interrupt throttling rate is 3230 * a multiple of 8. This is enforced by the register 3231 * definiton. 3232 */ 3233 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540) 3234 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3235 3236 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe, 3237 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP); 3238 } 3239 3240 static void 3241 ixgbe_init_params(ixgbe_t *ixgbe) 3242 { 3243 ixgbe->param_en_10000fdx_cap = 1; 3244 ixgbe->param_en_1000fdx_cap = 1; 3245 ixgbe->param_en_100fdx_cap = 1; 3246 ixgbe->param_adv_10000fdx_cap = 1; 3247 ixgbe->param_adv_1000fdx_cap = 1; 3248 ixgbe->param_adv_100fdx_cap = 1; 3249 3250 ixgbe->param_pause_cap = 1; 3251 ixgbe->param_asym_pause_cap = 1; 3252 ixgbe->param_rem_fault = 0; 3253 3254 ixgbe->param_adv_autoneg_cap = 1; 3255 ixgbe->param_adv_pause_cap = 1; 3256 ixgbe->param_adv_asym_pause_cap = 1; 3257 ixgbe->param_adv_rem_fault = 0; 3258 3259 ixgbe->param_lp_10000fdx_cap = 0; 3260 ixgbe->param_lp_1000fdx_cap = 0; 3261 ixgbe->param_lp_100fdx_cap = 0; 3262 ixgbe->param_lp_autoneg_cap = 0; 3263 ixgbe->param_lp_pause_cap = 0; 3264 ixgbe->param_lp_asym_pause_cap = 0; 3265 ixgbe->param_lp_rem_fault = 0; 3266 } 3267 3268 /* 3269 * ixgbe_get_prop - Get a property value out of the configuration file 3270 * ixgbe.conf. 3271 * 3272 * Caller provides the name of the property, a default value, a minimum 3273 * value, and a maximum value. 3274 * 3275 * Return configured value of the property, with default, minimum and 3276 * maximum properly applied. 3277 */ 3278 static int 3279 ixgbe_get_prop(ixgbe_t *ixgbe, 3280 char *propname, /* name of the property */ 3281 int minval, /* minimum acceptable value */ 3282 int maxval, /* maximim acceptable value */ 3283 int defval) /* default value */ 3284 { 3285 int value; 3286 3287 /* 3288 * Call ddi_prop_get_int() to read the conf settings 3289 */ 3290 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3291 DDI_PROP_DONTPASS, propname, defval); 3292 if (value > maxval) 3293 value = maxval; 3294 3295 if (value < minval) 3296 value = minval; 3297 3298 return (value); 3299 } 3300 3301 /* 3302 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3303 */ 3304 int 3305 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3306 { 3307 u32 autoneg_advertised = 0; 3308 3309 /* 3310 * No half duplex support with 10Gb parts 3311 */ 3312 if (ixgbe->param_adv_10000fdx_cap == 1) 3313 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3314 3315 if (ixgbe->param_adv_1000fdx_cap == 1) 3316 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3317 3318 if (ixgbe->param_adv_100fdx_cap == 1) 3319 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 3320 3321 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) { 3322 ixgbe_notice(ixgbe, "Invalid link settings. Setup link " 3323 "to autonegotiation with full link capabilities."); 3324 3325 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL | 3326 IXGBE_LINK_SPEED_1GB_FULL | 3327 IXGBE_LINK_SPEED_100_FULL; 3328 } 3329 3330 if (setup_hw) { 3331 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised, 3332 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) { 3333 ixgbe_notice(ixgbe, "Setup link failed on this " 3334 "device."); 3335 return (IXGBE_FAILURE); 3336 } 3337 } 3338 3339 return (IXGBE_SUCCESS); 3340 } 3341 3342 /* 3343 * ixgbe_driver_link_check - Link status processing. 3344 * 3345 * This function can be called in both kernel context and interrupt context 3346 */ 3347 static void 3348 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3349 { 3350 struct ixgbe_hw *hw = &ixgbe->hw; 3351 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3352 boolean_t link_up = B_FALSE; 3353 boolean_t link_changed = B_FALSE; 3354 3355 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3356 3357 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3358 if (link_up) { 3359 ixgbe->link_check_complete = B_TRUE; 3360 3361 /* Link is up, enable flow control settings */ 3362 (void) ixgbe_fc_enable(hw); 3363 3364 /* 3365 * The Link is up, check whether it was marked as down earlier 3366 */ 3367 if (ixgbe->link_state != LINK_STATE_UP) { 3368 switch (speed) { 3369 case IXGBE_LINK_SPEED_10GB_FULL: 3370 ixgbe->link_speed = SPEED_10GB; 3371 break; 3372 case IXGBE_LINK_SPEED_1GB_FULL: 3373 ixgbe->link_speed = SPEED_1GB; 3374 break; 3375 case IXGBE_LINK_SPEED_100_FULL: 3376 ixgbe->link_speed = SPEED_100; 3377 } 3378 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3379 ixgbe->link_state = LINK_STATE_UP; 3380 link_changed = B_TRUE; 3381 } 3382 } else { 3383 if (ixgbe->link_check_complete == B_TRUE || 3384 (ixgbe->link_check_complete == B_FALSE && 3385 gethrtime() >= ixgbe->link_check_hrtime)) { 3386 /* 3387 * The link is really down 3388 */ 3389 ixgbe->link_check_complete = B_TRUE; 3390 3391 if (ixgbe->link_state != LINK_STATE_DOWN) { 3392 ixgbe->link_speed = 0; 3393 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3394 ixgbe->link_state = LINK_STATE_DOWN; 3395 link_changed = B_TRUE; 3396 } 3397 } 3398 } 3399 3400 /* 3401 * If we are in an interrupt context, need to re-enable the 3402 * interrupt, which was automasked 3403 */ 3404 if (servicing_interrupt() != 0) { 3405 ixgbe->eims |= IXGBE_EICR_LSC; 3406 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3407 } 3408 3409 if (link_changed) { 3410 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3411 } 3412 } 3413 3414 /* 3415 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3416 */ 3417 static void 3418 ixgbe_sfp_check(void *arg) 3419 { 3420 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3421 uint32_t eicr = ixgbe->eicr; 3422 struct ixgbe_hw *hw = &ixgbe->hw; 3423 3424 mutex_enter(&ixgbe->gen_lock); 3425 if (eicr & IXGBE_EICR_GPI_SDP1) { 3426 /* clear the interrupt */ 3427 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 3428 3429 /* if link up, do multispeed fiber setup */ 3430 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3431 B_TRUE, B_TRUE); 3432 ixgbe_driver_link_check(ixgbe); 3433 ixgbe_get_hw_state(ixgbe); 3434 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 3435 /* clear the interrupt */ 3436 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 3437 3438 /* if link up, do sfp module setup */ 3439 (void) hw->mac.ops.setup_sfp(hw); 3440 3441 /* do multispeed fiber setup */ 3442 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3443 B_TRUE, B_TRUE); 3444 ixgbe_driver_link_check(ixgbe); 3445 ixgbe_get_hw_state(ixgbe); 3446 } 3447 mutex_exit(&ixgbe->gen_lock); 3448 3449 /* 3450 * We need to fully re-check the link later. 3451 */ 3452 ixgbe->link_check_complete = B_FALSE; 3453 ixgbe->link_check_hrtime = gethrtime() + 3454 (IXGBE_LINK_UP_TIME * 100000000ULL); 3455 } 3456 3457 /* 3458 * ixgbe_overtemp_check - overtemp module processing done in taskq 3459 * 3460 * This routine will only be called on adapters with temperature sensor. 3461 * The indication of over-temperature can be either SDP0 interrupt or the link 3462 * status change interrupt. 3463 */ 3464 static void 3465 ixgbe_overtemp_check(void *arg) 3466 { 3467 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3468 struct ixgbe_hw *hw = &ixgbe->hw; 3469 uint32_t eicr = ixgbe->eicr; 3470 ixgbe_link_speed speed; 3471 boolean_t link_up; 3472 3473 mutex_enter(&ixgbe->gen_lock); 3474 3475 /* make sure we know current state of link */ 3476 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3477 3478 /* check over-temp condition */ 3479 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) || 3480 (eicr & IXGBE_EICR_LSC)) { 3481 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3482 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3483 3484 /* 3485 * Disable the adapter interrupts 3486 */ 3487 ixgbe_disable_adapter_interrupts(ixgbe); 3488 3489 /* 3490 * Disable Rx/Tx units 3491 */ 3492 (void) ixgbe_stop_adapter(hw); 3493 3494 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3495 ixgbe_error(ixgbe, 3496 "Problem: Network adapter has been stopped " 3497 "because it has overheated"); 3498 ixgbe_error(ixgbe, 3499 "Action: Restart the computer. " 3500 "If the problem persists, power off the system " 3501 "and replace the adapter"); 3502 } 3503 } 3504 3505 /* write to clear the interrupt */ 3506 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3507 3508 mutex_exit(&ixgbe->gen_lock); 3509 } 3510 3511 /* 3512 * ixgbe_link_timer - timer for link status detection 3513 */ 3514 static void 3515 ixgbe_link_timer(void *arg) 3516 { 3517 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3518 3519 mutex_enter(&ixgbe->gen_lock); 3520 ixgbe_driver_link_check(ixgbe); 3521 mutex_exit(&ixgbe->gen_lock); 3522 } 3523 3524 /* 3525 * ixgbe_local_timer - Driver watchdog function. 3526 * 3527 * This function will handle the transmit stall check and other routines. 3528 */ 3529 static void 3530 ixgbe_local_timer(void *arg) 3531 { 3532 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3533 3534 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3535 goto out; 3536 3537 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3538 ixgbe->reset_count++; 3539 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3540 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3541 goto out; 3542 } 3543 3544 if (ixgbe_stall_check(ixgbe)) { 3545 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3546 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3547 3548 ixgbe->reset_count++; 3549 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3550 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3551 } 3552 3553 out: 3554 ixgbe_restart_watchdog_timer(ixgbe); 3555 } 3556 3557 /* 3558 * ixgbe_stall_check - Check for transmit stall. 3559 * 3560 * This function checks if the adapter is stalled (in transmit). 3561 * 3562 * It is called each time the watchdog timeout is invoked. 3563 * If the transmit descriptor reclaim continuously fails, 3564 * the watchdog value will increment by 1. If the watchdog 3565 * value exceeds the threshold, the ixgbe is assumed to 3566 * have stalled and need to be reset. 3567 */ 3568 static boolean_t 3569 ixgbe_stall_check(ixgbe_t *ixgbe) 3570 { 3571 ixgbe_tx_ring_t *tx_ring; 3572 boolean_t result; 3573 int i; 3574 3575 if (ixgbe->link_state != LINK_STATE_UP) 3576 return (B_FALSE); 3577 3578 /* 3579 * If any tx ring is stalled, we'll reset the chipset 3580 */ 3581 result = B_FALSE; 3582 for (i = 0; i < ixgbe->num_tx_rings; i++) { 3583 tx_ring = &ixgbe->tx_rings[i]; 3584 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 3585 tx_ring->tx_recycle(tx_ring); 3586 } 3587 3588 if (tx_ring->recycle_fail > 0) 3589 tx_ring->stall_watchdog++; 3590 else 3591 tx_ring->stall_watchdog = 0; 3592 3593 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3594 result = B_TRUE; 3595 break; 3596 } 3597 } 3598 3599 if (result) { 3600 tx_ring->stall_watchdog = 0; 3601 tx_ring->recycle_fail = 0; 3602 } 3603 3604 return (result); 3605 } 3606 3607 3608 /* 3609 * is_valid_mac_addr - Check if the mac address is valid. 3610 */ 3611 static boolean_t 3612 is_valid_mac_addr(uint8_t *mac_addr) 3613 { 3614 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3615 const uint8_t addr_test2[6] = 3616 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3617 3618 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3619 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3620 return (B_FALSE); 3621 3622 return (B_TRUE); 3623 } 3624 3625 static boolean_t 3626 ixgbe_find_mac_address(ixgbe_t *ixgbe) 3627 { 3628 #ifdef __sparc 3629 struct ixgbe_hw *hw = &ixgbe->hw; 3630 uchar_t *bytes; 3631 struct ether_addr sysaddr; 3632 uint_t nelts; 3633 int err; 3634 boolean_t found = B_FALSE; 3635 3636 /* 3637 * The "vendor's factory-set address" may already have 3638 * been extracted from the chip, but if the property 3639 * "local-mac-address" is set we use that instead. 3640 * 3641 * We check whether it looks like an array of 6 3642 * bytes (which it should, if OBP set it). If we can't 3643 * make sense of it this way, we'll ignore it. 3644 */ 3645 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3646 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3647 if (err == DDI_PROP_SUCCESS) { 3648 if (nelts == ETHERADDRL) { 3649 while (nelts--) 3650 hw->mac.addr[nelts] = bytes[nelts]; 3651 found = B_TRUE; 3652 } 3653 ddi_prop_free(bytes); 3654 } 3655 3656 /* 3657 * Look up the OBP property "local-mac-address?". If the user has set 3658 * 'local-mac-address? = false', use "the system address" instead. 3659 */ 3660 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 3661 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3662 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3663 if (localetheraddr(NULL, &sysaddr) != 0) { 3664 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3665 found = B_TRUE; 3666 } 3667 } 3668 ddi_prop_free(bytes); 3669 } 3670 3671 /* 3672 * Finally(!), if there's a valid "mac-address" property (created 3673 * if we netbooted from this interface), we must use this instead 3674 * of any of the above to ensure that the NFS/install server doesn't 3675 * get confused by the address changing as Solaris takes over! 3676 */ 3677 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3678 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3679 if (err == DDI_PROP_SUCCESS) { 3680 if (nelts == ETHERADDRL) { 3681 while (nelts--) 3682 hw->mac.addr[nelts] = bytes[nelts]; 3683 found = B_TRUE; 3684 } 3685 ddi_prop_free(bytes); 3686 } 3687 3688 if (found) { 3689 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3690 return (B_TRUE); 3691 } 3692 #else 3693 _NOTE(ARGUNUSED(ixgbe)); 3694 #endif 3695 3696 return (B_TRUE); 3697 } 3698 3699 #pragma inline(ixgbe_arm_watchdog_timer) 3700 static void 3701 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 3702 { 3703 /* 3704 * Fire a watchdog timer 3705 */ 3706 ixgbe->watchdog_tid = 3707 timeout(ixgbe_local_timer, 3708 (void *)ixgbe, 1 * drv_usectohz(1000000)); 3709 3710 } 3711 3712 /* 3713 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 3714 */ 3715 void 3716 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 3717 { 3718 mutex_enter(&ixgbe->watchdog_lock); 3719 3720 if (!ixgbe->watchdog_enable) { 3721 ixgbe->watchdog_enable = B_TRUE; 3722 ixgbe->watchdog_start = B_TRUE; 3723 ixgbe_arm_watchdog_timer(ixgbe); 3724 } 3725 3726 mutex_exit(&ixgbe->watchdog_lock); 3727 } 3728 3729 /* 3730 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 3731 */ 3732 void 3733 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 3734 { 3735 timeout_id_t tid; 3736 3737 mutex_enter(&ixgbe->watchdog_lock); 3738 3739 ixgbe->watchdog_enable = B_FALSE; 3740 ixgbe->watchdog_start = B_FALSE; 3741 tid = ixgbe->watchdog_tid; 3742 ixgbe->watchdog_tid = 0; 3743 3744 mutex_exit(&ixgbe->watchdog_lock); 3745 3746 if (tid != 0) 3747 (void) untimeout(tid); 3748 } 3749 3750 /* 3751 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 3752 */ 3753 void 3754 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 3755 { 3756 mutex_enter(&ixgbe->watchdog_lock); 3757 3758 if (ixgbe->watchdog_enable) { 3759 if (!ixgbe->watchdog_start) { 3760 ixgbe->watchdog_start = B_TRUE; 3761 ixgbe_arm_watchdog_timer(ixgbe); 3762 } 3763 } 3764 3765 mutex_exit(&ixgbe->watchdog_lock); 3766 } 3767 3768 /* 3769 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 3770 */ 3771 static void 3772 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 3773 { 3774 mutex_enter(&ixgbe->watchdog_lock); 3775 3776 if (ixgbe->watchdog_start) 3777 ixgbe_arm_watchdog_timer(ixgbe); 3778 3779 mutex_exit(&ixgbe->watchdog_lock); 3780 } 3781 3782 /* 3783 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 3784 */ 3785 void 3786 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 3787 { 3788 timeout_id_t tid; 3789 3790 mutex_enter(&ixgbe->watchdog_lock); 3791 3792 ixgbe->watchdog_start = B_FALSE; 3793 tid = ixgbe->watchdog_tid; 3794 ixgbe->watchdog_tid = 0; 3795 3796 mutex_exit(&ixgbe->watchdog_lock); 3797 3798 if (tid != 0) 3799 (void) untimeout(tid); 3800 } 3801 3802 /* 3803 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 3804 */ 3805 static void 3806 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 3807 { 3808 struct ixgbe_hw *hw = &ixgbe->hw; 3809 3810 /* 3811 * mask all interrupts off 3812 */ 3813 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 3814 3815 /* 3816 * for MSI-X, also disable autoclear 3817 */ 3818 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3819 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 3820 } 3821 3822 IXGBE_WRITE_FLUSH(hw); 3823 } 3824 3825 /* 3826 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 3827 */ 3828 static void 3829 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 3830 { 3831 struct ixgbe_hw *hw = &ixgbe->hw; 3832 uint32_t eiac, eiam; 3833 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 3834 3835 /* interrupt types to enable */ 3836 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 3837 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 3838 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 3839 3840 /* enable automask on "other" causes that this adapter can generate */ 3841 eiam = ixgbe->capab->other_intr; 3842 3843 /* 3844 * msi-x mode 3845 */ 3846 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3847 /* enable autoclear but not on bits 29:20 */ 3848 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 3849 3850 /* general purpose interrupt enable */ 3851 gpie |= (IXGBE_GPIE_MSIX_MODE 3852 | IXGBE_GPIE_PBA_SUPPORT 3853 | IXGBE_GPIE_OCD 3854 | IXGBE_GPIE_EIAME); 3855 /* 3856 * non-msi-x mode 3857 */ 3858 } else { 3859 3860 /* disable autoclear, leave gpie at default */ 3861 eiac = 0; 3862 3863 /* 3864 * General purpose interrupt enable. 3865 * For 82599 or X540, extended interrupt automask enable 3866 * only in MSI or MSI-X mode 3867 */ 3868 if ((hw->mac.type == ixgbe_mac_82598EB) || 3869 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 3870 gpie |= IXGBE_GPIE_EIAME; 3871 } 3872 } 3873 3874 /* Enable specific "other" interrupt types */ 3875 switch (hw->mac.type) { 3876 case ixgbe_mac_82598EB: 3877 gpie |= ixgbe->capab->other_gpie; 3878 break; 3879 3880 case ixgbe_mac_82599EB: 3881 case ixgbe_mac_X540: 3882 gpie |= ixgbe->capab->other_gpie; 3883 3884 /* Enable RSC Delay 8us when LRO enabled */ 3885 if (ixgbe->lro_enable) { 3886 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 3887 } 3888 break; 3889 3890 default: 3891 break; 3892 } 3893 3894 /* write to interrupt control registers */ 3895 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3896 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3897 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3898 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3899 IXGBE_WRITE_FLUSH(hw); 3900 } 3901 3902 /* 3903 * ixgbe_loopback_ioctl - Loopback support. 3904 */ 3905 enum ioc_reply 3906 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3907 { 3908 lb_info_sz_t *lbsp; 3909 lb_property_t *lbpp; 3910 uint32_t *lbmp; 3911 uint32_t size; 3912 uint32_t value; 3913 3914 if (mp->b_cont == NULL) 3915 return (IOC_INVAL); 3916 3917 switch (iocp->ioc_cmd) { 3918 default: 3919 return (IOC_INVAL); 3920 3921 case LB_GET_INFO_SIZE: 3922 size = sizeof (lb_info_sz_t); 3923 if (iocp->ioc_count != size) 3924 return (IOC_INVAL); 3925 3926 value = sizeof (lb_normal); 3927 value += sizeof (lb_mac); 3928 value += sizeof (lb_external); 3929 3930 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3931 *lbsp = value; 3932 break; 3933 3934 case LB_GET_INFO: 3935 value = sizeof (lb_normal); 3936 value += sizeof (lb_mac); 3937 value += sizeof (lb_external); 3938 3939 size = value; 3940 if (iocp->ioc_count != size) 3941 return (IOC_INVAL); 3942 3943 value = 0; 3944 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3945 3946 lbpp[value++] = lb_normal; 3947 lbpp[value++] = lb_mac; 3948 lbpp[value++] = lb_external; 3949 break; 3950 3951 case LB_GET_MODE: 3952 size = sizeof (uint32_t); 3953 if (iocp->ioc_count != size) 3954 return (IOC_INVAL); 3955 3956 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3957 *lbmp = ixgbe->loopback_mode; 3958 break; 3959 3960 case LB_SET_MODE: 3961 size = 0; 3962 if (iocp->ioc_count != sizeof (uint32_t)) 3963 return (IOC_INVAL); 3964 3965 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3966 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3967 return (IOC_INVAL); 3968 break; 3969 } 3970 3971 iocp->ioc_count = size; 3972 iocp->ioc_error = 0; 3973 3974 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3975 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3976 return (IOC_INVAL); 3977 } 3978 3979 return (IOC_REPLY); 3980 } 3981 3982 /* 3983 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3984 */ 3985 static boolean_t 3986 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3987 { 3988 if (mode == ixgbe->loopback_mode) 3989 return (B_TRUE); 3990 3991 ixgbe->loopback_mode = mode; 3992 3993 if (mode == IXGBE_LB_NONE) { 3994 /* 3995 * Reset the chip 3996 */ 3997 (void) ixgbe_reset(ixgbe); 3998 return (B_TRUE); 3999 } 4000 4001 mutex_enter(&ixgbe->gen_lock); 4002 4003 switch (mode) { 4004 default: 4005 mutex_exit(&ixgbe->gen_lock); 4006 return (B_FALSE); 4007 4008 case IXGBE_LB_EXTERNAL: 4009 break; 4010 4011 case IXGBE_LB_INTERNAL_MAC: 4012 ixgbe_set_internal_mac_loopback(ixgbe); 4013 break; 4014 } 4015 4016 mutex_exit(&ixgbe->gen_lock); 4017 4018 return (B_TRUE); 4019 } 4020 4021 /* 4022 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 4023 */ 4024 static void 4025 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 4026 { 4027 struct ixgbe_hw *hw; 4028 uint32_t reg; 4029 uint8_t atlas; 4030 4031 hw = &ixgbe->hw; 4032 4033 /* 4034 * Setup MAC loopback 4035 */ 4036 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 4037 reg |= IXGBE_HLREG0_LPBK; 4038 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 4039 4040 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4041 reg &= ~IXGBE_AUTOC_LMS_MASK; 4042 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4043 4044 /* 4045 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 4046 */ 4047 switch (hw->mac.type) { 4048 case ixgbe_mac_82598EB: 4049 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4050 &atlas); 4051 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 4052 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4053 atlas); 4054 4055 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4056 &atlas); 4057 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 4058 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4059 atlas); 4060 4061 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4062 &atlas); 4063 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 4064 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4065 atlas); 4066 4067 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4068 &atlas); 4069 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 4070 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4071 atlas); 4072 break; 4073 4074 case ixgbe_mac_82599EB: 4075 case ixgbe_mac_X540: 4076 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4077 reg |= (IXGBE_AUTOC_FLU | 4078 IXGBE_AUTOC_10G_KX4); 4079 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4080 4081 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 4082 B_FALSE, B_TRUE); 4083 break; 4084 4085 default: 4086 break; 4087 } 4088 } 4089 4090 #pragma inline(ixgbe_intr_rx_work) 4091 /* 4092 * ixgbe_intr_rx_work - RX processing of ISR. 4093 */ 4094 static void 4095 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 4096 { 4097 mblk_t *mp; 4098 4099 mutex_enter(&rx_ring->rx_lock); 4100 4101 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4102 mutex_exit(&rx_ring->rx_lock); 4103 4104 if (mp != NULL) 4105 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4106 rx_ring->ring_gen_num); 4107 } 4108 4109 #pragma inline(ixgbe_intr_tx_work) 4110 /* 4111 * ixgbe_intr_tx_work - TX processing of ISR. 4112 */ 4113 static void 4114 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 4115 { 4116 ixgbe_t *ixgbe = tx_ring->ixgbe; 4117 4118 /* 4119 * Recycle the tx descriptors 4120 */ 4121 tx_ring->tx_recycle(tx_ring); 4122 4123 /* 4124 * Schedule the re-transmit 4125 */ 4126 if (tx_ring->reschedule && 4127 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 4128 tx_ring->reschedule = B_FALSE; 4129 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 4130 tx_ring->ring_handle); 4131 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4132 } 4133 } 4134 4135 #pragma inline(ixgbe_intr_other_work) 4136 /* 4137 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4138 */ 4139 static void 4140 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4141 { 4142 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4143 4144 /* 4145 * handle link status change 4146 */ 4147 if (eicr & IXGBE_EICR_LSC) { 4148 ixgbe_driver_link_check(ixgbe); 4149 ixgbe_get_hw_state(ixgbe); 4150 } 4151 4152 /* 4153 * check for fan failure on adapters with fans 4154 */ 4155 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4156 (eicr & IXGBE_EICR_GPI_SDP1)) { 4157 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4158 4159 /* 4160 * Disable the adapter interrupts 4161 */ 4162 ixgbe_disable_adapter_interrupts(ixgbe); 4163 4164 /* 4165 * Disable Rx/Tx units 4166 */ 4167 (void) ixgbe_stop_adapter(&ixgbe->hw); 4168 4169 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4170 ixgbe_error(ixgbe, 4171 "Problem: Network adapter has been stopped " 4172 "because the fan has stopped.\n"); 4173 ixgbe_error(ixgbe, 4174 "Action: Replace the adapter.\n"); 4175 4176 /* re-enable the interrupt, which was automasked */ 4177 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4178 } 4179 4180 /* 4181 * Do SFP check for adapters with hot-plug capability 4182 */ 4183 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4184 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) { 4185 ixgbe->eicr = eicr; 4186 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4187 ixgbe_sfp_check, (void *)ixgbe, 4188 DDI_NOSLEEP)) != DDI_SUCCESS) { 4189 ixgbe_log(ixgbe, "No memory available to dispatch " 4190 "taskq for SFP check"); 4191 } 4192 } 4193 4194 /* 4195 * Do over-temperature check for adapters with temp sensor 4196 */ 4197 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4198 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) { 4199 ixgbe->eicr = eicr; 4200 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4201 ixgbe_overtemp_check, (void *)ixgbe, 4202 DDI_NOSLEEP)) != DDI_SUCCESS) { 4203 ixgbe_log(ixgbe, "No memory available to dispatch " 4204 "taskq for overtemp check"); 4205 } 4206 } 4207 } 4208 4209 /* 4210 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4211 */ 4212 static uint_t 4213 ixgbe_intr_legacy(void *arg1, void *arg2) 4214 { 4215 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4216 struct ixgbe_hw *hw = &ixgbe->hw; 4217 ixgbe_tx_ring_t *tx_ring; 4218 ixgbe_rx_ring_t *rx_ring; 4219 uint32_t eicr; 4220 mblk_t *mp; 4221 boolean_t tx_reschedule; 4222 uint_t result; 4223 4224 _NOTE(ARGUNUSED(arg2)); 4225 4226 mutex_enter(&ixgbe->gen_lock); 4227 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4228 mutex_exit(&ixgbe->gen_lock); 4229 return (DDI_INTR_UNCLAIMED); 4230 } 4231 4232 mp = NULL; 4233 tx_reschedule = B_FALSE; 4234 4235 /* 4236 * Any bit set in eicr: claim this interrupt 4237 */ 4238 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4239 4240 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4241 mutex_exit(&ixgbe->gen_lock); 4242 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4243 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4244 return (DDI_INTR_CLAIMED); 4245 } 4246 4247 if (eicr) { 4248 /* 4249 * For legacy interrupt, we have only one interrupt, 4250 * so we have only one rx ring and one tx ring enabled. 4251 */ 4252 ASSERT(ixgbe->num_rx_rings == 1); 4253 ASSERT(ixgbe->num_tx_rings == 1); 4254 4255 /* 4256 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4257 */ 4258 if (eicr & 0x1) { 4259 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4260 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4261 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4262 /* 4263 * Clean the rx descriptors 4264 */ 4265 rx_ring = &ixgbe->rx_rings[0]; 4266 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4267 } 4268 4269 /* 4270 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4271 */ 4272 if (eicr & 0x2) { 4273 /* 4274 * Recycle the tx descriptors 4275 */ 4276 tx_ring = &ixgbe->tx_rings[0]; 4277 tx_ring->tx_recycle(tx_ring); 4278 4279 /* 4280 * Schedule the re-transmit 4281 */ 4282 tx_reschedule = (tx_ring->reschedule && 4283 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4284 } 4285 4286 /* any interrupt type other than tx/rx */ 4287 if (eicr & ixgbe->capab->other_intr) { 4288 switch (hw->mac.type) { 4289 case ixgbe_mac_82598EB: 4290 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4291 break; 4292 4293 case ixgbe_mac_82599EB: 4294 case ixgbe_mac_X540: 4295 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4296 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4297 break; 4298 4299 default: 4300 break; 4301 } 4302 ixgbe_intr_other_work(ixgbe, eicr); 4303 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4304 } 4305 4306 mutex_exit(&ixgbe->gen_lock); 4307 4308 result = DDI_INTR_CLAIMED; 4309 } else { 4310 mutex_exit(&ixgbe->gen_lock); 4311 4312 /* 4313 * No interrupt cause bits set: don't claim this interrupt. 4314 */ 4315 result = DDI_INTR_UNCLAIMED; 4316 } 4317 4318 /* re-enable the interrupts which were automasked */ 4319 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4320 4321 /* 4322 * Do the following work outside of the gen_lock 4323 */ 4324 if (mp != NULL) { 4325 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4326 rx_ring->ring_gen_num); 4327 } 4328 4329 if (tx_reschedule) { 4330 tx_ring->reschedule = B_FALSE; 4331 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4332 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4333 } 4334 4335 return (result); 4336 } 4337 4338 /* 4339 * ixgbe_intr_msi - Interrupt handler for MSI. 4340 */ 4341 static uint_t 4342 ixgbe_intr_msi(void *arg1, void *arg2) 4343 { 4344 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4345 struct ixgbe_hw *hw = &ixgbe->hw; 4346 uint32_t eicr; 4347 4348 _NOTE(ARGUNUSED(arg2)); 4349 4350 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4351 4352 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4353 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4354 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4355 return (DDI_INTR_CLAIMED); 4356 } 4357 4358 /* 4359 * For MSI interrupt, we have only one vector, 4360 * so we have only one rx ring and one tx ring enabled. 4361 */ 4362 ASSERT(ixgbe->num_rx_rings == 1); 4363 ASSERT(ixgbe->num_tx_rings == 1); 4364 4365 /* 4366 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4367 */ 4368 if (eicr & 0x1) { 4369 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4370 } 4371 4372 /* 4373 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4374 */ 4375 if (eicr & 0x2) { 4376 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4377 } 4378 4379 /* any interrupt type other than tx/rx */ 4380 if (eicr & ixgbe->capab->other_intr) { 4381 mutex_enter(&ixgbe->gen_lock); 4382 switch (hw->mac.type) { 4383 case ixgbe_mac_82598EB: 4384 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4385 break; 4386 4387 case ixgbe_mac_82599EB: 4388 case ixgbe_mac_X540: 4389 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4390 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4391 break; 4392 4393 default: 4394 break; 4395 } 4396 ixgbe_intr_other_work(ixgbe, eicr); 4397 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4398 mutex_exit(&ixgbe->gen_lock); 4399 } 4400 4401 /* re-enable the interrupts which were automasked */ 4402 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4403 4404 return (DDI_INTR_CLAIMED); 4405 } 4406 4407 /* 4408 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4409 */ 4410 static uint_t 4411 ixgbe_intr_msix(void *arg1, void *arg2) 4412 { 4413 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4414 ixgbe_t *ixgbe = vect->ixgbe; 4415 struct ixgbe_hw *hw = &ixgbe->hw; 4416 uint32_t eicr; 4417 int r_idx = 0; 4418 4419 _NOTE(ARGUNUSED(arg2)); 4420 4421 /* 4422 * Clean each rx ring that has its bit set in the map 4423 */ 4424 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4425 while (r_idx >= 0) { 4426 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4427 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4428 (ixgbe->num_rx_rings - 1)); 4429 } 4430 4431 /* 4432 * Clean each tx ring that has its bit set in the map 4433 */ 4434 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4435 while (r_idx >= 0) { 4436 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4437 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4438 (ixgbe->num_tx_rings - 1)); 4439 } 4440 4441 4442 /* 4443 * Clean other interrupt (link change) that has its bit set in the map 4444 */ 4445 if (BT_TEST(vect->other_map, 0) == 1) { 4446 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4447 4448 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4449 DDI_FM_OK) { 4450 ddi_fm_service_impact(ixgbe->dip, 4451 DDI_SERVICE_DEGRADED); 4452 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4453 return (DDI_INTR_CLAIMED); 4454 } 4455 4456 /* 4457 * Check "other" cause bits: any interrupt type other than tx/rx 4458 */ 4459 if (eicr & ixgbe->capab->other_intr) { 4460 mutex_enter(&ixgbe->gen_lock); 4461 switch (hw->mac.type) { 4462 case ixgbe_mac_82598EB: 4463 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4464 ixgbe_intr_other_work(ixgbe, eicr); 4465 break; 4466 4467 case ixgbe_mac_82599EB: 4468 case ixgbe_mac_X540: 4469 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4470 ixgbe_intr_other_work(ixgbe, eicr); 4471 break; 4472 4473 default: 4474 break; 4475 } 4476 mutex_exit(&ixgbe->gen_lock); 4477 } 4478 4479 /* re-enable the interrupts which were automasked */ 4480 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4481 } 4482 4483 return (DDI_INTR_CLAIMED); 4484 } 4485 4486 /* 4487 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4488 * 4489 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4490 * if not successful, try Legacy. 4491 * ixgbe->intr_force can be used to force sequence to start with 4492 * any of the 3 types. 4493 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4494 */ 4495 static int 4496 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4497 { 4498 dev_info_t *devinfo; 4499 int intr_types; 4500 int rc; 4501 4502 devinfo = ixgbe->dip; 4503 4504 /* 4505 * Get supported interrupt types 4506 */ 4507 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4508 4509 if (rc != DDI_SUCCESS) { 4510 ixgbe_log(ixgbe, 4511 "Get supported interrupt types failed: %d", rc); 4512 return (IXGBE_FAILURE); 4513 } 4514 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4515 4516 ixgbe->intr_type = 0; 4517 4518 /* 4519 * Install MSI-X interrupts 4520 */ 4521 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4522 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4523 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4524 if (rc == IXGBE_SUCCESS) 4525 return (IXGBE_SUCCESS); 4526 4527 ixgbe_log(ixgbe, 4528 "Allocate MSI-X failed, trying MSI interrupts..."); 4529 } 4530 4531 /* 4532 * MSI-X not used, force rings and groups to 1 4533 */ 4534 ixgbe->num_rx_rings = 1; 4535 ixgbe->num_rx_groups = 1; 4536 ixgbe->num_tx_rings = 1; 4537 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4538 ixgbe_log(ixgbe, 4539 "MSI-X not used, force rings and groups number to 1"); 4540 4541 /* 4542 * Install MSI interrupts 4543 */ 4544 if ((intr_types & DDI_INTR_TYPE_MSI) && 4545 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 4546 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 4547 if (rc == IXGBE_SUCCESS) 4548 return (IXGBE_SUCCESS); 4549 4550 ixgbe_log(ixgbe, 4551 "Allocate MSI failed, trying Legacy interrupts..."); 4552 } 4553 4554 /* 4555 * Install legacy interrupts 4556 */ 4557 if (intr_types & DDI_INTR_TYPE_FIXED) { 4558 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 4559 if (rc == IXGBE_SUCCESS) 4560 return (IXGBE_SUCCESS); 4561 4562 ixgbe_log(ixgbe, 4563 "Allocate Legacy interrupts failed"); 4564 } 4565 4566 /* 4567 * If none of the 3 types succeeded, return failure 4568 */ 4569 return (IXGBE_FAILURE); 4570 } 4571 4572 /* 4573 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 4574 * 4575 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4576 * if fewer than 2 handles are available, return failure. 4577 * Upon success, this maps the vectors to rx and tx rings for 4578 * interrupts. 4579 */ 4580 static int 4581 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 4582 { 4583 dev_info_t *devinfo; 4584 int request, count, actual; 4585 int minimum; 4586 int rc; 4587 uint32_t ring_per_group; 4588 4589 devinfo = ixgbe->dip; 4590 4591 switch (intr_type) { 4592 case DDI_INTR_TYPE_FIXED: 4593 request = 1; /* Request 1 legacy interrupt handle */ 4594 minimum = 1; 4595 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 4596 break; 4597 4598 case DDI_INTR_TYPE_MSI: 4599 request = 1; /* Request 1 MSI interrupt handle */ 4600 minimum = 1; 4601 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 4602 break; 4603 4604 case DDI_INTR_TYPE_MSIX: 4605 /* 4606 * Best number of vectors for the adapter is 4607 * (# rx rings + # tx rings), however we will 4608 * limit the request number. 4609 */ 4610 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 4611 if (request > ixgbe->capab->max_ring_vect) 4612 request = ixgbe->capab->max_ring_vect; 4613 minimum = 1; 4614 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 4615 break; 4616 4617 default: 4618 ixgbe_log(ixgbe, 4619 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 4620 intr_type); 4621 return (IXGBE_FAILURE); 4622 } 4623 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 4624 request, minimum); 4625 4626 /* 4627 * Get number of supported interrupts 4628 */ 4629 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4630 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4631 ixgbe_log(ixgbe, 4632 "Get interrupt number failed. Return: %d, count: %d", 4633 rc, count); 4634 return (IXGBE_FAILURE); 4635 } 4636 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 4637 4638 actual = 0; 4639 ixgbe->intr_cnt = 0; 4640 ixgbe->intr_cnt_max = 0; 4641 ixgbe->intr_cnt_min = 0; 4642 4643 /* 4644 * Allocate an array of interrupt handles 4645 */ 4646 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 4647 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 4648 4649 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 4650 request, &actual, DDI_INTR_ALLOC_NORMAL); 4651 if (rc != DDI_SUCCESS) { 4652 ixgbe_log(ixgbe, "Allocate interrupts failed. " 4653 "return: %d, request: %d, actual: %d", 4654 rc, request, actual); 4655 goto alloc_handle_fail; 4656 } 4657 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 4658 4659 /* 4660 * upper/lower limit of interrupts 4661 */ 4662 ixgbe->intr_cnt = actual; 4663 ixgbe->intr_cnt_max = request; 4664 ixgbe->intr_cnt_min = minimum; 4665 4666 /* 4667 * rss number per group should not exceed the rx interrupt number, 4668 * else need to adjust rx ring number. 4669 */ 4670 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4671 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 4672 if (actual < ring_per_group) { 4673 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 4674 ixgbe_setup_vmdq_rss_conf(ixgbe); 4675 } 4676 4677 /* 4678 * Now we know the actual number of vectors. Here we map the vector 4679 * to other, rx rings and tx ring. 4680 */ 4681 if (actual < minimum) { 4682 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 4683 actual); 4684 goto alloc_handle_fail; 4685 } 4686 4687 /* 4688 * Get priority for first vector, assume remaining are all the same 4689 */ 4690 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 4691 if (rc != DDI_SUCCESS) { 4692 ixgbe_log(ixgbe, 4693 "Get interrupt priority failed: %d", rc); 4694 goto alloc_handle_fail; 4695 } 4696 4697 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 4698 if (rc != DDI_SUCCESS) { 4699 ixgbe_log(ixgbe, 4700 "Get interrupt cap failed: %d", rc); 4701 goto alloc_handle_fail; 4702 } 4703 4704 ixgbe->intr_type = intr_type; 4705 4706 return (IXGBE_SUCCESS); 4707 4708 alloc_handle_fail: 4709 ixgbe_rem_intrs(ixgbe); 4710 4711 return (IXGBE_FAILURE); 4712 } 4713 4714 /* 4715 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 4716 * 4717 * Before adding the interrupt handlers, the interrupt vectors have 4718 * been allocated, and the rx/tx rings have also been allocated. 4719 */ 4720 static int 4721 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 4722 { 4723 int vector = 0; 4724 int rc; 4725 4726 switch (ixgbe->intr_type) { 4727 case DDI_INTR_TYPE_MSIX: 4728 /* 4729 * Add interrupt handler for all vectors 4730 */ 4731 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 4732 /* 4733 * install pointer to vect_map[vector] 4734 */ 4735 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4736 (ddi_intr_handler_t *)ixgbe_intr_msix, 4737 (void *)&ixgbe->vect_map[vector], NULL); 4738 4739 if (rc != DDI_SUCCESS) { 4740 ixgbe_log(ixgbe, 4741 "Add interrupt handler failed. " 4742 "return: %d, vector: %d", rc, vector); 4743 for (vector--; vector >= 0; vector--) { 4744 (void) ddi_intr_remove_handler( 4745 ixgbe->htable[vector]); 4746 } 4747 return (IXGBE_FAILURE); 4748 } 4749 } 4750 4751 break; 4752 4753 case DDI_INTR_TYPE_MSI: 4754 /* 4755 * Add interrupt handlers for the only vector 4756 */ 4757 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4758 (ddi_intr_handler_t *)ixgbe_intr_msi, 4759 (void *)ixgbe, NULL); 4760 4761 if (rc != DDI_SUCCESS) { 4762 ixgbe_log(ixgbe, 4763 "Add MSI interrupt handler failed: %d", rc); 4764 return (IXGBE_FAILURE); 4765 } 4766 4767 break; 4768 4769 case DDI_INTR_TYPE_FIXED: 4770 /* 4771 * Add interrupt handlers for the only vector 4772 */ 4773 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4774 (ddi_intr_handler_t *)ixgbe_intr_legacy, 4775 (void *)ixgbe, NULL); 4776 4777 if (rc != DDI_SUCCESS) { 4778 ixgbe_log(ixgbe, 4779 "Add legacy interrupt handler failed: %d", rc); 4780 return (IXGBE_FAILURE); 4781 } 4782 4783 break; 4784 4785 default: 4786 return (IXGBE_FAILURE); 4787 } 4788 4789 return (IXGBE_SUCCESS); 4790 } 4791 4792 #pragma inline(ixgbe_map_rxring_to_vector) 4793 /* 4794 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 4795 */ 4796 static void 4797 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 4798 { 4799 /* 4800 * Set bit in map 4801 */ 4802 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4803 4804 /* 4805 * Count bits set 4806 */ 4807 ixgbe->vect_map[v_idx].rxr_cnt++; 4808 4809 /* 4810 * Remember bit position 4811 */ 4812 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 4813 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 4814 } 4815 4816 #pragma inline(ixgbe_map_txring_to_vector) 4817 /* 4818 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 4819 */ 4820 static void 4821 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 4822 { 4823 /* 4824 * Set bit in map 4825 */ 4826 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 4827 4828 /* 4829 * Count bits set 4830 */ 4831 ixgbe->vect_map[v_idx].txr_cnt++; 4832 4833 /* 4834 * Remember bit position 4835 */ 4836 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 4837 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 4838 } 4839 4840 /* 4841 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 4842 * allocation register (IVAR). 4843 * cause: 4844 * -1 : other cause 4845 * 0 : rx 4846 * 1 : tx 4847 */ 4848 static void 4849 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 4850 int8_t cause) 4851 { 4852 struct ixgbe_hw *hw = &ixgbe->hw; 4853 u32 ivar, index; 4854 4855 switch (hw->mac.type) { 4856 case ixgbe_mac_82598EB: 4857 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4858 if (cause == -1) { 4859 cause = 0; 4860 } 4861 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4862 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4863 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 4864 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 4865 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4866 break; 4867 4868 case ixgbe_mac_82599EB: 4869 case ixgbe_mac_X540: 4870 if (cause == -1) { 4871 /* other causes */ 4872 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4873 index = (intr_alloc_entry & 1) * 8; 4874 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4875 ivar &= ~(0xFF << index); 4876 ivar |= (msix_vector << index); 4877 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4878 } else { 4879 /* tx or rx causes */ 4880 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4881 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4882 ivar = IXGBE_READ_REG(hw, 4883 IXGBE_IVAR(intr_alloc_entry >> 1)); 4884 ivar &= ~(0xFF << index); 4885 ivar |= (msix_vector << index); 4886 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4887 ivar); 4888 } 4889 break; 4890 4891 default: 4892 break; 4893 } 4894 } 4895 4896 /* 4897 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 4898 * given interrupt vector allocation register (IVAR). 4899 * cause: 4900 * -1 : other cause 4901 * 0 : rx 4902 * 1 : tx 4903 */ 4904 static void 4905 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4906 { 4907 struct ixgbe_hw *hw = &ixgbe->hw; 4908 u32 ivar, index; 4909 4910 switch (hw->mac.type) { 4911 case ixgbe_mac_82598EB: 4912 if (cause == -1) { 4913 cause = 0; 4914 } 4915 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4916 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4917 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 4918 (intr_alloc_entry & 0x3))); 4919 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4920 break; 4921 4922 case ixgbe_mac_82599EB: 4923 case ixgbe_mac_X540: 4924 if (cause == -1) { 4925 /* other causes */ 4926 index = (intr_alloc_entry & 1) * 8; 4927 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4928 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4929 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4930 } else { 4931 /* tx or rx causes */ 4932 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4933 ivar = IXGBE_READ_REG(hw, 4934 IXGBE_IVAR(intr_alloc_entry >> 1)); 4935 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4936 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4937 ivar); 4938 } 4939 break; 4940 4941 default: 4942 break; 4943 } 4944 } 4945 4946 /* 4947 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 4948 * given interrupt vector allocation register (IVAR). 4949 * cause: 4950 * -1 : other cause 4951 * 0 : rx 4952 * 1 : tx 4953 */ 4954 static void 4955 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4956 { 4957 struct ixgbe_hw *hw = &ixgbe->hw; 4958 u32 ivar, index; 4959 4960 switch (hw->mac.type) { 4961 case ixgbe_mac_82598EB: 4962 if (cause == -1) { 4963 cause = 0; 4964 } 4965 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4966 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4967 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 4968 (intr_alloc_entry & 0x3))); 4969 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4970 break; 4971 4972 case ixgbe_mac_82599EB: 4973 case ixgbe_mac_X540: 4974 if (cause == -1) { 4975 /* other causes */ 4976 index = (intr_alloc_entry & 1) * 8; 4977 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4978 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4979 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4980 } else { 4981 /* tx or rx causes */ 4982 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4983 ivar = IXGBE_READ_REG(hw, 4984 IXGBE_IVAR(intr_alloc_entry >> 1)); 4985 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4986 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4987 ivar); 4988 } 4989 break; 4990 4991 default: 4992 break; 4993 } 4994 } 4995 4996 /* 4997 * Convert the rx ring index driver maintained to the rx ring index 4998 * in h/w. 4999 */ 5000 static uint32_t 5001 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 5002 { 5003 5004 struct ixgbe_hw *hw = &ixgbe->hw; 5005 uint32_t rx_ring_per_group, hw_rx_index; 5006 5007 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 5008 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 5009 return (sw_rx_index); 5010 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 5011 switch (hw->mac.type) { 5012 case ixgbe_mac_82598EB: 5013 return (sw_rx_index); 5014 5015 case ixgbe_mac_82599EB: 5016 case ixgbe_mac_X540: 5017 return (sw_rx_index * 2); 5018 5019 default: 5020 break; 5021 } 5022 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 5023 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5024 5025 switch (hw->mac.type) { 5026 case ixgbe_mac_82598EB: 5027 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 5028 16 + (sw_rx_index % rx_ring_per_group); 5029 return (hw_rx_index); 5030 5031 case ixgbe_mac_82599EB: 5032 case ixgbe_mac_X540: 5033 if (ixgbe->num_rx_groups > 32) { 5034 hw_rx_index = (sw_rx_index / 5035 rx_ring_per_group) * 2 + 5036 (sw_rx_index % rx_ring_per_group); 5037 } else { 5038 hw_rx_index = (sw_rx_index / 5039 rx_ring_per_group) * 4 + 5040 (sw_rx_index % rx_ring_per_group); 5041 } 5042 return (hw_rx_index); 5043 5044 default: 5045 break; 5046 } 5047 } 5048 5049 /* 5050 * Should never reach. Just to make compiler happy. 5051 */ 5052 return (sw_rx_index); 5053 } 5054 5055 /* 5056 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 5057 * 5058 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 5059 * to vector[0 - (intr_cnt -1)]. 5060 */ 5061 static int 5062 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 5063 { 5064 int i, vector = 0; 5065 5066 /* initialize vector map */ 5067 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 5068 for (i = 0; i < ixgbe->intr_cnt; i++) { 5069 ixgbe->vect_map[i].ixgbe = ixgbe; 5070 } 5071 5072 /* 5073 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 5074 * tx rings[0] on RTxQ[1]. 5075 */ 5076 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5077 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 5078 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 5079 return (IXGBE_SUCCESS); 5080 } 5081 5082 /* 5083 * Interrupts/vectors mapping for MSI-X 5084 */ 5085 5086 /* 5087 * Map other interrupt to vector 0, 5088 * Set bit in map and count the bits set. 5089 */ 5090 BT_SET(ixgbe->vect_map[vector].other_map, 0); 5091 ixgbe->vect_map[vector].other_cnt++; 5092 5093 /* 5094 * Map rx ring interrupts to vectors 5095 */ 5096 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5097 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 5098 vector = (vector +1) % ixgbe->intr_cnt; 5099 } 5100 5101 /* 5102 * Map tx ring interrupts to vectors 5103 */ 5104 for (i = 0; i < ixgbe->num_tx_rings; i++) { 5105 ixgbe_map_txring_to_vector(ixgbe, i, vector); 5106 vector = (vector +1) % ixgbe->intr_cnt; 5107 } 5108 5109 return (IXGBE_SUCCESS); 5110 } 5111 5112 /* 5113 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 5114 * 5115 * This relies on ring/vector mapping already set up in the 5116 * vect_map[] structures 5117 */ 5118 static void 5119 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 5120 { 5121 struct ixgbe_hw *hw = &ixgbe->hw; 5122 ixgbe_intr_vector_t *vect; /* vector bitmap */ 5123 int r_idx; /* ring index */ 5124 int v_idx; /* vector index */ 5125 uint32_t hw_index; 5126 5127 /* 5128 * Clear any previous entries 5129 */ 5130 switch (hw->mac.type) { 5131 case ixgbe_mac_82598EB: 5132 for (v_idx = 0; v_idx < 25; v_idx++) 5133 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5134 break; 5135 5136 case ixgbe_mac_82599EB: 5137 case ixgbe_mac_X540: 5138 for (v_idx = 0; v_idx < 64; v_idx++) 5139 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5140 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5141 break; 5142 5143 default: 5144 break; 5145 } 5146 5147 /* 5148 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5149 * tx rings[0] will use RTxQ[1]. 5150 */ 5151 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5152 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5153 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5154 return; 5155 } 5156 5157 /* 5158 * For MSI-X interrupt, "Other" is always on vector[0]. 5159 */ 5160 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5161 5162 /* 5163 * For each interrupt vector, populate the IVAR table 5164 */ 5165 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5166 vect = &ixgbe->vect_map[v_idx]; 5167 5168 /* 5169 * For each rx ring bit set 5170 */ 5171 r_idx = bt_getlowbit(vect->rx_map, 0, 5172 (ixgbe->num_rx_rings - 1)); 5173 5174 while (r_idx >= 0) { 5175 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5176 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5177 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5178 (ixgbe->num_rx_rings - 1)); 5179 } 5180 5181 /* 5182 * For each tx ring bit set 5183 */ 5184 r_idx = bt_getlowbit(vect->tx_map, 0, 5185 (ixgbe->num_tx_rings - 1)); 5186 5187 while (r_idx >= 0) { 5188 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5189 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5190 (ixgbe->num_tx_rings - 1)); 5191 } 5192 } 5193 } 5194 5195 /* 5196 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5197 */ 5198 static void 5199 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5200 { 5201 int i; 5202 int rc; 5203 5204 for (i = 0; i < ixgbe->intr_cnt; i++) { 5205 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5206 if (rc != DDI_SUCCESS) { 5207 IXGBE_DEBUGLOG_1(ixgbe, 5208 "Remove intr handler failed: %d", rc); 5209 } 5210 } 5211 } 5212 5213 /* 5214 * ixgbe_rem_intrs - Remove the allocated interrupts. 5215 */ 5216 static void 5217 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5218 { 5219 int i; 5220 int rc; 5221 5222 for (i = 0; i < ixgbe->intr_cnt; i++) { 5223 rc = ddi_intr_free(ixgbe->htable[i]); 5224 if (rc != DDI_SUCCESS) { 5225 IXGBE_DEBUGLOG_1(ixgbe, 5226 "Free intr failed: %d", rc); 5227 } 5228 } 5229 5230 kmem_free(ixgbe->htable, ixgbe->intr_size); 5231 ixgbe->htable = NULL; 5232 } 5233 5234 /* 5235 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5236 */ 5237 static int 5238 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5239 { 5240 int i; 5241 int rc; 5242 5243 /* 5244 * Enable interrupts 5245 */ 5246 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5247 /* 5248 * Call ddi_intr_block_enable() for MSI 5249 */ 5250 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5251 if (rc != DDI_SUCCESS) { 5252 ixgbe_log(ixgbe, 5253 "Enable block intr failed: %d", rc); 5254 return (IXGBE_FAILURE); 5255 } 5256 } else { 5257 /* 5258 * Call ddi_intr_enable() for Legacy/MSI non block enable 5259 */ 5260 for (i = 0; i < ixgbe->intr_cnt; i++) { 5261 rc = ddi_intr_enable(ixgbe->htable[i]); 5262 if (rc != DDI_SUCCESS) { 5263 ixgbe_log(ixgbe, 5264 "Enable intr failed: %d", rc); 5265 return (IXGBE_FAILURE); 5266 } 5267 } 5268 } 5269 5270 return (IXGBE_SUCCESS); 5271 } 5272 5273 /* 5274 * ixgbe_disable_intrs - Disable all the interrupts. 5275 */ 5276 static int 5277 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5278 { 5279 int i; 5280 int rc; 5281 5282 /* 5283 * Disable all interrupts 5284 */ 5285 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5286 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5287 if (rc != DDI_SUCCESS) { 5288 ixgbe_log(ixgbe, 5289 "Disable block intr failed: %d", rc); 5290 return (IXGBE_FAILURE); 5291 } 5292 } else { 5293 for (i = 0; i < ixgbe->intr_cnt; i++) { 5294 rc = ddi_intr_disable(ixgbe->htable[i]); 5295 if (rc != DDI_SUCCESS) { 5296 ixgbe_log(ixgbe, 5297 "Disable intr failed: %d", rc); 5298 return (IXGBE_FAILURE); 5299 } 5300 } 5301 } 5302 5303 return (IXGBE_SUCCESS); 5304 } 5305 5306 /* 5307 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5308 */ 5309 static void 5310 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5311 { 5312 struct ixgbe_hw *hw = &ixgbe->hw; 5313 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 5314 boolean_t link_up = B_FALSE; 5315 uint32_t pcs1g_anlp = 0; 5316 uint32_t pcs1g_ana = 0; 5317 boolean_t autoneg = B_FALSE; 5318 5319 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5320 ixgbe->param_lp_1000fdx_cap = 0; 5321 ixgbe->param_lp_100fdx_cap = 0; 5322 5323 /* check for link, don't wait */ 5324 (void) ixgbe_check_link(hw, &speed, &link_up, false); 5325 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 5326 5327 if (link_up) { 5328 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5329 5330 ixgbe->param_lp_1000fdx_cap = 5331 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5332 ixgbe->param_lp_100fdx_cap = 5333 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5334 } 5335 5336 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg); 5337 5338 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5339 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0; 5340 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5341 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0; 5342 } 5343 5344 /* 5345 * ixgbe_get_driver_control - Notify that driver is in control of device. 5346 */ 5347 static void 5348 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5349 { 5350 uint32_t ctrl_ext; 5351 5352 /* 5353 * Notify firmware that driver is in control of device 5354 */ 5355 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5356 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5357 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5358 } 5359 5360 /* 5361 * ixgbe_release_driver_control - Notify that driver is no longer in control 5362 * of device. 5363 */ 5364 static void 5365 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5366 { 5367 uint32_t ctrl_ext; 5368 5369 /* 5370 * Notify firmware that driver is no longer in control of device 5371 */ 5372 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5373 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5374 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5375 } 5376 5377 /* 5378 * ixgbe_atomic_reserve - Atomic decrease operation. 5379 */ 5380 int 5381 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5382 { 5383 uint32_t oldval; 5384 uint32_t newval; 5385 5386 /* 5387 * ATOMICALLY 5388 */ 5389 do { 5390 oldval = *count_p; 5391 if (oldval < n) 5392 return (-1); 5393 newval = oldval - n; 5394 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5395 5396 return (newval); 5397 } 5398 5399 /* 5400 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5401 */ 5402 static uint8_t * 5403 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5404 { 5405 uint8_t *addr = *upd_ptr; 5406 uint8_t *new_ptr; 5407 5408 _NOTE(ARGUNUSED(hw)); 5409 _NOTE(ARGUNUSED(vmdq)); 5410 5411 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5412 *upd_ptr = new_ptr; 5413 return (addr); 5414 } 5415 5416 /* 5417 * FMA support 5418 */ 5419 int 5420 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5421 { 5422 ddi_fm_error_t de; 5423 5424 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5425 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5426 return (de.fme_status); 5427 } 5428 5429 int 5430 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5431 { 5432 ddi_fm_error_t de; 5433 5434 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5435 return (de.fme_status); 5436 } 5437 5438 /* 5439 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5440 */ 5441 static int 5442 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5443 { 5444 _NOTE(ARGUNUSED(impl_data)); 5445 /* 5446 * as the driver can always deal with an error in any dma or 5447 * access handle, we can just return the fme_status value. 5448 */ 5449 pci_ereport_post(dip, err, NULL); 5450 return (err->fme_status); 5451 } 5452 5453 static void 5454 ixgbe_fm_init(ixgbe_t *ixgbe) 5455 { 5456 ddi_iblock_cookie_t iblk; 5457 int fma_dma_flag; 5458 5459 /* 5460 * Only register with IO Fault Services if we have some capability 5461 */ 5462 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5463 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5464 } else { 5465 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5466 } 5467 5468 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5469 fma_dma_flag = 1; 5470 } else { 5471 fma_dma_flag = 0; 5472 } 5473 5474 ixgbe_set_fma_flags(fma_dma_flag); 5475 5476 if (ixgbe->fm_capabilities) { 5477 5478 /* 5479 * Register capabilities with IO Fault Services 5480 */ 5481 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 5482 5483 /* 5484 * Initialize pci ereport capabilities if ereport capable 5485 */ 5486 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5487 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5488 pci_ereport_setup(ixgbe->dip); 5489 5490 /* 5491 * Register error callback if error callback capable 5492 */ 5493 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5494 ddi_fm_handler_register(ixgbe->dip, 5495 ixgbe_fm_error_cb, (void*) ixgbe); 5496 } 5497 } 5498 5499 static void 5500 ixgbe_fm_fini(ixgbe_t *ixgbe) 5501 { 5502 /* 5503 * Only unregister FMA capabilities if they are registered 5504 */ 5505 if (ixgbe->fm_capabilities) { 5506 5507 /* 5508 * Release any resources allocated by pci_ereport_setup() 5509 */ 5510 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5511 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5512 pci_ereport_teardown(ixgbe->dip); 5513 5514 /* 5515 * Un-register error callback if error callback capable 5516 */ 5517 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5518 ddi_fm_handler_unregister(ixgbe->dip); 5519 5520 /* 5521 * Unregister from IO Fault Service 5522 */ 5523 ddi_fm_fini(ixgbe->dip); 5524 } 5525 } 5526 5527 void 5528 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 5529 { 5530 uint64_t ena; 5531 char buf[FM_MAX_CLASS]; 5532 5533 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5534 ena = fm_ena_generate(0, FM_ENA_FMT1); 5535 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 5536 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 5537 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5538 } 5539 } 5540 5541 static int 5542 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 5543 { 5544 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 5545 5546 mutex_enter(&rx_ring->rx_lock); 5547 rx_ring->ring_gen_num = mr_gen_num; 5548 mutex_exit(&rx_ring->rx_lock); 5549 return (0); 5550 } 5551 5552 /* 5553 * Get the global ring index by a ring index within a group. 5554 */ 5555 static int 5556 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 5557 { 5558 ixgbe_rx_ring_t *rx_ring; 5559 int i; 5560 5561 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5562 rx_ring = &ixgbe->rx_rings[i]; 5563 if (rx_ring->group_index == gindex) 5564 rindex--; 5565 if (rindex < 0) 5566 return (i); 5567 } 5568 5569 return (-1); 5570 } 5571 5572 /* 5573 * Callback funtion for MAC layer to register all rings. 5574 */ 5575 /* ARGSUSED */ 5576 void 5577 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 5578 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 5579 { 5580 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5581 mac_intr_t *mintr = &infop->mri_intr; 5582 5583 switch (rtype) { 5584 case MAC_RING_TYPE_RX: { 5585 /* 5586 * 'index' is the ring index within the group. 5587 * Need to get the global ring index by searching in groups. 5588 */ 5589 int global_ring_index = ixgbe_get_rx_ring_index( 5590 ixgbe, group_index, ring_index); 5591 5592 ASSERT(global_ring_index >= 0); 5593 5594 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 5595 rx_ring->ring_handle = rh; 5596 5597 infop->mri_driver = (mac_ring_driver_t)rx_ring; 5598 infop->mri_start = ixgbe_ring_start; 5599 infop->mri_stop = NULL; 5600 infop->mri_poll = ixgbe_ring_rx_poll; 5601 infop->mri_stat = ixgbe_rx_ring_stat; 5602 5603 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 5604 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 5605 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 5606 if (ixgbe->intr_type & 5607 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5608 mintr->mi_ddi_handle = 5609 ixgbe->htable[rx_ring->intr_vector]; 5610 } 5611 5612 break; 5613 } 5614 case MAC_RING_TYPE_TX: { 5615 ASSERT(group_index == -1); 5616 ASSERT(ring_index < ixgbe->num_tx_rings); 5617 5618 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 5619 tx_ring->ring_handle = rh; 5620 5621 infop->mri_driver = (mac_ring_driver_t)tx_ring; 5622 infop->mri_start = NULL; 5623 infop->mri_stop = NULL; 5624 infop->mri_tx = ixgbe_ring_tx; 5625 infop->mri_stat = ixgbe_tx_ring_stat; 5626 if (ixgbe->intr_type & 5627 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5628 mintr->mi_ddi_handle = 5629 ixgbe->htable[tx_ring->intr_vector]; 5630 } 5631 break; 5632 } 5633 default: 5634 break; 5635 } 5636 } 5637 5638 /* 5639 * Callback funtion for MAC layer to register all groups. 5640 */ 5641 void 5642 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 5643 mac_group_info_t *infop, mac_group_handle_t gh) 5644 { 5645 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5646 5647 switch (rtype) { 5648 case MAC_RING_TYPE_RX: { 5649 ixgbe_rx_group_t *rx_group; 5650 5651 rx_group = &ixgbe->rx_groups[index]; 5652 rx_group->group_handle = gh; 5653 5654 infop->mgi_driver = (mac_group_driver_t)rx_group; 5655 infop->mgi_start = NULL; 5656 infop->mgi_stop = NULL; 5657 infop->mgi_addmac = ixgbe_addmac; 5658 infop->mgi_remmac = ixgbe_remmac; 5659 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 5660 5661 break; 5662 } 5663 case MAC_RING_TYPE_TX: 5664 break; 5665 default: 5666 break; 5667 } 5668 } 5669 5670 /* 5671 * Enable interrupt on the specificed rx ring. 5672 */ 5673 int 5674 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 5675 { 5676 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5677 ixgbe_t *ixgbe = rx_ring->ixgbe; 5678 int r_idx = rx_ring->index; 5679 int hw_r_idx = rx_ring->hw_index; 5680 int v_idx = rx_ring->intr_vector; 5681 5682 mutex_enter(&ixgbe->gen_lock); 5683 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5684 mutex_exit(&ixgbe->gen_lock); 5685 /* 5686 * Simply return 0. 5687 * Interrupts are being adjusted. ixgbe_intr_adjust() 5688 * will eventually re-enable the interrupt when it's 5689 * done with the adjustment. 5690 */ 5691 return (0); 5692 } 5693 5694 /* 5695 * To enable interrupt by setting the VAL bit of given interrupt 5696 * vector allocation register (IVAR). 5697 */ 5698 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 5699 5700 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5701 5702 /* 5703 * Trigger a Rx interrupt on this ring 5704 */ 5705 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 5706 IXGBE_WRITE_FLUSH(&ixgbe->hw); 5707 5708 mutex_exit(&ixgbe->gen_lock); 5709 5710 return (0); 5711 } 5712 5713 /* 5714 * Disable interrupt on the specificed rx ring. 5715 */ 5716 int 5717 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 5718 { 5719 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5720 ixgbe_t *ixgbe = rx_ring->ixgbe; 5721 int r_idx = rx_ring->index; 5722 int hw_r_idx = rx_ring->hw_index; 5723 int v_idx = rx_ring->intr_vector; 5724 5725 mutex_enter(&ixgbe->gen_lock); 5726 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5727 mutex_exit(&ixgbe->gen_lock); 5728 /* 5729 * Simply return 0. 5730 * In the rare case where an interrupt is being 5731 * disabled while interrupts are being adjusted, 5732 * we don't fail the operation. No interrupts will 5733 * be generated while they are adjusted, and 5734 * ixgbe_intr_adjust() will cause the interrupts 5735 * to be re-enabled once it completes. Note that 5736 * in this case, packets may be delivered to the 5737 * stack via interrupts before xgbe_rx_ring_intr_enable() 5738 * is called again. This is acceptable since interrupt 5739 * adjustment is infrequent, and the stack will be 5740 * able to handle these packets. 5741 */ 5742 return (0); 5743 } 5744 5745 /* 5746 * To disable interrupt by clearing the VAL bit of given interrupt 5747 * vector allocation register (IVAR). 5748 */ 5749 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 5750 5751 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 5752 5753 mutex_exit(&ixgbe->gen_lock); 5754 5755 return (0); 5756 } 5757 5758 /* 5759 * Add a mac address. 5760 */ 5761 static int 5762 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 5763 { 5764 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5765 ixgbe_t *ixgbe = rx_group->ixgbe; 5766 struct ixgbe_hw *hw = &ixgbe->hw; 5767 int slot, i; 5768 5769 mutex_enter(&ixgbe->gen_lock); 5770 5771 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5772 mutex_exit(&ixgbe->gen_lock); 5773 return (ECANCELED); 5774 } 5775 5776 if (ixgbe->unicst_avail == 0) { 5777 /* no slots available */ 5778 mutex_exit(&ixgbe->gen_lock); 5779 return (ENOSPC); 5780 } 5781 5782 /* 5783 * The first ixgbe->num_rx_groups slots are reserved for each respective 5784 * group. The rest slots are shared by all groups. While adding a 5785 * MAC address, reserved slots are firstly checked then the shared 5786 * slots are searched. 5787 */ 5788 slot = -1; 5789 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 5790 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 5791 if (ixgbe->unicst_addr[i].mac.set == 0) { 5792 slot = i; 5793 break; 5794 } 5795 } 5796 } else { 5797 slot = rx_group->index; 5798 } 5799 5800 if (slot == -1) { 5801 /* no slots available */ 5802 mutex_exit(&ixgbe->gen_lock); 5803 return (ENOSPC); 5804 } 5805 5806 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5807 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 5808 rx_group->index, IXGBE_RAH_AV); 5809 ixgbe->unicst_addr[slot].mac.set = 1; 5810 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 5811 ixgbe->unicst_avail--; 5812 5813 mutex_exit(&ixgbe->gen_lock); 5814 5815 return (0); 5816 } 5817 5818 /* 5819 * Remove a mac address. 5820 */ 5821 static int 5822 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 5823 { 5824 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5825 ixgbe_t *ixgbe = rx_group->ixgbe; 5826 struct ixgbe_hw *hw = &ixgbe->hw; 5827 int slot; 5828 5829 mutex_enter(&ixgbe->gen_lock); 5830 5831 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5832 mutex_exit(&ixgbe->gen_lock); 5833 return (ECANCELED); 5834 } 5835 5836 slot = ixgbe_unicst_find(ixgbe, mac_addr); 5837 if (slot == -1) { 5838 mutex_exit(&ixgbe->gen_lock); 5839 return (EINVAL); 5840 } 5841 5842 if (ixgbe->unicst_addr[slot].mac.set == 0) { 5843 mutex_exit(&ixgbe->gen_lock); 5844 return (EINVAL); 5845 } 5846 5847 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5848 (void) ixgbe_clear_rar(hw, slot); 5849 ixgbe->unicst_addr[slot].mac.set = 0; 5850 ixgbe->unicst_avail++; 5851 5852 mutex_exit(&ixgbe->gen_lock); 5853 5854 return (0); 5855 } 5856