1 /****************************************************************************** 2 3 Copyright (c) 2013-2020, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 34 #include "ixl_pf.h" 35 36 void 37 ixl_configure_tx_itr(struct ixl_pf *pf) 38 { 39 struct i40e_hw *hw = &pf->hw; 40 struct ixl_vsi *vsi = &pf->vsi; 41 struct ixl_tx_queue *que = vsi->tx_queues; 42 43 vsi->tx_itr_setting = pf->tx_itr; 44 45 for (int i = 0; i < vsi->num_tx_queues; i++, que++) { 46 struct tx_ring *txr = &que->txr; 47 48 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), 49 vsi->tx_itr_setting); 50 txr->itr = vsi->tx_itr_setting; 51 txr->latency = IXL_AVE_LATENCY; 52 } 53 } 54 55 void 56 ixl_configure_rx_itr(struct ixl_pf *pf) 57 { 58 struct i40e_hw *hw = &pf->hw; 59 struct ixl_vsi *vsi = &pf->vsi; 60 struct ixl_rx_queue *que = vsi->rx_queues; 61 62 vsi->rx_itr_setting = pf->rx_itr; 63 64 for (int i = 0; i < vsi->num_rx_queues; i++, que++) { 65 struct rx_ring *rxr = &que->rxr; 66 67 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), 68 vsi->rx_itr_setting); 69 rxr->itr = vsi->rx_itr_setting; 70 rxr->latency = IXL_AVE_LATENCY; 71 } 72 } 73 74 int 75 ixl_intr(void *arg) 76 { 77 struct ixl_pf *pf = arg; 78 struct i40e_hw *hw = &pf->hw; 79 struct ixl_vsi *vsi = &pf->vsi; 80 struct ixl_rx_queue *que = vsi->rx_queues; 81 u32 icr0; 82 83 ++que->irqs; 84 85 /* Clear PBA at start of ISR if using legacy interrupts */ 86 if (vsi->shared->isc_intr == IFLIB_INTR_LEGACY) 87 wr32(hw, I40E_PFINT_DYN_CTL0, 88 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 89 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 90 91 icr0 = rd32(hw, I40E_PFINT_ICR0); 92 93 94 #ifdef PCI_IOV 95 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) 96 iflib_iov_intr_deferred(vsi->ctx); 97 #endif 98 99 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) 100 iflib_admin_intr_deferred(vsi->ctx); 101 102 ixl_enable_intr0(hw); 103 104 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) 105 return (FILTER_SCHEDULE_THREAD); 106 else 107 return (FILTER_HANDLED); 108 } 109 110 /********************************************************************* 111 * 112 * MSI-X VSI Interrupt Service routine 113 * 114 **********************************************************************/ 115 int 116 ixl_msix_que(void *arg) 117 { 118 struct ixl_rx_queue *rx_que = arg; 119 120 ++rx_que->irqs; 121 122 ixl_set_queue_rx_itr(rx_que); 123 124 return (FILTER_SCHEDULE_THREAD); 125 } 126 127 /********************************************************************* 128 * 129 * MSI-X Admin Queue Interrupt Service routine 130 * 131 **********************************************************************/ 132 int 133 ixl_msix_adminq(void *arg) 134 { 135 struct ixl_pf *pf = arg; 136 struct i40e_hw *hw = &pf->hw; 137 device_t dev = pf->dev; 138 u32 reg, mask, rstat_reg; 139 bool do_task = FALSE; 140 141 DDPRINTF(dev, "begin"); 142 143 ++pf->admin_irq; 144 145 reg = rd32(hw, I40E_PFINT_ICR0); 146 /* 147 * For masking off interrupt causes that need to be handled before 148 * they can be re-enabled 149 */ 150 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 151 152 /* Check on the cause */ 153 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) { 154 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 155 do_task = TRUE; 156 } 157 158 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 159 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 160 ixl_set_state(&pf->state, IXL_STATE_MDD_PENDING); 161 do_task = TRUE; 162 } 163 164 if (reg & I40E_PFINT_ICR0_GRST_MASK) { 165 const char *reset_type; 166 mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 167 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT); 168 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 169 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 170 switch (rstat_reg) { 171 /* These others might be handled similarly to an EMPR reset */ 172 case I40E_RESET_CORER: 173 reset_type = "CORER"; 174 break; 175 case I40E_RESET_GLOBR: 176 reset_type = "GLOBR"; 177 break; 178 case I40E_RESET_EMPR: 179 reset_type = "EMPR"; 180 break; 181 default: 182 reset_type = "POR"; 183 break; 184 } 185 device_printf(dev, "Reset Requested! (%s)\n", reset_type); 186 /* overload admin queue task to check reset progress */ 187 ixl_set_state(&pf->state, IXL_STATE_RESETTING); 188 do_task = TRUE; 189 } 190 191 /* 192 * PE / PCI / ECC exceptions are all handled in the same way: 193 * mask out these three causes, then request a PF reset 194 */ 195 if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) 196 device_printf(dev, "ECC Error detected!\n"); 197 if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) 198 device_printf(dev, "PCI Exception detected!\n"); 199 if (reg & I40E_PFINT_ICR0_PE_CRITERR_MASK) 200 device_printf(dev, "Critical Protocol Engine Error detected!\n"); 201 /* Checks against the conditions above */ 202 if (reg & IXL_ICR0_CRIT_ERR_MASK) { 203 mask &= ~IXL_ICR0_CRIT_ERR_MASK; 204 ixl_set_state(&pf->state, IXL_STATE_PF_RESET_REQ); 205 ixl_set_state(&pf->state, IXL_STATE_PF_CRIT_ERR); 206 do_task = TRUE; 207 } 208 209 if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) { 210 reg = rd32(hw, I40E_PFHMC_ERRORINFO); 211 if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) { 212 device_printf(dev, "HMC Error detected!\n"); 213 device_printf(dev, "INFO 0x%08x\n", reg); 214 reg = rd32(hw, I40E_PFHMC_ERRORDATA); 215 device_printf(dev, "DATA 0x%08x\n", reg); 216 wr32(hw, I40E_PFHMC_ERRORINFO, 0); 217 } 218 } 219 220 #ifdef PCI_IOV 221 if (reg & I40E_PFINT_ICR0_VFLR_MASK) { 222 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 223 iflib_iov_intr_deferred(pf->vsi.ctx); 224 } 225 #endif 226 227 wr32(hw, I40E_PFINT_ICR0_ENA, mask); 228 ixl_enable_intr0(hw); 229 230 if (do_task) 231 return (FILTER_SCHEDULE_THREAD); 232 else 233 return (FILTER_HANDLED); 234 } 235 236 /* 237 * Configure queue interrupt cause registers in hardware. 238 * 239 * Linked list for each vector LNKLSTN(i) -> RQCTL(i) -> TQCTL(i) -> EOL 240 */ 241 void 242 ixl_configure_queue_intr_msix(struct ixl_pf *pf) 243 { 244 struct i40e_hw *hw = &pf->hw; 245 struct ixl_vsi *vsi = &pf->vsi; 246 u32 reg; 247 u16 vector = 1; 248 249 for (int i = 0; i < max(vsi->num_rx_queues, vsi->num_tx_queues); i++, vector++) { 250 /* Make sure interrupt is disabled */ 251 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); 252 /* Set linked list head to point to corresponding RX queue 253 * e.g. vector 1 (LNKLSTN register 0) points to queue pair 0's RX queue */ 254 reg = ((i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) 255 & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) | 256 ((I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT) 257 & I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK); 258 wr32(hw, I40E_PFINT_LNKLSTN(i), reg); 259 260 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 261 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 262 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 263 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 264 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 265 wr32(hw, I40E_QINT_RQCTL(i), reg); 266 267 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 268 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 269 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 270 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 271 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 272 wr32(hw, I40E_QINT_TQCTL(i), reg); 273 } 274 } 275 276 /* 277 * Configure for single interrupt vector operation 278 */ 279 void 280 ixl_configure_legacy(struct ixl_pf *pf) 281 { 282 struct i40e_hw *hw = &pf->hw; 283 struct ixl_vsi *vsi = &pf->vsi; 284 u32 reg; 285 286 vsi->rx_queues[0].rxr.itr = vsi->rx_itr_setting; 287 288 /* Setup "other" causes */ 289 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK 290 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK 291 | I40E_PFINT_ICR0_ENA_GRST_MASK 292 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK 293 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK 294 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK 295 | I40E_PFINT_ICR0_ENA_VFLR_MASK 296 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK 297 ; 298 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 299 300 /* No ITR for non-queue interrupts */ 301 wr32(hw, I40E_PFINT_STAT_CTL0, 302 IXL_ITR_NONE << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT); 303 304 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 305 wr32(hw, I40E_PFINT_LNKLST0, 0); 306 307 /* Associate the queue pair to the vector and enable the q int */ 308 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK 309 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) 310 | (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 311 wr32(hw, I40E_QINT_RQCTL(0), reg); 312 313 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK 314 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) 315 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 316 wr32(hw, I40E_QINT_TQCTL(0), reg); 317 } 318 319 void 320 ixl_free_pci_resources(struct ixl_pf *pf) 321 { 322 struct ixl_vsi *vsi = &pf->vsi; 323 device_t dev = iflib_get_dev(vsi->ctx); 324 struct ixl_rx_queue *rx_que = vsi->rx_queues; 325 326 /* We may get here before stations are set up */ 327 if (rx_que == NULL) 328 goto early; 329 330 /* 331 ** Release all MSI-X VSI resources: 332 */ 333 iflib_irq_free(vsi->ctx, &vsi->irq); 334 335 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) 336 iflib_irq_free(vsi->ctx, &rx_que->que_irq); 337 early: 338 if (pf->pci_mem != NULL) 339 bus_release_resource(dev, SYS_RES_MEMORY, 340 rman_get_rid(pf->pci_mem), pf->pci_mem); 341 } 342 343 /********************************************************************* 344 * 345 * Setup networking device structure and register an interface. 346 * 347 **********************************************************************/ 348 int 349 ixl_setup_interface(device_t dev, struct ixl_pf *pf) 350 { 351 struct ixl_vsi *vsi = &pf->vsi; 352 if_ctx_t ctx = vsi->ctx; 353 struct i40e_hw *hw = &pf->hw; 354 if_t ifp = iflib_get_ifp(ctx); 355 struct i40e_aq_get_phy_abilities_resp abilities; 356 enum i40e_status_code aq_error = 0; 357 358 INIT_DBG_DEV(dev, "begin"); 359 360 vsi->shared->isc_max_frame_size = 361 if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN 362 + ETHER_VLAN_ENCAP_LEN; 363 364 if (IXL_PF_IN_RECOVERY_MODE(pf)) 365 goto only_auto; 366 367 aq_error = i40e_aq_get_phy_capabilities(hw, 368 FALSE, TRUE, &abilities, NULL); 369 /* May need delay to detect fiber correctly */ 370 if (aq_error == I40E_ERR_UNKNOWN_PHY) { 371 i40e_msec_delay(200); 372 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, 373 TRUE, &abilities, NULL); 374 } 375 if (aq_error) { 376 if (aq_error == I40E_ERR_UNKNOWN_PHY) 377 device_printf(dev, "Unknown PHY type detected!\n"); 378 else 379 device_printf(dev, 380 "Error getting supported media types, err %d," 381 " AQ error %d\n", aq_error, hw->aq.asq_last_status); 382 } else { 383 pf->supported_speeds = abilities.link_speed; 384 if_setbaudrate(ifp, ixl_max_aq_speed_to_value(pf->supported_speeds)); 385 386 ixl_add_ifmedia(vsi->media, hw->phy.phy_types); 387 } 388 389 only_auto: 390 /* Use autoselect media by default */ 391 ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); 392 ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO); 393 394 return (0); 395 } 396 397 /* 398 ** Run when the Admin Queue gets a link state change interrupt. 399 */ 400 void 401 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) 402 { 403 struct i40e_hw *hw = &pf->hw; 404 device_t dev = iflib_get_dev(pf->vsi.ctx); 405 struct i40e_link_status *link_info = &hw->phy.link_info; 406 407 /* Driver needs to re-enable delivering of link status events 408 * by FW after each event reception. Call i40e_get_link_status 409 * to do that. To not lose information about link state changes, 410 * which happened between receiving an event and the call, 411 * do not rely on status from event but use most recent 412 * status information retrieved by the call. */ 413 hw->phy.get_link_info = TRUE; 414 i40e_get_link_status(hw, &pf->link_up); 415 416 /* Print out message if an unqualified module is found */ 417 if ((link_info->link_info & I40E_AQ_MEDIA_AVAILABLE) && 418 (pf->advertised_speed) && 419 (if_getflags(pf->vsi.ifp) & IFF_UP) && 420 (!(link_info->an_info & I40E_AQ_QUALIFIED_MODULE)) && 421 (!(link_info->link_info & I40E_AQ_LINK_UP))) 422 device_printf(dev, "Link failed because " 423 "an unqualified module was detected!\n"); 424 425 /* OS link info is updated elsewhere */ 426 } 427 428 /********************************************************************* 429 * 430 * Initialize the VSI: this handles contexts, which means things 431 * like the number of descriptors, buffer size, 432 * plus we init the rings thru this function. 433 * 434 **********************************************************************/ 435 int 436 ixl_initialize_vsi(struct ixl_vsi *vsi) 437 { 438 struct ixl_pf *pf = vsi->back; 439 if_softc_ctx_t scctx = iflib_get_softc_ctx(vsi->ctx); 440 struct ixl_tx_queue *tx_que = vsi->tx_queues; 441 struct ixl_rx_queue *rx_que = vsi->rx_queues; 442 device_t dev = iflib_get_dev(vsi->ctx); 443 struct i40e_hw *hw = vsi->hw; 444 struct i40e_vsi_context ctxt; 445 int tc_queues; 446 int err = 0; 447 448 memset(&ctxt, 0, sizeof(ctxt)); 449 ctxt.seid = vsi->seid; 450 if (pf->veb_seid != 0) 451 ctxt.uplink_seid = pf->veb_seid; 452 ctxt.pf_num = hw->pf_id; 453 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); 454 if (err) { 455 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d" 456 " aq_error %d\n", err, hw->aq.asq_last_status); 457 return (err); 458 } 459 ixl_dbg(pf, IXL_DBG_SWITCH_INFO, 460 "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " 461 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " 462 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, 463 ctxt.uplink_seid, ctxt.vsi_number, 464 ctxt.vsis_allocated, ctxt.vsis_unallocated, 465 ctxt.flags, ctxt.pf_num, ctxt.vf_num, 466 le16toh(ctxt.info.stat_counter_idx), ctxt.info.up_enable_bits); 467 /* 468 ** Set the queue and traffic class bits 469 ** - when multiple traffic classes are supported 470 ** this will need to be more robust. 471 */ 472 ctxt.info.valid_sections = 473 htole16(I40E_AQ_VSI_PROP_QUEUE_MAP_VALID); 474 ctxt.info.mapping_flags = htole16(I40E_AQ_VSI_QUE_MAP_CONTIG); 475 /* In contig mode, que_mapping[0] is first queue index used by this VSI */ 476 ctxt.info.queue_mapping[0] = 0; 477 /* 478 * This VSI will only use traffic class 0; start traffic class 0's 479 * queue allocation at queue 0, and assign it 2^tc_queues queues (though 480 * the driver may not use all of them). 481 */ 482 tc_queues = fls(pf->qtag.num_allocated) - 1; 483 ctxt.info.tc_mapping[0] = htole16( 484 ((pf->qtag.first_qidx << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) 485 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | 486 ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) 487 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK)); 488 489 /* Set VLAN receive stripping mode */ 490 ctxt.info.valid_sections |= htole16(I40E_AQ_VSI_PROP_VLAN_VALID); 491 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; 492 if (if_getcapenable(vsi->ifp) & IFCAP_VLAN_HWTAGGING) 493 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 494 else 495 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 496 497 #ifdef IXL_IW 498 /* Set TCP Enable for iWARP capable VSI */ 499 if (ixl_enable_iwarp && pf->iw_enabled) { 500 ctxt.info.valid_sections |= 501 htole16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 502 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 503 } 504 #endif 505 /* 506 * Save the VSI number and info for later. A changed counter index 507 * begins a new statistics epoch. 508 */ 509 if (vsi->stat_offsets_loaded && 510 vsi->info.stat_counter_idx != ctxt.info.stat_counter_idx) 511 ixl_vsi_reset_stats(vsi); 512 vsi->vsi_num = ctxt.vsi_number; 513 bcopy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 514 515 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 516 517 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 518 if (err) { 519 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d," 520 " aq_error %d\n", err, hw->aq.asq_last_status); 521 return (err); 522 } 523 524 for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { 525 struct tx_ring *txr = &tx_que->txr; 526 struct i40e_hmc_obj_txq tctx; 527 u32 txctl; 528 529 /* Setup the HMC TX Context */ 530 bzero(&tctx, sizeof(tctx)); 531 tctx.new_context = 1; 532 tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS); 533 tctx.qlen = scctx->isc_ntxd[0]; 534 tctx.fc_ena = 0; /* Disable FCoE */ 535 /* 536 * This value needs to pulled from the VSI that this queue 537 * is assigned to. Index into array is traffic class. 538 */ 539 tctx.rdylist = le16toh(vsi->info.qs_handle[0]); 540 /* 541 * Set these to enable Head Writeback 542 * - Address is last entry in TX ring (reserved for HWB index) 543 * Leave these as 0 for Descriptor Writeback 544 */ 545 if (vsi->enable_head_writeback) { 546 tctx.head_wb_ena = 1; 547 tctx.head_wb_addr = txr->tx_paddr + 548 (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc)); 549 } else { 550 tctx.head_wb_ena = 0; 551 tctx.head_wb_addr = 0; 552 } 553 tctx.rdylist_act = 0; 554 err = i40e_clear_lan_tx_queue_context(hw, i); 555 if (err) { 556 device_printf(dev, "Unable to clear TX context\n"); 557 break; 558 } 559 err = i40e_set_lan_tx_queue_context(hw, i, &tctx); 560 if (err) { 561 device_printf(dev, "Unable to set TX context\n"); 562 break; 563 } 564 /* Associate the ring with this PF */ 565 txctl = I40E_QTX_CTL_PF_QUEUE; 566 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 567 I40E_QTX_CTL_PF_INDX_MASK); 568 wr32(hw, I40E_QTX_CTL(i), txctl); 569 ixl_flush(hw); 570 571 /* Do ring (re)init */ 572 ixl_init_tx_ring(vsi, tx_que); 573 } 574 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { 575 struct rx_ring *rxr = &rx_que->rxr; 576 struct i40e_hmc_obj_rxq rctx; 577 578 /* Next setup the HMC RX Context */ 579 rxr->mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx); 580 581 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; 582 583 /* Set up an RX context for the HMC */ 584 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 585 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; 586 /* ignore header split for now */ 587 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 588 rctx.rxmax = (scctx->isc_max_frame_size < max_rxmax) ? 589 scctx->isc_max_frame_size : max_rxmax; 590 rctx.dtype = 0; 591 rctx.dsize = 1; /* do 32byte descriptors */ 592 rctx.hsplit_0 = 0; /* no header split */ 593 rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS); 594 rctx.qlen = scctx->isc_nrxd[0]; 595 rctx.tphrdesc_ena = 1; 596 rctx.tphwdesc_ena = 1; 597 rctx.tphdata_ena = 0; /* Header Split related */ 598 rctx.tphhead_ena = 0; /* Header Split related */ 599 rctx.lrxqthresh = 1; /* Interrupt at <64 desc avail */ 600 rctx.crcstrip = 1; 601 rctx.l2tsel = 1; 602 rctx.showiv = 1; /* Strip inner VLAN header */ 603 rctx.fc_ena = 0; /* Disable FCoE */ 604 rctx.prefena = 1; /* Prefetch descriptors */ 605 606 err = i40e_clear_lan_rx_queue_context(hw, i); 607 if (err) { 608 device_printf(dev, 609 "Unable to clear RX context %d\n", i); 610 break; 611 } 612 err = i40e_set_lan_rx_queue_context(hw, i, &rctx); 613 if (err) { 614 device_printf(dev, "Unable to set RX context %d\n", i); 615 break; 616 } 617 wr32(vsi->hw, I40E_QRX_TAIL(i), 0); 618 } 619 return (err); 620 } 621 622 623 /* 624 ** Provide a update to the queue RX 625 ** interrupt moderation value. 626 */ 627 void 628 ixl_set_queue_rx_itr(struct ixl_rx_queue *que) 629 { 630 struct ixl_vsi *vsi = que->vsi; 631 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 632 struct i40e_hw *hw = vsi->hw; 633 struct rx_ring *rxr = &que->rxr; 634 u16 rx_itr; 635 u16 rx_latency = 0; 636 int rx_bytes; 637 638 /* Idle, do nothing */ 639 if (rxr->bytes == 0) 640 return; 641 642 if (pf->dynamic_rx_itr) { 643 rx_bytes = rxr->bytes/rxr->itr; 644 rx_itr = rxr->itr; 645 646 /* Adjust latency range */ 647 switch (rxr->latency) { 648 case IXL_LOW_LATENCY: 649 if (rx_bytes > 10) { 650 rx_latency = IXL_AVE_LATENCY; 651 rx_itr = IXL_ITR_20K; 652 } 653 break; 654 case IXL_AVE_LATENCY: 655 if (rx_bytes > 20) { 656 rx_latency = IXL_BULK_LATENCY; 657 rx_itr = IXL_ITR_8K; 658 } else if (rx_bytes <= 10) { 659 rx_latency = IXL_LOW_LATENCY; 660 rx_itr = IXL_ITR_100K; 661 } 662 break; 663 case IXL_BULK_LATENCY: 664 if (rx_bytes <= 20) { 665 rx_latency = IXL_AVE_LATENCY; 666 rx_itr = IXL_ITR_20K; 667 } 668 break; 669 } 670 671 rxr->latency = rx_latency; 672 673 if (rx_itr != rxr->itr) { 674 /* do an exponential smoothing */ 675 rx_itr = (10 * rx_itr * rxr->itr) / 676 ((9 * rx_itr) + rxr->itr); 677 rxr->itr = min(rx_itr, IXL_MAX_ITR); 678 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 679 rxr->me), rxr->itr); 680 } 681 } else { /* We may have have toggled to non-dynamic */ 682 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) 683 vsi->rx_itr_setting = pf->rx_itr; 684 /* Update the hardware if needed */ 685 if (rxr->itr != vsi->rx_itr_setting) { 686 rxr->itr = vsi->rx_itr_setting; 687 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 688 rxr->me), rxr->itr); 689 } 690 } 691 rxr->bytes = 0; 692 rxr->packets = 0; 693 } 694 695 696 /* 697 ** Provide a update to the queue TX 698 ** interrupt moderation value. 699 */ 700 void 701 ixl_set_queue_tx_itr(struct ixl_tx_queue *que) 702 { 703 struct ixl_vsi *vsi = que->vsi; 704 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 705 struct i40e_hw *hw = vsi->hw; 706 struct tx_ring *txr = &que->txr; 707 u16 tx_itr; 708 u16 tx_latency = 0; 709 int tx_bytes; 710 711 712 /* Idle, do nothing */ 713 if (txr->bytes == 0) 714 return; 715 716 if (pf->dynamic_tx_itr) { 717 tx_bytes = txr->bytes/txr->itr; 718 tx_itr = txr->itr; 719 720 switch (txr->latency) { 721 case IXL_LOW_LATENCY: 722 if (tx_bytes > 10) { 723 tx_latency = IXL_AVE_LATENCY; 724 tx_itr = IXL_ITR_20K; 725 } 726 break; 727 case IXL_AVE_LATENCY: 728 if (tx_bytes > 20) { 729 tx_latency = IXL_BULK_LATENCY; 730 tx_itr = IXL_ITR_8K; 731 } else if (tx_bytes <= 10) { 732 tx_latency = IXL_LOW_LATENCY; 733 tx_itr = IXL_ITR_100K; 734 } 735 break; 736 case IXL_BULK_LATENCY: 737 if (tx_bytes <= 20) { 738 tx_latency = IXL_AVE_LATENCY; 739 tx_itr = IXL_ITR_20K; 740 } 741 break; 742 } 743 744 txr->latency = tx_latency; 745 746 if (tx_itr != txr->itr) { 747 /* do an exponential smoothing */ 748 tx_itr = (10 * tx_itr * txr->itr) / 749 ((9 * tx_itr) + txr->itr); 750 txr->itr = min(tx_itr, IXL_MAX_ITR); 751 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 752 txr->me), txr->itr); 753 } 754 755 } else { /* We may have have toggled to non-dynamic */ 756 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) 757 vsi->tx_itr_setting = pf->tx_itr; 758 /* Update the hardware if needed */ 759 if (txr->itr != vsi->tx_itr_setting) { 760 txr->itr = vsi->tx_itr_setting; 761 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 762 txr->me), txr->itr); 763 } 764 } 765 txr->bytes = 0; 766 txr->packets = 0; 767 return; 768 } 769 770 #ifdef IXL_DEBUG 771 /** 772 * ixl_sysctl_qtx_tail_handler 773 * Retrieves I40E_QTX_TAIL value from hardware 774 * for a sysctl. 775 */ 776 int 777 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) 778 { 779 struct ixl_tx_queue *tx_que; 780 int error; 781 u32 val; 782 783 tx_que = ((struct ixl_tx_queue *)oidp->oid_arg1); 784 if (!tx_que) return 0; 785 786 val = rd32(tx_que->vsi->hw, tx_que->txr.tail); 787 error = sysctl_handle_int(oidp, &val, 0, req); 788 if (error || !req->newptr) 789 return error; 790 return (0); 791 } 792 793 /** 794 * ixl_sysctl_qrx_tail_handler 795 * Retrieves I40E_QRX_TAIL value from hardware 796 * for a sysctl. 797 */ 798 int 799 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) 800 { 801 struct ixl_rx_queue *rx_que; 802 int error; 803 u32 val; 804 805 rx_que = ((struct ixl_rx_queue *)oidp->oid_arg1); 806 if (!rx_que) return 0; 807 808 val = rd32(rx_que->vsi->hw, rx_que->rxr.tail); 809 error = sysctl_handle_int(oidp, &val, 0, req); 810 if (error || !req->newptr) 811 return error; 812 return (0); 813 } 814 #endif 815 816 void 817 ixl_add_hw_stats(struct ixl_pf *pf) 818 { 819 struct ixl_vsi *vsi = &pf->vsi; 820 device_t dev = iflib_get_dev(vsi->ctx); 821 struct i40e_hw_port_stats *pf_stats = &pf->stats; 822 823 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 824 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 825 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 826 827 /* Driver statistics */ 828 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "admin_irq", 829 CTLFLAG_RD, &pf->admin_irq, 830 "Admin Queue IRQs received"); 831 832 sysctl_ctx_init(&vsi->sysctl_ctx); 833 ixl_vsi_add_sysctls(vsi, "pf", true); 834 835 ixl_add_sysctls_mac_stats(ctx, child, pf_stats); 836 } 837 838 void 839 ixl_set_rss_hlut(struct ixl_pf *pf) 840 { 841 struct i40e_hw *hw = &pf->hw; 842 struct ixl_vsi *vsi = &pf->vsi; 843 device_t dev = iflib_get_dev(vsi->ctx); 844 int i, que_id; 845 int lut_entry_width; 846 u32 lut = 0; 847 enum i40e_status_code status; 848 849 lut_entry_width = pf->hw.func_caps.rss_table_entry_width; 850 851 /* Populate the LUT with max no. of queues in round robin fashion */ 852 u8 hlut_buf[512]; 853 for (i = 0; i < pf->hw.func_caps.rss_table_size; i++) { 854 #ifdef RSS 855 /* 856 * Fetch the RSS bucket id for the given indirection entry. 857 * Cap it at the number of configured buckets (which is 858 * num_queues.) 859 */ 860 que_id = rss_get_indirection_to_bucket(i); 861 que_id = que_id % vsi->num_rx_queues; 862 #else 863 que_id = i % vsi->num_rx_queues; 864 #endif 865 lut = (que_id & ((0x1 << lut_entry_width) - 1)); 866 hlut_buf[i] = lut; 867 } 868 869 if (hw->mac.type == I40E_MAC_X722) { 870 status = i40e_aq_set_rss_lut(hw, vsi->vsi_num, TRUE, hlut_buf, sizeof(hlut_buf)); 871 if (status) 872 device_printf(dev, "i40e_aq_set_rss_lut status %s, error %s\n", 873 i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); 874 } else { 875 for (i = 0; i < pf->hw.func_caps.rss_table_size >> 2; i++) 876 wr32(hw, I40E_PFQF_HLUT(i), ((u32 *)hlut_buf)[i]); 877 ixl_flush(hw); 878 } 879 } 880 881 /* For PF VSI only */ 882 int 883 ixl_enable_rings(struct ixl_vsi *vsi) 884 { 885 struct ixl_pf *pf = vsi->back; 886 int error; 887 888 for (int i = 0; i < vsi->num_tx_queues; i++) { 889 error = ixl_enable_tx_ring(pf, &pf->qtag, i); 890 if (error != 0) 891 return (error); 892 } 893 894 for (int i = 0; i < vsi->num_rx_queues; i++) { 895 error = ixl_enable_rx_ring(pf, &pf->qtag, i); 896 if (error != 0) 897 return (error); 898 } 899 900 return (0); 901 } 902 903 int 904 ixl_disable_rings(struct ixl_pf *pf, struct ixl_vsi *vsi, struct ixl_pf_qtag *qtag) 905 { 906 int error, first_error; 907 908 first_error = 0; 909 for (int i = 0; i < vsi->num_tx_queues; i++) { 910 error = ixl_disable_tx_ring(pf, qtag, i); 911 if (error != 0 && first_error == 0) 912 first_error = error; 913 } 914 915 for (int i = 0; i < vsi->num_rx_queues; i++) { 916 error = ixl_disable_rx_ring(pf, qtag, i); 917 if (error != 0 && first_error == 0) 918 first_error = error; 919 } 920 921 return (first_error); 922 } 923 924 void 925 ixl_enable_intr(struct ixl_vsi *vsi) 926 { 927 struct i40e_hw *hw = vsi->hw; 928 struct ixl_rx_queue *que = vsi->rx_queues; 929 930 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 931 for (int i = 0; i < vsi->num_rx_queues; i++, que++) 932 ixl_enable_queue(hw, que->rxr.me); 933 } else 934 ixl_enable_intr0(hw); 935 } 936 937 void 938 ixl_disable_rings_intr(struct ixl_vsi *vsi) 939 { 940 struct i40e_hw *hw = vsi->hw; 941 struct ixl_rx_queue *que = vsi->rx_queues; 942 943 for (int i = 0; i < vsi->num_rx_queues; i++, que++) 944 ixl_disable_queue(hw, que->rxr.me); 945 } 946 947 int 948 ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up) 949 { 950 struct i40e_hw *hw = &pf->hw; 951 device_t dev = pf->dev; 952 int error, first_error; 953 954 first_error = 0; 955 /* Restore identification before the reset tears down AdminQ access. */ 956 ixl_led_restore(pf); 957 #ifdef PCI_IOV 958 ixl_notify_vfs_reset(pf); 959 error = ixl_quiesce_vfs_for_reset(pf); 960 if (error != 0) { 961 device_printf(dev, "Failed to quiesce one or more VFs: %d\n", 962 error); 963 first_error = error; 964 } 965 #endif 966 if (is_up) 967 ixl_if_stop(pf->vsi.ctx); 968 969 ixl_shutdown_hmc(pf); 970 971 ixl_disable_intr0(hw); 972 973 error = i40e_shutdown_adminq(hw); 974 if (error) { 975 device_printf(dev, 976 "Shutdown Admin queue failed with code %d\n", error); 977 if (first_error == 0) 978 first_error = error; 979 } 980 981 ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag); 982 return (first_error); 983 } 984 985 int 986 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf, bool is_up) 987 { 988 struct i40e_hw *hw = &pf->hw; 989 struct ixl_vsi *vsi = &pf->vsi; 990 device_t dev = pf->dev; 991 enum i40e_get_fw_lldp_status_resp lldp_status; 992 int error = 0; 993 994 device_printf(dev, "Rebuilding driver state...\n"); 995 996 /* Setup */ 997 error = i40e_init_adminq(hw); 998 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { 999 device_printf(dev, "Unable to initialize Admin Queue, error %d\n", 1000 error); 1001 goto ixl_rebuild_hw_structs_after_reset_err; 1002 } 1003 1004 if (IXL_PF_IN_RECOVERY_MODE(pf)) { 1005 /* Keep admin queue interrupts active while driver is loaded */ 1006 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 1007 ixl_configure_intr0_msix(pf); 1008 ixl_enable_intr0(hw); 1009 } 1010 1011 return (0); 1012 } 1013 1014 i40e_clear_pxe_mode(hw); 1015 1016 error = ixl_get_hw_capabilities(pf); 1017 if (error) { 1018 device_printf(dev, "ixl_get_hw_capabilities failed: %d\n", error); 1019 goto ixl_rebuild_hw_structs_after_reset_err; 1020 } 1021 1022 error = ixl_setup_hmc(pf); 1023 if (error) 1024 goto ixl_rebuild_hw_structs_after_reset_err; 1025 1026 /* reserve a contiguous allocation for the PF's VSI */ 1027 error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_tx_queues, &pf->qtag); 1028 if (error) { 1029 device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n", 1030 error); 1031 goto ixl_rebuild_hw_structs_after_reset_err; 1032 } 1033 1034 error = ixl_switch_config(pf); 1035 if (error) { 1036 device_printf(dev, "ixl_rebuild_hw_structs_after_reset: ixl_switch_config() failed: %d\n", 1037 error); 1038 error = EIO; 1039 goto ixl_rebuild_hw_structs_after_reset_err; 1040 } 1041 /* Firmware has rebuilt the port and VSI counter resources. */ 1042 ixl_pf_reset_stats(pf); 1043 ixl_vsi_reset_stats(vsi); 1044 1045 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, 1046 NULL); 1047 if (error) { 1048 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," 1049 " aq_err %d\n", error, hw->aq.asq_last_status); 1050 error = EIO; 1051 goto ixl_rebuild_hw_structs_after_reset_err; 1052 } 1053 1054 u8 set_fc_err_mask; 1055 error = i40e_set_fc(hw, &set_fc_err_mask, true); 1056 if (error) { 1057 device_printf(dev, "init: setting link flow control failed; retcode %d," 1058 " fc_err_mask 0x%02x\n", error, set_fc_err_mask); 1059 error = EIO; 1060 goto ixl_rebuild_hw_structs_after_reset_err; 1061 } 1062 1063 /* Remove default filters reinstalled by FW on reset */ 1064 ixl_del_default_hw_filters(vsi); 1065 1066 /* Receive broadcast Ethernet frames */ 1067 i40e_aq_set_vsi_broadcast(&pf->hw, vsi->seid, TRUE, NULL); 1068 1069 #ifdef PCI_IOV 1070 if (pf->num_vfs != 0) { 1071 error = ixl_rebuild_vfs_after_reset(pf); 1072 if (error != 0) 1073 device_printf(dev, 1074 "Failed to rebuild one or more VFs: %d\n", error); 1075 } 1076 #endif 1077 1078 /* Determine link state */ 1079 ixl_attach_get_link_status(pf); 1080 1081 i40e_aq_set_dcb_parameters(hw, TRUE, NULL); 1082 1083 /* Query device FW LLDP status */ 1084 if (i40e_get_fw_lldp_status(hw, &lldp_status) == I40E_SUCCESS) { 1085 if (lldp_status == I40E_GET_FW_LLDP_STATUS_DISABLED) { 1086 ixl_set_state(&pf->state, 1087 IXL_STATE_FW_LLDP_DISABLED); 1088 } else { 1089 ixl_clear_state(&pf->state, 1090 IXL_STATE_FW_LLDP_DISABLED); 1091 } 1092 } 1093 1094 /* Keep admin queue interrupts active while driver is loaded */ 1095 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 1096 ixl_configure_intr0_msix(pf); 1097 ixl_enable_intr0(hw); 1098 } 1099 1100 if (is_up) { 1101 iflib_request_reset(vsi->ctx); 1102 iflib_admin_intr_deferred(vsi->ctx); 1103 } 1104 1105 device_printf(dev, "Rebuilding driver state done.\n"); 1106 return (0); 1107 1108 ixl_rebuild_hw_structs_after_reset_err: 1109 device_printf(dev, "Reload the driver to recover\n"); 1110 return (error); 1111 } 1112 1113 /* 1114 ** Set flow control using sysctl: 1115 ** 0 - off 1116 ** 1 - rx pause 1117 ** 2 - tx pause 1118 ** 3 - full 1119 */ 1120 int 1121 ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS) 1122 { 1123 struct ixl_pf *pf = (struct ixl_pf *)arg1; 1124 struct i40e_hw *hw = &pf->hw; 1125 device_t dev = pf->dev; 1126 int requested_fc, error = 0; 1127 enum i40e_status_code aq_error = 0; 1128 u8 fc_aq_err = 0; 1129 1130 /* Get request */ 1131 requested_fc = pf->fc; 1132 error = sysctl_handle_int(oidp, &requested_fc, 0, req); 1133 if ((error) || (req->newptr == NULL)) 1134 return (error); 1135 if (requested_fc < 0 || requested_fc > 3) { 1136 device_printf(dev, 1137 "Invalid fc mode; valid modes are 0 through 3\n"); 1138 return (EINVAL); 1139 } 1140 1141 /* Set fc ability for port */ 1142 hw->fc.requested_mode = requested_fc; 1143 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); 1144 if (aq_error) { 1145 device_printf(dev, 1146 "%s: Error setting Flow Control mode %d; fc_err %#x\n", 1147 __func__, aq_error, fc_aq_err); 1148 return (EIO); 1149 } 1150 pf->fc = requested_fc; 1151 1152 return (0); 1153 } 1154