1 /****************************************************************************** 2 3 Copyright (c) 2013-2020, Intel Corporation 4 All rights reserved. 5 6 Redistribution and use in source and binary forms, with or without 7 modification, are permitted provided that the following conditions are met: 8 9 1. Redistributions of source code must retain the above copyright notice, 10 this list of conditions and the following disclaimer. 11 12 2. Redistributions in binary form must reproduce the above copyright 13 notice, this list of conditions and the following disclaimer in the 14 documentation and/or other materials provided with the distribution. 15 16 3. Neither the name of the Intel Corporation nor the names of its 17 contributors may be used to endorse or promote products derived from 18 this software without specific prior written permission. 19 20 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 24 LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 POSSIBILITY OF SUCH DAMAGE. 31 32 ******************************************************************************/ 33 34 #include "ixl_pf.h" 35 36 void ixl_configure_tx_itr(struct ixl_pf * pf)37 ixl_configure_tx_itr(struct ixl_pf *pf) 38 { 39 struct i40e_hw *hw = &pf->hw; 40 struct ixl_vsi *vsi = &pf->vsi; 41 struct ixl_tx_queue *que = vsi->tx_queues; 42 43 vsi->tx_itr_setting = pf->tx_itr; 44 45 for (int i = 0; i < vsi->num_tx_queues; i++, que++) { 46 struct tx_ring *txr = &que->txr; 47 48 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, i), 49 vsi->tx_itr_setting); 50 txr->itr = vsi->tx_itr_setting; 51 txr->latency = IXL_AVE_LATENCY; 52 } 53 } 54 55 void ixl_configure_rx_itr(struct ixl_pf * pf)56 ixl_configure_rx_itr(struct ixl_pf *pf) 57 { 58 struct i40e_hw *hw = &pf->hw; 59 struct ixl_vsi *vsi = &pf->vsi; 60 struct ixl_rx_queue *que = vsi->rx_queues; 61 62 vsi->rx_itr_setting = pf->rx_itr; 63 64 for (int i = 0; i < vsi->num_rx_queues; i++, que++) { 65 struct rx_ring *rxr = &que->rxr; 66 67 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, i), 68 vsi->rx_itr_setting); 69 rxr->itr = vsi->rx_itr_setting; 70 rxr->latency = IXL_AVE_LATENCY; 71 } 72 } 73 74 int ixl_intr(void * arg)75 ixl_intr(void *arg) 76 { 77 struct ixl_pf *pf = arg; 78 struct i40e_hw *hw = &pf->hw; 79 struct ixl_vsi *vsi = &pf->vsi; 80 struct ixl_rx_queue *que = vsi->rx_queues; 81 u32 icr0; 82 83 ++que->irqs; 84 85 /* Clear PBA at start of ISR if using legacy interrupts */ 86 if (vsi->shared->isc_intr == IFLIB_INTR_LEGACY) 87 wr32(hw, I40E_PFINT_DYN_CTL0, 88 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 89 (IXL_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT)); 90 91 icr0 = rd32(hw, I40E_PFINT_ICR0); 92 93 94 #ifdef PCI_IOV 95 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) 96 iflib_iov_intr_deferred(vsi->ctx); 97 #endif 98 99 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) 100 iflib_admin_intr_deferred(vsi->ctx); 101 102 ixl_enable_intr0(hw); 103 104 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) 105 return (FILTER_SCHEDULE_THREAD); 106 else 107 return (FILTER_HANDLED); 108 } 109 110 /********************************************************************* 111 * 112 * MSI-X VSI Interrupt Service routine 113 * 114 **********************************************************************/ 115 int ixl_msix_que(void * arg)116 ixl_msix_que(void *arg) 117 { 118 struct ixl_rx_queue *rx_que = arg; 119 120 ++rx_que->irqs; 121 122 ixl_set_queue_rx_itr(rx_que); 123 124 return (FILTER_SCHEDULE_THREAD); 125 } 126 127 /********************************************************************* 128 * 129 * MSI-X Admin Queue Interrupt Service routine 130 * 131 **********************************************************************/ 132 int ixl_msix_adminq(void * arg)133 ixl_msix_adminq(void *arg) 134 { 135 struct ixl_pf *pf = arg; 136 struct i40e_hw *hw = &pf->hw; 137 device_t dev = pf->dev; 138 u32 reg, mask, rstat_reg; 139 bool do_task = FALSE; 140 141 DDPRINTF(dev, "begin"); 142 143 ++pf->admin_irq; 144 145 reg = rd32(hw, I40E_PFINT_ICR0); 146 /* 147 * For masking off interrupt causes that need to be handled before 148 * they can be re-enabled 149 */ 150 mask = rd32(hw, I40E_PFINT_ICR0_ENA); 151 152 /* Check on the cause */ 153 if (reg & I40E_PFINT_ICR0_ADMINQ_MASK) { 154 mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 155 do_task = TRUE; 156 } 157 158 if (reg & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 159 mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 160 ixl_set_state(&pf->state, IXL_STATE_MDD_PENDING); 161 do_task = TRUE; 162 } 163 164 if (reg & I40E_PFINT_ICR0_GRST_MASK) { 165 const char *reset_type; 166 mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 167 rstat_reg = rd32(hw, I40E_GLGEN_RSTAT); 168 rstat_reg = (rstat_reg & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 169 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 170 switch (rstat_reg) { 171 /* These others might be handled similarly to an EMPR reset */ 172 case I40E_RESET_CORER: 173 reset_type = "CORER"; 174 break; 175 case I40E_RESET_GLOBR: 176 reset_type = "GLOBR"; 177 break; 178 case I40E_RESET_EMPR: 179 reset_type = "EMPR"; 180 break; 181 default: 182 reset_type = "POR"; 183 break; 184 } 185 device_printf(dev, "Reset Requested! (%s)\n", reset_type); 186 /* overload admin queue task to check reset progress */ 187 ixl_set_state(&pf->state, IXL_STATE_RESETTING); 188 do_task = TRUE; 189 } 190 191 /* 192 * PE / PCI / ECC exceptions are all handled in the same way: 193 * mask out these three causes, then request a PF reset 194 */ 195 if (reg & I40E_PFINT_ICR0_ECC_ERR_MASK) 196 device_printf(dev, "ECC Error detected!\n"); 197 if (reg & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) 198 device_printf(dev, "PCI Exception detected!\n"); 199 if (reg & I40E_PFINT_ICR0_PE_CRITERR_MASK) 200 device_printf(dev, "Critical Protocol Engine Error detected!\n"); 201 /* Checks against the conditions above */ 202 if (reg & IXL_ICR0_CRIT_ERR_MASK) { 203 mask &= ~IXL_ICR0_CRIT_ERR_MASK; 204 ixl_set_state(&pf->state, 205 IXL_STATE_PF_RESET_REQ | IXL_STATE_PF_CRIT_ERR); 206 do_task = TRUE; 207 } 208 209 if (reg & I40E_PFINT_ICR0_HMC_ERR_MASK) { 210 reg = rd32(hw, I40E_PFHMC_ERRORINFO); 211 if (reg & I40E_PFHMC_ERRORINFO_ERROR_DETECTED_MASK) { 212 device_printf(dev, "HMC Error detected!\n"); 213 device_printf(dev, "INFO 0x%08x\n", reg); 214 reg = rd32(hw, I40E_PFHMC_ERRORDATA); 215 device_printf(dev, "DATA 0x%08x\n", reg); 216 wr32(hw, I40E_PFHMC_ERRORINFO, 0); 217 } 218 } 219 220 #ifdef PCI_IOV 221 if (reg & I40E_PFINT_ICR0_VFLR_MASK) { 222 mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 223 iflib_iov_intr_deferred(pf->vsi.ctx); 224 } 225 #endif 226 227 wr32(hw, I40E_PFINT_ICR0_ENA, mask); 228 ixl_enable_intr0(hw); 229 230 if (do_task) 231 return (FILTER_SCHEDULE_THREAD); 232 else 233 return (FILTER_HANDLED); 234 } 235 236 /* 237 * Configure queue interrupt cause registers in hardware. 238 * 239 * Linked list for each vector LNKLSTN(i) -> RQCTL(i) -> TQCTL(i) -> EOL 240 */ 241 void ixl_configure_queue_intr_msix(struct ixl_pf * pf)242 ixl_configure_queue_intr_msix(struct ixl_pf *pf) 243 { 244 struct i40e_hw *hw = &pf->hw; 245 struct ixl_vsi *vsi = &pf->vsi; 246 u32 reg; 247 u16 vector = 1; 248 249 for (int i = 0; i < max(vsi->num_rx_queues, vsi->num_tx_queues); i++, vector++) { 250 /* Make sure interrupt is disabled */ 251 wr32(hw, I40E_PFINT_DYN_CTLN(i), 0); 252 /* Set linked list head to point to corresponding RX queue 253 * e.g. vector 1 (LNKLSTN register 0) points to queue pair 0's RX queue */ 254 reg = ((i << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT) 255 & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) | 256 ((I40E_QUEUE_TYPE_RX << I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_SHIFT) 257 & I40E_PFINT_LNKLSTN_FIRSTQ_TYPE_MASK); 258 wr32(hw, I40E_PFINT_LNKLSTN(i), reg); 259 260 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 261 (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 262 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 263 (i << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 264 (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 265 wr32(hw, I40E_QINT_RQCTL(i), reg); 266 267 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 268 (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 269 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 270 (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 271 (I40E_QUEUE_TYPE_RX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 272 wr32(hw, I40E_QINT_TQCTL(i), reg); 273 } 274 } 275 276 /* 277 * Configure for single interrupt vector operation 278 */ 279 void ixl_configure_legacy(struct ixl_pf * pf)280 ixl_configure_legacy(struct ixl_pf *pf) 281 { 282 struct i40e_hw *hw = &pf->hw; 283 struct ixl_vsi *vsi = &pf->vsi; 284 u32 reg; 285 286 vsi->rx_queues[0].rxr.itr = vsi->rx_itr_setting; 287 288 /* Setup "other" causes */ 289 reg = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK 290 | I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK 291 | I40E_PFINT_ICR0_ENA_GRST_MASK 292 | I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK 293 | I40E_PFINT_ICR0_ENA_HMC_ERR_MASK 294 | I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK 295 | I40E_PFINT_ICR0_ENA_VFLR_MASK 296 | I40E_PFINT_ICR0_ENA_ADMINQ_MASK 297 ; 298 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 299 300 /* No ITR for non-queue interrupts */ 301 wr32(hw, I40E_PFINT_STAT_CTL0, 302 IXL_ITR_NONE << I40E_PFINT_STAT_CTL0_OTHER_ITR_INDX_SHIFT); 303 304 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 305 wr32(hw, I40E_PFINT_LNKLST0, 0); 306 307 /* Associate the queue pair to the vector and enable the q int */ 308 reg = I40E_QINT_RQCTL_CAUSE_ENA_MASK 309 | (IXL_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) 310 | (I40E_QUEUE_TYPE_TX << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 311 wr32(hw, I40E_QINT_RQCTL(0), reg); 312 313 reg = I40E_QINT_TQCTL_CAUSE_ENA_MASK 314 | (IXL_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) 315 | (IXL_QUEUE_EOL << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 316 wr32(hw, I40E_QINT_TQCTL(0), reg); 317 } 318 319 void ixl_free_pci_resources(struct ixl_pf * pf)320 ixl_free_pci_resources(struct ixl_pf *pf) 321 { 322 struct ixl_vsi *vsi = &pf->vsi; 323 device_t dev = iflib_get_dev(vsi->ctx); 324 struct ixl_rx_queue *rx_que = vsi->rx_queues; 325 326 /* We may get here before stations are set up */ 327 if (rx_que == NULL) 328 goto early; 329 330 /* 331 ** Release all MSI-X VSI resources: 332 */ 333 iflib_irq_free(vsi->ctx, &vsi->irq); 334 335 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) 336 iflib_irq_free(vsi->ctx, &rx_que->que_irq); 337 early: 338 if (pf->pci_mem != NULL) 339 bus_release_resource(dev, SYS_RES_MEMORY, 340 rman_get_rid(pf->pci_mem), pf->pci_mem); 341 } 342 343 /********************************************************************* 344 * 345 * Setup networking device structure and register an interface. 346 * 347 **********************************************************************/ 348 int ixl_setup_interface(device_t dev,struct ixl_pf * pf)349 ixl_setup_interface(device_t dev, struct ixl_pf *pf) 350 { 351 struct ixl_vsi *vsi = &pf->vsi; 352 if_ctx_t ctx = vsi->ctx; 353 struct i40e_hw *hw = &pf->hw; 354 if_t ifp = iflib_get_ifp(ctx); 355 struct i40e_aq_get_phy_abilities_resp abilities; 356 enum i40e_status_code aq_error = 0; 357 358 INIT_DBG_DEV(dev, "begin"); 359 360 vsi->shared->isc_max_frame_size = 361 if_getmtu(ifp) + ETHER_HDR_LEN + ETHER_CRC_LEN 362 + ETHER_VLAN_ENCAP_LEN; 363 364 if (IXL_PF_IN_RECOVERY_MODE(pf)) 365 goto only_auto; 366 367 aq_error = i40e_aq_get_phy_capabilities(hw, 368 FALSE, TRUE, &abilities, NULL); 369 /* May need delay to detect fiber correctly */ 370 if (aq_error == I40E_ERR_UNKNOWN_PHY) { 371 i40e_msec_delay(200); 372 aq_error = i40e_aq_get_phy_capabilities(hw, FALSE, 373 TRUE, &abilities, NULL); 374 } 375 if (aq_error) { 376 if (aq_error == I40E_ERR_UNKNOWN_PHY) 377 device_printf(dev, "Unknown PHY type detected!\n"); 378 else 379 device_printf(dev, 380 "Error getting supported media types, err %d," 381 " AQ error %d\n", aq_error, hw->aq.asq_last_status); 382 } else { 383 pf->supported_speeds = abilities.link_speed; 384 if_setbaudrate(ifp, ixl_max_aq_speed_to_value(pf->supported_speeds)); 385 386 ixl_add_ifmedia(vsi->media, hw->phy.phy_types); 387 } 388 389 only_auto: 390 /* Use autoselect media by default */ 391 ifmedia_add(vsi->media, IFM_ETHER | IFM_AUTO, 0, NULL); 392 ifmedia_set(vsi->media, IFM_ETHER | IFM_AUTO); 393 394 return (0); 395 } 396 397 /* 398 ** Run when the Admin Queue gets a link state change interrupt. 399 */ 400 void ixl_link_event(struct ixl_pf * pf,struct i40e_arq_event_info * e)401 ixl_link_event(struct ixl_pf *pf, struct i40e_arq_event_info *e) 402 { 403 struct i40e_hw *hw = &pf->hw; 404 device_t dev = iflib_get_dev(pf->vsi.ctx); 405 struct i40e_link_status *link_info = &hw->phy.link_info; 406 407 /* Driver needs to re-enable delivering of link status events 408 * by FW after each event reception. Call i40e_get_link_status 409 * to do that. To not lose information about link state changes, 410 * which happened between receiving an event and the call, 411 * do not rely on status from event but use most recent 412 * status information retrieved by the call. */ 413 hw->phy.get_link_info = TRUE; 414 i40e_get_link_status(hw, &pf->link_up); 415 416 /* Print out message if an unqualified module is found */ 417 if ((link_info->link_info & I40E_AQ_MEDIA_AVAILABLE) && 418 (pf->advertised_speed) && 419 (if_getflags(pf->vsi.ifp) & IFF_UP) && 420 (!(link_info->an_info & I40E_AQ_QUALIFIED_MODULE)) && 421 (!(link_info->link_info & I40E_AQ_LINK_UP))) 422 device_printf(dev, "Link failed because " 423 "an unqualified module was detected!\n"); 424 425 /* OS link info is updated elsewhere */ 426 } 427 428 /********************************************************************* 429 * 430 * Initialize the VSI: this handles contexts, which means things 431 * like the number of descriptors, buffer size, 432 * plus we init the rings thru this function. 433 * 434 **********************************************************************/ 435 int ixl_initialize_vsi(struct ixl_vsi * vsi)436 ixl_initialize_vsi(struct ixl_vsi *vsi) 437 { 438 struct ixl_pf *pf = vsi->back; 439 if_softc_ctx_t scctx = iflib_get_softc_ctx(vsi->ctx); 440 struct ixl_tx_queue *tx_que = vsi->tx_queues; 441 struct ixl_rx_queue *rx_que = vsi->rx_queues; 442 device_t dev = iflib_get_dev(vsi->ctx); 443 struct i40e_hw *hw = vsi->hw; 444 struct i40e_vsi_context ctxt; 445 int tc_queues; 446 int err = 0; 447 448 memset(&ctxt, 0, sizeof(ctxt)); 449 ctxt.seid = vsi->seid; 450 if (pf->veb_seid != 0) 451 ctxt.uplink_seid = pf->veb_seid; 452 ctxt.pf_num = hw->pf_id; 453 err = i40e_aq_get_vsi_params(hw, &ctxt, NULL); 454 if (err) { 455 device_printf(dev, "i40e_aq_get_vsi_params() failed, error %d" 456 " aq_error %d\n", err, hw->aq.asq_last_status); 457 return (err); 458 } 459 ixl_dbg(pf, IXL_DBG_SWITCH_INFO, 460 "get_vsi_params: seid: %d, uplinkseid: %d, vsi_number: %d, " 461 "vsis_allocated: %d, vsis_unallocated: %d, flags: 0x%x, " 462 "pfnum: %d, vfnum: %d, stat idx: %d, enabled: %d\n", ctxt.seid, 463 ctxt.uplink_seid, ctxt.vsi_number, 464 ctxt.vsis_allocated, ctxt.vsis_unallocated, 465 ctxt.flags, ctxt.pf_num, ctxt.vf_num, 466 ctxt.info.stat_counter_idx, ctxt.info.up_enable_bits); 467 /* 468 ** Set the queue and traffic class bits 469 ** - when multiple traffic classes are supported 470 ** this will need to be more robust. 471 */ 472 ctxt.info.valid_sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 473 ctxt.info.mapping_flags |= I40E_AQ_VSI_QUE_MAP_CONTIG; 474 /* In contig mode, que_mapping[0] is first queue index used by this VSI */ 475 ctxt.info.queue_mapping[0] = 0; 476 /* 477 * This VSI will only use traffic class 0; start traffic class 0's 478 * queue allocation at queue 0, and assign it 2^tc_queues queues (though 479 * the driver may not use all of them). 480 */ 481 tc_queues = fls(pf->qtag.num_allocated) - 1; 482 ctxt.info.tc_mapping[0] = ((pf->qtag.first_qidx << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) 483 & I40E_AQ_VSI_TC_QUE_OFFSET_MASK) | 484 ((tc_queues << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT) 485 & I40E_AQ_VSI_TC_QUE_NUMBER_MASK); 486 487 /* Set VLAN receive stripping mode */ 488 ctxt.info.valid_sections |= I40E_AQ_VSI_PROP_VLAN_VALID; 489 ctxt.info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL; 490 if (if_getcapenable(vsi->ifp) & IFCAP_VLAN_HWTAGGING) 491 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 492 else 493 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 494 495 #ifdef IXL_IW 496 /* Set TCP Enable for iWARP capable VSI */ 497 if (ixl_enable_iwarp && pf->iw_enabled) { 498 ctxt.info.valid_sections |= 499 htole16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 500 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 501 } 502 #endif 503 /* Save VSI number and info for use later */ 504 vsi->vsi_num = ctxt.vsi_number; 505 bcopy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 506 507 ctxt.flags = htole16(I40E_AQ_VSI_TYPE_PF); 508 509 err = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 510 if (err) { 511 device_printf(dev, "i40e_aq_update_vsi_params() failed, error %d," 512 " aq_error %d\n", err, hw->aq.asq_last_status); 513 return (err); 514 } 515 516 for (int i = 0; i < vsi->num_tx_queues; i++, tx_que++) { 517 struct tx_ring *txr = &tx_que->txr; 518 struct i40e_hmc_obj_txq tctx; 519 u32 txctl; 520 521 /* Setup the HMC TX Context */ 522 bzero(&tctx, sizeof(tctx)); 523 tctx.new_context = 1; 524 tctx.base = (txr->tx_paddr/IXL_TX_CTX_BASE_UNITS); 525 tctx.qlen = scctx->isc_ntxd[0]; 526 tctx.fc_ena = 0; /* Disable FCoE */ 527 /* 528 * This value needs to pulled from the VSI that this queue 529 * is assigned to. Index into array is traffic class. 530 */ 531 tctx.rdylist = vsi->info.qs_handle[0]; 532 /* 533 * Set these to enable Head Writeback 534 * - Address is last entry in TX ring (reserved for HWB index) 535 * Leave these as 0 for Descriptor Writeback 536 */ 537 if (vsi->enable_head_writeback) { 538 tctx.head_wb_ena = 1; 539 tctx.head_wb_addr = txr->tx_paddr + 540 (scctx->isc_ntxd[0] * sizeof(struct i40e_tx_desc)); 541 } else { 542 tctx.head_wb_ena = 0; 543 tctx.head_wb_addr = 0; 544 } 545 tctx.rdylist_act = 0; 546 err = i40e_clear_lan_tx_queue_context(hw, i); 547 if (err) { 548 device_printf(dev, "Unable to clear TX context\n"); 549 break; 550 } 551 err = i40e_set_lan_tx_queue_context(hw, i, &tctx); 552 if (err) { 553 device_printf(dev, "Unable to set TX context\n"); 554 break; 555 } 556 /* Associate the ring with this PF */ 557 txctl = I40E_QTX_CTL_PF_QUEUE; 558 txctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 559 I40E_QTX_CTL_PF_INDX_MASK); 560 wr32(hw, I40E_QTX_CTL(i), txctl); 561 ixl_flush(hw); 562 563 /* Do ring (re)init */ 564 ixl_init_tx_ring(vsi, tx_que); 565 } 566 for (int i = 0; i < vsi->num_rx_queues; i++, rx_que++) { 567 struct rx_ring *rxr = &rx_que->rxr; 568 struct i40e_hmc_obj_rxq rctx; 569 570 /* Next setup the HMC RX Context */ 571 rxr->mbuf_sz = iflib_get_rx_mbuf_sz(vsi->ctx); 572 573 u16 max_rxmax = rxr->mbuf_sz * hw->func_caps.rx_buf_chain_len; 574 575 /* Set up an RX context for the HMC */ 576 memset(&rctx, 0, sizeof(struct i40e_hmc_obj_rxq)); 577 rctx.dbuff = rxr->mbuf_sz >> I40E_RXQ_CTX_DBUFF_SHIFT; 578 /* ignore header split for now */ 579 rctx.hbuff = 0 >> I40E_RXQ_CTX_HBUFF_SHIFT; 580 rctx.rxmax = (scctx->isc_max_frame_size < max_rxmax) ? 581 scctx->isc_max_frame_size : max_rxmax; 582 rctx.dtype = 0; 583 rctx.dsize = 1; /* do 32byte descriptors */ 584 rctx.hsplit_0 = 0; /* no header split */ 585 rctx.base = (rxr->rx_paddr/IXL_RX_CTX_BASE_UNITS); 586 rctx.qlen = scctx->isc_nrxd[0]; 587 rctx.tphrdesc_ena = 1; 588 rctx.tphwdesc_ena = 1; 589 rctx.tphdata_ena = 0; /* Header Split related */ 590 rctx.tphhead_ena = 0; /* Header Split related */ 591 rctx.lrxqthresh = 1; /* Interrupt at <64 desc avail */ 592 rctx.crcstrip = 1; 593 rctx.l2tsel = 1; 594 rctx.showiv = 1; /* Strip inner VLAN header */ 595 rctx.fc_ena = 0; /* Disable FCoE */ 596 rctx.prefena = 1; /* Prefetch descriptors */ 597 598 err = i40e_clear_lan_rx_queue_context(hw, i); 599 if (err) { 600 device_printf(dev, 601 "Unable to clear RX context %d\n", i); 602 break; 603 } 604 err = i40e_set_lan_rx_queue_context(hw, i, &rctx); 605 if (err) { 606 device_printf(dev, "Unable to set RX context %d\n", i); 607 break; 608 } 609 wr32(vsi->hw, I40E_QRX_TAIL(i), 0); 610 } 611 return (err); 612 } 613 614 615 /* 616 ** Provide a update to the queue RX 617 ** interrupt moderation value. 618 */ 619 void ixl_set_queue_rx_itr(struct ixl_rx_queue * que)620 ixl_set_queue_rx_itr(struct ixl_rx_queue *que) 621 { 622 struct ixl_vsi *vsi = que->vsi; 623 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 624 struct i40e_hw *hw = vsi->hw; 625 struct rx_ring *rxr = &que->rxr; 626 u16 rx_itr; 627 u16 rx_latency = 0; 628 int rx_bytes; 629 630 /* Idle, do nothing */ 631 if (rxr->bytes == 0) 632 return; 633 634 if (pf->dynamic_rx_itr) { 635 rx_bytes = rxr->bytes/rxr->itr; 636 rx_itr = rxr->itr; 637 638 /* Adjust latency range */ 639 switch (rxr->latency) { 640 case IXL_LOW_LATENCY: 641 if (rx_bytes > 10) { 642 rx_latency = IXL_AVE_LATENCY; 643 rx_itr = IXL_ITR_20K; 644 } 645 break; 646 case IXL_AVE_LATENCY: 647 if (rx_bytes > 20) { 648 rx_latency = IXL_BULK_LATENCY; 649 rx_itr = IXL_ITR_8K; 650 } else if (rx_bytes <= 10) { 651 rx_latency = IXL_LOW_LATENCY; 652 rx_itr = IXL_ITR_100K; 653 } 654 break; 655 case IXL_BULK_LATENCY: 656 if (rx_bytes <= 20) { 657 rx_latency = IXL_AVE_LATENCY; 658 rx_itr = IXL_ITR_20K; 659 } 660 break; 661 } 662 663 rxr->latency = rx_latency; 664 665 if (rx_itr != rxr->itr) { 666 /* do an exponential smoothing */ 667 rx_itr = (10 * rx_itr * rxr->itr) / 668 ((9 * rx_itr) + rxr->itr); 669 rxr->itr = min(rx_itr, IXL_MAX_ITR); 670 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 671 rxr->me), rxr->itr); 672 } 673 } else { /* We may have have toggled to non-dynamic */ 674 if (vsi->rx_itr_setting & IXL_ITR_DYNAMIC) 675 vsi->rx_itr_setting = pf->rx_itr; 676 /* Update the hardware if needed */ 677 if (rxr->itr != vsi->rx_itr_setting) { 678 rxr->itr = vsi->rx_itr_setting; 679 wr32(hw, I40E_PFINT_ITRN(IXL_RX_ITR, 680 rxr->me), rxr->itr); 681 } 682 } 683 rxr->bytes = 0; 684 rxr->packets = 0; 685 } 686 687 688 /* 689 ** Provide a update to the queue TX 690 ** interrupt moderation value. 691 */ 692 void ixl_set_queue_tx_itr(struct ixl_tx_queue * que)693 ixl_set_queue_tx_itr(struct ixl_tx_queue *que) 694 { 695 struct ixl_vsi *vsi = que->vsi; 696 struct ixl_pf *pf = (struct ixl_pf *)vsi->back; 697 struct i40e_hw *hw = vsi->hw; 698 struct tx_ring *txr = &que->txr; 699 u16 tx_itr; 700 u16 tx_latency = 0; 701 int tx_bytes; 702 703 704 /* Idle, do nothing */ 705 if (txr->bytes == 0) 706 return; 707 708 if (pf->dynamic_tx_itr) { 709 tx_bytes = txr->bytes/txr->itr; 710 tx_itr = txr->itr; 711 712 switch (txr->latency) { 713 case IXL_LOW_LATENCY: 714 if (tx_bytes > 10) { 715 tx_latency = IXL_AVE_LATENCY; 716 tx_itr = IXL_ITR_20K; 717 } 718 break; 719 case IXL_AVE_LATENCY: 720 if (tx_bytes > 20) { 721 tx_latency = IXL_BULK_LATENCY; 722 tx_itr = IXL_ITR_8K; 723 } else if (tx_bytes <= 10) { 724 tx_latency = IXL_LOW_LATENCY; 725 tx_itr = IXL_ITR_100K; 726 } 727 break; 728 case IXL_BULK_LATENCY: 729 if (tx_bytes <= 20) { 730 tx_latency = IXL_AVE_LATENCY; 731 tx_itr = IXL_ITR_20K; 732 } 733 break; 734 } 735 736 txr->latency = tx_latency; 737 738 if (tx_itr != txr->itr) { 739 /* do an exponential smoothing */ 740 tx_itr = (10 * tx_itr * txr->itr) / 741 ((9 * tx_itr) + txr->itr); 742 txr->itr = min(tx_itr, IXL_MAX_ITR); 743 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 744 txr->me), txr->itr); 745 } 746 747 } else { /* We may have have toggled to non-dynamic */ 748 if (vsi->tx_itr_setting & IXL_ITR_DYNAMIC) 749 vsi->tx_itr_setting = pf->tx_itr; 750 /* Update the hardware if needed */ 751 if (txr->itr != vsi->tx_itr_setting) { 752 txr->itr = vsi->tx_itr_setting; 753 wr32(hw, I40E_PFINT_ITRN(IXL_TX_ITR, 754 txr->me), txr->itr); 755 } 756 } 757 txr->bytes = 0; 758 txr->packets = 0; 759 return; 760 } 761 762 #ifdef IXL_DEBUG 763 /** 764 * ixl_sysctl_qtx_tail_handler 765 * Retrieves I40E_QTX_TAIL value from hardware 766 * for a sysctl. 767 */ 768 int ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS)769 ixl_sysctl_qtx_tail_handler(SYSCTL_HANDLER_ARGS) 770 { 771 struct ixl_tx_queue *tx_que; 772 int error; 773 u32 val; 774 775 tx_que = ((struct ixl_tx_queue *)oidp->oid_arg1); 776 if (!tx_que) return 0; 777 778 val = rd32(tx_que->vsi->hw, tx_que->txr.tail); 779 error = sysctl_handle_int(oidp, &val, 0, req); 780 if (error || !req->newptr) 781 return error; 782 return (0); 783 } 784 785 /** 786 * ixl_sysctl_qrx_tail_handler 787 * Retrieves I40E_QRX_TAIL value from hardware 788 * for a sysctl. 789 */ 790 int ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS)791 ixl_sysctl_qrx_tail_handler(SYSCTL_HANDLER_ARGS) 792 { 793 struct ixl_rx_queue *rx_que; 794 int error; 795 u32 val; 796 797 rx_que = ((struct ixl_rx_queue *)oidp->oid_arg1); 798 if (!rx_que) return 0; 799 800 val = rd32(rx_que->vsi->hw, rx_que->rxr.tail); 801 error = sysctl_handle_int(oidp, &val, 0, req); 802 if (error || !req->newptr) 803 return error; 804 return (0); 805 } 806 #endif 807 808 void ixl_add_hw_stats(struct ixl_pf * pf)809 ixl_add_hw_stats(struct ixl_pf *pf) 810 { 811 struct ixl_vsi *vsi = &pf->vsi; 812 device_t dev = iflib_get_dev(vsi->ctx); 813 struct i40e_hw_port_stats *pf_stats = &pf->stats; 814 815 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev); 816 struct sysctl_oid *tree = device_get_sysctl_tree(dev); 817 struct sysctl_oid_list *child = SYSCTL_CHILDREN(tree); 818 819 /* Driver statistics */ 820 SYSCTL_ADD_UQUAD(ctx, child, OID_AUTO, "admin_irq", 821 CTLFLAG_RD, &pf->admin_irq, 822 "Admin Queue IRQs received"); 823 824 sysctl_ctx_init(&vsi->sysctl_ctx); 825 ixl_vsi_add_sysctls(vsi, "pf", true); 826 827 ixl_add_sysctls_mac_stats(ctx, child, pf_stats); 828 } 829 830 void ixl_set_rss_hlut(struct ixl_pf * pf)831 ixl_set_rss_hlut(struct ixl_pf *pf) 832 { 833 struct i40e_hw *hw = &pf->hw; 834 struct ixl_vsi *vsi = &pf->vsi; 835 device_t dev = iflib_get_dev(vsi->ctx); 836 int i, que_id; 837 int lut_entry_width; 838 u32 lut = 0; 839 enum i40e_status_code status; 840 841 lut_entry_width = pf->hw.func_caps.rss_table_entry_width; 842 843 /* Populate the LUT with max no. of queues in round robin fashion */ 844 u8 hlut_buf[512]; 845 for (i = 0; i < pf->hw.func_caps.rss_table_size; i++) { 846 #ifdef RSS 847 /* 848 * Fetch the RSS bucket id for the given indirection entry. 849 * Cap it at the number of configured buckets (which is 850 * num_queues.) 851 */ 852 que_id = rss_get_indirection_to_bucket(i); 853 que_id = que_id % vsi->num_rx_queues; 854 #else 855 que_id = i % vsi->num_rx_queues; 856 #endif 857 lut = (que_id & ((0x1 << lut_entry_width) - 1)); 858 hlut_buf[i] = lut; 859 } 860 861 if (hw->mac.type == I40E_MAC_X722) { 862 status = i40e_aq_set_rss_lut(hw, vsi->vsi_num, TRUE, hlut_buf, sizeof(hlut_buf)); 863 if (status) 864 device_printf(dev, "i40e_aq_set_rss_lut status %s, error %s\n", 865 i40e_stat_str(hw, status), i40e_aq_str(hw, hw->aq.asq_last_status)); 866 } else { 867 for (i = 0; i < pf->hw.func_caps.rss_table_size >> 2; i++) 868 wr32(hw, I40E_PFQF_HLUT(i), ((u32 *)hlut_buf)[i]); 869 ixl_flush(hw); 870 } 871 } 872 873 /* For PF VSI only */ 874 int ixl_enable_rings(struct ixl_vsi * vsi)875 ixl_enable_rings(struct ixl_vsi *vsi) 876 { 877 struct ixl_pf *pf = vsi->back; 878 int error = 0; 879 880 for (int i = 0; i < vsi->num_tx_queues; i++) 881 error = ixl_enable_tx_ring(pf, &pf->qtag, i); 882 883 for (int i = 0; i < vsi->num_rx_queues; i++) 884 error = ixl_enable_rx_ring(pf, &pf->qtag, i); 885 886 return (error); 887 } 888 889 int ixl_disable_rings(struct ixl_pf * pf,struct ixl_vsi * vsi,struct ixl_pf_qtag * qtag)890 ixl_disable_rings(struct ixl_pf *pf, struct ixl_vsi *vsi, struct ixl_pf_qtag *qtag) 891 { 892 int error = 0; 893 894 for (int i = 0; i < vsi->num_tx_queues; i++) 895 error = ixl_disable_tx_ring(pf, qtag, i); 896 897 for (int i = 0; i < vsi->num_rx_queues; i++) 898 error = ixl_disable_rx_ring(pf, qtag, i); 899 900 return (error); 901 } 902 903 void ixl_enable_intr(struct ixl_vsi * vsi)904 ixl_enable_intr(struct ixl_vsi *vsi) 905 { 906 struct i40e_hw *hw = vsi->hw; 907 struct ixl_rx_queue *que = vsi->rx_queues; 908 909 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 910 for (int i = 0; i < vsi->num_rx_queues; i++, que++) 911 ixl_enable_queue(hw, que->rxr.me); 912 } else 913 ixl_enable_intr0(hw); 914 } 915 916 void ixl_disable_rings_intr(struct ixl_vsi * vsi)917 ixl_disable_rings_intr(struct ixl_vsi *vsi) 918 { 919 struct i40e_hw *hw = vsi->hw; 920 struct ixl_rx_queue *que = vsi->rx_queues; 921 922 for (int i = 0; i < vsi->num_rx_queues; i++, que++) 923 ixl_disable_queue(hw, que->rxr.me); 924 } 925 926 int ixl_prepare_for_reset(struct ixl_pf * pf,bool is_up)927 ixl_prepare_for_reset(struct ixl_pf *pf, bool is_up) 928 { 929 struct i40e_hw *hw = &pf->hw; 930 device_t dev = pf->dev; 931 int error = 0; 932 933 if (is_up) 934 ixl_if_stop(pf->vsi.ctx); 935 936 ixl_shutdown_hmc(pf); 937 938 ixl_disable_intr0(hw); 939 940 error = i40e_shutdown_adminq(hw); 941 if (error) 942 device_printf(dev, 943 "Shutdown Admin queue failed with code %d\n", error); 944 945 ixl_pf_qmgr_release(&pf->qmgr, &pf->qtag); 946 return (error); 947 } 948 949 int ixl_rebuild_hw_structs_after_reset(struct ixl_pf * pf,bool is_up)950 ixl_rebuild_hw_structs_after_reset(struct ixl_pf *pf, bool is_up) 951 { 952 struct i40e_hw *hw = &pf->hw; 953 struct ixl_vsi *vsi = &pf->vsi; 954 device_t dev = pf->dev; 955 enum i40e_get_fw_lldp_status_resp lldp_status; 956 int error = 0; 957 958 device_printf(dev, "Rebuilding driver state...\n"); 959 960 /* Setup */ 961 error = i40e_init_adminq(hw); 962 if (error != 0 && error != I40E_ERR_FIRMWARE_API_VERSION) { 963 device_printf(dev, "Unable to initialize Admin Queue, error %d\n", 964 error); 965 goto ixl_rebuild_hw_structs_after_reset_err; 966 } 967 968 if (IXL_PF_IN_RECOVERY_MODE(pf)) { 969 /* Keep admin queue interrupts active while driver is loaded */ 970 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 971 ixl_configure_intr0_msix(pf); 972 ixl_enable_intr0(hw); 973 } 974 975 return (0); 976 } 977 978 i40e_clear_pxe_mode(hw); 979 980 error = ixl_get_hw_capabilities(pf); 981 if (error) { 982 device_printf(dev, "ixl_get_hw_capabilities failed: %d\n", error); 983 goto ixl_rebuild_hw_structs_after_reset_err; 984 } 985 986 error = ixl_setup_hmc(pf); 987 if (error) 988 goto ixl_rebuild_hw_structs_after_reset_err; 989 990 /* reserve a contiguous allocation for the PF's VSI */ 991 error = ixl_pf_qmgr_alloc_contiguous(&pf->qmgr, vsi->num_tx_queues, &pf->qtag); 992 if (error) { 993 device_printf(dev, "Failed to reserve queues for PF LAN VSI, error %d\n", 994 error); 995 } 996 997 error = ixl_switch_config(pf); 998 if (error) { 999 device_printf(dev, "ixl_rebuild_hw_structs_after_reset: ixl_switch_config() failed: %d\n", 1000 error); 1001 error = EIO; 1002 goto ixl_rebuild_hw_structs_after_reset_err; 1003 } 1004 1005 error = i40e_aq_set_phy_int_mask(hw, IXL_DEFAULT_PHY_INT_MASK, 1006 NULL); 1007 if (error) { 1008 device_printf(dev, "init: i40e_aq_set_phy_mask() failed: err %d," 1009 " aq_err %d\n", error, hw->aq.asq_last_status); 1010 error = EIO; 1011 goto ixl_rebuild_hw_structs_after_reset_err; 1012 } 1013 1014 u8 set_fc_err_mask; 1015 error = i40e_set_fc(hw, &set_fc_err_mask, true); 1016 if (error) { 1017 device_printf(dev, "init: setting link flow control failed; retcode %d," 1018 " fc_err_mask 0x%02x\n", error, set_fc_err_mask); 1019 error = EIO; 1020 goto ixl_rebuild_hw_structs_after_reset_err; 1021 } 1022 1023 /* Remove default filters reinstalled by FW on reset */ 1024 ixl_del_default_hw_filters(vsi); 1025 1026 /* Receive broadcast Ethernet frames */ 1027 i40e_aq_set_vsi_broadcast(&pf->hw, vsi->seid, TRUE, NULL); 1028 1029 /* Determine link state */ 1030 ixl_attach_get_link_status(pf); 1031 1032 i40e_aq_set_dcb_parameters(hw, TRUE, NULL); 1033 1034 /* Query device FW LLDP status */ 1035 if (i40e_get_fw_lldp_status(hw, &lldp_status) == I40E_SUCCESS) { 1036 if (lldp_status == I40E_GET_FW_LLDP_STATUS_DISABLED) { 1037 ixl_set_state(&pf->state, 1038 IXL_STATE_FW_LLDP_DISABLED); 1039 } else { 1040 ixl_clear_state(&pf->state, 1041 IXL_STATE_FW_LLDP_DISABLED); 1042 } 1043 } 1044 1045 /* Keep admin queue interrupts active while driver is loaded */ 1046 if (vsi->shared->isc_intr == IFLIB_INTR_MSIX) { 1047 ixl_configure_intr0_msix(pf); 1048 ixl_enable_intr0(hw); 1049 } 1050 1051 if (is_up) { 1052 iflib_request_reset(vsi->ctx); 1053 iflib_admin_intr_deferred(vsi->ctx); 1054 } 1055 1056 device_printf(dev, "Rebuilding driver state done.\n"); 1057 return (0); 1058 1059 ixl_rebuild_hw_structs_after_reset_err: 1060 device_printf(dev, "Reload the driver to recover\n"); 1061 return (error); 1062 } 1063 1064 /* 1065 ** Set flow control using sysctl: 1066 ** 0 - off 1067 ** 1 - rx pause 1068 ** 2 - tx pause 1069 ** 3 - full 1070 */ 1071 int ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS)1072 ixl_sysctl_set_flowcntl(SYSCTL_HANDLER_ARGS) 1073 { 1074 struct ixl_pf *pf = (struct ixl_pf *)arg1; 1075 struct i40e_hw *hw = &pf->hw; 1076 device_t dev = pf->dev; 1077 int requested_fc, error = 0; 1078 enum i40e_status_code aq_error = 0; 1079 u8 fc_aq_err = 0; 1080 1081 /* Get request */ 1082 requested_fc = pf->fc; 1083 error = sysctl_handle_int(oidp, &requested_fc, 0, req); 1084 if ((error) || (req->newptr == NULL)) 1085 return (error); 1086 if (requested_fc < 0 || requested_fc > 3) { 1087 device_printf(dev, 1088 "Invalid fc mode; valid modes are 0 through 3\n"); 1089 return (EINVAL); 1090 } 1091 1092 /* Set fc ability for port */ 1093 hw->fc.requested_mode = requested_fc; 1094 aq_error = i40e_set_fc(hw, &fc_aq_err, TRUE); 1095 if (aq_error) { 1096 device_printf(dev, 1097 "%s: Error setting Flow Control mode %d; fc_err %#x\n", 1098 __func__, aq_error, fc_aq_err); 1099 return (EIO); 1100 } 1101 pf->fc = requested_fc; 1102 1103 return (0); 1104 } 1105