1 /*- 2 * Copyright (c) 2012-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include <sys/cdefs.h> 32 __FBSDID("$FreeBSD$"); 33 34 #include "efx.h" 35 #include "efx_impl.h" 36 #if EFSYS_OPT_MON_STATS 37 #include "mcdi_mon.h" 38 #endif 39 40 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 41 42 #if EFSYS_OPT_QSTATS 43 #define EFX_EV_QSTAT_INCR(_eep, _stat) \ 44 do { \ 45 (_eep)->ee_stat[_stat]++; \ 46 _NOTE(CONSTANTCONDITION) \ 47 } while (B_FALSE) 48 #else 49 #define EFX_EV_QSTAT_INCR(_eep, _stat) 50 #endif 51 52 53 static __checkReturn boolean_t 54 ef10_ev_rx( 55 __in efx_evq_t *eep, 56 __in efx_qword_t *eqp, 57 __in const efx_ev_callbacks_t *eecp, 58 __in_opt void *arg); 59 60 static __checkReturn boolean_t 61 ef10_ev_tx( 62 __in efx_evq_t *eep, 63 __in efx_qword_t *eqp, 64 __in const efx_ev_callbacks_t *eecp, 65 __in_opt void *arg); 66 67 static __checkReturn boolean_t 68 ef10_ev_driver( 69 __in efx_evq_t *eep, 70 __in efx_qword_t *eqp, 71 __in const efx_ev_callbacks_t *eecp, 72 __in_opt void *arg); 73 74 static __checkReturn boolean_t 75 ef10_ev_drv_gen( 76 __in efx_evq_t *eep, 77 __in efx_qword_t *eqp, 78 __in const efx_ev_callbacks_t *eecp, 79 __in_opt void *arg); 80 81 static __checkReturn boolean_t 82 ef10_ev_mcdi( 83 __in efx_evq_t *eep, 84 __in efx_qword_t *eqp, 85 __in const efx_ev_callbacks_t *eecp, 86 __in_opt void *arg); 87 88 89 static __checkReturn efx_rc_t 90 efx_mcdi_init_evq( 91 __in efx_nic_t *enp, 92 __in unsigned int instance, 93 __in efsys_mem_t *esmp, 94 __in size_t nevs, 95 __in uint32_t irq) 96 { 97 efx_mcdi_req_t req; 98 uint8_t payload[ 99 MAX(MC_CMD_INIT_EVQ_IN_LEN(EFX_EVQ_NBUFS(EFX_EVQ_MAXNEVS)), 100 MC_CMD_INIT_EVQ_OUT_LEN)]; 101 efx_qword_t *dma_addr; 102 uint64_t addr; 103 int npages; 104 int i; 105 int supports_rx_batching; 106 efx_rc_t rc; 107 108 npages = EFX_EVQ_NBUFS(nevs); 109 if (MC_CMD_INIT_EVQ_IN_LEN(npages) > MC_CMD_INIT_EVQ_IN_LENMAX) { 110 rc = EINVAL; 111 goto fail1; 112 } 113 114 (void) memset(payload, 0, sizeof (payload)); 115 req.emr_cmd = MC_CMD_INIT_EVQ; 116 req.emr_in_buf = payload; 117 req.emr_in_length = MC_CMD_INIT_EVQ_IN_LEN(npages); 118 req.emr_out_buf = payload; 119 req.emr_out_length = MC_CMD_INIT_EVQ_OUT_LEN; 120 121 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_SIZE, nevs); 122 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_INSTANCE, instance); 123 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_IRQ_NUM, irq); 124 125 /* 126 * On Huntington RX and TX event batching can only be requested 127 * together (even if the datapath firmware doesn't actually support RX 128 * batching). 129 * Cut through is incompatible with RX batching and so enabling cut 130 * through disables RX batching (but it does not affect TX batching). 131 * 132 * So always enable RX and TX event batching, and enable cut through 133 * if RX event batching isn't supported (i.e. on low latency firmware). 134 */ 135 supports_rx_batching = enp->en_nic_cfg.enc_rx_batching_enabled ? 1 : 0; 136 MCDI_IN_POPULATE_DWORD_6(req, INIT_EVQ_IN_FLAGS, 137 INIT_EVQ_IN_FLAG_INTERRUPTING, 1, 138 INIT_EVQ_IN_FLAG_RPTR_DOS, 0, 139 INIT_EVQ_IN_FLAG_INT_ARMD, 0, 140 INIT_EVQ_IN_FLAG_CUT_THRU, !supports_rx_batching, 141 INIT_EVQ_IN_FLAG_RX_MERGE, 1, 142 INIT_EVQ_IN_FLAG_TX_MERGE, 1); 143 144 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_MODE, 145 MC_CMD_INIT_EVQ_IN_TMR_MODE_DIS); 146 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_LOAD, 0); 147 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_TMR_RELOAD, 0); 148 149 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_COUNT_MODE, 150 MC_CMD_INIT_EVQ_IN_COUNT_MODE_DIS); 151 MCDI_IN_SET_DWORD(req, INIT_EVQ_IN_COUNT_THRSHLD, 0); 152 153 dma_addr = MCDI_IN2(req, efx_qword_t, INIT_EVQ_IN_DMA_ADDR); 154 addr = EFSYS_MEM_ADDR(esmp); 155 156 for (i = 0; i < npages; i++) { 157 EFX_POPULATE_QWORD_2(*dma_addr, 158 EFX_DWORD_1, (uint32_t)(addr >> 32), 159 EFX_DWORD_0, (uint32_t)(addr & 0xffffffff)); 160 161 dma_addr++; 162 addr += EFX_BUF_SIZE; 163 } 164 165 efx_mcdi_execute(enp, &req); 166 167 if (req.emr_rc != 0) { 168 rc = req.emr_rc; 169 goto fail2; 170 } 171 172 if (req.emr_out_length_used < MC_CMD_INIT_EVQ_OUT_LEN) { 173 rc = EMSGSIZE; 174 goto fail3; 175 } 176 177 /* NOTE: ignore the returned IRQ param as firmware does not set it. */ 178 179 return (0); 180 181 fail3: 182 EFSYS_PROBE(fail3); 183 fail2: 184 EFSYS_PROBE(fail2); 185 fail1: 186 EFSYS_PROBE1(fail1, efx_rc_t, rc); 187 188 return (rc); 189 } 190 191 static __checkReturn efx_rc_t 192 efx_mcdi_fini_evq( 193 __in efx_nic_t *enp, 194 __in uint32_t instance) 195 { 196 efx_mcdi_req_t req; 197 uint8_t payload[MAX(MC_CMD_FINI_EVQ_IN_LEN, 198 MC_CMD_FINI_EVQ_OUT_LEN)]; 199 efx_rc_t rc; 200 201 (void) memset(payload, 0, sizeof (payload)); 202 req.emr_cmd = MC_CMD_FINI_EVQ; 203 req.emr_in_buf = payload; 204 req.emr_in_length = MC_CMD_FINI_EVQ_IN_LEN; 205 req.emr_out_buf = payload; 206 req.emr_out_length = MC_CMD_FINI_EVQ_OUT_LEN; 207 208 MCDI_IN_SET_DWORD(req, FINI_EVQ_IN_INSTANCE, instance); 209 210 efx_mcdi_execute_quiet(enp, &req); 211 212 if (req.emr_rc != 0) { 213 rc = req.emr_rc; 214 goto fail1; 215 } 216 217 return (0); 218 219 fail1: 220 EFSYS_PROBE1(fail1, efx_rc_t, rc); 221 222 return (rc); 223 } 224 225 226 227 __checkReturn efx_rc_t 228 ef10_ev_init( 229 __in efx_nic_t *enp) 230 { 231 _NOTE(ARGUNUSED(enp)) 232 return (0); 233 } 234 235 void 236 ef10_ev_fini( 237 __in efx_nic_t *enp) 238 { 239 _NOTE(ARGUNUSED(enp)) 240 } 241 242 __checkReturn efx_rc_t 243 ef10_ev_qcreate( 244 __in efx_nic_t *enp, 245 __in unsigned int index, 246 __in efsys_mem_t *esmp, 247 __in size_t n, 248 __in uint32_t id, 249 __in efx_evq_t *eep) 250 { 251 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 252 uint32_t irq; 253 efx_rc_t rc; 254 255 _NOTE(ARGUNUSED(id)) /* buftbl id managed by MC */ 256 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); 257 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); 258 259 if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { 260 rc = EINVAL; 261 goto fail1; 262 } 263 264 if (index >= encp->enc_evq_limit) { 265 rc = EINVAL; 266 goto fail2; 267 } 268 269 /* Set up the handler table */ 270 eep->ee_rx = ef10_ev_rx; 271 eep->ee_tx = ef10_ev_tx; 272 eep->ee_driver = ef10_ev_driver; 273 eep->ee_drv_gen = ef10_ev_drv_gen; 274 eep->ee_mcdi = ef10_ev_mcdi; 275 276 /* Set up the event queue */ 277 irq = index; /* INIT_EVQ expects function-relative vector number */ 278 if ((rc = efx_mcdi_init_evq(enp, index, esmp, n, irq)) != 0) 279 goto fail3; 280 281 return (0); 282 283 fail3: 284 EFSYS_PROBE(fail3); 285 fail2: 286 EFSYS_PROBE(fail2); 287 fail1: 288 EFSYS_PROBE1(fail1, efx_rc_t, rc); 289 290 return (rc); 291 } 292 293 void 294 ef10_ev_qdestroy( 295 __in efx_evq_t *eep) 296 { 297 efx_nic_t *enp = eep->ee_enp; 298 299 EFSYS_ASSERT(enp->en_family == EFX_FAMILY_HUNTINGTON || 300 enp->en_family == EFX_FAMILY_MEDFORD); 301 302 (void) efx_mcdi_fini_evq(eep->ee_enp, eep->ee_index); 303 } 304 305 __checkReturn efx_rc_t 306 ef10_ev_qprime( 307 __in efx_evq_t *eep, 308 __in unsigned int count) 309 { 310 efx_nic_t *enp = eep->ee_enp; 311 uint32_t rptr; 312 efx_dword_t dword; 313 314 rptr = count & eep->ee_mask; 315 316 if (enp->en_nic_cfg.enc_bug35388_workaround) { 317 EFX_STATIC_ASSERT(EFX_EVQ_MINNEVS > 318 (1 << ERF_DD_EVQ_IND_RPTR_WIDTH)); 319 EFX_STATIC_ASSERT(EFX_EVQ_MAXNEVS < 320 (1 << 2 * ERF_DD_EVQ_IND_RPTR_WIDTH)); 321 322 EFX_POPULATE_DWORD_2(dword, 323 ERF_DD_EVQ_IND_RPTR_FLAGS, 324 EFE_DD_EVQ_IND_RPTR_FLAGS_HIGH, 325 ERF_DD_EVQ_IND_RPTR, 326 (rptr >> ERF_DD_EVQ_IND_RPTR_WIDTH)); 327 EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, eep->ee_index, 328 &dword, B_FALSE); 329 330 EFX_POPULATE_DWORD_2(dword, 331 ERF_DD_EVQ_IND_RPTR_FLAGS, 332 EFE_DD_EVQ_IND_RPTR_FLAGS_LOW, 333 ERF_DD_EVQ_IND_RPTR, 334 rptr & ((1 << ERF_DD_EVQ_IND_RPTR_WIDTH) - 1)); 335 EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, eep->ee_index, 336 &dword, B_FALSE); 337 } else { 338 EFX_POPULATE_DWORD_1(dword, ERF_DZ_EVQ_RPTR, rptr); 339 EFX_BAR_TBL_WRITED(enp, ER_DZ_EVQ_RPTR_REG, eep->ee_index, 340 &dword, B_FALSE); 341 } 342 343 return (0); 344 } 345 346 static __checkReturn efx_rc_t 347 efx_mcdi_driver_event( 348 __in efx_nic_t *enp, 349 __in uint32_t evq, 350 __in efx_qword_t data) 351 { 352 efx_mcdi_req_t req; 353 uint8_t payload[MAX(MC_CMD_DRIVER_EVENT_IN_LEN, 354 MC_CMD_DRIVER_EVENT_OUT_LEN)]; 355 efx_rc_t rc; 356 357 req.emr_cmd = MC_CMD_DRIVER_EVENT; 358 req.emr_in_buf = payload; 359 req.emr_in_length = MC_CMD_DRIVER_EVENT_IN_LEN; 360 req.emr_out_buf = payload; 361 req.emr_out_length = MC_CMD_DRIVER_EVENT_OUT_LEN; 362 363 MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_EVQ, evq); 364 365 MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_DATA_LO, 366 EFX_QWORD_FIELD(data, EFX_DWORD_0)); 367 MCDI_IN_SET_DWORD(req, DRIVER_EVENT_IN_DATA_HI, 368 EFX_QWORD_FIELD(data, EFX_DWORD_1)); 369 370 efx_mcdi_execute(enp, &req); 371 372 if (req.emr_rc != 0) { 373 rc = req.emr_rc; 374 goto fail1; 375 } 376 377 return (0); 378 379 fail1: 380 EFSYS_PROBE1(fail1, efx_rc_t, rc); 381 382 return (rc); 383 } 384 385 void 386 ef10_ev_qpost( 387 __in efx_evq_t *eep, 388 __in uint16_t data) 389 { 390 efx_nic_t *enp = eep->ee_enp; 391 efx_qword_t event; 392 393 EFX_POPULATE_QWORD_3(event, 394 ESF_DZ_DRV_CODE, ESE_DZ_EV_CODE_DRV_GEN_EV, 395 ESF_DZ_DRV_SUB_CODE, 0, 396 ESF_DZ_DRV_SUB_DATA_DW0, (uint32_t)data); 397 398 (void) efx_mcdi_driver_event(enp, eep->ee_index, event); 399 } 400 401 __checkReturn efx_rc_t 402 ef10_ev_qmoderate( 403 __in efx_evq_t *eep, 404 __in unsigned int us) 405 { 406 efx_nic_t *enp = eep->ee_enp; 407 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 408 efx_dword_t dword; 409 uint32_t timer_val, mode; 410 efx_rc_t rc; 411 412 if (us > encp->enc_evq_timer_max_us) { 413 rc = EINVAL; 414 goto fail1; 415 } 416 417 /* If the value is zero then disable the timer */ 418 if (us == 0) { 419 timer_val = 0; 420 mode = FFE_CZ_TIMER_MODE_DIS; 421 } else { 422 /* Calculate the timer value in quanta */ 423 timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns; 424 425 /* Moderation value is base 0 so we need to deduct 1 */ 426 if (timer_val > 0) 427 timer_val--; 428 429 mode = FFE_CZ_TIMER_MODE_INT_HLDOFF; 430 } 431 432 if (encp->enc_bug35388_workaround) { 433 EFX_POPULATE_DWORD_3(dword, 434 ERF_DD_EVQ_IND_TIMER_FLAGS, 435 EFE_DD_EVQ_IND_TIMER_FLAGS, 436 ERF_DD_EVQ_IND_TIMER_MODE, mode, 437 ERF_DD_EVQ_IND_TIMER_VAL, timer_val); 438 EFX_BAR_TBL_WRITED(enp, ER_DD_EVQ_INDIRECT, 439 eep->ee_index, &dword, 0); 440 } else { 441 EFX_POPULATE_DWORD_2(dword, 442 ERF_DZ_TC_TIMER_MODE, mode, 443 ERF_DZ_TC_TIMER_VAL, timer_val); 444 EFX_BAR_TBL_WRITED(enp, ER_DZ_EVQ_TMR_REG, 445 eep->ee_index, &dword, 0); 446 } 447 448 return (0); 449 450 fail1: 451 EFSYS_PROBE1(fail1, efx_rc_t, rc); 452 453 return (rc); 454 } 455 456 457 #if EFSYS_OPT_QSTATS 458 void 459 ef10_ev_qstats_update( 460 __in efx_evq_t *eep, 461 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 462 { 463 unsigned int id; 464 465 for (id = 0; id < EV_NQSTATS; id++) { 466 efsys_stat_t *essp = &stat[id]; 467 468 EFSYS_STAT_INCR(essp, eep->ee_stat[id]); 469 eep->ee_stat[id] = 0; 470 } 471 } 472 #endif /* EFSYS_OPT_QSTATS */ 473 474 475 static __checkReturn boolean_t 476 ef10_ev_rx( 477 __in efx_evq_t *eep, 478 __in efx_qword_t *eqp, 479 __in const efx_ev_callbacks_t *eecp, 480 __in_opt void *arg) 481 { 482 efx_nic_t *enp = eep->ee_enp; 483 uint32_t size; 484 uint32_t label; 485 uint32_t mac_class; 486 uint32_t eth_tag_class; 487 uint32_t l3_class; 488 uint32_t l4_class; 489 uint32_t next_read_lbits; 490 uint16_t flags; 491 boolean_t cont; 492 boolean_t should_abort; 493 efx_evq_rxq_state_t *eersp; 494 unsigned int desc_count; 495 unsigned int last_used_id; 496 497 EFX_EV_QSTAT_INCR(eep, EV_RX); 498 499 /* Discard events after RXQ/TXQ errors */ 500 if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR)) 501 return (B_FALSE); 502 503 /* Basic packet information */ 504 size = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_BYTES); 505 next_read_lbits = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DSC_PTR_LBITS); 506 label = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_QLABEL); 507 eth_tag_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ETH_TAG_CLASS); 508 mac_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_MAC_CLASS); 509 l3_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L3_CLASS); 510 l4_class = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_L4_CLASS); 511 cont = EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_CONT); 512 513 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_DROP_EVENT) != 0) { 514 /* Drop this event */ 515 return (B_FALSE); 516 } 517 flags = 0; 518 519 if (cont != 0) { 520 /* 521 * This may be part of a scattered frame, or it may be a 522 * truncated frame if scatter is disabled on this RXQ. 523 * Overlength frames can be received if e.g. a VF is configured 524 * for 1500 MTU but connected to a port set to 9000 MTU 525 * (see bug56567). 526 * FIXME: There is not yet any driver that supports scatter on 527 * Huntington. Scatter support is required for OSX. 528 */ 529 flags |= EFX_PKT_CONT; 530 } 531 532 if (mac_class == ESE_DZ_MAC_CLASS_UCAST) 533 flags |= EFX_PKT_UNICAST; 534 535 /* Increment the count of descriptors read */ 536 eersp = &eep->ee_rxq_state[label]; 537 desc_count = (next_read_lbits - eersp->eers_rx_read_ptr) & 538 EFX_MASK32(ESF_DZ_RX_DSC_PTR_LBITS); 539 eersp->eers_rx_read_ptr += desc_count; 540 541 /* 542 * FIXME: add error checking to make sure this a batched event. 543 * This could also be an aborted scatter, see Bug36629. 544 */ 545 if (desc_count > 1) { 546 EFX_EV_QSTAT_INCR(eep, EV_RX_BATCH); 547 flags |= EFX_PKT_PREFIX_LEN; 548 } 549 550 /* Calculate the index of the last descriptor consumed */ 551 last_used_id = (eersp->eers_rx_read_ptr - 1) & eersp->eers_rx_mask; 552 553 /* Check for errors that invalidate checksum and L3/L4 fields */ 554 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECC_ERR) != 0) { 555 /* RX frame truncated (error flag is misnamed) */ 556 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); 557 flags |= EFX_DISCARD; 558 goto deliver; 559 } 560 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_ECRC_ERR) != 0) { 561 /* Bad Ethernet frame CRC */ 562 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); 563 flags |= EFX_DISCARD; 564 goto deliver; 565 } 566 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_PARSE_INCOMPLETE)) { 567 /* 568 * Hardware parse failed, due to malformed headers 569 * or headers that are too long for the parser. 570 * Headers and checksums must be validated by the host. 571 */ 572 // TODO: EFX_EV_QSTAT_INCR(eep, EV_RX_PARSE_INCOMPLETE); 573 goto deliver; 574 } 575 576 if ((eth_tag_class == ESE_DZ_ETH_TAG_CLASS_VLAN1) || 577 (eth_tag_class == ESE_DZ_ETH_TAG_CLASS_VLAN2)) { 578 flags |= EFX_PKT_VLAN_TAGGED; 579 } 580 581 switch (l3_class) { 582 case ESE_DZ_L3_CLASS_IP4: 583 case ESE_DZ_L3_CLASS_IP4_FRAG: 584 flags |= EFX_PKT_IPV4; 585 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_IPCKSUM_ERR)) { 586 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); 587 } else { 588 flags |= EFX_CKSUM_IPV4; 589 } 590 591 if (l4_class == ESE_DZ_L4_CLASS_TCP) { 592 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); 593 flags |= EFX_PKT_TCP; 594 } else if (l4_class == ESE_DZ_L4_CLASS_UDP) { 595 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); 596 flags |= EFX_PKT_UDP; 597 } else { 598 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); 599 } 600 break; 601 602 case ESE_DZ_L3_CLASS_IP6: 603 case ESE_DZ_L3_CLASS_IP6_FRAG: 604 flags |= EFX_PKT_IPV6; 605 606 if (l4_class == ESE_DZ_L4_CLASS_TCP) { 607 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); 608 flags |= EFX_PKT_TCP; 609 } else if (l4_class == ESE_DZ_L4_CLASS_UDP) { 610 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); 611 flags |= EFX_PKT_UDP; 612 } else { 613 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); 614 } 615 break; 616 617 default: 618 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); 619 break; 620 } 621 622 if (flags & (EFX_PKT_TCP | EFX_PKT_UDP)) { 623 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_RX_TCPUDP_CKSUM_ERR)) { 624 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); 625 } else { 626 flags |= EFX_CKSUM_TCPUDP; 627 } 628 } 629 630 deliver: 631 /* If we're not discarding the packet then it is ok */ 632 if (~flags & EFX_DISCARD) 633 EFX_EV_QSTAT_INCR(eep, EV_RX_OK); 634 635 EFSYS_ASSERT(eecp->eec_rx != NULL); 636 should_abort = eecp->eec_rx(arg, label, last_used_id, size, flags); 637 638 return (should_abort); 639 } 640 641 static __checkReturn boolean_t 642 ef10_ev_tx( 643 __in efx_evq_t *eep, 644 __in efx_qword_t *eqp, 645 __in const efx_ev_callbacks_t *eecp, 646 __in_opt void *arg) 647 { 648 efx_nic_t *enp = eep->ee_enp; 649 uint32_t id; 650 uint32_t label; 651 boolean_t should_abort; 652 653 EFX_EV_QSTAT_INCR(eep, EV_TX); 654 655 /* Discard events after RXQ/TXQ errors */ 656 if (enp->en_reset_flags & (EFX_RESET_RXQ_ERR | EFX_RESET_TXQ_ERR)) 657 return (B_FALSE); 658 659 if (EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DROP_EVENT) != 0) { 660 /* Drop this event */ 661 return (B_FALSE); 662 } 663 664 /* Per-packet TX completion (was per-descriptor for Falcon/Siena) */ 665 id = EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_DESCR_INDX); 666 label = EFX_QWORD_FIELD(*eqp, ESF_DZ_TX_QLABEL); 667 668 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); 669 670 EFSYS_ASSERT(eecp->eec_tx != NULL); 671 should_abort = eecp->eec_tx(arg, label, id); 672 673 return (should_abort); 674 } 675 676 static __checkReturn boolean_t 677 ef10_ev_driver( 678 __in efx_evq_t *eep, 679 __in efx_qword_t *eqp, 680 __in const efx_ev_callbacks_t *eecp, 681 __in_opt void *arg) 682 { 683 unsigned int code; 684 boolean_t should_abort; 685 686 EFX_EV_QSTAT_INCR(eep, EV_DRIVER); 687 should_abort = B_FALSE; 688 689 code = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_SUB_CODE); 690 switch (code) { 691 case ESE_DZ_DRV_TIMER_EV: { 692 uint32_t id; 693 694 id = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_TMR_ID); 695 696 EFSYS_ASSERT(eecp->eec_timer != NULL); 697 should_abort = eecp->eec_timer(arg, id); 698 break; 699 } 700 701 case ESE_DZ_DRV_WAKE_UP_EV: { 702 uint32_t id; 703 704 id = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_EVQ_ID); 705 706 EFSYS_ASSERT(eecp->eec_wake_up != NULL); 707 should_abort = eecp->eec_wake_up(arg, id); 708 break; 709 } 710 711 case ESE_DZ_DRV_START_UP_EV: 712 EFSYS_ASSERT(eecp->eec_initialized != NULL); 713 should_abort = eecp->eec_initialized(arg); 714 break; 715 716 default: 717 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 718 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 719 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 720 break; 721 } 722 723 return (should_abort); 724 } 725 726 static __checkReturn boolean_t 727 ef10_ev_drv_gen( 728 __in efx_evq_t *eep, 729 __in efx_qword_t *eqp, 730 __in const efx_ev_callbacks_t *eecp, 731 __in_opt void *arg) 732 { 733 uint32_t data; 734 boolean_t should_abort; 735 736 EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); 737 should_abort = B_FALSE; 738 739 data = EFX_QWORD_FIELD(*eqp, ESF_DZ_DRV_SUB_DATA_DW0); 740 if (data >= ((uint32_t)1 << 16)) { 741 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 742 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 743 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 744 745 return (B_TRUE); 746 } 747 748 EFSYS_ASSERT(eecp->eec_software != NULL); 749 should_abort = eecp->eec_software(arg, (uint16_t)data); 750 751 return (should_abort); 752 } 753 754 static __checkReturn boolean_t 755 ef10_ev_mcdi( 756 __in efx_evq_t *eep, 757 __in efx_qword_t *eqp, 758 __in const efx_ev_callbacks_t *eecp, 759 __in_opt void *arg) 760 { 761 efx_nic_t *enp = eep->ee_enp; 762 unsigned code; 763 boolean_t should_abort = B_FALSE; 764 765 EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); 766 767 code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); 768 switch (code) { 769 case MCDI_EVENT_CODE_BADSSERT: 770 efx_mcdi_ev_death(enp, EINTR); 771 break; 772 773 case MCDI_EVENT_CODE_CMDDONE: 774 efx_mcdi_ev_cpl(enp, 775 MCDI_EV_FIELD(eqp, CMDDONE_SEQ), 776 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), 777 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); 778 break; 779 780 #if EFSYS_OPT_MCDI_PROXY_AUTH 781 case MCDI_EVENT_CODE_PROXY_RESPONSE: 782 /* 783 * This event notifies a function that an authorization request 784 * has been processed. If the request was authorized then the 785 * function can now re-send the original MCDI request. 786 * See SF-113652-SW "SR-IOV Proxied Network Access Control". 787 */ 788 efx_mcdi_ev_proxy_response(enp, 789 MCDI_EV_FIELD(eqp, PROXY_RESPONSE_HANDLE), 790 MCDI_EV_FIELD(eqp, PROXY_RESPONSE_RC)); 791 break; 792 #endif /* EFSYS_OPT_MCDI_PROXY_AUTH */ 793 794 case MCDI_EVENT_CODE_LINKCHANGE: { 795 efx_link_mode_t link_mode; 796 797 ef10_phy_link_ev(enp, eqp, &link_mode); 798 should_abort = eecp->eec_link_change(arg, link_mode); 799 break; 800 } 801 802 case MCDI_EVENT_CODE_SENSOREVT: { 803 #if EFSYS_OPT_MON_STATS 804 efx_mon_stat_t id; 805 efx_mon_stat_value_t value; 806 efx_rc_t rc; 807 808 /* Decode monitor stat for MCDI sensor (if supported) */ 809 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) { 810 /* Report monitor stat change */ 811 should_abort = eecp->eec_monitor(arg, id, value); 812 } else if (rc == ENOTSUP) { 813 should_abort = eecp->eec_exception(arg, 814 EFX_EXCEPTION_UNKNOWN_SENSOREVT, 815 MCDI_EV_FIELD(eqp, DATA)); 816 } else { 817 EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ 818 } 819 #endif 820 break; 821 } 822 823 case MCDI_EVENT_CODE_SCHEDERR: 824 /* Informational only */ 825 break; 826 827 case MCDI_EVENT_CODE_REBOOT: 828 /* Falcon/Siena only (should not been seen with Huntington). */ 829 efx_mcdi_ev_death(enp, EIO); 830 break; 831 832 case MCDI_EVENT_CODE_MC_REBOOT: 833 /* MC_REBOOT event is used for Huntington (EF10) and later. */ 834 efx_mcdi_ev_death(enp, EIO); 835 break; 836 837 case MCDI_EVENT_CODE_MAC_STATS_DMA: 838 #if EFSYS_OPT_MAC_STATS 839 if (eecp->eec_mac_stats != NULL) { 840 eecp->eec_mac_stats(arg, 841 MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); 842 } 843 #endif 844 break; 845 846 case MCDI_EVENT_CODE_FWALERT: { 847 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); 848 849 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) 850 should_abort = eecp->eec_exception(arg, 851 EFX_EXCEPTION_FWALERT_SRAM, 852 MCDI_EV_FIELD(eqp, FWALERT_DATA)); 853 else 854 should_abort = eecp->eec_exception(arg, 855 EFX_EXCEPTION_UNKNOWN_FWALERT, 856 MCDI_EV_FIELD(eqp, DATA)); 857 break; 858 } 859 860 case MCDI_EVENT_CODE_TX_ERR: { 861 /* 862 * After a TXQ error is detected, firmware sends a TX_ERR event. 863 * This may be followed by TX completions (which we discard), 864 * and then finally by a TX_FLUSH event. Firmware destroys the 865 * TXQ automatically after sending the TX_FLUSH event. 866 */ 867 enp->en_reset_flags |= EFX_RESET_TXQ_ERR; 868 869 EFSYS_PROBE2(tx_descq_err, 870 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 871 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 872 873 /* Inform the driver that a reset is required. */ 874 eecp->eec_exception(arg, EFX_EXCEPTION_TX_ERROR, 875 MCDI_EV_FIELD(eqp, TX_ERR_DATA)); 876 break; 877 } 878 879 case MCDI_EVENT_CODE_TX_FLUSH: { 880 uint32_t txq_index = MCDI_EV_FIELD(eqp, TX_FLUSH_TXQ); 881 882 /* 883 * EF10 firmware sends two TX_FLUSH events: one to the txq's 884 * event queue, and one to evq 0 (with TX_FLUSH_TO_DRIVER set). 885 * We want to wait for all completions, so ignore the events 886 * with TX_FLUSH_TO_DRIVER. 887 */ 888 if (MCDI_EV_FIELD(eqp, TX_FLUSH_TO_DRIVER) != 0) { 889 should_abort = B_FALSE; 890 break; 891 } 892 893 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); 894 895 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); 896 897 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); 898 should_abort = eecp->eec_txq_flush_done(arg, txq_index); 899 break; 900 } 901 902 case MCDI_EVENT_CODE_RX_ERR: { 903 /* 904 * After an RXQ error is detected, firmware sends an RX_ERR 905 * event. This may be followed by RX events (which we discard), 906 * and then finally by an RX_FLUSH event. Firmware destroys the 907 * RXQ automatically after sending the RX_FLUSH event. 908 */ 909 enp->en_reset_flags |= EFX_RESET_RXQ_ERR; 910 911 EFSYS_PROBE2(rx_descq_err, 912 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 913 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 914 915 /* Inform the driver that a reset is required. */ 916 eecp->eec_exception(arg, EFX_EXCEPTION_RX_ERROR, 917 MCDI_EV_FIELD(eqp, RX_ERR_DATA)); 918 break; 919 } 920 921 case MCDI_EVENT_CODE_RX_FLUSH: { 922 uint32_t rxq_index = MCDI_EV_FIELD(eqp, RX_FLUSH_RXQ); 923 924 /* 925 * EF10 firmware sends two RX_FLUSH events: one to the rxq's 926 * event queue, and one to evq 0 (with RX_FLUSH_TO_DRIVER set). 927 * We want to wait for all completions, so ignore the events 928 * with RX_FLUSH_TO_DRIVER. 929 */ 930 if (MCDI_EV_FIELD(eqp, RX_FLUSH_TO_DRIVER) != 0) { 931 should_abort = B_FALSE; 932 break; 933 } 934 935 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); 936 937 EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); 938 939 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); 940 should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); 941 break; 942 } 943 944 default: 945 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 946 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 947 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 948 break; 949 } 950 951 return (should_abort); 952 } 953 954 void 955 ef10_ev_rxlabel_init( 956 __in efx_evq_t *eep, 957 __in efx_rxq_t *erp, 958 __in unsigned int label) 959 { 960 efx_evq_rxq_state_t *eersp; 961 962 EFSYS_ASSERT3U(label, <, EFX_ARRAY_SIZE(eep->ee_rxq_state)); 963 eersp = &eep->ee_rxq_state[label]; 964 965 EFSYS_ASSERT3U(eersp->eers_rx_mask, ==, 0); 966 967 eersp->eers_rx_read_ptr = 0; 968 eersp->eers_rx_mask = erp->er_mask; 969 } 970 971 void 972 ef10_ev_rxlabel_fini( 973 __in efx_evq_t *eep, 974 __in unsigned int label) 975 { 976 efx_evq_rxq_state_t *eersp; 977 978 EFSYS_ASSERT3U(label, <, EFX_ARRAY_SIZE(eep->ee_rxq_state)); 979 eersp = &eep->ee_rxq_state[label]; 980 981 EFSYS_ASSERT3U(eersp->eers_rx_mask, !=, 0); 982 983 eersp->eers_rx_read_ptr = 0; 984 eersp->eers_rx_mask = 0; 985 } 986 987 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 988