1 /* 2 * Copyright (c) 2007-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions are met: 7 * 8 * 1. Redistributions of source code must retain the above copyright notice, 9 * this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright notice, 11 * this list of conditions and the following disclaimer in the documentation 12 * and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 16 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 17 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 18 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 19 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 20 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 21 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 22 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 23 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 24 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * The views and conclusions contained in the software and documentation are 27 * those of the authors and should not be interpreted as representing official 28 * policies, either expressed or implied, of the FreeBSD Project. 29 */ 30 31 #include "efx.h" 32 #include "efx_impl.h" 33 #if EFSYS_OPT_MON_MCDI 34 #include "mcdi_mon.h" 35 #endif 36 37 #if EFSYS_OPT_QSTATS 38 #define EFX_EV_QSTAT_INCR(_eep, _stat) \ 39 do { \ 40 (_eep)->ee_stat[_stat]++; \ 41 _NOTE(CONSTANTCONDITION) \ 42 } while (B_FALSE) 43 #else 44 #define EFX_EV_QSTAT_INCR(_eep, _stat) 45 #endif 46 47 #define EFX_EV_PRESENT(_qword) \ 48 (EFX_QWORD_FIELD((_qword), EFX_DWORD_0) != 0xffffffff && \ 49 EFX_QWORD_FIELD((_qword), EFX_DWORD_1) != 0xffffffff) 50 51 52 53 #if EFSYS_OPT_SIENA 54 55 static __checkReturn efx_rc_t 56 siena_ev_init( 57 __in efx_nic_t *enp); 58 59 static void 60 siena_ev_fini( 61 __in efx_nic_t *enp); 62 63 static __checkReturn efx_rc_t 64 siena_ev_qcreate( 65 __in efx_nic_t *enp, 66 __in unsigned int index, 67 __in efsys_mem_t *esmp, 68 __in size_t n, 69 __in uint32_t id, 70 __in efx_evq_t *eep); 71 72 static void 73 siena_ev_qdestroy( 74 __in efx_evq_t *eep); 75 76 static __checkReturn efx_rc_t 77 siena_ev_qprime( 78 __in efx_evq_t *eep, 79 __in unsigned int count); 80 81 static void 82 siena_ev_qpoll( 83 __in efx_evq_t *eep, 84 __inout unsigned int *countp, 85 __in const efx_ev_callbacks_t *eecp, 86 __in_opt void *arg); 87 88 static void 89 siena_ev_qpost( 90 __in efx_evq_t *eep, 91 __in uint16_t data); 92 93 static __checkReturn efx_rc_t 94 siena_ev_qmoderate( 95 __in efx_evq_t *eep, 96 __in unsigned int us); 97 98 #if EFSYS_OPT_QSTATS 99 static void 100 siena_ev_qstats_update( 101 __in efx_evq_t *eep, 102 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat); 103 104 #endif 105 106 #endif /* EFSYS_OPT_SIENA */ 107 108 #if EFSYS_OPT_SIENA 109 static const efx_ev_ops_t __efx_ev_siena_ops = { 110 siena_ev_init, /* eevo_init */ 111 siena_ev_fini, /* eevo_fini */ 112 siena_ev_qcreate, /* eevo_qcreate */ 113 siena_ev_qdestroy, /* eevo_qdestroy */ 114 siena_ev_qprime, /* eevo_qprime */ 115 siena_ev_qpost, /* eevo_qpost */ 116 siena_ev_qmoderate, /* eevo_qmoderate */ 117 #if EFSYS_OPT_QSTATS 118 siena_ev_qstats_update, /* eevo_qstats_update */ 119 #endif 120 }; 121 #endif /* EFSYS_OPT_SIENA */ 122 123 #if EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD 124 static const efx_ev_ops_t __efx_ev_ef10_ops = { 125 ef10_ev_init, /* eevo_init */ 126 ef10_ev_fini, /* eevo_fini */ 127 ef10_ev_qcreate, /* eevo_qcreate */ 128 ef10_ev_qdestroy, /* eevo_qdestroy */ 129 ef10_ev_qprime, /* eevo_qprime */ 130 ef10_ev_qpost, /* eevo_qpost */ 131 ef10_ev_qmoderate, /* eevo_qmoderate */ 132 #if EFSYS_OPT_QSTATS 133 ef10_ev_qstats_update, /* eevo_qstats_update */ 134 #endif 135 }; 136 #endif /* EFSYS_OPT_HUNTINGTON || EFSYS_OPT_MEDFORD */ 137 138 139 __checkReturn efx_rc_t 140 efx_ev_init( 141 __in efx_nic_t *enp) 142 { 143 const efx_ev_ops_t *eevop; 144 efx_rc_t rc; 145 146 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 147 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 148 149 if (enp->en_mod_flags & EFX_MOD_EV) { 150 rc = EINVAL; 151 goto fail1; 152 } 153 154 switch (enp->en_family) { 155 #if EFSYS_OPT_SIENA 156 case EFX_FAMILY_SIENA: 157 eevop = &__efx_ev_siena_ops; 158 break; 159 #endif /* EFSYS_OPT_SIENA */ 160 161 #if EFSYS_OPT_HUNTINGTON 162 case EFX_FAMILY_HUNTINGTON: 163 eevop = &__efx_ev_ef10_ops; 164 break; 165 #endif /* EFSYS_OPT_HUNTINGTON */ 166 167 #if EFSYS_OPT_MEDFORD 168 case EFX_FAMILY_MEDFORD: 169 eevop = &__efx_ev_ef10_ops; 170 break; 171 #endif /* EFSYS_OPT_MEDFORD */ 172 173 default: 174 EFSYS_ASSERT(0); 175 rc = ENOTSUP; 176 goto fail1; 177 } 178 179 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 180 181 if ((rc = eevop->eevo_init(enp)) != 0) 182 goto fail2; 183 184 enp->en_eevop = eevop; 185 enp->en_mod_flags |= EFX_MOD_EV; 186 return (0); 187 188 fail2: 189 EFSYS_PROBE(fail2); 190 191 fail1: 192 EFSYS_PROBE1(fail1, efx_rc_t, rc); 193 194 enp->en_eevop = NULL; 195 enp->en_mod_flags &= ~EFX_MOD_EV; 196 return (rc); 197 } 198 199 void 200 efx_ev_fini( 201 __in efx_nic_t *enp) 202 { 203 const efx_ev_ops_t *eevop = enp->en_eevop; 204 205 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 206 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_INTR); 207 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 208 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_RX)); 209 EFSYS_ASSERT(!(enp->en_mod_flags & EFX_MOD_TX)); 210 EFSYS_ASSERT3U(enp->en_ev_qcount, ==, 0); 211 212 eevop->eevo_fini(enp); 213 214 enp->en_eevop = NULL; 215 enp->en_mod_flags &= ~EFX_MOD_EV; 216 } 217 218 219 __checkReturn efx_rc_t 220 efx_ev_qcreate( 221 __in efx_nic_t *enp, 222 __in unsigned int index, 223 __in efsys_mem_t *esmp, 224 __in size_t n, 225 __in uint32_t id, 226 __deref_out efx_evq_t **eepp) 227 { 228 const efx_ev_ops_t *eevop = enp->en_eevop; 229 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 230 efx_evq_t *eep; 231 efx_rc_t rc; 232 233 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 234 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_EV); 235 236 EFSYS_ASSERT3U(enp->en_ev_qcount + 1, <, encp->enc_evq_limit); 237 238 /* Allocate an EVQ object */ 239 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_evq_t), eep); 240 if (eep == NULL) { 241 rc = ENOMEM; 242 goto fail1; 243 } 244 245 eep->ee_magic = EFX_EVQ_MAGIC; 246 eep->ee_enp = enp; 247 eep->ee_index = index; 248 eep->ee_mask = n - 1; 249 eep->ee_esmp = esmp; 250 251 if ((rc = eevop->eevo_qcreate(enp, index, esmp, n, id, eep)) != 0) 252 goto fail2; 253 254 enp->en_ev_qcount++; 255 *eepp = eep; 256 257 return (0); 258 259 fail2: 260 EFSYS_PROBE(fail2); 261 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 262 fail1: 263 EFSYS_PROBE1(fail1, efx_rc_t, rc); 264 return (rc); 265 } 266 267 void 268 efx_ev_qdestroy( 269 __in efx_evq_t *eep) 270 { 271 efx_nic_t *enp = eep->ee_enp; 272 const efx_ev_ops_t *eevop = enp->en_eevop; 273 274 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 275 276 EFSYS_ASSERT(enp->en_ev_qcount != 0); 277 --enp->en_ev_qcount; 278 279 eevop->eevo_qdestroy(eep); 280 281 /* Free the EVQ object */ 282 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_evq_t), eep); 283 } 284 285 __checkReturn efx_rc_t 286 efx_ev_qprime( 287 __in efx_evq_t *eep, 288 __in unsigned int count) 289 { 290 efx_nic_t *enp = eep->ee_enp; 291 const efx_ev_ops_t *eevop = enp->en_eevop; 292 efx_rc_t rc; 293 294 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 295 296 if (!(enp->en_mod_flags & EFX_MOD_INTR)) { 297 rc = EINVAL; 298 goto fail1; 299 } 300 301 if ((rc = eevop->eevo_qprime(eep, count)) != 0) 302 goto fail2; 303 304 return (0); 305 306 fail2: 307 EFSYS_PROBE(fail2); 308 fail1: 309 EFSYS_PROBE1(fail1, efx_rc_t, rc); 310 return (rc); 311 } 312 313 __checkReturn boolean_t 314 efx_ev_qpending( 315 __in efx_evq_t *eep, 316 __in unsigned int count) 317 { 318 size_t offset; 319 efx_qword_t qword; 320 321 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 322 323 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 324 EFSYS_MEM_READQ(eep->ee_esmp, offset, &qword); 325 326 return (EFX_EV_PRESENT(qword)); 327 } 328 329 #if EFSYS_OPT_EV_PREFETCH 330 331 void 332 efx_ev_qprefetch( 333 __in efx_evq_t *eep, 334 __in unsigned int count) 335 { 336 efx_nic_t *enp = eep->ee_enp; 337 unsigned int offset; 338 339 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 340 341 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 342 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 343 } 344 345 #endif /* EFSYS_OPT_EV_PREFETCH */ 346 347 void 348 efx_ev_qpoll( 349 __in efx_evq_t *eep, 350 __inout unsigned int *countp, 351 __in const efx_ev_callbacks_t *eecp, 352 __in_opt void *arg) 353 { 354 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 355 356 /* 357 * FIXME: Huntington will require support for hardware event batching 358 * and merging, which will need a different ev_qpoll implementation. 359 * 360 * Without those features the Falcon/Siena code can be used unchanged. 361 */ 362 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_LBN == FSF_AZ_EV_CODE_LBN); 363 EFX_STATIC_ASSERT(ESF_DZ_EV_CODE_WIDTH == FSF_AZ_EV_CODE_WIDTH); 364 365 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_RX_EV == FSE_AZ_EV_CODE_RX_EV); 366 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_TX_EV == FSE_AZ_EV_CODE_TX_EV); 367 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRIVER_EV == FSE_AZ_EV_CODE_DRIVER_EV); 368 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_DRV_GEN_EV == 369 FSE_AZ_EV_CODE_DRV_GEN_EV); 370 #if EFSYS_OPT_MCDI 371 EFX_STATIC_ASSERT(ESE_DZ_EV_CODE_MCDI_EV == 372 FSE_AZ_EV_CODE_MCDI_EVRESPONSE); 373 #endif 374 siena_ev_qpoll(eep, countp, eecp, arg); 375 } 376 377 void 378 efx_ev_qpost( 379 __in efx_evq_t *eep, 380 __in uint16_t data) 381 { 382 efx_nic_t *enp = eep->ee_enp; 383 const efx_ev_ops_t *eevop = enp->en_eevop; 384 385 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 386 387 EFSYS_ASSERT(eevop != NULL && 388 eevop->eevo_qpost != NULL); 389 390 eevop->eevo_qpost(eep, data); 391 } 392 393 __checkReturn efx_rc_t 394 efx_ev_qmoderate( 395 __in efx_evq_t *eep, 396 __in unsigned int us) 397 { 398 efx_nic_t *enp = eep->ee_enp; 399 const efx_ev_ops_t *eevop = enp->en_eevop; 400 efx_rc_t rc; 401 402 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 403 404 if ((rc = eevop->eevo_qmoderate(eep, us)) != 0) 405 goto fail1; 406 407 return (0); 408 409 fail1: 410 EFSYS_PROBE1(fail1, efx_rc_t, rc); 411 return (rc); 412 } 413 414 #if EFSYS_OPT_QSTATS 415 void 416 efx_ev_qstats_update( 417 __in efx_evq_t *eep, 418 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 419 420 { efx_nic_t *enp = eep->ee_enp; 421 const efx_ev_ops_t *eevop = enp->en_eevop; 422 423 EFSYS_ASSERT3U(eep->ee_magic, ==, EFX_EVQ_MAGIC); 424 425 eevop->eevo_qstats_update(eep, stat); 426 } 427 428 #endif /* EFSYS_OPT_QSTATS */ 429 430 #if EFSYS_OPT_SIENA 431 432 static __checkReturn efx_rc_t 433 siena_ev_init( 434 __in efx_nic_t *enp) 435 { 436 efx_oword_t oword; 437 438 /* 439 * Program the event queue for receive and transmit queue 440 * flush events. 441 */ 442 EFX_BAR_READO(enp, FR_AZ_DP_CTRL_REG, &oword); 443 EFX_SET_OWORD_FIELD(oword, FRF_AZ_FLS_EVQ_ID, 0); 444 EFX_BAR_WRITEO(enp, FR_AZ_DP_CTRL_REG, &oword); 445 446 return (0); 447 448 } 449 450 static __checkReturn boolean_t 451 siena_ev_rx_not_ok( 452 __in efx_evq_t *eep, 453 __in efx_qword_t *eqp, 454 __in uint32_t label, 455 __in uint32_t id, 456 __inout uint16_t *flagsp) 457 { 458 boolean_t ignore = B_FALSE; 459 460 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TOBE_DISC) != 0) { 461 EFX_EV_QSTAT_INCR(eep, EV_RX_TOBE_DISC); 462 EFSYS_PROBE(tobe_disc); 463 /* 464 * Assume this is a unicast address mismatch, unless below 465 * we find either FSF_AZ_RX_EV_ETH_CRC_ERR or 466 * EV_RX_PAUSE_FRM_ERR is set. 467 */ 468 (*flagsp) |= EFX_ADDR_MISMATCH; 469 } 470 471 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_FRM_TRUNC) != 0) { 472 EFSYS_PROBE2(frm_trunc, uint32_t, label, uint32_t, id); 473 EFX_EV_QSTAT_INCR(eep, EV_RX_FRM_TRUNC); 474 (*flagsp) |= EFX_DISCARD; 475 476 #if EFSYS_OPT_RX_SCATTER 477 /* 478 * Lookout for payload queue ran dry errors and ignore them. 479 * 480 * Sadly for the header/data split cases, the descriptor 481 * pointer in this event refers to the header queue and 482 * therefore cannot be easily detected as duplicate. 483 * So we drop these and rely on the receive processing seeing 484 * a subsequent packet with FSF_AZ_RX_EV_SOP set to discard 485 * the partially received packet. 486 */ 487 if ((EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) == 0) && 488 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) == 0) && 489 (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT) == 0)) 490 ignore = B_TRUE; 491 #endif /* EFSYS_OPT_RX_SCATTER */ 492 } 493 494 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_ETH_CRC_ERR) != 0) { 495 EFX_EV_QSTAT_INCR(eep, EV_RX_ETH_CRC_ERR); 496 EFSYS_PROBE(crc_err); 497 (*flagsp) &= ~EFX_ADDR_MISMATCH; 498 (*flagsp) |= EFX_DISCARD; 499 } 500 501 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PAUSE_FRM_ERR) != 0) { 502 EFX_EV_QSTAT_INCR(eep, EV_RX_PAUSE_FRM_ERR); 503 EFSYS_PROBE(pause_frm_err); 504 (*flagsp) &= ~EFX_ADDR_MISMATCH; 505 (*flagsp) |= EFX_DISCARD; 506 } 507 508 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BUF_OWNER_ID_ERR) != 0) { 509 EFX_EV_QSTAT_INCR(eep, EV_RX_BUF_OWNER_ID_ERR); 510 EFSYS_PROBE(owner_id_err); 511 (*flagsp) |= EFX_DISCARD; 512 } 513 514 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_HDR_CHKSUM_ERR) != 0) { 515 EFX_EV_QSTAT_INCR(eep, EV_RX_IPV4_HDR_CHKSUM_ERR); 516 EFSYS_PROBE(ipv4_err); 517 (*flagsp) &= ~EFX_CKSUM_IPV4; 518 } 519 520 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_TCP_UDP_CHKSUM_ERR) != 0) { 521 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_UDP_CHKSUM_ERR); 522 EFSYS_PROBE(udp_chk_err); 523 (*flagsp) &= ~EFX_CKSUM_TCPUDP; 524 } 525 526 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_IP_FRAG_ERR) != 0) { 527 EFX_EV_QSTAT_INCR(eep, EV_RX_IP_FRAG_ERR); 528 529 /* 530 * If IP is fragmented FSF_AZ_RX_EV_IP_FRAG_ERR is set. This 531 * causes FSF_AZ_RX_EV_PKT_OK to be clear. This is not an error 532 * condition. 533 */ 534 (*flagsp) &= ~(EFX_PKT_TCP | EFX_PKT_UDP | EFX_CKSUM_TCPUDP); 535 } 536 537 return (ignore); 538 } 539 540 static __checkReturn boolean_t 541 siena_ev_rx( 542 __in efx_evq_t *eep, 543 __in efx_qword_t *eqp, 544 __in const efx_ev_callbacks_t *eecp, 545 __in_opt void *arg) 546 { 547 uint32_t id; 548 uint32_t size; 549 uint32_t label; 550 boolean_t ok; 551 #if EFSYS_OPT_RX_SCATTER 552 boolean_t sop; 553 boolean_t jumbo_cont; 554 #endif /* EFSYS_OPT_RX_SCATTER */ 555 uint32_t hdr_type; 556 boolean_t is_v6; 557 uint16_t flags; 558 boolean_t ignore; 559 boolean_t should_abort; 560 561 EFX_EV_QSTAT_INCR(eep, EV_RX); 562 563 /* Basic packet information */ 564 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_DESC_PTR); 565 size = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_BYTE_CNT); 566 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_Q_LABEL); 567 ok = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_OK) != 0); 568 569 #if EFSYS_OPT_RX_SCATTER 570 sop = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_SOP) != 0); 571 jumbo_cont = (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_JUMBO_CONT) != 0); 572 #endif /* EFSYS_OPT_RX_SCATTER */ 573 574 hdr_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_HDR_TYPE); 575 576 is_v6 = (EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_IPV6_PKT) != 0); 577 578 /* 579 * If packet is marked as OK and packet type is TCP/IP or 580 * UDP/IP or other IP, then we can rely on the hardware checksums. 581 */ 582 switch (hdr_type) { 583 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_TCP: 584 flags = EFX_PKT_TCP | EFX_CKSUM_TCPUDP; 585 if (is_v6) { 586 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV6); 587 flags |= EFX_PKT_IPV6; 588 } else { 589 EFX_EV_QSTAT_INCR(eep, EV_RX_TCP_IPV4); 590 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 591 } 592 break; 593 594 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_UDP: 595 flags = EFX_PKT_UDP | EFX_CKSUM_TCPUDP; 596 if (is_v6) { 597 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV6); 598 flags |= EFX_PKT_IPV6; 599 } else { 600 EFX_EV_QSTAT_INCR(eep, EV_RX_UDP_IPV4); 601 flags |= EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 602 } 603 break; 604 605 case FSE_AZ_RX_EV_HDR_TYPE_IPV4V6_OTHER: 606 if (is_v6) { 607 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV6); 608 flags = EFX_PKT_IPV6; 609 } else { 610 EFX_EV_QSTAT_INCR(eep, EV_RX_OTHER_IPV4); 611 flags = EFX_PKT_IPV4 | EFX_CKSUM_IPV4; 612 } 613 break; 614 615 case FSE_AZ_RX_EV_HDR_TYPE_OTHER: 616 EFX_EV_QSTAT_INCR(eep, EV_RX_NON_IP); 617 flags = 0; 618 break; 619 620 default: 621 EFSYS_ASSERT(B_FALSE); 622 flags = 0; 623 break; 624 } 625 626 #if EFSYS_OPT_RX_SCATTER 627 /* Report scatter and header/lookahead split buffer flags */ 628 if (sop) 629 flags |= EFX_PKT_START; 630 if (jumbo_cont) 631 flags |= EFX_PKT_CONT; 632 #endif /* EFSYS_OPT_RX_SCATTER */ 633 634 /* Detect errors included in the FSF_AZ_RX_EV_PKT_OK indication */ 635 if (!ok) { 636 ignore = siena_ev_rx_not_ok(eep, eqp, label, id, &flags); 637 if (ignore) { 638 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 639 uint32_t, size, uint16_t, flags); 640 641 return (B_FALSE); 642 } 643 } 644 645 /* If we're not discarding the packet then it is ok */ 646 if (~flags & EFX_DISCARD) 647 EFX_EV_QSTAT_INCR(eep, EV_RX_OK); 648 649 /* Detect multicast packets that didn't match the filter */ 650 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_PKT) != 0) { 651 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_PKT); 652 653 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_MCAST_HASH_MATCH) != 0) { 654 EFX_EV_QSTAT_INCR(eep, EV_RX_MCAST_HASH_MATCH); 655 } else { 656 EFSYS_PROBE(mcast_mismatch); 657 flags |= EFX_ADDR_MISMATCH; 658 } 659 } else { 660 flags |= EFX_PKT_UNICAST; 661 } 662 663 /* 664 * The packet parser in Siena can abort parsing packets under 665 * certain error conditions, setting the PKT_NOT_PARSED bit 666 * (which clears PKT_OK). If this is set, then don't trust 667 * the PKT_TYPE field. 668 */ 669 if (!ok) { 670 uint32_t parse_err; 671 672 parse_err = EFX_QWORD_FIELD(*eqp, FSF_CZ_RX_EV_PKT_NOT_PARSED); 673 if (parse_err != 0) 674 flags |= EFX_CHECK_VLAN; 675 } 676 677 if (~flags & EFX_CHECK_VLAN) { 678 uint32_t pkt_type; 679 680 pkt_type = EFX_QWORD_FIELD(*eqp, FSF_AZ_RX_EV_PKT_TYPE); 681 if (pkt_type >= FSE_AZ_RX_EV_PKT_TYPE_VLAN) 682 flags |= EFX_PKT_VLAN_TAGGED; 683 } 684 685 EFSYS_PROBE4(rx_complete, uint32_t, label, uint32_t, id, 686 uint32_t, size, uint16_t, flags); 687 688 EFSYS_ASSERT(eecp->eec_rx != NULL); 689 should_abort = eecp->eec_rx(arg, label, id, size, flags); 690 691 return (should_abort); 692 } 693 694 static __checkReturn boolean_t 695 siena_ev_tx( 696 __in efx_evq_t *eep, 697 __in efx_qword_t *eqp, 698 __in const efx_ev_callbacks_t *eecp, 699 __in_opt void *arg) 700 { 701 uint32_t id; 702 uint32_t label; 703 boolean_t should_abort; 704 705 EFX_EV_QSTAT_INCR(eep, EV_TX); 706 707 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0 && 708 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) == 0 && 709 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) == 0 && 710 EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) == 0) { 711 712 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_DESC_PTR); 713 label = EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_Q_LABEL); 714 715 EFSYS_PROBE2(tx_complete, uint32_t, label, uint32_t, id); 716 717 EFSYS_ASSERT(eecp->eec_tx != NULL); 718 should_abort = eecp->eec_tx(arg, label, id); 719 720 return (should_abort); 721 } 722 723 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_COMP) != 0) 724 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 725 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 726 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 727 728 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_ERR) != 0) 729 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_ERR); 730 731 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_PKT_TOO_BIG) != 0) 732 EFX_EV_QSTAT_INCR(eep, EV_TX_PKT_TOO_BIG); 733 734 if (EFX_QWORD_FIELD(*eqp, FSF_AZ_TX_EV_WQ_FF_FULL) != 0) 735 EFX_EV_QSTAT_INCR(eep, EV_TX_WQ_FF_FULL); 736 737 EFX_EV_QSTAT_INCR(eep, EV_TX_UNEXPECTED); 738 return (B_FALSE); 739 } 740 741 static __checkReturn boolean_t 742 siena_ev_global( 743 __in efx_evq_t *eep, 744 __in efx_qword_t *eqp, 745 __in const efx_ev_callbacks_t *eecp, 746 __in_opt void *arg) 747 { 748 _NOTE(ARGUNUSED(eqp, eecp, arg)) 749 750 EFX_EV_QSTAT_INCR(eep, EV_GLOBAL); 751 752 return (B_FALSE); 753 } 754 755 static __checkReturn boolean_t 756 siena_ev_driver( 757 __in efx_evq_t *eep, 758 __in efx_qword_t *eqp, 759 __in const efx_ev_callbacks_t *eecp, 760 __in_opt void *arg) 761 { 762 boolean_t should_abort; 763 764 EFX_EV_QSTAT_INCR(eep, EV_DRIVER); 765 should_abort = B_FALSE; 766 767 switch (EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBCODE)) { 768 case FSE_AZ_TX_DESCQ_FLS_DONE_EV: { 769 uint32_t txq_index; 770 771 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DESCQ_FLS_DONE); 772 773 txq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 774 775 EFSYS_PROBE1(tx_descq_fls_done, uint32_t, txq_index); 776 777 EFSYS_ASSERT(eecp->eec_txq_flush_done != NULL); 778 should_abort = eecp->eec_txq_flush_done(arg, txq_index); 779 780 break; 781 } 782 case FSE_AZ_RX_DESCQ_FLS_DONE_EV: { 783 uint32_t rxq_index; 784 uint32_t failed; 785 786 rxq_index = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_DESCQ_ID); 787 failed = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_RX_FLUSH_FAIL); 788 789 EFSYS_ASSERT(eecp->eec_rxq_flush_done != NULL); 790 EFSYS_ASSERT(eecp->eec_rxq_flush_failed != NULL); 791 792 if (failed) { 793 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_FAILED); 794 795 EFSYS_PROBE1(rx_descq_fls_failed, uint32_t, rxq_index); 796 797 should_abort = eecp->eec_rxq_flush_failed(arg, 798 rxq_index); 799 } else { 800 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DESCQ_FLS_DONE); 801 802 EFSYS_PROBE1(rx_descq_fls_done, uint32_t, rxq_index); 803 804 should_abort = eecp->eec_rxq_flush_done(arg, rxq_index); 805 } 806 807 break; 808 } 809 case FSE_AZ_EVQ_INIT_DONE_EV: 810 EFSYS_ASSERT(eecp->eec_initialized != NULL); 811 should_abort = eecp->eec_initialized(arg); 812 813 break; 814 815 case FSE_AZ_EVQ_NOT_EN_EV: 816 EFSYS_PROBE(evq_not_en); 817 break; 818 819 case FSE_AZ_SRM_UPD_DONE_EV: { 820 uint32_t code; 821 822 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_SRM_UPD_DONE); 823 824 code = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 825 826 EFSYS_ASSERT(eecp->eec_sram != NULL); 827 should_abort = eecp->eec_sram(arg, code); 828 829 break; 830 } 831 case FSE_AZ_WAKE_UP_EV: { 832 uint32_t id; 833 834 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 835 836 EFSYS_ASSERT(eecp->eec_wake_up != NULL); 837 should_abort = eecp->eec_wake_up(arg, id); 838 839 break; 840 } 841 case FSE_AZ_TX_PKT_NON_TCP_UDP: 842 EFSYS_PROBE(tx_pkt_non_tcp_udp); 843 break; 844 845 case FSE_AZ_TIMER_EV: { 846 uint32_t id; 847 848 id = EFX_QWORD_FIELD(*eqp, FSF_AZ_DRIVER_EV_SUBDATA); 849 850 EFSYS_ASSERT(eecp->eec_timer != NULL); 851 should_abort = eecp->eec_timer(arg, id); 852 853 break; 854 } 855 case FSE_AZ_RX_DSC_ERROR_EV: 856 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_RX_DSC_ERROR); 857 858 EFSYS_PROBE(rx_dsc_error); 859 860 EFSYS_ASSERT(eecp->eec_exception != NULL); 861 should_abort = eecp->eec_exception(arg, 862 EFX_EXCEPTION_RX_DSC_ERROR, 0); 863 864 break; 865 866 case FSE_AZ_TX_DSC_ERROR_EV: 867 EFX_EV_QSTAT_INCR(eep, EV_DRIVER_TX_DSC_ERROR); 868 869 EFSYS_PROBE(tx_dsc_error); 870 871 EFSYS_ASSERT(eecp->eec_exception != NULL); 872 should_abort = eecp->eec_exception(arg, 873 EFX_EXCEPTION_TX_DSC_ERROR, 0); 874 875 break; 876 877 default: 878 break; 879 } 880 881 return (should_abort); 882 } 883 884 static __checkReturn boolean_t 885 siena_ev_drv_gen( 886 __in efx_evq_t *eep, 887 __in efx_qword_t *eqp, 888 __in const efx_ev_callbacks_t *eecp, 889 __in_opt void *arg) 890 { 891 uint32_t data; 892 boolean_t should_abort; 893 894 EFX_EV_QSTAT_INCR(eep, EV_DRV_GEN); 895 896 data = EFX_QWORD_FIELD(*eqp, FSF_AZ_EV_DATA_DW0); 897 if (data >= ((uint32_t)1 << 16)) { 898 EFSYS_PROBE3(bad_event, unsigned int, eep->ee_index, 899 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_1), 900 uint32_t, EFX_QWORD_FIELD(*eqp, EFX_DWORD_0)); 901 return (B_TRUE); 902 } 903 904 EFSYS_ASSERT(eecp->eec_software != NULL); 905 should_abort = eecp->eec_software(arg, (uint16_t)data); 906 907 return (should_abort); 908 } 909 910 #if EFSYS_OPT_MCDI 911 912 static __checkReturn boolean_t 913 siena_ev_mcdi( 914 __in efx_evq_t *eep, 915 __in efx_qword_t *eqp, 916 __in const efx_ev_callbacks_t *eecp, 917 __in_opt void *arg) 918 { 919 efx_nic_t *enp = eep->ee_enp; 920 unsigned code; 921 boolean_t should_abort = B_FALSE; 922 923 EFSYS_ASSERT3U(enp->en_family, ==, EFX_FAMILY_SIENA); 924 925 if (enp->en_family != EFX_FAMILY_SIENA) 926 goto out; 927 928 EFSYS_ASSERT(eecp->eec_link_change != NULL); 929 EFSYS_ASSERT(eecp->eec_exception != NULL); 930 #if EFSYS_OPT_MON_STATS 931 EFSYS_ASSERT(eecp->eec_monitor != NULL); 932 #endif 933 934 EFX_EV_QSTAT_INCR(eep, EV_MCDI_RESPONSE); 935 936 code = EFX_QWORD_FIELD(*eqp, MCDI_EVENT_CODE); 937 switch (code) { 938 case MCDI_EVENT_CODE_BADSSERT: 939 efx_mcdi_ev_death(enp, EINTR); 940 break; 941 942 case MCDI_EVENT_CODE_CMDDONE: 943 efx_mcdi_ev_cpl(enp, 944 MCDI_EV_FIELD(eqp, CMDDONE_SEQ), 945 MCDI_EV_FIELD(eqp, CMDDONE_DATALEN), 946 MCDI_EV_FIELD(eqp, CMDDONE_ERRNO)); 947 break; 948 949 case MCDI_EVENT_CODE_LINKCHANGE: { 950 efx_link_mode_t link_mode; 951 952 siena_phy_link_ev(enp, eqp, &link_mode); 953 should_abort = eecp->eec_link_change(arg, link_mode); 954 break; 955 } 956 case MCDI_EVENT_CODE_SENSOREVT: { 957 #if EFSYS_OPT_MON_STATS 958 efx_mon_stat_t id; 959 efx_mon_stat_value_t value; 960 efx_rc_t rc; 961 962 if ((rc = mcdi_mon_ev(enp, eqp, &id, &value)) == 0) 963 should_abort = eecp->eec_monitor(arg, id, value); 964 else if (rc == ENOTSUP) { 965 should_abort = eecp->eec_exception(arg, 966 EFX_EXCEPTION_UNKNOWN_SENSOREVT, 967 MCDI_EV_FIELD(eqp, DATA)); 968 } else 969 EFSYS_ASSERT(rc == ENODEV); /* Wrong port */ 970 #else 971 should_abort = B_FALSE; 972 #endif 973 break; 974 } 975 case MCDI_EVENT_CODE_SCHEDERR: 976 /* Informational only */ 977 break; 978 979 case MCDI_EVENT_CODE_REBOOT: 980 efx_mcdi_ev_death(enp, EIO); 981 break; 982 983 case MCDI_EVENT_CODE_MAC_STATS_DMA: 984 #if EFSYS_OPT_MAC_STATS 985 if (eecp->eec_mac_stats != NULL) { 986 eecp->eec_mac_stats(arg, 987 MCDI_EV_FIELD(eqp, MAC_STATS_DMA_GENERATION)); 988 } 989 #endif 990 break; 991 992 case MCDI_EVENT_CODE_FWALERT: { 993 uint32_t reason = MCDI_EV_FIELD(eqp, FWALERT_REASON); 994 995 if (reason == MCDI_EVENT_FWALERT_REASON_SRAM_ACCESS) 996 should_abort = eecp->eec_exception(arg, 997 EFX_EXCEPTION_FWALERT_SRAM, 998 MCDI_EV_FIELD(eqp, FWALERT_DATA)); 999 else 1000 should_abort = eecp->eec_exception(arg, 1001 EFX_EXCEPTION_UNKNOWN_FWALERT, 1002 MCDI_EV_FIELD(eqp, DATA)); 1003 break; 1004 } 1005 1006 default: 1007 EFSYS_PROBE1(mc_pcol_error, int, code); 1008 break; 1009 } 1010 1011 out: 1012 return (should_abort); 1013 } 1014 1015 #endif /* EFSYS_OPT_MCDI */ 1016 1017 static __checkReturn efx_rc_t 1018 siena_ev_qprime( 1019 __in efx_evq_t *eep, 1020 __in unsigned int count) 1021 { 1022 efx_nic_t *enp = eep->ee_enp; 1023 uint32_t rptr; 1024 efx_dword_t dword; 1025 1026 rptr = count & eep->ee_mask; 1027 1028 EFX_POPULATE_DWORD_1(dword, FRF_AZ_EVQ_RPTR, rptr); 1029 1030 EFX_BAR_TBL_WRITED(enp, FR_AZ_EVQ_RPTR_REG, eep->ee_index, 1031 &dword, B_FALSE); 1032 1033 return (0); 1034 } 1035 1036 #define EFX_EV_BATCH 8 1037 1038 static void 1039 siena_ev_qpoll( 1040 __in efx_evq_t *eep, 1041 __inout unsigned int *countp, 1042 __in const efx_ev_callbacks_t *eecp, 1043 __in_opt void *arg) 1044 { 1045 efx_qword_t ev[EFX_EV_BATCH]; 1046 unsigned int batch; 1047 unsigned int total; 1048 unsigned int count; 1049 unsigned int index; 1050 size_t offset; 1051 1052 EFSYS_ASSERT(countp != NULL); 1053 EFSYS_ASSERT(eecp != NULL); 1054 1055 count = *countp; 1056 do { 1057 /* Read up until the end of the batch period */ 1058 batch = EFX_EV_BATCH - (count & (EFX_EV_BATCH - 1)); 1059 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 1060 for (total = 0; total < batch; ++total) { 1061 EFSYS_MEM_READQ(eep->ee_esmp, offset, &(ev[total])); 1062 1063 if (!EFX_EV_PRESENT(ev[total])) 1064 break; 1065 1066 EFSYS_PROBE3(event, unsigned int, eep->ee_index, 1067 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_1), 1068 uint32_t, EFX_QWORD_FIELD(ev[total], EFX_DWORD_0)); 1069 1070 offset += sizeof (efx_qword_t); 1071 } 1072 1073 #if EFSYS_OPT_EV_PREFETCH && (EFSYS_OPT_EV_PREFETCH_PERIOD > 1) 1074 /* 1075 * Prefetch the next batch when we get within PREFETCH_PERIOD 1076 * of a completed batch. If the batch is smaller, then prefetch 1077 * immediately. 1078 */ 1079 if (total == batch && total < EFSYS_OPT_EV_PREFETCH_PERIOD) 1080 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 1081 #endif /* EFSYS_OPT_EV_PREFETCH */ 1082 1083 /* Process the batch of events */ 1084 for (index = 0; index < total; ++index) { 1085 boolean_t should_abort; 1086 uint32_t code; 1087 1088 #if EFSYS_OPT_EV_PREFETCH 1089 /* Prefetch if we've now reached the batch period */ 1090 if (total == batch && 1091 index + EFSYS_OPT_EV_PREFETCH_PERIOD == total) { 1092 offset = (count + batch) & eep->ee_mask; 1093 offset *= sizeof (efx_qword_t); 1094 1095 EFSYS_MEM_PREFETCH(eep->ee_esmp, offset); 1096 } 1097 #endif /* EFSYS_OPT_EV_PREFETCH */ 1098 1099 EFX_EV_QSTAT_INCR(eep, EV_ALL); 1100 1101 code = EFX_QWORD_FIELD(ev[index], FSF_AZ_EV_CODE); 1102 switch (code) { 1103 case FSE_AZ_EV_CODE_RX_EV: 1104 should_abort = eep->ee_rx(eep, 1105 &(ev[index]), eecp, arg); 1106 break; 1107 case FSE_AZ_EV_CODE_TX_EV: 1108 should_abort = eep->ee_tx(eep, 1109 &(ev[index]), eecp, arg); 1110 break; 1111 case FSE_AZ_EV_CODE_DRIVER_EV: 1112 should_abort = eep->ee_driver(eep, 1113 &(ev[index]), eecp, arg); 1114 break; 1115 case FSE_AZ_EV_CODE_DRV_GEN_EV: 1116 should_abort = eep->ee_drv_gen(eep, 1117 &(ev[index]), eecp, arg); 1118 break; 1119 #if EFSYS_OPT_MCDI 1120 case FSE_AZ_EV_CODE_MCDI_EVRESPONSE: 1121 should_abort = eep->ee_mcdi(eep, 1122 &(ev[index]), eecp, arg); 1123 break; 1124 #endif 1125 case FSE_AZ_EV_CODE_GLOBAL_EV: 1126 if (eep->ee_global) { 1127 should_abort = eep->ee_global(eep, 1128 &(ev[index]), eecp, arg); 1129 break; 1130 } 1131 /* else fallthrough */ 1132 default: 1133 EFSYS_PROBE3(bad_event, 1134 unsigned int, eep->ee_index, 1135 uint32_t, 1136 EFX_QWORD_FIELD(ev[index], EFX_DWORD_1), 1137 uint32_t, 1138 EFX_QWORD_FIELD(ev[index], EFX_DWORD_0)); 1139 1140 EFSYS_ASSERT(eecp->eec_exception != NULL); 1141 (void) eecp->eec_exception(arg, 1142 EFX_EXCEPTION_EV_ERROR, code); 1143 should_abort = B_TRUE; 1144 } 1145 if (should_abort) { 1146 /* Ignore subsequent events */ 1147 total = index + 1; 1148 break; 1149 } 1150 } 1151 1152 /* 1153 * Now that the hardware has most likely moved onto dma'ing 1154 * into the next cache line, clear the processed events. Take 1155 * care to only clear out events that we've processed 1156 */ 1157 EFX_SET_QWORD(ev[0]); 1158 offset = (count & eep->ee_mask) * sizeof (efx_qword_t); 1159 for (index = 0; index < total; ++index) { 1160 EFSYS_MEM_WRITEQ(eep->ee_esmp, offset, &(ev[0])); 1161 offset += sizeof (efx_qword_t); 1162 } 1163 1164 count += total; 1165 1166 } while (total == batch); 1167 1168 *countp = count; 1169 } 1170 1171 static void 1172 siena_ev_qpost( 1173 __in efx_evq_t *eep, 1174 __in uint16_t data) 1175 { 1176 efx_nic_t *enp = eep->ee_enp; 1177 efx_qword_t ev; 1178 efx_oword_t oword; 1179 1180 EFX_POPULATE_QWORD_2(ev, FSF_AZ_EV_CODE, FSE_AZ_EV_CODE_DRV_GEN_EV, 1181 FSF_AZ_EV_DATA_DW0, (uint32_t)data); 1182 1183 EFX_POPULATE_OWORD_3(oword, FRF_AZ_DRV_EV_QID, eep->ee_index, 1184 EFX_DWORD_0, EFX_QWORD_FIELD(ev, EFX_DWORD_0), 1185 EFX_DWORD_1, EFX_QWORD_FIELD(ev, EFX_DWORD_1)); 1186 1187 EFX_BAR_WRITEO(enp, FR_AZ_DRV_EV_REG, &oword); 1188 } 1189 1190 static __checkReturn efx_rc_t 1191 siena_ev_qmoderate( 1192 __in efx_evq_t *eep, 1193 __in unsigned int us) 1194 { 1195 efx_nic_t *enp = eep->ee_enp; 1196 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1197 unsigned int locked; 1198 efx_dword_t dword; 1199 efx_rc_t rc; 1200 1201 if (us > encp->enc_evq_timer_max_us) { 1202 rc = EINVAL; 1203 goto fail1; 1204 } 1205 1206 /* If the value is zero then disable the timer */ 1207 if (us == 0) { 1208 EFX_POPULATE_DWORD_2(dword, 1209 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_DIS, 1210 FRF_CZ_TC_TIMER_VAL, 0); 1211 } else { 1212 uint32_t timer_val; 1213 1214 /* Calculate the timer value in quanta */ 1215 timer_val = us * 1000 / encp->enc_evq_timer_quantum_ns; 1216 1217 /* Moderation value is base 0 so we need to deduct 1 */ 1218 if (timer_val > 0) 1219 timer_val--; 1220 1221 EFX_POPULATE_DWORD_2(dword, 1222 FRF_CZ_TC_TIMER_MODE, FFE_CZ_TIMER_MODE_INT_HLDOFF, 1223 FRF_CZ_TC_TIMER_VAL, timer_val); 1224 } 1225 1226 locked = (eep->ee_index == 0) ? 1 : 0; 1227 1228 EFX_BAR_TBL_WRITED(enp, FR_BZ_TIMER_COMMAND_REGP0, 1229 eep->ee_index, &dword, locked); 1230 1231 return (0); 1232 1233 fail1: 1234 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1235 1236 return (rc); 1237 } 1238 1239 static __checkReturn efx_rc_t 1240 siena_ev_qcreate( 1241 __in efx_nic_t *enp, 1242 __in unsigned int index, 1243 __in efsys_mem_t *esmp, 1244 __in size_t n, 1245 __in uint32_t id, 1246 __in efx_evq_t *eep) 1247 { 1248 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1249 uint32_t size; 1250 efx_oword_t oword; 1251 efx_rc_t rc; 1252 1253 _NOTE(ARGUNUSED(esmp)); 1254 1255 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MAXNEVS)); 1256 EFX_STATIC_ASSERT(ISP2(EFX_EVQ_MINNEVS)); 1257 1258 if (!ISP2(n) || (n < EFX_EVQ_MINNEVS) || (n > EFX_EVQ_MAXNEVS)) { 1259 rc = EINVAL; 1260 goto fail1; 1261 } 1262 if (index >= encp->enc_evq_limit) { 1263 rc = EINVAL; 1264 goto fail2; 1265 } 1266 #if EFSYS_OPT_RX_SCALE 1267 if (enp->en_intr.ei_type == EFX_INTR_LINE && 1268 index >= EFX_MAXRSS_LEGACY) { 1269 rc = EINVAL; 1270 goto fail3; 1271 } 1272 #endif 1273 for (size = 0; (1 << size) <= (EFX_EVQ_MAXNEVS / EFX_EVQ_MINNEVS); 1274 size++) 1275 if ((1 << size) == (int)(n / EFX_EVQ_MINNEVS)) 1276 break; 1277 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1278 rc = EINVAL; 1279 goto fail4; 1280 } 1281 1282 /* Set up the handler table */ 1283 eep->ee_rx = siena_ev_rx; 1284 eep->ee_tx = siena_ev_tx; 1285 eep->ee_driver = siena_ev_driver; 1286 eep->ee_global = siena_ev_global; 1287 eep->ee_drv_gen = siena_ev_drv_gen; 1288 #if EFSYS_OPT_MCDI 1289 eep->ee_mcdi = siena_ev_mcdi; 1290 #endif /* EFSYS_OPT_MCDI */ 1291 1292 /* Set up the new event queue */ 1293 EFX_POPULATE_OWORD_1(oword, FRF_CZ_TIMER_Q_EN, 1); 1294 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, index, &oword, B_TRUE); 1295 1296 EFX_POPULATE_OWORD_3(oword, FRF_AZ_EVQ_EN, 1, FRF_AZ_EVQ_SIZE, size, 1297 FRF_AZ_EVQ_BUF_BASE_ID, id); 1298 1299 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, index, &oword, B_TRUE); 1300 1301 return (0); 1302 1303 fail4: 1304 EFSYS_PROBE(fail4); 1305 #if EFSYS_OPT_RX_SCALE 1306 fail3: 1307 EFSYS_PROBE(fail3); 1308 #endif 1309 fail2: 1310 EFSYS_PROBE(fail2); 1311 fail1: 1312 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1313 1314 return (rc); 1315 } 1316 1317 #endif /* EFSYS_OPT_SIENA */ 1318 1319 #if EFSYS_OPT_QSTATS 1320 #if EFSYS_OPT_NAMES 1321 /* START MKCONFIG GENERATED EfxEventQueueStatNamesBlock b693ddf85aee1bfd */ 1322 static const char *__efx_ev_qstat_name[] = { 1323 "all", 1324 "rx", 1325 "rx_ok", 1326 "rx_frm_trunc", 1327 "rx_tobe_disc", 1328 "rx_pause_frm_err", 1329 "rx_buf_owner_id_err", 1330 "rx_ipv4_hdr_chksum_err", 1331 "rx_tcp_udp_chksum_err", 1332 "rx_eth_crc_err", 1333 "rx_ip_frag_err", 1334 "rx_mcast_pkt", 1335 "rx_mcast_hash_match", 1336 "rx_tcp_ipv4", 1337 "rx_tcp_ipv6", 1338 "rx_udp_ipv4", 1339 "rx_udp_ipv6", 1340 "rx_other_ipv4", 1341 "rx_other_ipv6", 1342 "rx_non_ip", 1343 "rx_batch", 1344 "tx", 1345 "tx_wq_ff_full", 1346 "tx_pkt_err", 1347 "tx_pkt_too_big", 1348 "tx_unexpected", 1349 "global", 1350 "global_mnt", 1351 "driver", 1352 "driver_srm_upd_done", 1353 "driver_tx_descq_fls_done", 1354 "driver_rx_descq_fls_done", 1355 "driver_rx_descq_fls_failed", 1356 "driver_rx_dsc_error", 1357 "driver_tx_dsc_error", 1358 "drv_gen", 1359 "mcdi_response", 1360 }; 1361 /* END MKCONFIG GENERATED EfxEventQueueStatNamesBlock */ 1362 1363 const char * 1364 efx_ev_qstat_name( 1365 __in efx_nic_t *enp, 1366 __in unsigned int id) 1367 { 1368 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1369 EFSYS_ASSERT3U(id, <, EV_NQSTATS); 1370 1371 return (__efx_ev_qstat_name[id]); 1372 } 1373 #endif /* EFSYS_OPT_NAMES */ 1374 #endif /* EFSYS_OPT_QSTATS */ 1375 1376 #if EFSYS_OPT_SIENA 1377 1378 #if EFSYS_OPT_QSTATS 1379 static void 1380 siena_ev_qstats_update( 1381 __in efx_evq_t *eep, 1382 __inout_ecount(EV_NQSTATS) efsys_stat_t *stat) 1383 { 1384 unsigned int id; 1385 1386 for (id = 0; id < EV_NQSTATS; id++) { 1387 efsys_stat_t *essp = &stat[id]; 1388 1389 EFSYS_STAT_INCR(essp, eep->ee_stat[id]); 1390 eep->ee_stat[id] = 0; 1391 } 1392 } 1393 #endif /* EFSYS_OPT_QSTATS */ 1394 1395 static void 1396 siena_ev_qdestroy( 1397 __in efx_evq_t *eep) 1398 { 1399 efx_nic_t *enp = eep->ee_enp; 1400 efx_oword_t oword; 1401 1402 /* Purge event queue */ 1403 EFX_ZERO_OWORD(oword); 1404 1405 EFX_BAR_TBL_WRITEO(enp, FR_AZ_EVQ_PTR_TBL, 1406 eep->ee_index, &oword, B_TRUE); 1407 1408 EFX_ZERO_OWORD(oword); 1409 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TIMER_TBL, eep->ee_index, &oword, B_TRUE); 1410 } 1411 1412 static void 1413 siena_ev_fini( 1414 __in efx_nic_t *enp) 1415 { 1416 _NOTE(ARGUNUSED(enp)) 1417 } 1418 1419 #endif /* EFSYS_OPT_SIENA */ 1420