1 /*- 2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "efsys.h" 27 #include "efx.h" 28 #include "efx_types.h" 29 #include "efx_regs.h" 30 #include "efx_impl.h" 31 32 #if EFSYS_OPT_QSTATS 33 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 34 do { \ 35 (_etp)->et_stat[_stat]++; \ 36 _NOTE(CONSTANTCONDITION) \ 37 } while (B_FALSE) 38 #else 39 #define EFX_TX_QSTAT_INCR(_etp, _stat) 40 #endif 41 42 __checkReturn int 43 efx_tx_init( 44 __in efx_nic_t *enp) 45 { 46 efx_oword_t oword; 47 int rc; 48 49 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 50 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 51 52 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 53 rc = EINVAL; 54 goto fail1; 55 } 56 57 if (enp->en_mod_flags & EFX_MOD_TX) { 58 rc = EINVAL; 59 goto fail2; 60 } 61 62 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 63 64 /* 65 * Disable the timer-based TX DMA backoff and allow TX DMA to be 66 * controlled by the RX FIFO fill level (although always allow a 67 * minimal trickle). 68 */ 69 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 70 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 71 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 72 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 77 78 /* 79 * Filter all packets less than 14 bytes to avoid parsing 80 * errors. 81 */ 82 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 83 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 84 85 /* 86 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 87 * descriptors (which is bad). 88 */ 89 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 90 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 91 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 92 93 enp->en_mod_flags |= EFX_MOD_TX; 94 return (0); 95 96 fail2: 97 EFSYS_PROBE(fail2); 98 fail1: 99 EFSYS_PROBE1(fail1, int, rc); 100 101 return (rc); 102 } 103 104 #if EFSYS_OPT_FILTER 105 extern __checkReturn int 106 efx_tx_filter_insert( 107 __in efx_txq_t *etp, 108 __inout efx_filter_spec_t *spec) 109 { 110 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 111 EFSYS_ASSERT3P(spec, !=, NULL); 112 113 spec->efs_dmaq_id = (uint16_t)etp->et_index; 114 return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE); 115 } 116 #endif 117 118 #if EFSYS_OPT_FILTER 119 extern __checkReturn int 120 efx_tx_filter_remove( 121 __in efx_txq_t *etp, 122 __inout efx_filter_spec_t *spec) 123 { 124 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 125 EFSYS_ASSERT3P(spec, !=, NULL); 126 127 spec->efs_dmaq_id = (uint16_t)etp->et_index; 128 return efx_filter_remove_filter(etp->et_enp, spec); 129 } 130 #endif 131 132 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 133 do { \ 134 unsigned int id; \ 135 size_t offset; \ 136 efx_qword_t qword; \ 137 \ 138 id = (_added)++ & (_etp)->et_mask; \ 139 offset = id * sizeof (efx_qword_t); \ 140 \ 141 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 142 unsigned int, id, efsys_dma_addr_t, (_addr), \ 143 size_t, (_size), boolean_t, (_eop)); \ 144 \ 145 EFX_POPULATE_QWORD_4(qword, \ 146 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 147 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 148 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 149 (uint32_t)((_addr) & 0xffffffff), \ 150 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 151 (uint32_t)((_addr) >> 32)); \ 152 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 153 \ 154 _NOTE(CONSTANTCONDITION) \ 155 } while (B_FALSE) 156 157 __checkReturn int 158 efx_tx_qpost( 159 __in efx_txq_t *etp, 160 __in_ecount(n) efx_buffer_t *eb, 161 __in unsigned int n, 162 __in unsigned int completed, 163 __inout unsigned int *addedp) 164 { 165 unsigned int added = *addedp; 166 unsigned int i; 167 int rc = ENOSPC; 168 169 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 170 171 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 172 goto fail1; 173 174 for (i = 0; i < n; i++) { 175 efx_buffer_t *ebp = &eb[i]; 176 efsys_dma_addr_t start = ebp->eb_addr; 177 size_t size = ebp->eb_size; 178 efsys_dma_addr_t end = start + size; 179 180 /* Fragments must not span 4k boundaries. */ 181 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 182 183 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 184 } 185 186 EFX_TX_QSTAT_INCR(etp, TX_POST); 187 188 *addedp = added; 189 return (0); 190 191 fail1: 192 EFSYS_PROBE1(fail1, int, rc); 193 194 return (rc); 195 } 196 197 void 198 efx_tx_qpush( 199 __in efx_txq_t *etp, 200 __in unsigned int added) 201 { 202 efx_nic_t *enp = etp->et_enp; 203 uint32_t wptr; 204 efx_dword_t dword; 205 efx_oword_t oword; 206 207 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 208 209 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 210 EFSYS_PIO_WRITE_BARRIER(); 211 212 /* Push the populated descriptors out */ 213 wptr = added & etp->et_mask; 214 215 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 216 217 /* Only write the third DWORD */ 218 EFX_POPULATE_DWORD_1(dword, 219 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 220 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 221 etp->et_index, &dword, B_FALSE); 222 } 223 224 void 225 efx_tx_qflush( 226 __in efx_txq_t *etp) 227 { 228 efx_nic_t *enp = etp->et_enp; 229 efx_oword_t oword; 230 uint32_t label; 231 232 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 233 234 label = etp->et_index; 235 236 /* Flush the queue */ 237 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 238 FRF_AZ_TX_FLUSH_DESCQ, label); 239 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 240 } 241 242 void 243 efx_tx_qenable( 244 __in efx_txq_t *etp) 245 { 246 efx_nic_t *enp = etp->et_enp; 247 efx_oword_t oword; 248 249 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 250 251 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 252 etp->et_index, &oword); 253 254 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 255 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 256 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 257 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 259 260 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 261 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 262 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 263 264 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 265 etp->et_index, &oword); 266 } 267 268 __checkReturn int 269 efx_tx_qcreate( 270 __in efx_nic_t *enp, 271 __in unsigned int index, 272 __in unsigned int label, 273 __in efsys_mem_t *esmp, 274 __in size_t n, 275 __in uint32_t id, 276 __in uint16_t flags, 277 __in efx_evq_t *eep, 278 __deref_out efx_txq_t **etpp) 279 { 280 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 281 efx_txq_t *etp; 282 efx_oword_t oword; 283 uint32_t size; 284 int rc; 285 286 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 287 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 288 289 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 290 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 291 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 292 293 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) { 294 rc = EINVAL; 295 goto fail1; 296 } 297 if (index >= encp->enc_txq_limit) { 298 rc = EINVAL; 299 goto fail2; 300 } 301 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS); 302 size++) 303 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 304 break; 305 if (id + (1 << size) >= encp->enc_buftbl_limit) { 306 rc = EINVAL; 307 goto fail3; 308 } 309 310 /* Allocate an TXQ object */ 311 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 312 313 if (etp == NULL) { 314 rc = ENOMEM; 315 goto fail4; 316 } 317 318 etp->et_magic = EFX_TXQ_MAGIC; 319 etp->et_enp = enp; 320 etp->et_index = index; 321 etp->et_mask = n - 1; 322 etp->et_esmp = esmp; 323 324 /* Set up the new descriptor queue */ 325 EFX_POPULATE_OWORD_6(oword, 326 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 327 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 328 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 329 FRF_AZ_TX_DESCQ_LABEL, label, 330 FRF_AZ_TX_DESCQ_SIZE, size, 331 FRF_AZ_TX_DESCQ_TYPE, 0); 332 333 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 334 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 335 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 336 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 337 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 338 339 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 340 etp->et_index, &oword); 341 342 enp->en_tx_qcount++; 343 *etpp = etp; 344 return (0); 345 346 fail4: 347 EFSYS_PROBE(fail4); 348 fail3: 349 EFSYS_PROBE(fail3); 350 fail2: 351 EFSYS_PROBE(fail2); 352 fail1: 353 EFSYS_PROBE1(fail1, int, rc); 354 355 return (rc); 356 } 357 358 #if EFSYS_OPT_NAMES 359 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */ 360 static const char __cs * __cs __efx_tx_qstat_name[] = { 361 "post", 362 "unaligned_split", 363 }; 364 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 365 366 const char __cs * 367 efx_tx_qstat_name( 368 __in efx_nic_t *enp, 369 __in unsigned int id) 370 { 371 _NOTE(ARGUNUSED(enp)) 372 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 373 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 374 375 return (__efx_tx_qstat_name[id]); 376 } 377 #endif /* EFSYS_OPT_NAMES */ 378 379 #if EFSYS_OPT_QSTATS 380 void 381 efx_tx_qstats_update( 382 __in efx_txq_t *etp, 383 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 384 { 385 unsigned int id; 386 387 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 388 389 for (id = 0; id < TX_NQSTATS; id++) { 390 efsys_stat_t *essp = &stat[id]; 391 392 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 393 etp->et_stat[id] = 0; 394 } 395 } 396 #endif /* EFSYS_OPT_QSTATS */ 397 398 void 399 efx_tx_qdestroy( 400 __in efx_txq_t *etp) 401 { 402 efx_nic_t *enp = etp->et_enp; 403 efx_oword_t oword; 404 405 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 406 407 EFSYS_ASSERT(enp->en_tx_qcount != 0); 408 --enp->en_tx_qcount; 409 410 /* Purge descriptor queue */ 411 EFX_ZERO_OWORD(oword); 412 413 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 414 etp->et_index, &oword); 415 416 /* Free the TXQ object */ 417 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 418 } 419 420 void 421 efx_tx_fini( 422 __in efx_nic_t *enp) 423 { 424 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 425 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 426 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 427 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 428 429 enp->en_mod_flags &= ~EFX_MOD_TX; 430 } 431