1 /*- 2 * Copyright 2007-2009 Solarflare Communications Inc. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/cdefs.h> 27 __FBSDID("$FreeBSD$"); 28 29 #include "efsys.h" 30 #include "efx.h" 31 #include "efx_types.h" 32 #include "efx_regs.h" 33 #include "efx_impl.h" 34 35 #if EFSYS_OPT_QSTATS 36 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 37 do { \ 38 (_etp)->et_stat[_stat]++; \ 39 _NOTE(CONSTANTCONDITION) \ 40 } while (B_FALSE) 41 #else 42 #define EFX_TX_QSTAT_INCR(_etp, _stat) 43 #endif 44 45 __checkReturn int 46 efx_tx_init( 47 __in efx_nic_t *enp) 48 { 49 efx_oword_t oword; 50 int rc; 51 52 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 53 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 54 55 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 56 rc = EINVAL; 57 goto fail1; 58 } 59 60 if (enp->en_mod_flags & EFX_MOD_TX) { 61 rc = EINVAL; 62 goto fail2; 63 } 64 65 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 66 67 /* 68 * Disable the timer-based TX DMA backoff and allow TX DMA to be 69 * controlled by the RX FIFO fill level (although always allow a 70 * minimal trickle). 71 */ 72 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 73 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 74 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 75 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 76 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 77 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 78 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 79 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 80 81 /* 82 * Filter all packets less than 14 bytes to avoid parsing 83 * errors. 84 */ 85 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 86 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 87 88 /* 89 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 90 * descriptors (which is bad). 91 */ 92 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 93 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 94 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 95 96 enp->en_mod_flags |= EFX_MOD_TX; 97 return (0); 98 99 fail2: 100 EFSYS_PROBE(fail2); 101 fail1: 102 EFSYS_PROBE1(fail1, int, rc); 103 104 return (rc); 105 } 106 107 #if EFSYS_OPT_FILTER 108 extern __checkReturn int 109 efx_tx_filter_insert( 110 __in efx_txq_t *etp, 111 __inout efx_filter_spec_t *spec) 112 { 113 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 114 EFSYS_ASSERT3P(spec, !=, NULL); 115 116 spec->efs_dmaq_id = (uint16_t)etp->et_index; 117 return efx_filter_insert_filter(etp->et_enp, spec, B_FALSE); 118 } 119 #endif 120 121 #if EFSYS_OPT_FILTER 122 extern __checkReturn int 123 efx_tx_filter_remove( 124 __in efx_txq_t *etp, 125 __inout efx_filter_spec_t *spec) 126 { 127 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 128 EFSYS_ASSERT3P(spec, !=, NULL); 129 130 spec->efs_dmaq_id = (uint16_t)etp->et_index; 131 return efx_filter_remove_filter(etp->et_enp, spec); 132 } 133 #endif 134 135 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 136 do { \ 137 unsigned int id; \ 138 size_t offset; \ 139 efx_qword_t qword; \ 140 \ 141 id = (_added)++ & (_etp)->et_mask; \ 142 offset = id * sizeof (efx_qword_t); \ 143 \ 144 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 145 unsigned int, id, efsys_dma_addr_t, (_addr), \ 146 size_t, (_size), boolean_t, (_eop)); \ 147 \ 148 EFX_POPULATE_QWORD_4(qword, \ 149 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 150 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 151 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 152 (uint32_t)((_addr) & 0xffffffff), \ 153 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 154 (uint32_t)((_addr) >> 32)); \ 155 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 156 \ 157 _NOTE(CONSTANTCONDITION) \ 158 } while (B_FALSE) 159 160 __checkReturn int 161 efx_tx_qpost( 162 __in efx_txq_t *etp, 163 __in_ecount(n) efx_buffer_t *eb, 164 __in unsigned int n, 165 __in unsigned int completed, 166 __inout unsigned int *addedp) 167 { 168 unsigned int added = *addedp; 169 unsigned int i; 170 int rc = ENOSPC; 171 172 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 173 174 if (added - completed + n > EFX_TXQ_LIMIT(etp->et_mask + 1)) 175 goto fail1; 176 177 for (i = 0; i < n; i++) { 178 efx_buffer_t *ebp = &eb[i]; 179 efsys_dma_addr_t start = ebp->eb_addr; 180 size_t size = ebp->eb_size; 181 efsys_dma_addr_t end = start + size; 182 183 /* Fragments must not span 4k boundaries. */ 184 EFSYS_ASSERT(P2ROUNDUP(start + 1, 4096) >= end); 185 186 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 187 } 188 189 EFX_TX_QSTAT_INCR(etp, TX_POST); 190 191 *addedp = added; 192 return (0); 193 194 fail1: 195 EFSYS_PROBE1(fail1, int, rc); 196 197 return (rc); 198 } 199 200 void 201 efx_tx_qpush( 202 __in efx_txq_t *etp, 203 __in unsigned int added) 204 { 205 efx_nic_t *enp = etp->et_enp; 206 uint32_t wptr; 207 efx_dword_t dword; 208 efx_oword_t oword; 209 210 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 211 212 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 213 EFSYS_PIO_WRITE_BARRIER(); 214 215 /* Push the populated descriptors out */ 216 wptr = added & etp->et_mask; 217 218 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 219 220 /* Only write the third DWORD */ 221 EFX_POPULATE_DWORD_1(dword, 222 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 223 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 224 etp->et_index, &dword, B_FALSE); 225 } 226 227 void 228 efx_tx_qflush( 229 __in efx_txq_t *etp) 230 { 231 efx_nic_t *enp = etp->et_enp; 232 efx_oword_t oword; 233 uint32_t label; 234 235 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 236 237 label = etp->et_index; 238 239 /* Flush the queue */ 240 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 241 FRF_AZ_TX_FLUSH_DESCQ, label); 242 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 243 } 244 245 void 246 efx_tx_qenable( 247 __in efx_txq_t *etp) 248 { 249 efx_nic_t *enp = etp->et_enp; 250 efx_oword_t oword; 251 252 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 253 254 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 255 etp->et_index, &oword); 256 257 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 258 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 259 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 260 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 261 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 262 263 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 264 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 265 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 266 267 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 268 etp->et_index, &oword); 269 } 270 271 __checkReturn int 272 efx_tx_qcreate( 273 __in efx_nic_t *enp, 274 __in unsigned int index, 275 __in unsigned int label, 276 __in efsys_mem_t *esmp, 277 __in size_t n, 278 __in uint32_t id, 279 __in uint16_t flags, 280 __in efx_evq_t *eep, 281 __deref_out efx_txq_t **etpp) 282 { 283 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 284 efx_txq_t *etp; 285 efx_oword_t oword; 286 uint32_t size; 287 int rc; 288 289 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 290 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 291 292 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 293 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 294 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, encp->enc_txq_limit); 295 296 if (!ISP2(n) || !(n & EFX_TXQ_NDESCS_MASK)) { 297 rc = EINVAL; 298 goto fail1; 299 } 300 if (index >= encp->enc_txq_limit) { 301 rc = EINVAL; 302 goto fail2; 303 } 304 for (size = 0; (1 << size) <= (EFX_TXQ_MAXNDESCS / EFX_TXQ_MINNDESCS); 305 size++) 306 if ((1 << size) == (int)(n / EFX_TXQ_MINNDESCS)) 307 break; 308 if (id + (1 << size) >= encp->enc_buftbl_limit) { 309 rc = EINVAL; 310 goto fail3; 311 } 312 313 /* Allocate an TXQ object */ 314 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 315 316 if (etp == NULL) { 317 rc = ENOMEM; 318 goto fail4; 319 } 320 321 etp->et_magic = EFX_TXQ_MAGIC; 322 etp->et_enp = enp; 323 etp->et_index = index; 324 etp->et_mask = n - 1; 325 etp->et_esmp = esmp; 326 327 /* Set up the new descriptor queue */ 328 EFX_POPULATE_OWORD_6(oword, 329 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 330 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 331 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 332 FRF_AZ_TX_DESCQ_LABEL, label, 333 FRF_AZ_TX_DESCQ_SIZE, size, 334 FRF_AZ_TX_DESCQ_TYPE, 0); 335 336 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 337 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 338 (flags & EFX_CKSUM_IPV4) ? 0 : 1); 339 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 340 (flags & EFX_CKSUM_TCPUDP) ? 0 : 1); 341 342 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 343 etp->et_index, &oword); 344 345 enp->en_tx_qcount++; 346 *etpp = etp; 347 return (0); 348 349 fail4: 350 EFSYS_PROBE(fail4); 351 fail3: 352 EFSYS_PROBE(fail3); 353 fail2: 354 EFSYS_PROBE(fail2); 355 fail1: 356 EFSYS_PROBE1(fail1, int, rc); 357 358 return (rc); 359 } 360 361 #if EFSYS_OPT_NAMES 362 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 78ca9ab00287fffb */ 363 static const char __cs * __cs __efx_tx_qstat_name[] = { 364 "post", 365 "unaligned_split", 366 }; 367 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 368 369 const char __cs * 370 efx_tx_qstat_name( 371 __in efx_nic_t *enp, 372 __in unsigned int id) 373 { 374 _NOTE(ARGUNUSED(enp)) 375 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 376 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 377 378 return (__efx_tx_qstat_name[id]); 379 } 380 #endif /* EFSYS_OPT_NAMES */ 381 382 #if EFSYS_OPT_QSTATS 383 void 384 efx_tx_qstats_update( 385 __in efx_txq_t *etp, 386 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 387 { 388 unsigned int id; 389 390 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 391 392 for (id = 0; id < TX_NQSTATS; id++) { 393 efsys_stat_t *essp = &stat[id]; 394 395 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 396 etp->et_stat[id] = 0; 397 } 398 } 399 #endif /* EFSYS_OPT_QSTATS */ 400 401 void 402 efx_tx_qdestroy( 403 __in efx_txq_t *etp) 404 { 405 efx_nic_t *enp = etp->et_enp; 406 efx_oword_t oword; 407 408 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 409 410 EFSYS_ASSERT(enp->en_tx_qcount != 0); 411 --enp->en_tx_qcount; 412 413 /* Purge descriptor queue */ 414 EFX_ZERO_OWORD(oword); 415 416 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 417 etp->et_index, &oword); 418 419 /* Free the TXQ object */ 420 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 421 } 422 423 void 424 efx_tx_fini( 425 __in efx_nic_t *enp) 426 { 427 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 428 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 429 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 430 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 431 432 enp->en_mod_flags &= ~EFX_MOD_TX; 433 } 434