1 /* 2 * aQuantia Corporation Network Driver 3 * Copyright (C) 2014-2018 aQuantia Corporation. All rights reserved 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * (1) Redistributions of source code must retain the above 10 * copyright notice, this list of conditions and the following 11 * disclaimer. 12 * 13 * (2) Redistributions in binary form must reproduce the above 14 * copyright notice, this list of conditions and the following 15 * disclaimer in the documentation and/or other materials provided 16 * with the distribution. 17 * 18 * (3)The name of the author may not be used to endorse or promote 19 * products derived from this software without specific prior 20 * written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS 23 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 24 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY 26 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 28 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 31 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 32 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 */ 34 35 #include <sys/param.h> 36 #include <sys/bitstring.h> 37 #include <sys/kernel.h> 38 #include <sys/socket.h> 39 #include <machine/param.h> 40 #include <net/ethernet.h> 41 #include <net/if.h> 42 #include <net/if_var.h> 43 #include <net/iflib.h> 44 #include <netinet/in.h> 45 46 #include "aq_common.h" 47 48 #include "aq_ring.h" 49 #include "aq_dbg.h" 50 #include "aq_device.h" 51 #include "aq_hw.h" 52 #include "aq_hw_llh.h" 53 54 /* iflib txrx interface prototypes */ 55 static int aq_isc_txd_encap(void *arg, if_pkt_info_t pi); 56 static void aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 57 static int aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 58 static void aq_ring_rx_refill(void* arg, if_rxd_update_t iru); 59 static void aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); 60 static int aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); 61 static int aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 62 63 struct if_txrx aq_txrx = { 64 .ift_txd_encap = aq_isc_txd_encap, 65 .ift_txd_flush = aq_isc_txd_flush, 66 .ift_txd_credits_update = aq_isc_txd_credits_update, 67 .ift_rxd_available = aq_isc_rxd_available, 68 .ift_rxd_pkt_get = aq_isc_rxd_pkt_get, 69 .ift_rxd_refill = aq_ring_rx_refill, 70 .ift_rxd_flush = aq_isc_rxd_flush, 71 .ift_legacy_intr = NULL 72 }; 73 74 75 static inline uint32_t 76 aq_next(uint32_t i, uint32_t lim) 77 { 78 return (i == lim) ? 0 : i + 1; 79 } 80 81 int 82 aq_ring_rx_init(struct aq_hw *hw, struct aq_ring *ring) 83 { 84 int err; 85 uint32_t dma_desc_addr_lsw = (uint32_t)ring->rx_descs_phys & 0xffffffff; 86 uint32_t dma_desc_addr_msw = (uint32_t)(ring->rx_descs_phys >> 32); 87 88 AQ_DBG_ENTERA("[%d]", ring->index); 89 90 rdm_rx_desc_en_set(hw, false, ring->index); 91 92 rdm_rx_desc_head_splitting_set(hw, 0U, ring->index); 93 94 reg_rx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index); 95 96 reg_rx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index); 97 98 rdm_rx_desc_len_set(hw, ring->rx_size / 8U, ring->index); 99 100 rdm_rx_desc_data_buff_size_set(hw, ring->rx_buf_size / 1024U, 101 ring->index); 102 103 rdm_rx_desc_head_buff_size_set(hw, 0U, ring->index); 104 rdm_rx_desc_head_splitting_set(hw, 0U, ring->index); 105 rpo_rx_desc_vlan_stripping_set(hw, 106 (if_getcapenable(iflib_get_ifp(ring->dev->ctx)) & 107 IFCAP_VLAN_HWTAGGING) != 0 ? 1U : 0U, ring->index); 108 109 /* Rx ring set mode */ 110 111 /* Mapping interrupt vector */ 112 itr_irq_map_rx_set(hw, ring->msix, ring->index); 113 itr_irq_map_en_rx_set(hw, true, ring->index); 114 115 rdm_cpu_id_set(hw, 0, ring->index); 116 rdm_rx_desc_dca_en_set(hw, 0U, ring->index); 117 rdm_rx_head_dca_en_set(hw, 0U, ring->index); 118 rdm_rx_pld_dca_en_set(hw, 0U, ring->index); 119 120 err = aq_hw_err_from_flags(hw); 121 AQ_DBG_EXIT(err); 122 return (err); 123 } 124 125 int 126 aq_ring_tx_init(struct aq_hw *hw, struct aq_ring *ring) 127 { 128 int err; 129 uint32_t dma_desc_addr_lsw = (uint32_t)ring->tx_descs_phys & 0xffffffff; 130 uint32_t dma_desc_addr_msw = (uint32_t)(ring->tx_descs_phys >> 32); 131 132 AQ_DBG_ENTERA("[%d]", ring->index); 133 134 tdm_tx_desc_en_set(hw, 0U, ring->index); 135 136 reg_tx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index); 137 138 reg_tx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index); 139 140 tdm_tx_desc_len_set(hw, ring->tx_size / 8U, ring->index); 141 142 aq_ring_tx_tail_update(hw, ring, 0U); 143 144 /* Set Tx threshold */ 145 tdm_tx_desc_wr_wb_threshold_set(hw, 0U, ring->index); 146 147 /* Mapping interrupt vector */ 148 itr_irq_map_tx_set(hw, ring->msix, ring->index); 149 itr_irq_map_en_tx_set(hw, true, ring->index); 150 151 tdm_cpu_id_set(hw, 0, ring->index); 152 tdm_tx_desc_dca_en_set(hw, 0U, ring->index); 153 154 err = aq_hw_err_from_flags(hw); 155 AQ_DBG_EXIT(err); 156 return (err); 157 } 158 159 int 160 aq_ring_tx_tail_update(struct aq_hw *hw, struct aq_ring *ring, uint32_t tail) 161 { 162 AQ_DBG_ENTERA("[%d]", ring->index); 163 reg_tx_dma_desc_tail_ptr_set(hw, tail, ring->index); 164 AQ_DBG_EXIT(0); 165 return (0); 166 } 167 168 int 169 aq_ring_tx_start(struct aq_hw *hw, struct aq_ring *ring) 170 { 171 int err; 172 173 AQ_DBG_ENTERA("[%d]", ring->index); 174 tdm_tx_desc_en_set(hw, 1U, ring->index); 175 err = aq_hw_err_from_flags(hw); 176 AQ_DBG_EXIT(err); 177 return (err); 178 } 179 180 int 181 aq_ring_rx_start(struct aq_hw *hw, struct aq_ring *ring) 182 { 183 int err; 184 185 AQ_DBG_ENTERA("[%d]", ring->index); 186 rdm_rx_desc_en_set(hw, 1U, ring->index); 187 err = aq_hw_err_from_flags(hw); 188 AQ_DBG_EXIT(err); 189 return (err); 190 } 191 192 int 193 aq_ring_tx_stop(struct aq_hw *hw, struct aq_ring *ring) 194 { 195 int err; 196 197 AQ_DBG_ENTERA("[%d]", ring->index); 198 tdm_tx_desc_en_set(hw, 0U, ring->index); 199 err = aq_hw_err_from_flags(hw); 200 AQ_DBG_EXIT(err); 201 return (err); 202 } 203 204 int 205 aq_ring_rx_stop(struct aq_hw *hw, struct aq_ring *ring) 206 { 207 int err; 208 209 AQ_DBG_ENTERA("[%d]", ring->index); 210 rdm_rx_desc_en_set(hw, 0U, ring->index); 211 err = aq_hw_err_from_flags(hw); 212 AQ_DBG_EXIT(err); 213 return (err); 214 } 215 216 static void 217 aq_ring_rx_refill(void* arg, if_rxd_update_t iru) 218 { 219 struct aq_dev *aq_dev = arg; 220 volatile struct aq_rx_desc *rx_desc; 221 struct aq_ring *ring; 222 qidx_t i, pidx; 223 224 AQ_DBG_ENTERA("ring=%d iru_pidx=%d iru_count=%d iru->iru_buf_size=%d", 225 iru->iru_qsidx, iru->iru_pidx, iru->iru_count, iru->iru_buf_size); 226 227 ring = aq_dev->rx_rings[iru->iru_qsidx]; 228 pidx = iru->iru_pidx; 229 230 for (i = 0; i < iru->iru_count; i++) { 231 rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[pidx]; 232 rx_desc->read.buf_addr = htole64(iru->iru_paddrs[i]); 233 rx_desc->read.hdr_addr = 0; 234 235 pidx=aq_next(pidx, ring->rx_size - 1); 236 } 237 238 AQ_DBG_EXIT(0); 239 } 240 241 static void 242 aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) 243 { 244 struct aq_dev *aq_dev = arg; 245 struct aq_ring *ring = aq_dev->rx_rings[rxqid]; 246 247 AQ_DBG_ENTERA("[%d] tail=%u", ring->index, pidx); 248 reg_rx_dma_desc_tail_ptr_set(&aq_dev->hw, pidx, ring->index); 249 AQ_DBG_EXIT(0); 250 } 251 252 static int 253 aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 254 { 255 struct aq_dev *aq_dev = arg; 256 struct aq_ring *ring = aq_dev->rx_rings[rxqid]; 257 volatile struct aq_rx_desc *rx_desc = (volatile struct aq_rx_desc *) ring->rx_descs; 258 int cnt, i, iter; 259 260 AQ_DBG_ENTERA("[%d] head=%u, budget %d", ring->index, idx, budget); 261 262 for (iter = 0, cnt = 0, i = idx; 263 iter < ring->rx_size && cnt <= budget;) { 264 trace_aq_rx_descr(&aq_dev->hw, ring->index, i, 265 (volatile uint64_t*)&rx_desc[i]); 266 if (!rx_desc[i].wb.dd) 267 break; 268 269 if (rx_desc[i].wb.eop) 270 cnt++; 271 iter++; 272 i = aq_next(i, ring->rx_size - 1); 273 } 274 275 AQ_DBG_EXIT(cnt); 276 return (cnt); 277 } 278 279 static void 280 aq_rx_set_cso_flags(volatile struct aq_rx_desc *rx_desc, if_rxd_info_t ri) 281 { 282 if ((rx_desc->wb.pkt_type & 0x3) == 0) { // IPv4 283 if (rx_desc->wb.rx_cntl & BIT(0)) { // IPv4 csum checked 284 ri->iri_csum_flags |= CSUM_IP_CHECKED; 285 if (!(rx_desc->wb.rx_stat & BIT(1))) 286 ri->iri_csum_flags |= CSUM_IP_VALID; 287 } 288 } 289 if (rx_desc->wb.rx_cntl & BIT(1)) { // TCP/UDP csum checked 290 ri->iri_csum_flags |= CSUM_L4_CALC; 291 if (!(rx_desc->wb.rx_stat & BIT(2)) && // L4 csum error 292 (rx_desc->wb.rx_stat & BIT(3))) { // L4 csum valid 293 ri->iri_csum_flags |= CSUM_L4_VALID; 294 ri->iri_csum_data = htons(0xffff); 295 } 296 } 297 } 298 299 static uint8_t bsd_rss_type[16] = { 300 [AQ_RX_RSS_TYPE_IPV4] = M_HASHTYPE_RSS_IPV4, 301 [AQ_RX_RSS_TYPE_IPV6] = M_HASHTYPE_RSS_IPV6, 302 [AQ_RX_RSS_TYPE_IPV4_TCP] = M_HASHTYPE_RSS_TCP_IPV4, 303 [AQ_RX_RSS_TYPE_IPV6_TCP] = M_HASHTYPE_RSS_TCP_IPV6, 304 [AQ_RX_RSS_TYPE_IPV4_UDP] = M_HASHTYPE_RSS_UDP_IPV4, 305 [AQ_RX_RSS_TYPE_IPV6_UDP] = M_HASHTYPE_RSS_UDP_IPV6, 306 }; 307 308 309 310 static int 311 aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 312 { 313 struct aq_dev *aq_dev = arg; 314 struct aq_ring *ring = aq_dev->rx_rings[ri->iri_qsidx]; 315 volatile struct aq_rx_desc *rx_desc; 316 if_t ifp; 317 int cidx, rc = 0, i; 318 size_t len, total_len; 319 bool is_error = false; 320 321 AQ_DBG_ENTERA("[%d] start=%d", ring->index, ri->iri_cidx); 322 cidx = ri->iri_cidx; 323 ifp = iflib_get_ifp(aq_dev->ctx); 324 i = 0; 325 total_len = 0; 326 327 do { 328 if (i >= aq_dev->sctx->isc_rx_nsegments) 329 goto rx_err; 330 331 rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[cidx]; 332 333 trace_aq_rx_descr(&aq_dev->hw, ring->index, cidx, 334 (volatile uint64_t *)rx_desc); 335 336 /* MAC error (rx_stat) or RX-DMA fault (rdm_err) -> drop. */ 337 if ((rx_desc->wb.rx_stat & BIT(0)) != 0 || rx_desc->wb.rdm_err) 338 is_error = true; 339 340 if (!rx_desc->wb.eop) { 341 len = ring->rx_buf_size; 342 } else if (is_error) { 343 total_len = le32toh(rx_desc->wb.pkt_len); 344 len = 0; 345 } else { 346 total_len = le32toh(rx_desc->wb.pkt_len); 347 if (total_len < (size_t)i * ring->rx_buf_size) 348 goto rx_err; 349 len = total_len - (size_t)i * ring->rx_buf_size; 350 if (len > (size_t)ring->rx_buf_size) 351 goto rx_err; 352 } 353 ri->iri_frags[i].irf_flid = 0; 354 ri->iri_frags[i].irf_idx = cidx; 355 ri->iri_frags[i].irf_len = len; 356 357 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0 && 358 (rx_desc->wb.pkt_type & 0x60) != 0) { 359 ri->iri_flags |= M_VLANTAG; 360 ri->iri_vtag = le32toh(rx_desc->wb.vlan); 361 } 362 363 i++; 364 cidx = aq_next(cidx, ring->rx_size - 1); 365 } while (!rx_desc->wb.eop); 366 367 ri->iri_nfrags = i; 368 ri->iri_len = total_len; 369 370 /* Per-frame RX error: drop (zero-length frags), don't reset. */ 371 if (is_error) { 372 counter_u64_add(ring->stats.rx_err, 1); 373 for (i = 0; i < ri->iri_nfrags; i++) 374 ri->iri_frags[i].irf_len = 0; 375 if (ri->iri_len == 0) 376 ri->iri_len = 1; /* iflib asserts iri_len != 0 */ 377 goto exit; 378 } 379 380 if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) { 381 aq_rx_set_cso_flags(rx_desc, ri); 382 } 383 ri->iri_rsstype = bsd_rss_type[rx_desc->wb.rss_type & 0xF]; 384 if (ri->iri_rsstype != M_HASHTYPE_NONE) { 385 ri->iri_flowid = le32toh(rx_desc->wb.rss_hash); 386 } 387 388 counter_u64_add(ring->stats.rx_bytes, total_len); 389 counter_u64_add(ring->stats.rx_pkts, 1); 390 goto exit; 391 392 rx_err: 393 counter_u64_add(ring->stats.rx_err, 1); 394 rc = EBADMSG; 395 exit: 396 AQ_DBG_EXIT(rc); 397 return (rc); 398 } 399 400 /*****************************************************************************/ 401 /* */ 402 /*****************************************************************************/ 403 404 static void 405 aq_setup_offloads(struct aq_dev *aq_dev, if_pkt_info_t pi, volatile struct aq_tx_desc *txd, 406 uint32_t tx_cmd) 407 { 408 AQ_DBG_ENTER(); 409 txd->cmd |= tx_desc_cmd_fcs; 410 txd->cmd |= ((pi->ipi_flags & IPI_TX_IPV4) && 411 (pi->ipi_csum_flags & (CSUM_IP | CSUM_IP_TSO))) ? 412 tx_desc_cmd_ipv4 : 0; 413 txd->cmd |= (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP | 414 CSUM_IP_UDP | CSUM_IP6_UDP)) ? tx_desc_cmd_l4cs : 0; 415 txd->cmd |= (pi->ipi_flags & IPI_TX_INTR) ? tx_desc_cmd_wb : 0; 416 txd->cmd |= tx_cmd; 417 AQ_DBG_EXIT(0); 418 } 419 420 static int 421 aq_ring_tso_setup(struct aq_dev *aq_dev, if_pkt_info_t pi, uint32_t *hdrlen, 422 volatile union aq_txc_desc *txc) 423 { 424 uint32_t tx_cmd = 0; 425 426 AQ_DBG_ENTER(); 427 if (pi->ipi_csum_flags & CSUM_TSO) { 428 AQ_DBG_PRINT("aq_tso_setup(): TSO enabled"); 429 tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs; 430 431 if (pi->ipi_ipproto != IPPROTO_TCP) { 432 AQ_DBG_PRINT("aq_tso_setup not a tcp"); 433 AQ_DBG_EXIT(0); 434 return (0); 435 } 436 437 txc->cmd = 0x4; /* TCP */ 438 439 if (pi->ipi_csum_flags & CSUM_IP6_TCP) 440 txc->cmd |= 0x2; 441 442 txc->l2_len = pi->ipi_ehdrlen; 443 txc->l3_len = pi->ipi_ip_hlen; 444 txc->l4_len = pi->ipi_tcp_hlen; 445 txc->mss_len = pi->ipi_tso_segsz; 446 *hdrlen = txc->l2_len + txc->l3_len + txc->l4_len; 447 } 448 449 // Set VLAN tag 450 if (pi->ipi_mflags & M_VLANTAG) { 451 tx_cmd |= tx_desc_cmd_vlan; 452 txc->vlan_tag = htole16(pi->ipi_vtag); 453 } 454 455 if (tx_cmd) { 456 txc->type = tx_desc_type_ctx; 457 txc->idx = 0; 458 } 459 460 AQ_DBG_EXIT(tx_cmd); 461 return (tx_cmd); 462 } 463 464 static int 465 aq_isc_txd_encap(void *arg, if_pkt_info_t pi) 466 { 467 struct aq_dev *aq_dev = arg; 468 struct aq_ring *ring; 469 volatile union aq_txc_desc *txc; 470 volatile struct aq_tx_desc *txd = NULL; 471 bus_dma_segment_t *segs; 472 qidx_t pidx; 473 uint32_t hdrlen=0, pay_len; 474 uint8_t tx_cmd = 0; 475 int i, desc_count = 0; 476 477 AQ_DBG_ENTERA("[%d] start=%d", pi->ipi_qsidx, pi->ipi_pidx); 478 ring = aq_dev->tx_rings[pi->ipi_qsidx]; 479 480 segs = pi->ipi_segs; 481 pidx = pi->ipi_pidx; 482 txc = (volatile union aq_txc_desc *)&ring->tx_descs[pidx]; 483 AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p len %d", txc, txd, pi->ipi_len); 484 485 pay_len = pi->ipi_len; 486 487 txc->flags1 = 0U; 488 txc->flags2 = 0U; 489 490 tx_cmd = aq_ring_tso_setup(aq_dev, pi, &hdrlen, txc); 491 AQ_DBG_PRINT("tx_cmd = 0x%x", tx_cmd); 492 493 if (tx_cmd) { 494 trace_aq_tx_context_descr(&aq_dev->hw, ring->index, pidx, 495 (volatile void*)txc); 496 /* We've consumed the first desc, adjust counters */ 497 pidx = aq_next(pidx, ring->tx_size - 1); 498 499 txd = &ring->tx_descs[pidx]; 500 txd->flags = 0U; 501 } else { 502 txd = (volatile struct aq_tx_desc *)txc; 503 } 504 AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p", txc, txd); 505 506 txd->ct_en = !!tx_cmd; 507 508 txd->type = tx_desc_type_desc; 509 510 aq_setup_offloads(aq_dev, pi, txd, tx_cmd); 511 512 if (tx_cmd) { 513 txd->ct_idx = 0; 514 } 515 516 pay_len -= hdrlen; 517 518 txd->pay_len = pay_len; 519 520 AQ_DBG_PRINT("num_frag[%d] pay_len[%d]", pi->ipi_nsegs, pay_len); 521 for (i = 0; i < pi->ipi_nsegs; i++) { 522 if (desc_count > 0) { 523 txd = &ring->tx_descs[pidx]; 524 txd->flags = 0U; 525 } 526 527 txd->buf_addr = htole64(segs[i].ds_addr); 528 529 txd->type = tx_desc_type_desc; 530 txd->len = segs[i].ds_len; 531 txd->pay_len = pay_len; 532 if (i < pi->ipi_nsegs - 1) 533 trace_aq_tx_descr(&aq_dev->hw, ring->index, pidx, 534 (volatile void*)txd); 535 536 pidx = aq_next(pidx, ring->tx_size - 1); 537 538 desc_count++; 539 } 540 // Last descriptor requires EOP and WB 541 txd->eop = 1U; 542 543 AQ_DBG_DUMP_DESC(txd); 544 trace_aq_tx_descr(&aq_dev->hw, ring->index, pidx, (volatile void*)txd); 545 ring->tx_tail = pidx; 546 547 counter_u64_add(ring->stats.tx_pkts, 1); 548 counter_u64_add(ring->stats.tx_bytes, pay_len); 549 550 pi->ipi_new_pidx = pidx; 551 552 AQ_DBG_EXIT(0); 553 return (0); 554 } 555 556 static void 557 aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 558 { 559 struct aq_dev *aq_dev = arg; 560 struct aq_ring *ring = aq_dev->tx_rings[txqid]; 561 AQ_DBG_ENTERA("[%d] tail=%d", ring->index, pidx); 562 563 // Update the write pointer - submits packet for transmission 564 aq_ring_tx_tail_update(&aq_dev->hw, ring, pidx); 565 AQ_DBG_EXIT(0); 566 } 567 568 569 static inline unsigned int 570 aq_avail_desc(int a, int b, int size) 571 { 572 return (((b >= a)) ? ((size) - b + a) : (a - b)); 573 } 574 575 static int 576 aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 577 { 578 struct aq_dev *aq_dev = arg; 579 struct aq_ring *ring = aq_dev->tx_rings[txqid]; 580 uint32_t head; 581 int avail; 582 583 AQ_DBG_ENTERA("[%d] clear=%d", ring->index, clear); 584 avail = 0; 585 head = tdm_tx_desc_head_ptr_get(&aq_dev->hw, ring->index); 586 AQ_DBG_PRINT("swhead %d hwhead %d", ring->tx_head, head); 587 588 if (head >= ring->tx_size) { 589 /* Malformed HW head; report no completions. */ 590 goto done; 591 } 592 593 if (ring->tx_head == head) { 594 avail = 0; 595 goto done; 596 } 597 598 avail = aq_avail_desc(head, ring->tx_head, ring->tx_size); 599 if (clear) 600 ring->tx_head = head; 601 602 done: 603 AQ_DBG_EXIT(avail); 604 return (avail); 605 } 606