1 /*
2 * aQuantia Corporation Network Driver
3 * Copyright (C) 2014-2018 aQuantia Corporation. All rights reserved
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * (1) Redistributions of source code must retain the above
10 * copyright notice, this list of conditions and the following
11 * disclaimer.
12 *
13 * (2) Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following
15 * disclaimer in the documentation and/or other materials provided
16 * with the distribution.
17 *
18 * (3)The name of the author may not be used to endorse or promote
19 * products derived from this software without specific prior
20 * written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
23 * OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
24 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
26 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
28 * GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
30 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
31 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
32 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/param.h>
36 #include <sys/bitstring.h>
37 #include <sys/kernel.h>
38 #include <sys/socket.h>
39 #include <machine/param.h>
40 #include <net/ethernet.h>
41 #include <net/if.h>
42 #include <net/if_var.h>
43 #include <net/iflib.h>
44 #include <netinet/in.h>
45
46 #include "aq_common.h"
47
48 #include "aq_ring.h"
49 #include "aq_dbg.h"
50 #include "aq_device.h"
51 #include "aq_hw.h"
52 #include "aq_hw_llh.h"
53
54 /* iflib txrx interface prototypes */
55 static int aq_isc_txd_encap(void *arg, if_pkt_info_t pi);
56 static void aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
57 static int aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
58 static void aq_ring_rx_refill(void* arg, if_rxd_update_t iru);
59 static void aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx);
60 static int aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget);
61 static int aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
62
63 struct if_txrx aq_txrx = {
64 .ift_txd_encap = aq_isc_txd_encap,
65 .ift_txd_flush = aq_isc_txd_flush,
66 .ift_txd_credits_update = aq_isc_txd_credits_update,
67 .ift_rxd_available = aq_isc_rxd_available,
68 .ift_rxd_pkt_get = aq_isc_rxd_pkt_get,
69 .ift_rxd_refill = aq_ring_rx_refill,
70 .ift_rxd_flush = aq_isc_rxd_flush,
71 .ift_legacy_intr = NULL
72 };
73
74
75 static inline uint32_t
aq_next(uint32_t i,uint32_t lim)76 aq_next(uint32_t i, uint32_t lim)
77 {
78 return (i == lim) ? 0 : i + 1;
79 }
80
81 int
aq_ring_rx_init(struct aq_hw * hw,struct aq_ring * ring)82 aq_ring_rx_init(struct aq_hw *hw, struct aq_ring *ring)
83 {
84 int err;
85 uint32_t dma_desc_addr_lsw = (uint32_t)ring->rx_descs_phys & 0xffffffff;
86 uint32_t dma_desc_addr_msw = (uint32_t)(ring->rx_descs_phys >> 32);
87
88 AQ_DBG_ENTERA("[%d]", ring->index);
89
90 rdm_rx_desc_en_set(hw, false, ring->index);
91
92 rdm_rx_desc_head_splitting_set(hw, 0U, ring->index);
93
94 reg_rx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index);
95
96 reg_rx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index);
97
98 rdm_rx_desc_len_set(hw, ring->rx_size / 8U, ring->index);
99
100 rdm_rx_desc_data_buff_size_set(hw, ring->rx_buf_size / 1024U,
101 ring->index);
102
103 rdm_rx_desc_head_buff_size_set(hw, 0U, ring->index);
104 rdm_rx_desc_head_splitting_set(hw, 0U, ring->index);
105 rpo_rx_desc_vlan_stripping_set(hw,
106 (if_getcapenable(iflib_get_ifp(ring->dev->ctx)) &
107 IFCAP_VLAN_HWTAGGING) != 0 ? 1U : 0U, ring->index);
108
109 /* Rx ring set mode */
110
111 /* Mapping interrupt vector */
112 itr_irq_map_rx_set(hw, ring->msix, ring->index);
113 itr_irq_map_en_rx_set(hw, true, ring->index);
114
115 rdm_cpu_id_set(hw, 0, ring->index);
116 rdm_rx_desc_dca_en_set(hw, 0U, ring->index);
117 rdm_rx_head_dca_en_set(hw, 0U, ring->index);
118 rdm_rx_pld_dca_en_set(hw, 0U, ring->index);
119
120 err = aq_hw_err_from_flags(hw);
121 AQ_DBG_EXIT(err);
122 return (err);
123 }
124
125 int
aq_ring_tx_init(struct aq_hw * hw,struct aq_ring * ring)126 aq_ring_tx_init(struct aq_hw *hw, struct aq_ring *ring)
127 {
128 int err;
129 uint32_t dma_desc_addr_lsw = (uint32_t)ring->tx_descs_phys & 0xffffffff;
130 uint32_t dma_desc_addr_msw = (uint32_t)(ring->tx_descs_phys >> 32);
131
132 AQ_DBG_ENTERA("[%d]", ring->index);
133
134 tdm_tx_desc_en_set(hw, 0U, ring->index);
135
136 reg_tx_dma_desc_base_addresslswset(hw, dma_desc_addr_lsw, ring->index);
137
138 reg_tx_dma_desc_base_addressmswset(hw, dma_desc_addr_msw, ring->index);
139
140 tdm_tx_desc_len_set(hw, ring->tx_size / 8U, ring->index);
141
142 aq_ring_tx_tail_update(hw, ring, 0U);
143
144 /* Set Tx threshold */
145 tdm_tx_desc_wr_wb_threshold_set(hw, 0U, ring->index);
146
147 /* Mapping interrupt vector */
148 itr_irq_map_tx_set(hw, ring->msix, ring->index);
149 itr_irq_map_en_tx_set(hw, true, ring->index);
150
151 tdm_cpu_id_set(hw, 0, ring->index);
152 tdm_tx_desc_dca_en_set(hw, 0U, ring->index);
153
154 err = aq_hw_err_from_flags(hw);
155 AQ_DBG_EXIT(err);
156 return (err);
157 }
158
159 int
aq_ring_tx_tail_update(struct aq_hw * hw,struct aq_ring * ring,uint32_t tail)160 aq_ring_tx_tail_update(struct aq_hw *hw, struct aq_ring *ring, uint32_t tail)
161 {
162 AQ_DBG_ENTERA("[%d]", ring->index);
163 reg_tx_dma_desc_tail_ptr_set(hw, tail, ring->index);
164 AQ_DBG_EXIT(0);
165 return (0);
166 }
167
168 int
aq_ring_tx_start(struct aq_hw * hw,struct aq_ring * ring)169 aq_ring_tx_start(struct aq_hw *hw, struct aq_ring *ring)
170 {
171 int err;
172
173 AQ_DBG_ENTERA("[%d]", ring->index);
174 tdm_tx_desc_en_set(hw, 1U, ring->index);
175 err = aq_hw_err_from_flags(hw);
176 AQ_DBG_EXIT(err);
177 return (err);
178 }
179
180 int
aq_ring_rx_start(struct aq_hw * hw,struct aq_ring * ring)181 aq_ring_rx_start(struct aq_hw *hw, struct aq_ring *ring)
182 {
183 int err;
184
185 AQ_DBG_ENTERA("[%d]", ring->index);
186 rdm_rx_desc_en_set(hw, 1U, ring->index);
187 err = aq_hw_err_from_flags(hw);
188 AQ_DBG_EXIT(err);
189 return (err);
190 }
191
192 int
aq_ring_tx_stop(struct aq_hw * hw,struct aq_ring * ring)193 aq_ring_tx_stop(struct aq_hw *hw, struct aq_ring *ring)
194 {
195 int err;
196
197 AQ_DBG_ENTERA("[%d]", ring->index);
198 tdm_tx_desc_en_set(hw, 0U, ring->index);
199 err = aq_hw_err_from_flags(hw);
200 AQ_DBG_EXIT(err);
201 return (err);
202 }
203
204 int
aq_ring_rx_stop(struct aq_hw * hw,struct aq_ring * ring)205 aq_ring_rx_stop(struct aq_hw *hw, struct aq_ring *ring)
206 {
207 int err;
208
209 AQ_DBG_ENTERA("[%d]", ring->index);
210 rdm_rx_desc_en_set(hw, 0U, ring->index);
211 err = aq_hw_err_from_flags(hw);
212 AQ_DBG_EXIT(err);
213 return (err);
214 }
215
216 static void
aq_ring_rx_refill(void * arg,if_rxd_update_t iru)217 aq_ring_rx_refill(void* arg, if_rxd_update_t iru)
218 {
219 struct aq_dev *aq_dev = arg;
220 volatile struct aq_rx_desc *rx_desc;
221 struct aq_ring *ring;
222 qidx_t i, pidx;
223
224 AQ_DBG_ENTERA("ring=%d iru_pidx=%d iru_count=%d iru->iru_buf_size=%d",
225 iru->iru_qsidx, iru->iru_pidx, iru->iru_count, iru->iru_buf_size);
226
227 ring = aq_dev->rx_rings[iru->iru_qsidx];
228 pidx = iru->iru_pidx;
229
230 for (i = 0; i < iru->iru_count; i++) {
231 rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[pidx];
232 rx_desc->read.buf_addr = htole64(iru->iru_paddrs[i]);
233 rx_desc->read.hdr_addr = 0;
234
235 pidx=aq_next(pidx, ring->rx_size - 1);
236 }
237
238 AQ_DBG_EXIT(0);
239 }
240
241 static void
aq_isc_rxd_flush(void * arg,uint16_t rxqid,uint8_t flid __unused,qidx_t pidx)242 aq_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
243 {
244 struct aq_dev *aq_dev = arg;
245 struct aq_ring *ring = aq_dev->rx_rings[rxqid];
246
247 AQ_DBG_ENTERA("[%d] tail=%u", ring->index, pidx);
248 reg_rx_dma_desc_tail_ptr_set(&aq_dev->hw, pidx, ring->index);
249 AQ_DBG_EXIT(0);
250 }
251
252 static int
aq_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)253 aq_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
254 {
255 struct aq_dev *aq_dev = arg;
256 struct aq_ring *ring = aq_dev->rx_rings[rxqid];
257 volatile struct aq_rx_desc *rx_desc = (volatile struct aq_rx_desc *) ring->rx_descs;
258 int cnt, i, iter;
259
260 AQ_DBG_ENTERA("[%d] head=%u, budget %d", ring->index, idx, budget);
261
262 for (iter = 0, cnt = 0, i = idx;
263 iter < ring->rx_size && cnt <= budget;) {
264 trace_aq_rx_descr(&aq_dev->hw, ring->index, i,
265 (volatile uint64_t*)&rx_desc[i]);
266 if (!rx_desc[i].wb.dd)
267 break;
268
269 if (rx_desc[i].wb.eop)
270 cnt++;
271 iter++;
272 i = aq_next(i, ring->rx_size - 1);
273 }
274
275 AQ_DBG_EXIT(cnt);
276 return (cnt);
277 }
278
279 static void
aq_rx_set_cso_flags(volatile struct aq_rx_desc * rx_desc,if_rxd_info_t ri)280 aq_rx_set_cso_flags(volatile struct aq_rx_desc *rx_desc, if_rxd_info_t ri)
281 {
282 if ((rx_desc->wb.pkt_type & 0x3) == 0) { // IPv4
283 if (rx_desc->wb.rx_cntl & BIT(0)) { // IPv4 csum checked
284 ri->iri_csum_flags |= CSUM_IP_CHECKED;
285 if (!(rx_desc->wb.rx_stat & BIT(1)))
286 ri->iri_csum_flags |= CSUM_IP_VALID;
287 }
288 }
289 if (rx_desc->wb.rx_cntl & BIT(1)) { // TCP/UDP csum checked
290 ri->iri_csum_flags |= CSUM_L4_CALC;
291 if (!(rx_desc->wb.rx_stat & BIT(2)) && // L4 csum error
292 (rx_desc->wb.rx_stat & BIT(3))) { // L4 csum valid
293 ri->iri_csum_flags |= CSUM_L4_VALID;
294 ri->iri_csum_data = htons(0xffff);
295 }
296 }
297 }
298
299 static uint8_t bsd_rss_type[16] = {
300 [AQ_RX_RSS_TYPE_IPV4] = M_HASHTYPE_RSS_IPV4,
301 [AQ_RX_RSS_TYPE_IPV6] = M_HASHTYPE_RSS_IPV6,
302 [AQ_RX_RSS_TYPE_IPV4_TCP] = M_HASHTYPE_RSS_TCP_IPV4,
303 [AQ_RX_RSS_TYPE_IPV6_TCP] = M_HASHTYPE_RSS_TCP_IPV6,
304 [AQ_RX_RSS_TYPE_IPV4_UDP] = M_HASHTYPE_RSS_UDP_IPV4,
305 [AQ_RX_RSS_TYPE_IPV6_UDP] = M_HASHTYPE_RSS_UDP_IPV6,
306 };
307
308
309
310 static int
aq_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)311 aq_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
312 {
313 struct aq_dev *aq_dev = arg;
314 struct aq_ring *ring = aq_dev->rx_rings[ri->iri_qsidx];
315 volatile struct aq_rx_desc *rx_desc;
316 if_t ifp;
317 int cidx, rc = 0, i;
318 size_t len, total_len;
319 bool is_error = false;
320
321 AQ_DBG_ENTERA("[%d] start=%d", ring->index, ri->iri_cidx);
322 cidx = ri->iri_cidx;
323 ifp = iflib_get_ifp(aq_dev->ctx);
324 i = 0;
325 total_len = 0;
326
327 do {
328 if (i >= aq_dev->sctx->isc_rx_nsegments)
329 goto rx_err;
330
331 rx_desc = (volatile struct aq_rx_desc *) &ring->rx_descs[cidx];
332
333 trace_aq_rx_descr(&aq_dev->hw, ring->index, cidx,
334 (volatile uint64_t *)rx_desc);
335
336 /* MAC error (rx_stat) or RX-DMA fault (rdm_err) -> drop. */
337 if ((rx_desc->wb.rx_stat & BIT(0)) != 0 || rx_desc->wb.rdm_err)
338 is_error = true;
339
340 if (!rx_desc->wb.eop) {
341 len = ring->rx_buf_size;
342 } else if (is_error) {
343 total_len = le32toh(rx_desc->wb.pkt_len);
344 len = 0;
345 } else {
346 total_len = le32toh(rx_desc->wb.pkt_len);
347 if (total_len < (size_t)i * ring->rx_buf_size)
348 goto rx_err;
349 len = total_len - (size_t)i * ring->rx_buf_size;
350 if (len > (size_t)ring->rx_buf_size)
351 goto rx_err;
352 }
353 ri->iri_frags[i].irf_flid = 0;
354 ri->iri_frags[i].irf_idx = cidx;
355 ri->iri_frags[i].irf_len = len;
356
357 if ((if_getcapenable(ifp) & IFCAP_VLAN_HWTAGGING) != 0 &&
358 (rx_desc->wb.pkt_type & 0x60) != 0) {
359 ri->iri_flags |= M_VLANTAG;
360 ri->iri_vtag = le32toh(rx_desc->wb.vlan);
361 }
362
363 i++;
364 cidx = aq_next(cidx, ring->rx_size - 1);
365 } while (!rx_desc->wb.eop);
366
367 ri->iri_nfrags = i;
368 ri->iri_len = total_len;
369
370 /* Per-frame RX error: drop (zero-length frags), don't reset. */
371 if (is_error) {
372 counter_u64_add(ring->stats.rx_err, 1);
373 for (i = 0; i < ri->iri_nfrags; i++)
374 ri->iri_frags[i].irf_len = 0;
375 if (ri->iri_len == 0)
376 ri->iri_len = 1; /* iflib asserts iri_len != 0 */
377 goto exit;
378 }
379
380 if ((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) {
381 aq_rx_set_cso_flags(rx_desc, ri);
382 }
383 ri->iri_rsstype = bsd_rss_type[rx_desc->wb.rss_type & 0xF];
384 if (ri->iri_rsstype != M_HASHTYPE_NONE) {
385 ri->iri_flowid = le32toh(rx_desc->wb.rss_hash);
386 }
387
388 counter_u64_add(ring->stats.rx_bytes, total_len);
389 counter_u64_add(ring->stats.rx_pkts, 1);
390 goto exit;
391
392 rx_err:
393 counter_u64_add(ring->stats.rx_err, 1);
394 rc = EBADMSG;
395 exit:
396 AQ_DBG_EXIT(rc);
397 return (rc);
398 }
399
400 /*****************************************************************************/
401 /* */
402 /*****************************************************************************/
403
404 static void
aq_setup_offloads(struct aq_dev * aq_dev,if_pkt_info_t pi,volatile struct aq_tx_desc * txd,uint32_t tx_cmd)405 aq_setup_offloads(struct aq_dev *aq_dev, if_pkt_info_t pi, volatile struct aq_tx_desc *txd,
406 uint32_t tx_cmd)
407 {
408 AQ_DBG_ENTER();
409 txd->cmd |= tx_desc_cmd_fcs;
410 txd->cmd |= ((pi->ipi_flags & IPI_TX_IPV4) &&
411 (pi->ipi_csum_flags & (CSUM_IP | CSUM_IP_TSO))) ?
412 tx_desc_cmd_ipv4 : 0;
413 txd->cmd |= (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP |
414 CSUM_IP_UDP | CSUM_IP6_UDP)) ? tx_desc_cmd_l4cs : 0;
415 txd->cmd |= (pi->ipi_flags & IPI_TX_INTR) ? tx_desc_cmd_wb : 0;
416 txd->cmd |= tx_cmd;
417 AQ_DBG_EXIT(0);
418 }
419
420 static int
aq_ring_tso_setup(struct aq_dev * aq_dev,if_pkt_info_t pi,uint32_t * hdrlen,volatile union aq_txc_desc * txc)421 aq_ring_tso_setup(struct aq_dev *aq_dev, if_pkt_info_t pi, uint32_t *hdrlen,
422 volatile union aq_txc_desc *txc)
423 {
424 uint32_t tx_cmd = 0;
425
426 AQ_DBG_ENTER();
427 if (pi->ipi_csum_flags & CSUM_TSO) {
428 AQ_DBG_PRINT("aq_tso_setup(): TSO enabled");
429 tx_cmd |= tx_desc_cmd_lso | tx_desc_cmd_l4cs;
430
431 if (pi->ipi_ipproto != IPPROTO_TCP) {
432 AQ_DBG_PRINT("aq_tso_setup not a tcp");
433 AQ_DBG_EXIT(0);
434 return (0);
435 }
436
437 txc->cmd = 0x4; /* TCP */
438
439 if (pi->ipi_csum_flags & CSUM_IP6_TCP)
440 txc->cmd |= 0x2;
441
442 txc->l2_len = pi->ipi_ehdrlen;
443 txc->l3_len = pi->ipi_ip_hlen;
444 txc->l4_len = pi->ipi_tcp_hlen;
445 txc->mss_len = pi->ipi_tso_segsz;
446 *hdrlen = txc->l2_len + txc->l3_len + txc->l4_len;
447 }
448
449 // Set VLAN tag
450 if (pi->ipi_mflags & M_VLANTAG) {
451 tx_cmd |= tx_desc_cmd_vlan;
452 txc->vlan_tag = htole16(pi->ipi_vtag);
453 }
454
455 if (tx_cmd) {
456 txc->type = tx_desc_type_ctx;
457 txc->idx = 0;
458 }
459
460 AQ_DBG_EXIT(tx_cmd);
461 return (tx_cmd);
462 }
463
464 static int
aq_isc_txd_encap(void * arg,if_pkt_info_t pi)465 aq_isc_txd_encap(void *arg, if_pkt_info_t pi)
466 {
467 struct aq_dev *aq_dev = arg;
468 struct aq_ring *ring;
469 volatile union aq_txc_desc *txc;
470 volatile struct aq_tx_desc *txd = NULL;
471 bus_dma_segment_t *segs;
472 qidx_t pidx;
473 uint32_t hdrlen=0, pay_len;
474 uint8_t tx_cmd = 0;
475 int i, desc_count = 0;
476
477 AQ_DBG_ENTERA("[%d] start=%d", pi->ipi_qsidx, pi->ipi_pidx);
478 ring = aq_dev->tx_rings[pi->ipi_qsidx];
479
480 segs = pi->ipi_segs;
481 pidx = pi->ipi_pidx;
482 txc = (volatile union aq_txc_desc *)&ring->tx_descs[pidx];
483 AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p len %d", txc, txd, pi->ipi_len);
484
485 pay_len = pi->ipi_len;
486
487 txc->flags1 = 0U;
488 txc->flags2 = 0U;
489
490 tx_cmd = aq_ring_tso_setup(aq_dev, pi, &hdrlen, txc);
491 AQ_DBG_PRINT("tx_cmd = 0x%x", tx_cmd);
492
493 if (tx_cmd) {
494 trace_aq_tx_context_descr(&aq_dev->hw, ring->index, pidx,
495 (volatile void*)txc);
496 /* We've consumed the first desc, adjust counters */
497 pidx = aq_next(pidx, ring->tx_size - 1);
498
499 txd = &ring->tx_descs[pidx];
500 txd->flags = 0U;
501 } else {
502 txd = (volatile struct aq_tx_desc *)txc;
503 }
504 AQ_DBG_PRINT("txc at 0x%p, txd at 0x%p", txc, txd);
505
506 txd->ct_en = !!tx_cmd;
507
508 txd->type = tx_desc_type_desc;
509
510 aq_setup_offloads(aq_dev, pi, txd, tx_cmd);
511
512 if (tx_cmd) {
513 txd->ct_idx = 0;
514 }
515
516 pay_len -= hdrlen;
517
518 txd->pay_len = pay_len;
519
520 AQ_DBG_PRINT("num_frag[%d] pay_len[%d]", pi->ipi_nsegs, pay_len);
521 for (i = 0; i < pi->ipi_nsegs; i++) {
522 if (desc_count > 0) {
523 txd = &ring->tx_descs[pidx];
524 txd->flags = 0U;
525 }
526
527 txd->buf_addr = htole64(segs[i].ds_addr);
528
529 txd->type = tx_desc_type_desc;
530 txd->len = segs[i].ds_len;
531 txd->pay_len = pay_len;
532 if (i < pi->ipi_nsegs - 1)
533 trace_aq_tx_descr(&aq_dev->hw, ring->index, pidx,
534 (volatile void*)txd);
535
536 pidx = aq_next(pidx, ring->tx_size - 1);
537
538 desc_count++;
539 }
540 // Last descriptor requires EOP and WB
541 txd->eop = 1U;
542
543 AQ_DBG_DUMP_DESC(txd);
544 trace_aq_tx_descr(&aq_dev->hw, ring->index, pidx, (volatile void*)txd);
545 ring->tx_tail = pidx;
546
547 counter_u64_add(ring->stats.tx_pkts, 1);
548 counter_u64_add(ring->stats.tx_bytes, pay_len);
549
550 pi->ipi_new_pidx = pidx;
551
552 AQ_DBG_EXIT(0);
553 return (0);
554 }
555
556 static void
aq_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)557 aq_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
558 {
559 struct aq_dev *aq_dev = arg;
560 struct aq_ring *ring = aq_dev->tx_rings[txqid];
561 AQ_DBG_ENTERA("[%d] tail=%d", ring->index, pidx);
562
563 // Update the write pointer - submits packet for transmission
564 aq_ring_tx_tail_update(&aq_dev->hw, ring, pidx);
565 AQ_DBG_EXIT(0);
566 }
567
568
569 static inline unsigned int
aq_avail_desc(int a,int b,int size)570 aq_avail_desc(int a, int b, int size)
571 {
572 return (((b >= a)) ? ((size) - b + a) : (a - b));
573 }
574
575 static int
aq_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)576 aq_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
577 {
578 struct aq_dev *aq_dev = arg;
579 struct aq_ring *ring = aq_dev->tx_rings[txqid];
580 uint32_t head;
581 int avail;
582
583 AQ_DBG_ENTERA("[%d] clear=%d", ring->index, clear);
584 avail = 0;
585 head = tdm_tx_desc_head_ptr_get(&aq_dev->hw, ring->index);
586 AQ_DBG_PRINT("swhead %d hwhead %d", ring->tx_head, head);
587
588 if (head >= ring->tx_size) {
589 /* Malformed HW head; report no completions. */
590 goto done;
591 }
592
593 if (ring->tx_head == head) {
594 avail = 0;
595 goto done;
596 }
597
598 avail = aq_avail_desc(head, ring->tx_head, ring->tx_size);
599 if (clear)
600 ring->tx_head = head;
601
602 done:
603 AQ_DBG_EXIT(avail);
604 return (avail);
605 }
606