1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /* Copyright © 2003-2011 Emulex. All rights reserved. */
23
24 /*
25 * Source file containing the implementation of the Transmit
26 * Path
27 */
28
29 #include <oce_impl.h>
30
31 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed);
32 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed,
33 mblk_t *mp, uint32_t pkt_len);
34 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
35 uint32_t pkt_len);
36 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd);
37 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq,
38 size_t size, int flags);
39 static inline oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq);
40 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd);
41
42 static void oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
43 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
44 static oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq);
45 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq);
46 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
47 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed);
48 static void oce_remove_vtag(mblk_t *mp);
49 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag);
50 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm);
51
52
53 static ddi_dma_attr_t tx_map_dma_attr = {
54 DMA_ATTR_V0, /* version number */
55 0x0000000000000000ull, /* low address */
56 0xFFFFFFFFFFFFFFFFull, /* high address */
57 0x0000000000010000ull, /* dma counter max */
58 OCE_TXMAP_ALIGN, /* alignment */
59 0x7FF, /* burst sizes */
60 0x00000001, /* minimum transfer size */
61 0x00000000FFFFFFFFull, /* maximum transfer size */
62 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */
63 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */
64 0x00000001, /* granularity */
65 DDI_DMA_FLAGERR /* dma_attr_flags */
66 };
67
68
69 ddi_dma_attr_t oce_tx_dma_buf_attr = {
70 DMA_ATTR_V0, /* version number */
71 0x0000000000000000ull, /* low address */
72 0xFFFFFFFFFFFFFFFFull, /* high address */
73 0x00000000FFFFFFFFull, /* dma counter max */
74 OCE_DMA_ALIGNMENT, /* alignment */
75 0x000007FF, /* burst sizes */
76 0x00000001, /* minimum transfer size */
77 0x00000000FFFFFFFFull, /* maximum transfer size */
78 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */
79 1, /* scatter/gather list length */
80 0x00000001, /* granularity */
81 DDI_DMA_FLAGERR /* dma_attr_flags */
82 };
83
84 /*
85 * WQ map handle destructor
86 *
87 * wq - Pointer to WQ structure
88 * wqmd - pointer to WQE mapping handle descriptor
89 *
90 * return none
91 */
92
93 static void
oce_wqm_dtor(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)94 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
95 {
96 _NOTE(ARGUNUSED(wq));
97 /* Free the DMA handle */
98 if (wqmd->dma_handle != NULL)
99 (void) ddi_dma_free_handle(&(wqmd->dma_handle));
100 wqmd->dma_handle = NULL;
101 } /* oce_wqm_dtor */
102
103 /*
104 * WQ map handles contructor
105 *
106 * wqmd - pointer to WQE mapping handle descriptor
107 * wq - Pointer to WQ structure
108 *
109 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
110 */
111 static int
oce_wqm_ctor(oce_wq_mdesc_t * wqmd,struct oce_wq * wq)112 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq)
113 {
114 struct oce_dev *dev;
115 int ret;
116
117 dev = wq->parent;
118 /* Allocate DMA handle */
119 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr,
120 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle);
121
122 return (ret);
123 } /* oce_wqm_ctor */
124
125 /*
126 * function to create WQ mapping handles cache
127 *
128 * wq - pointer to WQ structure
129 *
130 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
131 */
132 int
oce_wqm_cache_create(struct oce_wq * wq)133 oce_wqm_cache_create(struct oce_wq *wq)
134 {
135 struct oce_dev *dev = wq->parent;
136 int size;
137 int cnt;
138 int ret;
139
140 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t);
141 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP);
142 if (wq->wq_mdesc_array == NULL) {
143 return (DDI_FAILURE);
144 }
145
146 /* Create the free buffer list */
147 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri));
148
149 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) {
150 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq);
151 if (ret != DDI_SUCCESS) {
152 goto wqm_fail;
153 }
154 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list,
155 &wq->wq_mdesc_array[cnt]);
156 }
157 return (DDI_SUCCESS);
158
159 wqm_fail:
160 oce_wqm_cache_destroy(wq);
161 return (DDI_FAILURE);
162 }
163
164 /*
165 * function to destroy WQ mapping handles cache
166 *
167 * wq - pointer to WQ structure
168 *
169 * return none
170 */
171 void
oce_wqm_cache_destroy(struct oce_wq * wq)172 oce_wqm_cache_destroy(struct oce_wq *wq)
173 {
174 oce_wq_mdesc_t *wqmd;
175
176 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) {
177 oce_wqm_dtor(wq, wqmd);
178 }
179
180 kmem_free(wq->wq_mdesc_array,
181 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t));
182
183 OCE_LIST_DESTROY(&wq->wq_mdesc_list);
184 }
185
186 /*
187 * function to create WQ buffer cache
188 *
189 * wq - pointer to WQ structure
190 * buf_size - size of the buffer
191 *
192 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
193 */
194 int
oce_wqb_cache_create(struct oce_wq * wq,size_t buf_size)195 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size)
196 {
197 struct oce_dev *dev = wq->parent;
198 int size;
199 int cnt;
200 int ret;
201
202 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t);
203 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP);
204 if (wq->wq_bdesc_array == NULL) {
205 return (DDI_FAILURE);
206 }
207
208 /* Create the free buffer list */
209 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri));
210
211 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) {
212 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt],
213 wq, buf_size, DDI_DMA_STREAMING);
214 if (ret != DDI_SUCCESS) {
215 goto wqb_fail;
216 }
217 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list,
218 &wq->wq_bdesc_array[cnt]);
219 }
220 return (DDI_SUCCESS);
221
222 wqb_fail:
223 oce_wqb_cache_destroy(wq);
224 return (DDI_FAILURE);
225 }
226
227 /*
228 * function to destroy WQ buffer cache
229 *
230 * wq - pointer to WQ structure
231 *
232 * return none
233 */
234 void
oce_wqb_cache_destroy(struct oce_wq * wq)235 oce_wqb_cache_destroy(struct oce_wq *wq)
236 {
237 oce_wq_bdesc_t *wqbd;
238 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) {
239 oce_wqb_dtor(wq, wqbd);
240 }
241 kmem_free(wq->wq_bdesc_array,
242 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t));
243 OCE_LIST_DESTROY(&wq->wq_buf_list);
244 }
245
246 /*
247 * WQ buffer constructor
248 *
249 * wqbd - pointer to WQ buffer descriptor
250 * wq - pointer to WQ structure
251 * size - size of the buffer
252 * flags - KM_SLEEP or KM_NOSLEEP
253 *
254 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
255 */
256 static int
oce_wqb_ctor(oce_wq_bdesc_t * wqbd,struct oce_wq * wq,size_t size,int flags)257 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags)
258 {
259 struct oce_dev *dev;
260 dev = wq->parent;
261
262 wqbd->wqb = oce_alloc_dma_buffer(dev, size, &oce_tx_dma_buf_attr,
263 flags);
264 if (wqbd->wqb == NULL) {
265 return (DDI_FAILURE);
266 }
267 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr);
268 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr);
269 return (DDI_SUCCESS);
270 }
271
272 /*
273 * WQ buffer destructor
274 *
275 * wq - pointer to WQ structure
276 * wqbd - pointer to WQ buffer descriptor
277 *
278 * return none
279 */
280 static void
oce_wqb_dtor(struct oce_wq * wq,oce_wq_bdesc_t * wqbd)281 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd)
282 {
283 oce_free_dma_buffer(wq->parent, wqbd->wqb);
284 }
285
286 /*
287 * function to alloc WQE buffer descriptor
288 *
289 * wq - pointer to WQ structure
290 *
291 * return pointer to WQE buffer descriptor
292 */
293 static inline oce_wq_bdesc_t *
oce_wqb_alloc(struct oce_wq * wq)294 oce_wqb_alloc(struct oce_wq *wq)
295 {
296 return (OCE_LIST_REM_HEAD(&wq->wq_buf_list));
297 }
298
299 /*
300 * function to free WQE buffer descriptor
301 *
302 * wq - pointer to WQ structure
303 * wqbd - pointer to WQ buffer descriptor
304 *
305 * return none
306 */
307 static inline void
oce_wqb_free(struct oce_wq * wq,oce_wq_bdesc_t * wqbd)308 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd)
309 {
310 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd);
311 } /* oce_wqb_free */
312
313 /*
314 * function to allocate WQE mapping descriptor
315 *
316 * wq - pointer to WQ structure
317 *
318 * return pointer to WQE mapping descriptor
319 */
320 static inline oce_wq_mdesc_t *
oce_wqm_alloc(struct oce_wq * wq)321 oce_wqm_alloc(struct oce_wq *wq)
322 {
323 return (OCE_LIST_REM_HEAD(&wq->wq_mdesc_list));
324 } /* oce_wqm_alloc */
325
326 /*
327 * function to insert WQE mapping descriptor to the list
328 *
329 * wq - pointer to WQ structure
330 * wqmd - Pointer to WQ mapping descriptor
331 *
332 * return none
333 */
334 static inline void
oce_wqm_free(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)335 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
336 {
337 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd);
338 }
339
340 /*
341 * function to free WQE mapping descriptor
342 *
343 * wq - pointer to WQ structure
344 * wqmd - Pointer to WQ mapping descriptor
345 *
346 * return none
347 */
348 static void
oce_wqmd_free(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)349 oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
350 {
351 if (wqmd == NULL) {
352 return;
353 }
354 (void) ddi_dma_unbind_handle(wqmd->dma_handle);
355 oce_wqm_free(wq, wqmd);
356 }
357
358 /*
359 * WQED kmem_cache constructor
360 *
361 * buf - pointer to WQE descriptor
362 *
363 * return DDI_SUCCESS
364 */
365 int
oce_wqe_desc_ctor(void * buf,void * arg,int kmflags)366 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags)
367 {
368 _NOTE(ARGUNUSED(buf));
369 _NOTE(ARGUNUSED(arg));
370 _NOTE(ARGUNUSED(kmflags));
371
372 return (DDI_SUCCESS);
373 }
374
375 /*
376 * WQED kmem_cache destructor
377 *
378 * buf - pointer to WQE descriptor
379 *
380 * return none
381 */
382 void
oce_wqe_desc_dtor(void * buf,void * arg)383 oce_wqe_desc_dtor(void *buf, void *arg)
384 {
385 _NOTE(ARGUNUSED(buf));
386 _NOTE(ARGUNUSED(arg));
387 }
388
389 /*
390 * function to choose a WQ given a mblk depending on priority, flowID etc.
391 *
392 * dev - software handle to device
393 * mp - the mblk to send
394 *
395 * return pointer to the WQ selected
396 */
397 static uint8_t oce_tx_hash_policy = 0x4;
398 struct oce_wq *
oce_get_wq(struct oce_dev * dev,mblk_t * mp)399 oce_get_wq(struct oce_dev *dev, mblk_t *mp)
400 {
401 struct oce_wq *wq;
402 int qidx = 0;
403 if (dev->nwqs > 1) {
404 qidx = mac_pkt_hash(DL_ETHER, mp, oce_tx_hash_policy, B_TRUE);
405 qidx = qidx % dev->nwqs;
406
407 } else {
408 qidx = 0;
409 }
410 wq = dev->wq[qidx];
411 /* for the time being hardcode */
412 return (wq);
413 } /* oce_get_wq */
414
415 /*
416 * function to populate the single WQE
417 *
418 * wq - pointer to wq
419 * wqed - pointer to WQ entry descriptor
420 *
421 * return none
422 */
423 static void
oce_fill_ring_descs(struct oce_wq * wq,oce_wqe_desc_t * wqed)424 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed)
425 {
426
427 struct oce_nic_frag_wqe *wqe;
428 int i;
429 /* Copy the precreate WQE descs to the ring desc */
430 for (i = 0; i < wqed->wqe_cnt; i++) {
431 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring,
432 struct oce_nic_frag_wqe);
433
434 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE);
435 RING_PUT(wq->ring, 1);
436 }
437 } /* oce_fill_ring_descs */
438
439 /*
440 * function to copy the packet to preallocated Tx buffer
441 *
442 * wq - pointer to WQ
443 * wqed - Pointer to WQE descriptor
444 * mp - Pointer to packet chain
445 * pktlen - Size of the packet
446 *
447 * return 0=>success, error code otherwise
448 */
449 static int
oce_bcopy_wqe(struct oce_wq * wq,oce_wqe_desc_t * wqed,mblk_t * mp,uint32_t pkt_len)450 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
451 uint32_t pkt_len)
452 {
453 oce_wq_bdesc_t *wqbd;
454 caddr_t buf_va;
455 struct oce_dev *dev = wq->parent;
456 int len = 0;
457
458 wqbd = oce_wqb_alloc(wq);
459 if (wqbd == NULL) {
460 atomic_inc_32(&dev->tx_noxmtbuf);
461 oce_log(dev, CE_WARN, MOD_TX, "%s",
462 "wqb pool empty");
463 return (ENOMEM);
464 }
465
466 /* create a fragment wqe for the packet */
467 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi;
468 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo;
469 buf_va = DBUF_VA(wqbd->wqb);
470
471 /* copy pkt into buffer */
472 for (len = 0; mp != NULL && len < pkt_len; mp = mp->b_cont) {
473 bcopy(mp->b_rptr, buf_va, MBLKL(mp));
474 buf_va += MBLKL(mp);
475 len += MBLKL(mp);
476 }
477
478 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len,
479 DDI_DMA_SYNC_FORDEV);
480
481 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) {
482 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
483 /* Free the buffer */
484 oce_wqb_free(wq, wqbd);
485 return (EIO);
486 }
487 wqed->frag[wqed->frag_idx].u0.s.frag_len = pkt_len;
488 wqed->hdesc[wqed->nhdl].hdl = (void *)(wqbd);
489 wqed->hdesc[wqed->nhdl].type = COPY_WQE;
490 wqed->frag_cnt++;
491 wqed->frag_idx++;
492 wqed->nhdl++;
493 return (0);
494 } /* oce_bcopy_wqe */
495
496 /*
497 * function to copy the packet or dma map on the fly depending on size
498 *
499 * wq - pointer to WQ
500 * wqed - Pointer to WQE descriptor
501 * mp - Pointer to packet chain
502 *
503 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
504 */
505 static int
oce_map_wqe(struct oce_wq * wq,oce_wqe_desc_t * wqed,mblk_t * mp,uint32_t pkt_len)506 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
507 uint32_t pkt_len)
508 {
509 ddi_dma_cookie_t cookie;
510 oce_wq_mdesc_t *wqmd;
511 uint32_t ncookies;
512 int ret;
513 struct oce_dev *dev = wq->parent;
514
515 wqmd = oce_wqm_alloc(wq);
516 if (wqmd == NULL) {
517 oce_log(dev, CE_WARN, MOD_TX, "%s",
518 "wqm pool empty");
519 return (ENOMEM);
520 }
521
522 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle,
523 (struct as *)0, (caddr_t)mp->b_rptr,
524 pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING,
525 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies);
526 if (ret != DDI_DMA_MAPPED) {
527 oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d",
528 ret);
529 /* free the last one */
530 oce_wqm_free(wq, wqmd);
531 return (ENOMEM);
532 }
533 do {
534 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi =
535 ADDR_HI(cookie.dmac_laddress);
536 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo =
537 ADDR_LO(cookie.dmac_laddress);
538 wqed->frag[wqed->frag_idx].u0.s.frag_len =
539 (uint32_t)cookie.dmac_size;
540 wqed->frag_cnt++;
541 wqed->frag_idx++;
542 if (--ncookies > 0)
543 ddi_dma_nextcookie(wqmd->dma_handle,
544 &cookie);
545 else break;
546 } while (ncookies > 0);
547
548 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd;
549 wqed->hdesc[wqed->nhdl].type = MAPPED_WQE;
550 wqed->nhdl++;
551 return (0);
552 } /* oce_map_wqe */
553
554 static inline int
oce_process_tx_compl(struct oce_wq * wq,boolean_t rearm)555 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm)
556 {
557 struct oce_nic_tx_cqe *cqe;
558 uint16_t num_cqe = 0;
559 struct oce_cq *cq;
560 oce_wqe_desc_t *wqed;
561 int wqe_freed = 0;
562 struct oce_dev *dev;
563
564 cq = wq->cq;
565 dev = wq->parent;
566 (void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0,
567 DDI_DMA_SYNC_FORKERNEL);
568
569 mutex_enter(&wq->txc_lock);
570 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
571 while (WQ_CQE_VALID(cqe)) {
572
573 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe));
574
575 /* update stats */
576 if (cqe->u0.s.status != 0) {
577 atomic_inc_32(&dev->tx_errors);
578 }
579
580 /* complete the WQEs */
581 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list);
582
583 wqe_freed = wqed->wqe_cnt;
584 oce_free_wqed(wq, wqed);
585 RING_GET(wq->ring, wqe_freed);
586 atomic_add_32(&wq->wq_free, wqe_freed);
587 /* clear the valid bit and progress cqe */
588 WQ_CQE_INVALIDATE(cqe);
589 RING_GET(cq->ring, 1);
590 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
591 struct oce_nic_tx_cqe);
592 num_cqe++;
593 } /* for all valid CQE */
594 mutex_exit(&wq->txc_lock);
595 if (num_cqe)
596 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm);
597 return (num_cqe);
598 } /* oce_process_tx_completion */
599
600 /*
601 * function to drain a TxCQ and process its CQEs
602 *
603 * dev - software handle to the device
604 * cq - pointer to the cq to drain
605 *
606 * return the number of CQEs processed
607 */
608 uint16_t
oce_drain_wq_cq(void * arg)609 oce_drain_wq_cq(void *arg)
610 {
611 uint16_t num_cqe = 0;
612 struct oce_dev *dev;
613 struct oce_wq *wq;
614
615 wq = (struct oce_wq *)arg;
616 dev = wq->parent;
617
618 /* do while we do not reach a cqe that is not valid */
619 num_cqe = oce_process_tx_compl(wq, B_FALSE);
620
621 /* check if we need to restart Tx */
622 if (wq->resched && num_cqe) {
623 wq->resched = B_FALSE;
624 mac_tx_update(dev->mac_handle);
625 }
626
627 return (num_cqe);
628 } /* oce_process_wq_cqe */
629
630 /*
631 * function to insert vtag to packet
632 *
633 * mp - mblk pointer
634 * vlan_tag - tag to be inserted
635 *
636 * return none
637 */
638 static inline void
oce_insert_vtag(mblk_t * mp,uint16_t vlan_tag)639 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag)
640 {
641 struct ether_vlan_header *evh;
642 (void) memmove(mp->b_rptr - VTAG_SIZE,
643 mp->b_rptr, 2 * ETHERADDRL);
644 mp->b_rptr -= VTAG_SIZE;
645 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
646 evh->ether_tpid = htons(VLAN_TPID);
647 evh->ether_tci = htons(vlan_tag);
648 }
649
650 /*
651 * function to strip vtag from packet
652 *
653 * mp - mblk pointer
654 *
655 * return none
656 */
657
658 static inline void
oce_remove_vtag(mblk_t * mp)659 oce_remove_vtag(mblk_t *mp)
660 {
661 (void) memmove(mp->b_rptr + VTAG_SIZE, mp->b_rptr,
662 ETHERADDRL * 2);
663 mp->b_rptr += VTAG_SIZE;
664 }
665
666 /*
667 * function to xmit Single packet over the wire
668 *
669 * wq - pointer to WQ
670 * mp - Pointer to packet chain
671 *
672 * return pointer to the packet
673 */
674 mblk_t *
oce_send_packet(struct oce_wq * wq,mblk_t * mp)675 oce_send_packet(struct oce_wq *wq, mblk_t *mp)
676 {
677 struct oce_nic_hdr_wqe *wqeh;
678 struct oce_dev *dev;
679 struct ether_header *eh;
680 struct ether_vlan_header *evh;
681 int32_t num_wqes;
682 uint16_t etype;
683 uint32_t ip_offset;
684 uint32_t csum_flags = 0;
685 boolean_t use_copy = B_FALSE;
686 boolean_t tagged = B_FALSE;
687 uint16_t vlan_tag;
688 uint32_t reg_value = 0;
689 oce_wqe_desc_t *wqed = NULL;
690 mblk_t *nmp = NULL;
691 mblk_t *tmp = NULL;
692 uint32_t pkt_len = 0;
693 int num_mblks = 0;
694 int ret = 0;
695 uint32_t mss = 0;
696 uint32_t flags = 0;
697 int len = 0;
698
699 /* retrieve the adap priv struct ptr */
700 dev = wq->parent;
701
702 /* check if we have enough free slots */
703 if (wq->wq_free < dev->tx_reclaim_threshold) {
704 (void) oce_process_tx_compl(wq, B_FALSE);
705 }
706 if (wq->wq_free < OCE_MAX_TX_HDL) {
707 return (mp);
708 }
709
710 /* check if we should copy */
711 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) {
712 pkt_len += MBLKL(tmp);
713 num_mblks++;
714 }
715
716 if (pkt_len == 0 || num_mblks == 0) {
717 freemsg(mp);
718 return (NULL);
719 }
720
721 /* retrieve LSO information */
722 mac_lso_get(mp, &mss, &flags);
723
724 /* get the offload flags */
725 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags);
726
727 /* restrict the mapped segment to wat we support */
728 if (num_mblks > OCE_MAX_TX_HDL) {
729 nmp = msgpullup(mp, -1);
730 if (nmp == NULL) {
731 atomic_inc_32(&wq->pkt_drops);
732 freemsg(mp);
733 return (NULL);
734 }
735 /* Reset it to new collapsed mp */
736 freemsg(mp);
737 mp = nmp;
738 }
739
740 /* Get the packet descriptor for Tx */
741 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP);
742 if (wqed == NULL) {
743 atomic_inc_32(&wq->pkt_drops);
744 freemsg(mp);
745 return (NULL);
746 }
747 eh = (struct ether_header *)(void *)mp->b_rptr;
748 if (ntohs(eh->ether_type) == VLAN_TPID) {
749 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
750 tagged = B_TRUE;
751 etype = ntohs(evh->ether_type);
752 ip_offset = sizeof (struct ether_vlan_header);
753 pkt_len -= VTAG_SIZE;
754 vlan_tag = ntohs(evh->ether_tci);
755 oce_remove_vtag(mp);
756 } else {
757 etype = ntohs(eh->ether_type);
758 ip_offset = sizeof (struct ether_header);
759 }
760
761 /* Save the WQ pointer */
762 wqed->wq = wq;
763 wqed->frag_idx = 1; /* index zero is always header */
764 wqed->frag_cnt = 0;
765 wqed->nhdl = 0;
766 wqed->mp = NULL;
767 OCE_LIST_LINK_INIT(&wqed->link);
768
769 /* If entire packet is less than the copy limit just do copy */
770 if (pkt_len < dev->tx_bcopy_limit) {
771 use_copy = B_TRUE;
772 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len);
773 } else {
774 /* copy or dma map the individual fragments */
775 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
776 len = MBLKL(nmp);
777 if (len == 0) {
778 continue;
779 }
780 if (len < dev->tx_bcopy_limit) {
781 ret = oce_bcopy_wqe(wq, wqed, nmp, len);
782 } else {
783 ret = oce_map_wqe(wq, wqed, nmp, len);
784 }
785 if (ret != 0)
786 break;
787 }
788 }
789
790 /*
791 * Any failure other than insufficient Q entries
792 * drop the packet
793 */
794 if (ret != 0) {
795 oce_free_wqed(wq, wqed);
796 atomic_inc_32(&wq->pkt_drops);
797 freemsg(mp);
798 return (NULL);
799 }
800
801 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0];
802 bzero(wqeh, sizeof (struct oce_nic_hdr_wqe));
803
804 /* fill rest of wqe header fields based on packet */
805 if (flags & HW_LSO) {
806 wqeh->u0.s.lso = B_TRUE;
807 wqeh->u0.s.lso_mss = mss;
808 }
809 if (csum_flags & HCK_FULLCKSUM) {
810 uint8_t *proto;
811 if (etype == ETHERTYPE_IP) {
812 proto = (uint8_t *)(void *)
813 (mp->b_rptr + ip_offset);
814 if (proto[9] == 6)
815 /* IPPROTO_TCP */
816 wqeh->u0.s.tcpcs = B_TRUE;
817 else if (proto[9] == 17)
818 /* IPPROTO_UDP */
819 wqeh->u0.s.udpcs = B_TRUE;
820 }
821 }
822
823 if (csum_flags & HCK_IPV4_HDRCKSUM)
824 wqeh->u0.s.ipcs = B_TRUE;
825 if (tagged) {
826 wqeh->u0.s.vlan = B_TRUE;
827 wqeh->u0.s.vlan_tag = vlan_tag;
828 }
829
830 wqeh->u0.s.complete = B_TRUE;
831 wqeh->u0.s.event = B_TRUE;
832 wqeh->u0.s.crc = B_TRUE;
833 wqeh->u0.s.total_length = pkt_len;
834
835 num_wqes = wqed->frag_cnt + 1;
836
837 /* h/w expects even no. of WQEs */
838 if (num_wqes & 0x1) {
839 bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe));
840 num_wqes++;
841 }
842 wqed->wqe_cnt = (uint16_t)num_wqes;
843 wqeh->u0.s.num_wqe = num_wqes;
844 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE));
845
846 mutex_enter(&wq->tx_lock);
847 if (num_wqes > wq->wq_free) {
848 atomic_inc_32(&wq->tx_deferd);
849 mutex_exit(&wq->tx_lock);
850 goto wqe_fail;
851 }
852 atomic_add_32(&wq->wq_free, -num_wqes);
853
854 /* fill the wq for adapter */
855 oce_fill_ring_descs(wq, wqed);
856
857 /* Set the mp pointer in the wqe descriptor */
858 if (use_copy == B_FALSE) {
859 wqed->mp = mp;
860 }
861 /* Add the packet desc to list to be retrieved during cmpl */
862 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed);
863 (void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0,
864 DDI_DMA_SYNC_FORDEV);
865
866 /* ring tx doorbell */
867 reg_value = (num_wqes << 16) | wq->wq_id;
868 /* Ring the door bell */
869 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value);
870 mutex_exit(&wq->tx_lock);
871 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) {
872 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
873 }
874
875 /* free mp if copied or packet chain collapsed */
876 if (use_copy == B_TRUE) {
877 freemsg(mp);
878 }
879 return (NULL);
880
881 wqe_fail:
882
883 if (tagged) {
884 oce_insert_vtag(mp, vlan_tag);
885 }
886 oce_free_wqed(wq, wqed);
887 return (mp);
888 } /* oce_send_packet */
889
890 /*
891 * function to free the WQE descriptor
892 *
893 * wq - pointer to WQ
894 * wqed - Pointer to WQE descriptor
895 *
896 * return none
897 */
898 static void
oce_free_wqed(struct oce_wq * wq,oce_wqe_desc_t * wqed)899 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed)
900 {
901 int i = 0;
902 if (wqed == NULL) {
903 return;
904 }
905
906 for (i = 0; i < wqed->nhdl; i++) {
907 if (wqed->hdesc[i].type == COPY_WQE) {
908 oce_wqb_free(wq, wqed->hdesc[i].hdl);
909 } else if (wqed->hdesc[i].type == MAPPED_WQE) {
910 oce_wqmd_free(wq, wqed->hdesc[i].hdl);
911 }
912 }
913 if (wqed->mp)
914 freemsg(wqed->mp);
915 kmem_cache_free(wq->wqed_cache, wqed);
916 } /* oce_free_wqed */
917
918 /*
919 * function to start the WQ
920 *
921 * wq - pointer to WQ
922 *
923 * return DDI_SUCCESS
924 */
925
926 int
oce_start_wq(struct oce_wq * wq)927 oce_start_wq(struct oce_wq *wq)
928 {
929 _NOTE(ARGUNUSED(wq));
930 return (DDI_SUCCESS);
931 } /* oce_start_wq */
932
933 /*
934 * function to stop the WQ
935 *
936 * wq - pointer to WQ
937 *
938 * return none
939 */
940 void
oce_clean_wq(struct oce_wq * wq)941 oce_clean_wq(struct oce_wq *wq)
942 {
943 oce_wqe_desc_t *wqed;
944 int ti;
945
946 /* Wait for already posted Tx to complete */
947
948 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) {
949 (void) oce_process_tx_compl(wq, B_FALSE);
950 OCE_MSDELAY(1);
951 }
952
953 /* Free the remaining descriptors */
954 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) {
955 atomic_add_32(&wq->wq_free, wqed->wqe_cnt);
956 oce_free_wqed(wq, wqed);
957 }
958 oce_drain_eq(wq->cq->eq);
959 } /* oce_stop_wq */
960
961 /*
962 * function to set the tx mapping handle fma attr
963 *
964 * fm_caps - capability flags
965 *
966 * return none
967 */
968
969 void
oce_set_tx_map_dma_fma_flags(int fm_caps)970 oce_set_tx_map_dma_fma_flags(int fm_caps)
971 {
972 if (fm_caps == DDI_FM_NOT_CAPABLE) {
973 return;
974 }
975
976 if (DDI_FM_DMA_ERR_CAP(fm_caps)) {
977 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
978 } else {
979 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
980 }
981 } /* oce_set_tx_map_dma_fma_flags */
982