1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /* Copyright © 2003-2011 Emulex. All rights reserved. */
23
24 /*
25 * Source file containing the implementation of the Transmit
26 * Path
27 */
28
29 #include <oce_impl.h>
30
31 static void oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed);
32 static int oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed,
33 mblk_t *mp, uint32_t pkt_len);
34 static int oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
35 uint32_t pkt_len);
36 static void oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd);
37 static int oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq,
38 size_t size, int flags);
39 static inline oce_wq_bdesc_t *oce_wqb_alloc(struct oce_wq *wq);
40 static void oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd);
41
42 static void oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
43 static void oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
44 static oce_wq_mdesc_t *oce_wqm_alloc(struct oce_wq *wq);
45 static int oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq);
46 static void oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd);
47 static void oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed);
48 static void oce_remove_vtag(mblk_t *mp);
49 static void oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag);
50 static inline int oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm);
51
52
53 static ddi_dma_attr_t tx_map_dma_attr = {
54 DMA_ATTR_V0, /* version number */
55 0x0000000000000000ull, /* low address */
56 0xFFFFFFFFFFFFFFFFull, /* high address */
57 0x0000000000010000ull, /* dma counter max */
58 OCE_TXMAP_ALIGN, /* alignment */
59 0x7FF, /* burst sizes */
60 0x00000001, /* minimum transfer size */
61 0x00000000FFFFFFFFull, /* maximum transfer size */
62 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */
63 OCE_MAX_TXDMA_COOKIES, /* scatter/gather list length */
64 0x00000001, /* granularity */
65 DDI_DMA_FLAGERR /* dma_attr_flags */
66 };
67
68
69 ddi_dma_attr_t oce_tx_dma_buf_attr = {
70 DMA_ATTR_V0, /* version number */
71 0x0000000000000000ull, /* low address */
72 0xFFFFFFFFFFFFFFFFull, /* high address */
73 0x00000000FFFFFFFFull, /* dma counter max */
74 OCE_DMA_ALIGNMENT, /* alignment */
75 0x000007FF, /* burst sizes */
76 0x00000001, /* minimum transfer size */
77 0x00000000FFFFFFFFull, /* maximum transfer size */
78 0xFFFFFFFFFFFFFFFFull, /* maximum segment size */
79 1, /* scatter/gather list length */
80 0x00000001, /* granularity */
81 DDI_DMA_FLAGERR /* dma_attr_flags */
82 };
83
84 /*
85 * WQ map handle destructor
86 *
87 * wq - Pointer to WQ structure
88 * wqmd - pointer to WQE mapping handle descriptor
89 *
90 * return none
91 */
92
93 static void
oce_wqm_dtor(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)94 oce_wqm_dtor(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
95 {
96 _NOTE(ARGUNUSED(wq));
97 /* Free the DMA handle */
98 if (wqmd->dma_handle != NULL)
99 (void) ddi_dma_free_handle(&(wqmd->dma_handle));
100 wqmd->dma_handle = NULL;
101 } /* oce_wqm_dtor */
102
103 /*
104 * WQ map handles contructor
105 *
106 * wqmd - pointer to WQE mapping handle descriptor
107 * wq - Pointer to WQ structure
108 *
109 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
110 */
111 static int
oce_wqm_ctor(oce_wq_mdesc_t * wqmd,struct oce_wq * wq)112 oce_wqm_ctor(oce_wq_mdesc_t *wqmd, struct oce_wq *wq)
113 {
114 struct oce_dev *dev;
115 int ret;
116
117 dev = wq->parent;
118 /* Allocate DMA handle */
119 ret = ddi_dma_alloc_handle(dev->dip, &tx_map_dma_attr,
120 DDI_DMA_DONTWAIT, NULL, &wqmd->dma_handle);
121
122 return (ret);
123 } /* oce_wqm_ctor */
124
125 /*
126 * function to create WQ mapping handles cache
127 *
128 * wq - pointer to WQ structure
129 *
130 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
131 */
132 int
oce_wqm_cache_create(struct oce_wq * wq)133 oce_wqm_cache_create(struct oce_wq *wq)
134 {
135 struct oce_dev *dev = wq->parent;
136 int size;
137 int cnt;
138 int ret;
139
140 size = wq->cfg.nhdl * sizeof (oce_wq_mdesc_t);
141 wq->wq_mdesc_array = kmem_zalloc(size, KM_NOSLEEP);
142 if (wq->wq_mdesc_array == NULL) {
143 return (DDI_FAILURE);
144 }
145
146 /* Create the free buffer list */
147 OCE_LIST_CREATE(&wq->wq_mdesc_list, DDI_INTR_PRI(dev->intr_pri));
148
149 for (cnt = 0; cnt < wq->cfg.nhdl; cnt++) {
150 ret = oce_wqm_ctor(&wq->wq_mdesc_array[cnt], wq);
151 if (ret != DDI_SUCCESS) {
152 goto wqm_fail;
153 }
154 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list,
155 &wq->wq_mdesc_array[cnt]);
156 }
157 return (DDI_SUCCESS);
158
159 wqm_fail:
160 oce_wqm_cache_destroy(wq);
161 return (DDI_FAILURE);
162 }
163
164 /*
165 * function to destroy WQ mapping handles cache
166 *
167 * wq - pointer to WQ structure
168 *
169 * return none
170 */
171 void
oce_wqm_cache_destroy(struct oce_wq * wq)172 oce_wqm_cache_destroy(struct oce_wq *wq)
173 {
174 oce_wq_mdesc_t *wqmd;
175
176 while ((wqmd = OCE_LIST_REM_HEAD(&wq->wq_mdesc_list)) != NULL) {
177 oce_wqm_dtor(wq, wqmd);
178 }
179
180 kmem_free(wq->wq_mdesc_array,
181 wq->cfg.nhdl * sizeof (oce_wq_mdesc_t));
182
183 OCE_LIST_DESTROY(&wq->wq_mdesc_list);
184 }
185
186 /*
187 * function to create WQ buffer cache
188 *
189 * wq - pointer to WQ structure
190 * buf_size - size of the buffer
191 *
192 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
193 */
194 int
oce_wqb_cache_create(struct oce_wq * wq,size_t buf_size)195 oce_wqb_cache_create(struct oce_wq *wq, size_t buf_size)
196 {
197 struct oce_dev *dev = wq->parent;
198 int size;
199 int cnt;
200 int ret;
201
202 size = wq->cfg.nbufs * sizeof (oce_wq_bdesc_t);
203 wq->wq_bdesc_array = kmem_zalloc(size, KM_NOSLEEP);
204 if (wq->wq_bdesc_array == NULL) {
205 return (DDI_FAILURE);
206 }
207
208 /* Create the free buffer list */
209 OCE_LIST_CREATE(&wq->wq_buf_list, DDI_INTR_PRI(dev->intr_pri));
210
211 for (cnt = 0; cnt < wq->cfg.nbufs; cnt++) {
212 ret = oce_wqb_ctor(&wq->wq_bdesc_array[cnt],
213 wq, buf_size, DDI_DMA_STREAMING);
214 if (ret != DDI_SUCCESS) {
215 goto wqb_fail;
216 }
217 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list,
218 &wq->wq_bdesc_array[cnt]);
219 }
220 return (DDI_SUCCESS);
221
222 wqb_fail:
223 oce_wqb_cache_destroy(wq);
224 return (DDI_FAILURE);
225 }
226
227 /*
228 * function to destroy WQ buffer cache
229 *
230 * wq - pointer to WQ structure
231 *
232 * return none
233 */
234 void
oce_wqb_cache_destroy(struct oce_wq * wq)235 oce_wqb_cache_destroy(struct oce_wq *wq)
236 {
237 oce_wq_bdesc_t *wqbd;
238 while ((wqbd = OCE_LIST_REM_HEAD(&wq->wq_buf_list)) != NULL) {
239 oce_wqb_dtor(wq, wqbd);
240 }
241 kmem_free(wq->wq_bdesc_array,
242 wq->cfg.nbufs * sizeof (oce_wq_bdesc_t));
243 OCE_LIST_DESTROY(&wq->wq_buf_list);
244 }
245
246 /*
247 * WQ buffer constructor
248 *
249 * wqbd - pointer to WQ buffer descriptor
250 * wq - pointer to WQ structure
251 * size - size of the buffer
252 * flags - KM_SLEEP or KM_NOSLEEP
253 *
254 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
255 */
256 static int
oce_wqb_ctor(oce_wq_bdesc_t * wqbd,struct oce_wq * wq,size_t size,int flags)257 oce_wqb_ctor(oce_wq_bdesc_t *wqbd, struct oce_wq *wq, size_t size, int flags)
258 {
259 struct oce_dev *dev;
260 dev = wq->parent;
261
262 wqbd->wqb = oce_alloc_dma_buffer(dev, size, &oce_tx_dma_buf_attr,
263 flags);
264 if (wqbd->wqb == NULL) {
265 return (DDI_FAILURE);
266 }
267 wqbd->frag_addr.dw.addr_lo = ADDR_LO(wqbd->wqb->addr);
268 wqbd->frag_addr.dw.addr_hi = ADDR_HI(wqbd->wqb->addr);
269 return (DDI_SUCCESS);
270 }
271
272 /*
273 * WQ buffer destructor
274 *
275 * wq - pointer to WQ structure
276 * wqbd - pointer to WQ buffer descriptor
277 *
278 * return none
279 */
280 static void
oce_wqb_dtor(struct oce_wq * wq,oce_wq_bdesc_t * wqbd)281 oce_wqb_dtor(struct oce_wq *wq, oce_wq_bdesc_t *wqbd)
282 {
283 oce_free_dma_buffer(wq->parent, wqbd->wqb);
284 }
285
286 /*
287 * function to alloc WQE buffer descriptor
288 *
289 * wq - pointer to WQ structure
290 *
291 * return pointer to WQE buffer descriptor
292 */
293 static inline oce_wq_bdesc_t *
oce_wqb_alloc(struct oce_wq * wq)294 oce_wqb_alloc(struct oce_wq *wq)
295 {
296 return (OCE_LIST_REM_HEAD(&wq->wq_buf_list));
297 }
298
299 /*
300 * function to free WQE buffer descriptor
301 *
302 * wq - pointer to WQ structure
303 * wqbd - pointer to WQ buffer descriptor
304 *
305 * return none
306 */
307 static inline void
oce_wqb_free(struct oce_wq * wq,oce_wq_bdesc_t * wqbd)308 oce_wqb_free(struct oce_wq *wq, oce_wq_bdesc_t *wqbd)
309 {
310 OCE_LIST_INSERT_TAIL(&wq->wq_buf_list, wqbd);
311 } /* oce_wqb_free */
312
313 /*
314 * function to allocate WQE mapping descriptor
315 *
316 * wq - pointer to WQ structure
317 *
318 * return pointer to WQE mapping descriptor
319 */
320 static inline oce_wq_mdesc_t *
oce_wqm_alloc(struct oce_wq * wq)321 oce_wqm_alloc(struct oce_wq *wq)
322 {
323 return (OCE_LIST_REM_HEAD(&wq->wq_mdesc_list));
324 } /* oce_wqm_alloc */
325
326 /*
327 * function to insert WQE mapping descriptor to the list
328 *
329 * wq - pointer to WQ structure
330 * wqmd - Pointer to WQ mapping descriptor
331 *
332 * return none
333 */
334 static inline void
oce_wqm_free(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)335 oce_wqm_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
336 {
337 OCE_LIST_INSERT_TAIL(&wq->wq_mdesc_list, wqmd);
338 }
339
340 /*
341 * function to free WQE mapping descriptor
342 *
343 * wq - pointer to WQ structure
344 * wqmd - Pointer to WQ mapping descriptor
345 *
346 * return none
347 */
348 static void
oce_wqmd_free(struct oce_wq * wq,oce_wq_mdesc_t * wqmd)349 oce_wqmd_free(struct oce_wq *wq, oce_wq_mdesc_t *wqmd)
350 {
351 if (wqmd == NULL) {
352 return;
353 }
354 (void) ddi_dma_unbind_handle(wqmd->dma_handle);
355 oce_wqm_free(wq, wqmd);
356 }
357
358 /*
359 * WQED kmem_cache constructor
360 *
361 * buf - pointer to WQE descriptor
362 *
363 * return DDI_SUCCESS
364 */
365 int
oce_wqe_desc_ctor(void * buf,void * arg,int kmflags)366 oce_wqe_desc_ctor(void *buf, void *arg, int kmflags)
367 {
368 _NOTE(ARGUNUSED(buf));
369 _NOTE(ARGUNUSED(arg));
370 _NOTE(ARGUNUSED(kmflags));
371
372 return (DDI_SUCCESS);
373 }
374
375 /*
376 * WQED kmem_cache destructor
377 *
378 * buf - pointer to WQE descriptor
379 *
380 * return none
381 */
382 void
oce_wqe_desc_dtor(void * buf,void * arg)383 oce_wqe_desc_dtor(void *buf, void *arg)
384 {
385 _NOTE(ARGUNUSED(buf));
386 _NOTE(ARGUNUSED(arg));
387 }
388
389 /*
390 * function to choose a WQ given a mblk depending on priority, flowID etc.
391 *
392 * dev - software handle to device
393 * mp - the mblk to send
394 *
395 * return pointer to the WQ selected
396 */
397 static uint8_t oce_tx_hash_policy = 0x4;
398 struct oce_wq *
oce_get_wq(struct oce_dev * dev,mblk_t * mp)399 oce_get_wq(struct oce_dev *dev, mblk_t *mp)
400 {
401 struct oce_wq *wq;
402 int qidx = 0;
403 if (dev->nwqs > 1) {
404 qidx = mac_pkt_hash(DL_ETHER, mp, oce_tx_hash_policy, B_TRUE);
405 qidx = qidx % dev->nwqs;
406
407 } else {
408 qidx = 0;
409 }
410 wq = dev->wq[qidx];
411 /* for the time being hardcode */
412 return (wq);
413 } /* oce_get_wq */
414
415 /*
416 * function to populate the single WQE
417 *
418 * wq - pointer to wq
419 * wqed - pointer to WQ entry descriptor
420 *
421 * return none
422 */
423 #pragma inline(oce_fill_ring_descs)
424 static void
oce_fill_ring_descs(struct oce_wq * wq,oce_wqe_desc_t * wqed)425 oce_fill_ring_descs(struct oce_wq *wq, oce_wqe_desc_t *wqed)
426 {
427
428 struct oce_nic_frag_wqe *wqe;
429 int i;
430 /* Copy the precreate WQE descs to the ring desc */
431 for (i = 0; i < wqed->wqe_cnt; i++) {
432 wqe = RING_GET_PRODUCER_ITEM_VA(wq->ring,
433 struct oce_nic_frag_wqe);
434
435 bcopy(&wqed->frag[i], wqe, NIC_WQE_SIZE);
436 RING_PUT(wq->ring, 1);
437 }
438 } /* oce_fill_ring_descs */
439
440 /*
441 * function to copy the packet to preallocated Tx buffer
442 *
443 * wq - pointer to WQ
444 * wqed - Pointer to WQE descriptor
445 * mp - Pointer to packet chain
446 * pktlen - Size of the packet
447 *
448 * return 0=>success, error code otherwise
449 */
450 static int
oce_bcopy_wqe(struct oce_wq * wq,oce_wqe_desc_t * wqed,mblk_t * mp,uint32_t pkt_len)451 oce_bcopy_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
452 uint32_t pkt_len)
453 {
454 oce_wq_bdesc_t *wqbd;
455 caddr_t buf_va;
456 struct oce_dev *dev = wq->parent;
457 int len = 0;
458
459 wqbd = oce_wqb_alloc(wq);
460 if (wqbd == NULL) {
461 atomic_inc_32(&dev->tx_noxmtbuf);
462 oce_log(dev, CE_WARN, MOD_TX, "%s",
463 "wqb pool empty");
464 return (ENOMEM);
465 }
466
467 /* create a fragment wqe for the packet */
468 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi = wqbd->frag_addr.dw.addr_hi;
469 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo = wqbd->frag_addr.dw.addr_lo;
470 buf_va = DBUF_VA(wqbd->wqb);
471
472 /* copy pkt into buffer */
473 for (len = 0; mp != NULL && len < pkt_len; mp = mp->b_cont) {
474 bcopy(mp->b_rptr, buf_va, MBLKL(mp));
475 buf_va += MBLKL(mp);
476 len += MBLKL(mp);
477 }
478
479 (void) ddi_dma_sync(DBUF_DHDL(wqbd->wqb), 0, pkt_len,
480 DDI_DMA_SYNC_FORDEV);
481
482 if (oce_fm_check_dma_handle(dev, DBUF_DHDL(wqbd->wqb))) {
483 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
484 /* Free the buffer */
485 oce_wqb_free(wq, wqbd);
486 return (EIO);
487 }
488 wqed->frag[wqed->frag_idx].u0.s.frag_len = pkt_len;
489 wqed->hdesc[wqed->nhdl].hdl = (void *)(wqbd);
490 wqed->hdesc[wqed->nhdl].type = COPY_WQE;
491 wqed->frag_cnt++;
492 wqed->frag_idx++;
493 wqed->nhdl++;
494 return (0);
495 } /* oce_bcopy_wqe */
496
497 /*
498 * function to copy the packet or dma map on the fly depending on size
499 *
500 * wq - pointer to WQ
501 * wqed - Pointer to WQE descriptor
502 * mp - Pointer to packet chain
503 *
504 * return DDI_SUCCESS=>success, DDI_FAILURE=>error
505 */
506 static int
oce_map_wqe(struct oce_wq * wq,oce_wqe_desc_t * wqed,mblk_t * mp,uint32_t pkt_len)507 oce_map_wqe(struct oce_wq *wq, oce_wqe_desc_t *wqed, mblk_t *mp,
508 uint32_t pkt_len)
509 {
510 ddi_dma_cookie_t cookie;
511 oce_wq_mdesc_t *wqmd;
512 uint32_t ncookies;
513 int ret;
514 struct oce_dev *dev = wq->parent;
515
516 wqmd = oce_wqm_alloc(wq);
517 if (wqmd == NULL) {
518 oce_log(dev, CE_WARN, MOD_TX, "%s",
519 "wqm pool empty");
520 return (ENOMEM);
521 }
522
523 ret = ddi_dma_addr_bind_handle(wqmd->dma_handle,
524 (struct as *)0, (caddr_t)mp->b_rptr,
525 pkt_len, DDI_DMA_WRITE | DDI_DMA_STREAMING,
526 DDI_DMA_DONTWAIT, NULL, &cookie, &ncookies);
527 if (ret != DDI_DMA_MAPPED) {
528 oce_log(dev, CE_WARN, MOD_TX, "MAP FAILED %d",
529 ret);
530 /* free the last one */
531 oce_wqm_free(wq, wqmd);
532 return (ENOMEM);
533 }
534 do {
535 wqed->frag[wqed->frag_idx].u0.s.frag_pa_hi =
536 ADDR_HI(cookie.dmac_laddress);
537 wqed->frag[wqed->frag_idx].u0.s.frag_pa_lo =
538 ADDR_LO(cookie.dmac_laddress);
539 wqed->frag[wqed->frag_idx].u0.s.frag_len =
540 (uint32_t)cookie.dmac_size;
541 wqed->frag_cnt++;
542 wqed->frag_idx++;
543 if (--ncookies > 0)
544 ddi_dma_nextcookie(wqmd->dma_handle,
545 &cookie);
546 else break;
547 } while (ncookies > 0);
548
549 wqed->hdesc[wqed->nhdl].hdl = (void *)wqmd;
550 wqed->hdesc[wqed->nhdl].type = MAPPED_WQE;
551 wqed->nhdl++;
552 return (0);
553 } /* oce_map_wqe */
554
555 static inline int
oce_process_tx_compl(struct oce_wq * wq,boolean_t rearm)556 oce_process_tx_compl(struct oce_wq *wq, boolean_t rearm)
557 {
558 struct oce_nic_tx_cqe *cqe;
559 uint16_t num_cqe = 0;
560 struct oce_cq *cq;
561 oce_wqe_desc_t *wqed;
562 int wqe_freed = 0;
563 struct oce_dev *dev;
564
565 cq = wq->cq;
566 dev = wq->parent;
567 (void) ddi_dma_sync(cq->ring->dbuf->dma_handle, 0, 0,
568 DDI_DMA_SYNC_FORKERNEL);
569
570 mutex_enter(&wq->txc_lock);
571 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
572 while (WQ_CQE_VALID(cqe)) {
573
574 DW_SWAP(u32ptr(cqe), sizeof (struct oce_nic_tx_cqe));
575
576 /* update stats */
577 if (cqe->u0.s.status != 0) {
578 atomic_inc_32(&dev->tx_errors);
579 }
580
581 /* complete the WQEs */
582 wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list);
583
584 wqe_freed = wqed->wqe_cnt;
585 oce_free_wqed(wq, wqed);
586 RING_GET(wq->ring, wqe_freed);
587 atomic_add_32(&wq->wq_free, wqe_freed);
588 /* clear the valid bit and progress cqe */
589 WQ_CQE_INVALIDATE(cqe);
590 RING_GET(cq->ring, 1);
591 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring,
592 struct oce_nic_tx_cqe);
593 num_cqe++;
594 } /* for all valid CQE */
595 mutex_exit(&wq->txc_lock);
596 if (num_cqe)
597 oce_arm_cq(wq->parent, cq->cq_id, num_cqe, rearm);
598 return (num_cqe);
599 } /* oce_process_tx_completion */
600
601 /*
602 * function to drain a TxCQ and process its CQEs
603 *
604 * dev - software handle to the device
605 * cq - pointer to the cq to drain
606 *
607 * return the number of CQEs processed
608 */
609 uint16_t
oce_drain_wq_cq(void * arg)610 oce_drain_wq_cq(void *arg)
611 {
612 uint16_t num_cqe = 0;
613 struct oce_dev *dev;
614 struct oce_wq *wq;
615
616 wq = (struct oce_wq *)arg;
617 dev = wq->parent;
618
619 /* do while we do not reach a cqe that is not valid */
620 num_cqe = oce_process_tx_compl(wq, B_FALSE);
621
622 /* check if we need to restart Tx */
623 if (wq->resched && num_cqe) {
624 wq->resched = B_FALSE;
625 mac_tx_update(dev->mac_handle);
626 }
627
628 return (num_cqe);
629 } /* oce_process_wq_cqe */
630
631 /*
632 * function to insert vtag to packet
633 *
634 * mp - mblk pointer
635 * vlan_tag - tag to be inserted
636 *
637 * return none
638 */
639 static inline void
oce_insert_vtag(mblk_t * mp,uint16_t vlan_tag)640 oce_insert_vtag(mblk_t *mp, uint16_t vlan_tag)
641 {
642 struct ether_vlan_header *evh;
643 (void) memmove(mp->b_rptr - VTAG_SIZE,
644 mp->b_rptr, 2 * ETHERADDRL);
645 mp->b_rptr -= VTAG_SIZE;
646 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
647 evh->ether_tpid = htons(VLAN_TPID);
648 evh->ether_tci = htons(vlan_tag);
649 }
650
651 /*
652 * function to strip vtag from packet
653 *
654 * mp - mblk pointer
655 *
656 * return none
657 */
658
659 static inline void
oce_remove_vtag(mblk_t * mp)660 oce_remove_vtag(mblk_t *mp)
661 {
662 (void) memmove(mp->b_rptr + VTAG_SIZE, mp->b_rptr,
663 ETHERADDRL * 2);
664 mp->b_rptr += VTAG_SIZE;
665 }
666
667 /*
668 * function to xmit Single packet over the wire
669 *
670 * wq - pointer to WQ
671 * mp - Pointer to packet chain
672 *
673 * return pointer to the packet
674 */
675 mblk_t *
oce_send_packet(struct oce_wq * wq,mblk_t * mp)676 oce_send_packet(struct oce_wq *wq, mblk_t *mp)
677 {
678 struct oce_nic_hdr_wqe *wqeh;
679 struct oce_dev *dev;
680 struct ether_header *eh;
681 struct ether_vlan_header *evh;
682 int32_t num_wqes;
683 uint16_t etype;
684 uint32_t ip_offset;
685 uint32_t csum_flags = 0;
686 boolean_t use_copy = B_FALSE;
687 boolean_t tagged = B_FALSE;
688 uint16_t vlan_tag;
689 uint32_t reg_value = 0;
690 oce_wqe_desc_t *wqed = NULL;
691 mblk_t *nmp = NULL;
692 mblk_t *tmp = NULL;
693 uint32_t pkt_len = 0;
694 int num_mblks = 0;
695 int ret = 0;
696 uint32_t mss = 0;
697 uint32_t flags = 0;
698 int len = 0;
699
700 /* retrieve the adap priv struct ptr */
701 dev = wq->parent;
702
703 /* check if we have enough free slots */
704 if (wq->wq_free < dev->tx_reclaim_threshold) {
705 (void) oce_process_tx_compl(wq, B_FALSE);
706 }
707 if (wq->wq_free < OCE_MAX_TX_HDL) {
708 return (mp);
709 }
710
711 /* check if we should copy */
712 for (tmp = mp; tmp != NULL; tmp = tmp->b_cont) {
713 pkt_len += MBLKL(tmp);
714 num_mblks++;
715 }
716
717 if (pkt_len == 0 || num_mblks == 0) {
718 freemsg(mp);
719 return (NULL);
720 }
721
722 /* retrieve LSO information */
723 mac_lso_get(mp, &mss, &flags);
724
725 /* get the offload flags */
726 mac_hcksum_get(mp, NULL, NULL, NULL, NULL, &csum_flags);
727
728 /* restrict the mapped segment to wat we support */
729 if (num_mblks > OCE_MAX_TX_HDL) {
730 nmp = msgpullup(mp, -1);
731 if (nmp == NULL) {
732 atomic_inc_32(&wq->pkt_drops);
733 freemsg(mp);
734 return (NULL);
735 }
736 /* Reset it to new collapsed mp */
737 freemsg(mp);
738 mp = nmp;
739 }
740
741 /* Get the packet descriptor for Tx */
742 wqed = kmem_cache_alloc(wq->wqed_cache, KM_NOSLEEP);
743 if (wqed == NULL) {
744 atomic_inc_32(&wq->pkt_drops);
745 freemsg(mp);
746 return (NULL);
747 }
748 eh = (struct ether_header *)(void *)mp->b_rptr;
749 if (ntohs(eh->ether_type) == VLAN_TPID) {
750 evh = (struct ether_vlan_header *)(void *)mp->b_rptr;
751 tagged = B_TRUE;
752 etype = ntohs(evh->ether_type);
753 ip_offset = sizeof (struct ether_vlan_header);
754 pkt_len -= VTAG_SIZE;
755 vlan_tag = ntohs(evh->ether_tci);
756 oce_remove_vtag(mp);
757 } else {
758 etype = ntohs(eh->ether_type);
759 ip_offset = sizeof (struct ether_header);
760 }
761
762 /* Save the WQ pointer */
763 wqed->wq = wq;
764 wqed->frag_idx = 1; /* index zero is always header */
765 wqed->frag_cnt = 0;
766 wqed->nhdl = 0;
767 wqed->mp = NULL;
768 OCE_LIST_LINK_INIT(&wqed->link);
769
770 /* If entire packet is less than the copy limit just do copy */
771 if (pkt_len < dev->tx_bcopy_limit) {
772 use_copy = B_TRUE;
773 ret = oce_bcopy_wqe(wq, wqed, mp, pkt_len);
774 } else {
775 /* copy or dma map the individual fragments */
776 for (nmp = mp; nmp != NULL; nmp = nmp->b_cont) {
777 len = MBLKL(nmp);
778 if (len == 0) {
779 continue;
780 }
781 if (len < dev->tx_bcopy_limit) {
782 ret = oce_bcopy_wqe(wq, wqed, nmp, len);
783 } else {
784 ret = oce_map_wqe(wq, wqed, nmp, len);
785 }
786 if (ret != 0)
787 break;
788 }
789 }
790
791 /*
792 * Any failure other than insufficient Q entries
793 * drop the packet
794 */
795 if (ret != 0) {
796 oce_free_wqed(wq, wqed);
797 atomic_inc_32(&wq->pkt_drops);
798 freemsg(mp);
799 return (NULL);
800 }
801
802 wqeh = (struct oce_nic_hdr_wqe *)&wqed->frag[0];
803 bzero(wqeh, sizeof (struct oce_nic_hdr_wqe));
804
805 /* fill rest of wqe header fields based on packet */
806 if (flags & HW_LSO) {
807 wqeh->u0.s.lso = B_TRUE;
808 wqeh->u0.s.lso_mss = mss;
809 }
810 if (csum_flags & HCK_FULLCKSUM) {
811 uint8_t *proto;
812 if (etype == ETHERTYPE_IP) {
813 proto = (uint8_t *)(void *)
814 (mp->b_rptr + ip_offset);
815 if (proto[9] == 6)
816 /* IPPROTO_TCP */
817 wqeh->u0.s.tcpcs = B_TRUE;
818 else if (proto[9] == 17)
819 /* IPPROTO_UDP */
820 wqeh->u0.s.udpcs = B_TRUE;
821 }
822 }
823
824 if (csum_flags & HCK_IPV4_HDRCKSUM)
825 wqeh->u0.s.ipcs = B_TRUE;
826 if (tagged) {
827 wqeh->u0.s.vlan = B_TRUE;
828 wqeh->u0.s.vlan_tag = vlan_tag;
829 }
830
831 wqeh->u0.s.complete = B_TRUE;
832 wqeh->u0.s.event = B_TRUE;
833 wqeh->u0.s.crc = B_TRUE;
834 wqeh->u0.s.total_length = pkt_len;
835
836 num_wqes = wqed->frag_cnt + 1;
837
838 /* h/w expects even no. of WQEs */
839 if (num_wqes & 0x1) {
840 bzero(&wqed->frag[num_wqes], sizeof (struct oce_nic_frag_wqe));
841 num_wqes++;
842 }
843 wqed->wqe_cnt = (uint16_t)num_wqes;
844 wqeh->u0.s.num_wqe = num_wqes;
845 DW_SWAP(u32ptr(&wqed->frag[0]), (wqed->wqe_cnt * NIC_WQE_SIZE));
846
847 mutex_enter(&wq->tx_lock);
848 if (num_wqes > wq->wq_free) {
849 atomic_inc_32(&wq->tx_deferd);
850 mutex_exit(&wq->tx_lock);
851 goto wqe_fail;
852 }
853 atomic_add_32(&wq->wq_free, -num_wqes);
854
855 /* fill the wq for adapter */
856 oce_fill_ring_descs(wq, wqed);
857
858 /* Set the mp pointer in the wqe descriptor */
859 if (use_copy == B_FALSE) {
860 wqed->mp = mp;
861 }
862 /* Add the packet desc to list to be retrieved during cmpl */
863 OCE_LIST_INSERT_TAIL(&wq->wqe_desc_list, wqed);
864 (void) ddi_dma_sync(wq->ring->dbuf->dma_handle, 0, 0,
865 DDI_DMA_SYNC_FORDEV);
866
867 /* ring tx doorbell */
868 reg_value = (num_wqes << 16) | wq->wq_id;
869 /* Ring the door bell */
870 OCE_DB_WRITE32(dev, PD_TXULP_DB, reg_value);
871 mutex_exit(&wq->tx_lock);
872 if (oce_fm_check_acc_handle(dev, dev->db_handle) != DDI_FM_OK) {
873 ddi_fm_service_impact(dev->dip, DDI_SERVICE_DEGRADED);
874 }
875
876 /* free mp if copied or packet chain collapsed */
877 if (use_copy == B_TRUE) {
878 freemsg(mp);
879 }
880 return (NULL);
881
882 wqe_fail:
883
884 if (tagged) {
885 oce_insert_vtag(mp, vlan_tag);
886 }
887 oce_free_wqed(wq, wqed);
888 return (mp);
889 } /* oce_send_packet */
890
891 /*
892 * function to free the WQE descriptor
893 *
894 * wq - pointer to WQ
895 * wqed - Pointer to WQE descriptor
896 *
897 * return none
898 */
899 #pragma inline(oce_free_wqed)
900 static void
oce_free_wqed(struct oce_wq * wq,oce_wqe_desc_t * wqed)901 oce_free_wqed(struct oce_wq *wq, oce_wqe_desc_t *wqed)
902 {
903 int i = 0;
904 if (wqed == NULL) {
905 return;
906 }
907
908 for (i = 0; i < wqed->nhdl; i++) {
909 if (wqed->hdesc[i].type == COPY_WQE) {
910 oce_wqb_free(wq, wqed->hdesc[i].hdl);
911 } else if (wqed->hdesc[i].type == MAPPED_WQE) {
912 oce_wqmd_free(wq, wqed->hdesc[i].hdl);
913 }
914 }
915 if (wqed->mp)
916 freemsg(wqed->mp);
917 kmem_cache_free(wq->wqed_cache, wqed);
918 } /* oce_free_wqed */
919
920 /*
921 * function to start the WQ
922 *
923 * wq - pointer to WQ
924 *
925 * return DDI_SUCCESS
926 */
927
928 int
oce_start_wq(struct oce_wq * wq)929 oce_start_wq(struct oce_wq *wq)
930 {
931 _NOTE(ARGUNUSED(wq));
932 return (DDI_SUCCESS);
933 } /* oce_start_wq */
934
935 /*
936 * function to stop the WQ
937 *
938 * wq - pointer to WQ
939 *
940 * return none
941 */
942 void
oce_clean_wq(struct oce_wq * wq)943 oce_clean_wq(struct oce_wq *wq)
944 {
945 oce_wqe_desc_t *wqed;
946 int ti;
947
948 /* Wait for already posted Tx to complete */
949
950 for (ti = 0; ti < DEFAULT_DRAIN_TIME; ti++) {
951 (void) oce_process_tx_compl(wq, B_FALSE);
952 OCE_MSDELAY(1);
953 }
954
955 /* Free the remaining descriptors */
956 while ((wqed = OCE_LIST_REM_HEAD(&wq->wqe_desc_list)) != NULL) {
957 atomic_add_32(&wq->wq_free, wqed->wqe_cnt);
958 oce_free_wqed(wq, wqed);
959 }
960 oce_drain_eq(wq->cq->eq);
961 } /* oce_stop_wq */
962
963 /*
964 * function to set the tx mapping handle fma attr
965 *
966 * fm_caps - capability flags
967 *
968 * return none
969 */
970
971 void
oce_set_tx_map_dma_fma_flags(int fm_caps)972 oce_set_tx_map_dma_fma_flags(int fm_caps)
973 {
974 if (fm_caps == DDI_FM_NOT_CAPABLE) {
975 return;
976 }
977
978 if (DDI_FM_DMA_ERR_CAP(fm_caps)) {
979 tx_map_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
980 } else {
981 tx_map_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
982 }
983 } /* oce_set_tx_map_dma_fma_flags */
984