xref: /titanic_44/usr/src/uts/common/io/nge/nge_rx.c (revision 0dc2366f7b9f9f36e10909b1e95edbf2a261c2ac)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #include "nge.h"
28 
29 #undef	NGE_DBG
30 #define	NGE_DBG		NGE_DBG_RECV
31 
32 #define	RXD_END		0x20000000
33 #define	RXD_ERR		0x40000000
34 #define	RXD_OWN		0x80000000
35 #define	RXD_CSUM_MSK	0x1C000000
36 #define	RXD_BCNT_MSK	0x00003FFF
37 
38 #define	RXD_CK8G_NO_HSUM	0x0
39 #define	RXD_CK8G_TCP_SUM_ERR	0x04000000
40 #define	RXD_CK8G_UDP_SUM_ERR	0x08000000
41 #define	RXD_CK8G_IP_HSUM_ERR	0x0C000000
42 #define	RXD_CK8G_IP_HSUM	0x10000000
43 #define	RXD_CK8G_TCP_SUM	0x14000000
44 #define	RXD_CK8G_UDP_SUM	0x18000000
45 #define	RXD_CK8G_RESV		0x1C000000
46 
47 extern ddi_device_acc_attr_t nge_data_accattr;
48 
49 /*
50  * Callback code invoked from STREAMs when the recv data buffer is free for
51  * recycling.
52  *
53  * The following table describes function behaviour:
54  *
55  *                      | mac stopped | mac running
56  * ---------------------------------------------------
57  * buffer delivered     | free buffer | recycle buffer
58  * buffer not delivered | do nothing  | recycle buffer (*)
59  *
60  * Note (*):
61  *   Recycle buffer only if mac state did not change during execution of
62  *   function. Otherwise if mac state changed, set buffer delivered & re-enter
63  *   function by calling freemsg().
64  */
65 
66 void
nge_recv_recycle(caddr_t arg)67 nge_recv_recycle(caddr_t arg)
68 {
69 	boolean_t val;
70 	boolean_t valid;
71 	nge_t *ngep;
72 	dma_area_t *bufp;
73 	buff_ring_t *brp;
74 	nge_sw_statistics_t *sw_stp;
75 
76 	bufp = (dma_area_t *)arg;
77 	ngep = (nge_t *)bufp->private;
78 	brp = ngep->buff;
79 	sw_stp = &ngep->statistics.sw_statistics;
80 
81 	/*
82 	 * Free the buffer directly if the buffer was allocated
83 	 * previously or mac was stopped.
84 	 */
85 	if (bufp->signature != brp->buf_sign) {
86 		if (bufp->rx_delivered == B_TRUE) {
87 			nge_free_dma_mem(bufp);
88 			kmem_free(bufp, sizeof (dma_area_t));
89 			val = nge_atomic_decrease(&brp->rx_hold, 1);
90 			ASSERT(val == B_TRUE);
91 		}
92 		return;
93 	}
94 
95 	/*
96 	 * recycle the data buffer again and fill them in free ring
97 	 */
98 	bufp->rx_recycle.free_func = nge_recv_recycle;
99 	bufp->rx_recycle.free_arg = (caddr_t)bufp;
100 
101 	bufp->mp = desballoc(DMA_VPTR(*bufp),
102 	    ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle);
103 
104 	if (bufp->mp == NULL) {
105 		sw_stp->mp_alloc_err++;
106 		sw_stp->recy_free++;
107 		nge_free_dma_mem(bufp);
108 		kmem_free(bufp, sizeof (dma_area_t));
109 		val = nge_atomic_decrease(&brp->rx_hold, 1);
110 		ASSERT(val == B_TRUE);
111 	} else {
112 
113 		mutex_enter(brp->recycle_lock);
114 		if (bufp->signature != brp->buf_sign)
115 			valid = B_TRUE;
116 		else
117 			valid = B_FALSE;
118 		bufp->rx_delivered = valid;
119 		if (bufp->rx_delivered == B_FALSE)  {
120 			bufp->next = brp->recycle_list;
121 			brp->recycle_list = bufp;
122 		}
123 		mutex_exit(brp->recycle_lock);
124 		if (valid == B_TRUE)
125 			/* call nge_rx_recycle again to free it */
126 			freemsg(bufp->mp);
127 		else {
128 			val = nge_atomic_decrease(&brp->rx_hold, 1);
129 			ASSERT(val == B_TRUE);
130 		}
131 	}
132 }
133 
134 /*
135  * Checking the rx's BDs (one or more) to receive
136  * one complete packet.
137  * start_index: the start indexer of BDs for one packet.
138  * end_index: the end indexer of BDs for one packet.
139  */
140 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len);
141 #pragma	inline(nge_recv_packet)
142 
143 static mblk_t *
nge_recv_packet(nge_t * ngep,uint32_t start_index,size_t len)144 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len)
145 {
146 	uint8_t *rptr;
147 	uint32_t minsize;
148 	uint32_t maxsize;
149 	mblk_t *mp;
150 	buff_ring_t *brp;
151 	sw_rx_sbd_t *srbdp;
152 	dma_area_t *bufp;
153 	nge_sw_statistics_t *sw_stp;
154 	void *hw_bd_p;
155 
156 	brp = ngep->buff;
157 	minsize = ETHERMIN;
158 	maxsize = ngep->max_sdu;
159 	sw_stp = &ngep->statistics.sw_statistics;
160 	mp = NULL;
161 
162 	srbdp = &brp->sw_rbds[start_index];
163 	DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL);
164 	hw_bd_p = DMA_VPTR(srbdp->desc);
165 
166 	/*
167 	 * First check the free_list, if it is NULL,
168 	 * make the recycle_list be free_list.
169 	 */
170 	if (brp->free_list == NULL) {
171 		mutex_enter(brp->recycle_lock);
172 		brp->free_list = brp->recycle_list;
173 		brp->recycle_list = NULL;
174 		mutex_exit(brp->recycle_lock);
175 	}
176 	bufp = brp->free_list;
177 	/* If it's not a qualified packet, delete it */
178 	if (len > maxsize || len < minsize) {
179 		ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
180 		    srbdp->bufp->alength);
181 		srbdp->flags = CONTROLER_OWN;
182 		return (NULL);
183 	}
184 
185 	/*
186 	 * If receive packet size is smaller than RX bcopy threshold,
187 	 * or there is no available buffer in free_list or recycle list,
188 	 * we use bcopy directly.
189 	 */
190 	if (len <= ngep->param_rxbcopy_threshold || bufp == NULL)
191 		brp->rx_bcopy = B_TRUE;
192 	else
193 		brp->rx_bcopy = B_FALSE;
194 
195 	if (brp->rx_bcopy) {
196 		mp = allocb(len + NGE_HEADROOM, 0);
197 		if (mp == NULL) {
198 			sw_stp->mp_alloc_err++;
199 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
200 			    srbdp->bufp->alength);
201 			srbdp->flags = CONTROLER_OWN;
202 			return (NULL);
203 		}
204 		rptr = DMA_VPTR(*srbdp->bufp);
205 		mp->b_rptr = mp->b_rptr + NGE_HEADROOM;
206 		bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len);
207 		mp->b_wptr = mp->b_rptr + len;
208 	} else {
209 		mp = srbdp->bufp->mp;
210 		/*
211 		 * Make sure the packet *contents* 4-byte aligned
212 		 */
213 		mp->b_rptr += NGE_HEADROOM;
214 		mp->b_wptr = mp->b_rptr + len;
215 		mp->b_next = mp->b_cont = NULL;
216 		srbdp->bufp->rx_delivered = B_TRUE;
217 		srbdp->bufp = NULL;
218 		nge_atomic_increase(&brp->rx_hold, 1);
219 
220 		/* Fill the buffer from free_list */
221 		srbdp->bufp = bufp;
222 		brp->free_list = bufp->next;
223 		bufp->next = NULL;
224 	}
225 
226 	/* replenish the buffer for hardware descriptor */
227 	ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
228 	    srbdp->bufp->alength);
229 	srbdp->flags = CONTROLER_OWN;
230 	sw_stp->rbytes += len;
231 	sw_stp->recv_count++;
232 
233 	return (mp);
234 }
235 
236 
237 #define	RX_HW_ERR	0x01
238 #define	RX_SUM_NO	0x02
239 #define	RX_SUM_ERR	0x04
240 
241 /*
242  * Statistic the rx's error
243  * and generate a log msg for these.
244  * Note:
245  * RXE, Parity Error, Symbo error, CRC error
246  * have been recored by nvidia's  hardware
247  * statistics part (nge_statistics). So it is uncessary to record them by
248  * driver in this place.
249  */
250 static uint32_t
251 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags);
252 #pragma	inline(nge_rxsta_handle)
253 
254 static uint32_t
nge_rxsta_handle(nge_t * ngep,uint32_t stflag,uint32_t * pflags)255 nge_rxsta_handle(nge_t *ngep,  uint32_t stflag, uint32_t *pflags)
256 {
257 	uint32_t errors;
258 	uint32_t err_flag;
259 	nge_sw_statistics_t *sw_stp;
260 
261 	err_flag = 0;
262 	sw_stp = &ngep->statistics.sw_statistics;
263 
264 	if ((RXD_END & stflag) == 0)
265 		return (RX_HW_ERR);
266 
267 	errors = stflag & RXD_CSUM_MSK;
268 	switch (errors) {
269 	default:
270 	break;
271 
272 	case RXD_CK8G_TCP_SUM:
273 	case RXD_CK8G_UDP_SUM:
274 		*pflags |= HCK_IPV4_HDRCKSUM_OK;
275 		*pflags |= HCK_FULLCKSUM_OK;
276 		break;
277 
278 	case RXD_CK8G_TCP_SUM_ERR:
279 	case RXD_CK8G_UDP_SUM_ERR:
280 		sw_stp->tcp_hwsum_err++;
281 		*pflags |= HCK_IPV4_HDRCKSUM_OK;
282 		break;
283 
284 	case RXD_CK8G_IP_HSUM:
285 		*pflags |= HCK_IPV4_HDRCKSUM_OK;
286 		break;
287 
288 	case RXD_CK8G_NO_HSUM:
289 		err_flag |= RX_SUM_NO;
290 		break;
291 
292 	case RXD_CK8G_IP_HSUM_ERR:
293 		sw_stp->ip_hwsum_err++;
294 		err_flag |=  RX_SUM_ERR;
295 		break;
296 	}
297 
298 	if ((stflag & RXD_ERR) != 0)	{
299 
300 		err_flag |= RX_HW_ERR;
301 		NGE_DEBUG(("Receive desc error, status: 0x%x", stflag));
302 	}
303 
304 	return (err_flag);
305 }
306 
307 static mblk_t *
nge_recv_ring(nge_t * ngep)308 nge_recv_ring(nge_t *ngep)
309 {
310 	uint32_t stflag;
311 	uint32_t flag_err;
312 	uint32_t sum_flags;
313 	size_t len;
314 	uint64_t end_index;
315 	uint64_t sync_start;
316 	mblk_t *mp;
317 	mblk_t **tail;
318 	mblk_t *head;
319 	recv_ring_t *rrp;
320 	buff_ring_t *brp;
321 	sw_rx_sbd_t *srbdp;
322 	void * hw_bd_p;
323 	nge_mode_cntl mode_cntl;
324 
325 	mp = NULL;
326 	head = NULL;
327 	tail = &head;
328 	rrp = ngep->recv;
329 	brp = ngep->buff;
330 
331 	end_index = sync_start = rrp->prod_index;
332 	/* Sync the descriptor for kernel */
333 	if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) {
334 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
335 		    sync_start * ngep->desc_attr.rxd_size,
336 		    ngep->param_recv_max_packet * ngep->desc_attr.rxd_size,
337 		    DDI_DMA_SYNC_FORKERNEL);
338 	} else {
339 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
340 		    sync_start * ngep->desc_attr.rxd_size,
341 		    0,
342 		    DDI_DMA_SYNC_FORKERNEL);
343 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
344 		    0,
345 		    (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) *
346 		    ngep->desc_attr.rxd_size,
347 		    DDI_DMA_SYNC_FORKERNEL);
348 	}
349 
350 	/*
351 	 * Looking through the rx's ring to find the good packets
352 	 * and try to receive more and more packets in rx's ring
353 	 */
354 	for (;;) {
355 		sum_flags = 0;
356 		flag_err = 0;
357 		end_index = rrp->prod_index;
358 		srbdp = &brp->sw_rbds[end_index];
359 		hw_bd_p = DMA_VPTR(srbdp->desc);
360 		stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len);
361 		/*
362 		 * If there is no packet in receving ring
363 		 * break the loop
364 		 */
365 		if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags)
366 			break;
367 
368 		ngep->recv_count++;
369 		flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags);
370 		if ((flag_err & RX_HW_ERR) == 0) {
371 			srbdp->flags = NGE_END_PACKET;
372 			mp = nge_recv_packet(ngep, end_index, len);
373 		} else {
374 			/* Hardware error, re-use the buffer */
375 			ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie,
376 			    srbdp->bufp->alength);
377 			srbdp->flags = CONTROLER_OWN;
378 		}
379 		if (mp != NULL) {
380 			if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) {
381 				mac_hcksum_set(mp, 0, 0, 0, 0, sum_flags);
382 			}
383 			*tail = mp;
384 			tail = &mp->b_next;
385 			mp = NULL;
386 		}
387 		rrp->prod_index = NEXT(end_index, rrp->desc.nslots);
388 		if (ngep->recv_count >= ngep->param_recv_max_packet)
389 			break;
390 	}
391 
392 	/* Sync the descriptors for device */
393 	if (sync_start + ngep->recv_count <= ngep->rx_desc) {
394 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
395 		    sync_start * ngep->desc_attr.rxd_size,
396 		    ngep->recv_count * ngep->desc_attr.rxd_size,
397 		    DDI_DMA_SYNC_FORDEV);
398 	} else {
399 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
400 		    sync_start * ngep->desc_attr.rxd_size,
401 		    0,
402 		    DDI_DMA_SYNC_FORDEV);
403 		(void) ddi_dma_sync(rrp->desc.dma_hdl,
404 		    0,
405 		    (ngep->recv_count + sync_start - ngep->rx_desc) *
406 		    ngep->desc_attr.rxd_size,
407 		    DDI_DMA_SYNC_FORDEV);
408 	}
409 	mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL);
410 	mode_cntl.mode_bits.rxdm = NGE_SET;
411 	mode_cntl.mode_bits.tx_rcom_en = NGE_SET;
412 	nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val);
413 
414 	return (head);
415 }
416 
417 void
nge_receive(nge_t * ngep)418 nge_receive(nge_t *ngep)
419 {
420 	mblk_t *mp;
421 	recv_ring_t *rrp;
422 	rrp = ngep->recv;
423 
424 	mp = nge_recv_ring(ngep);
425 	mutex_exit(ngep->genlock);
426 	if (mp != NULL)
427 		mac_rx(ngep->mh, rrp->handle, mp);
428 	mutex_enter(ngep->genlock);
429 }
430 
431 void
nge_hot_rxd_fill(void * hwd,const ddi_dma_cookie_t * cookie,size_t len)432 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
433 {
434 	uint64_t dmac_addr;
435 	hot_rx_bd * hw_bd_p;
436 
437 	hw_bd_p = (hot_rx_bd *)hwd;
438 	dmac_addr = cookie->dmac_laddress + NGE_HEADROOM;
439 
440 	hw_bd_p->cntl_status.cntl_val = 0;
441 
442 	hw_bd_p->host_buf_addr_hi = dmac_addr >> 32;
443 	hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr;
444 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
445 
446 	membar_producer();
447 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
448 }
449 
450 void
nge_sum_rxd_fill(void * hwd,const ddi_dma_cookie_t * cookie,size_t len)451 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len)
452 {
453 	sum_rx_bd * hw_bd_p;
454 
455 	hw_bd_p = hwd;
456 
457 	hw_bd_p->cntl_status.cntl_val = 0;
458 
459 	hw_bd_p->host_buf_addr =
460 	    (uint32_t)(cookie->dmac_address + NGE_HEADROOM);
461 	hw_bd_p->cntl_status.control_bits.bcnt = len - 1;
462 
463 	membar_producer();
464 	hw_bd_p->cntl_status.control_bits.own = NGE_SET;
465 }
466 
467 uint32_t
nge_hot_rxd_check(const void * hwd,size_t * len)468 nge_hot_rxd_check(const void *hwd, size_t *len)
469 {
470 	uint32_t err_flag;
471 	const hot_rx_bd * hrbdp;
472 
473 	hrbdp = hwd;
474 	err_flag = hrbdp->cntl_status.cntl_val;
475 	*len = err_flag & RXD_BCNT_MSK;
476 	return (err_flag);
477 }
478 
479 uint32_t
nge_sum_rxd_check(const void * hwd,size_t * len)480 nge_sum_rxd_check(const void *hwd, size_t *len)
481 {
482 	uint32_t err_flag;
483 	const sum_rx_bd * hrbdp;
484 
485 	hrbdp = hwd;
486 
487 	err_flag = hrbdp->cntl_status.cntl_val;
488 	*len = err_flag & RXD_BCNT_MSK;
489 	return (err_flag);
490 }
491