xref: /titanic_50/usr/src/uts/common/io/chxge/sge.c (revision 2a52a1627973d84afe3af6de87c1fe00ac3d087e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * This file is part of the Chelsio T1 Ethernet driver.
29  *
30  * Copyright (C) 2003-2005 Chelsio Communications.  All rights reserved.
31  */
32 
33 #pragma ident	"%Z%%M%	%I%	%E% SMI"
34 
35 #include <sys/types.h>
36 #include <sys/param.h>
37 #include <sys/cmn_err.h>
38 #include <sys/sunddi.h>
39 #include <sys/kmem.h>
40 #include <sys/cmn_err.h>
41 #include <sys/byteorder.h>
42 #include <sys/atomic.h>
43 #include <sys/stropts.h>
44 #include <sys/stream.h>
45 #include <sys/strsubr.h>
46 #include <sys/dlpi.h>
47 #include <sys/kstat.h>
48 #include <sys/ethernet.h>
49 #include <netinet/in.h>
50 #include <netinet/udp.h>
51 #include <inet/common.h>
52 #include <inet/nd.h>
53 #include <inet/ip.h>
54 #include <inet/tcp.h>
55 #include <netinet/udp.h>
56 #include <sys/gld.h>
57 #include "ostypes.h"
58 #include "common.h"
59 #ifdef CONFIG_CHELSIO_T1_1G
60 #include "fpga_defs.h"
61 #endif
62 #include "regs.h"
63 #include "suni1x10gexp_regs.h"
64 #include "sge.h"
65 #include "espi.h"
66 
67 #include "ch.h"
68 
69 extern uint32_t buffers_in_use[];
70 
71 uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N;
72 uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N;
73 uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N;
74 uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N;
75 uint32_t sge_respq_cnt = SGE_RESPQ_E_N;
76 
77 uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N;
78 uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N;
79 uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N;
80 uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N;
81 uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N;
82 
83 #ifdef HOST_PAUSE
84 uint32_t do_host_pause = 1;
85 uint32_t flq_pause_window = 64;
86 #endif
87 
88 static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb,
89     ulong_t *dh);
90 void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t);
91 
92 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q,
93     unsigned int len, unsigned int offload);
94 #ifdef HOST_PAUSE
95 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
96 #endif
97 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
98 static void freelQs_empty(pesge *sge);
99 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
100 static int alloc_rx_resources(pesge *sge, struct sge_params *p);
101 static int alloc_tx_resources(pesge *sge, struct sge_params *p);
102 static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size,
103     int base_reg_lo, int base_reg_hi, int size_reg);
104 static void configure_sge(pesge *sge, struct sge_params *p);
105 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
106 static void free_rx_resources(pesge *sge);
107 static void free_tx_resources(pesge *sge);
108 static inline unsigned int jumbo_payload_capacity(pesge *sge);
109 #ifdef SUN_KSTATS
110 static int sge_kstat_setup(pesge *);
111 static void sge_kstat_remove(pesge *);
112 static int sge_kstat_update(p_kstat_t, int);
113 #endif
114 static uint16_t calc_ocsum(mblk_t *, int);
115 
116 /*
117  * Local routines.
118  */
119 static inline void sge_ring_doorbell(pesge *sge, u32 control_reg);
120 
121 static inline void
122 sge_ring_doorbell(pesge *sge, u32 control_reg)
123 {
124 	membar_producer();
125 	t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg);
126 }
127 
128 /*
129  * DESC:
130  *
131  * NOTES:   Must have at least 1 command queue and 1 freelist queue.
132  *
133  */
134 pesge *
135 t1_sge_create(ch_t *sa, struct sge_params *p)
136 {
137 	pesge *sge;
138 
139 	sge = t1_os_malloc_wait_zero(sizeof (pesge));
140 
141 	if (sge == NULL)
142 		goto error_no_mem;
143 
144 	memset(sge, 0, sizeof (*sge));
145 
146 	/*
147 	 * PR2928 & PR3309
148 	 * set default timeout value - 20 msec
149 	 * we set the initial value to 2 which gurantees at least one tick.
150 	 */
151 	if (is_T2(sa))
152 		sge->ptimeout = 1;
153 
154 	sge->obj = sa;
155 #ifdef SUN_KSTATS
156 	if (sge_kstat_setup(sge) != 0)
157 		goto t1_sge_create_fail1;
158 #endif
159 	p->cmdQ_size[0] = sge_cmdq0_cnt;
160 	p->cmdQ_size[1] = sge_cmdq1_cnt;
161 
162 	/* note that jumbo frame index is inverted for T2 */
163 	if (is_T2(sa)) {
164 		p->freelQ_size[1] = sge_flq0_cnt;
165 		p->freelQ_size[0] = sge_flq1_cnt;
166 	} else {
167 		p->freelQ_size[0] = sge_flq0_cnt;
168 		p->freelQ_size[1] = sge_flq1_cnt;
169 	}
170 
171 #if CH_DEBUG
172 	/* DEBUG only */
173 	cmn_err(CE_NOTE, "sge: %p\n", sge);
174 	cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]);
175 	cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]);
176 	cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]);
177 	cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ);
178 	cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt);
179 #endif
180 #ifdef SUN_KSTATS
181 	goto error_no_mem;
182 
183 t1_sge_create_fail1:
184 	t1_os_free(sge, sizeof (pesge));
185 	sge = NULL;
186 #endif
187 error_no_mem:
188 	return (sge);
189 }
190 
191 int
192 t1_sge_destroy(pesge* sge)
193 {
194 	if (sge != NULL) {
195 		free_tx_resources(sge);
196 		free_rx_resources(sge);
197 
198 		/* PR2928 & PR3309 */
199 		if ((is_T2(sge->obj)) && (sge->pskb))
200 			pe_free_fake_arp(sge->pskb);
201 #ifdef SUN_KSTATS
202 		sge_kstat_remove(sge);
203 #endif
204 		t1_os_free(sge, sizeof (pesge));
205 	}
206 	return (0);
207 }
208 
209 /*
210  * PR2928 & PR3309
211  * call out event from timeout
212  *
213  * there is a potential race between the timeout and the close.
214  * unless we protect the timeout, the close could occur at the
215  * same time. Then if the timeout service routine was slow or
216  * interrupted, the sge_stop() could complete with a timeoutID
217  * that has expired, thus letting another timeout occur. If the
218  * service routine was delayed still further, a detach could occur.
219  * the second time could then end up accessing memory that has been
220  * released back to the system. Bad things could then occur. We
221  * set a flag in sge_stop() to tell the service routine not to
222  * issue further timeouts. sge_stop() will block until a timeout
223  * has occured. If the command Q is full then we shouldn't put out
224  * an arp.
225  */
226 
227 void
228 t1_espi_workaround(ch_t *adapter)
229 {
230 	pesge *sge = adapter->sge;
231 	ch_t *chp = (ch_t *)sge->obj;
232 	int rv = 1;
233 
234 	if ((chp->ch_state == PERUNNING) &&
235 	    atomic_read(&sge->cmdQ[0].cq_asleep)) {
236 		u32 seop;
237 		seop = t1_espi_get_mon(adapter, 0x930, 0);
238 		if ((seop & 0xfff0fff) == 0xfff) {
239 			/* after first arp */
240 			if (sge->pskb)
241 				rv = pe_start(adapter, (mblk_t *)sge->pskb,
242 				    CH_ARP);
243 				if (!rv)
244 					sge->intr_cnt.arp_sent++;
245 		}
246 	}
247 #ifdef HOST_PAUSE
248 	/*
249 	 * If we are already in sge_data_in, then we can skip calling
250 	 * t1_sge_check_pause() this clock cycle. lockstat showed that
251 	 * we were blocking on the mutex ~ 2% of the time.
252 	 */
253 	if (mutex_tryenter(&adapter->ch_intr)) {
254 		t1_sge_check_pause(sge, &sge->freelQ[0]);
255 		t1_sge_check_pause(sge, &sge->freelQ[1]);
256 		mutex_exit(&adapter->ch_intr);
257 	}
258 #endif
259 }
260 
261 int
262 sge_start(pesge *sge)
263 {
264 	t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control);
265 	/* PR2928 & PR3309, also need to avoid Pause deadlock */
266 	ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic,
267 	    (void (*)(void *))t1_espi_workaround, sge->obj);
268 	ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout);
269 	return (0);
270 }
271 
272 /*
273  * Disables SGE queues.
274  */
275 int
276 sge_stop(pesge *sge)
277 {
278 	uint32_t status;
279 	int loops;
280 
281 	DBGASSERT(sge);
282 
283 	/* PR2928 & PR3309, also need to avoid Pause deadlock */
284 	t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0);
285 
286 	/* wait until there's no more outstanding interrupts pending */
287 	loops = 0;
288 	do {
289 		status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE);
290 		t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status);
291 		drv_usecwait(125);
292 		loops++;
293 	} while (status && (loops < 1000));
294 
295 	ch_stop_cyclic(&sge->espi_wa_cyclic);
296 
297 	return (0);
298 }
299 
300 uint32_t sge_cmdq_send_fail;
301 
302 int
303 sge_data_out(pesge* sge, int qid, mblk_t *m0,
304 			cmdQ_ce_t *cmp, int count, uint32_t flg)
305 {
306 	struct cmdQ *Q = &sge->cmdQ[qid];
307 	ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh;
308 	spinlock_t *qlock = &Q->cq_qlock;
309 	cmdQ_e *e;
310 	cmdQ_e *q = Q->cq_entries;
311 	uint32_t credits;
312 	uint32_t pidx;
313 	uint32_t genbit;
314 	uint32_t entries_n = Q->cq_entries_n;
315 	cmdQ_ce_t *ce;
316 	cmdQ_ce_t *cq = Q->cq_centries;
317 	dma_addr_t mapping;
318 	uint32_t j = 0;
319 	uint32_t offset;
320 #if defined(TX_CKSUM_FIX)
321 	uint16_t csum;
322 	uint16_t *csum_loc;
323 #endif
324 #ifdef TX_THREAD_RECLAIM
325 	uint32_t reclaim_cnt;
326 #endif
327 
328 	/*
329 	 * We must exit if we don't have enough free command queue entries
330 	 * available.
331 	 */
332 
333 	spin_lock(qlock);
334 
335 #if defined(TX_CKSUM_FIX)
336 	/*
337 	 * This checksum fix will address a fragmented datagram
338 	 * checksum error. Which will lead to the next packet after
339 	 * the last packet with the More fragment bit set having its
340 	 * checksum corrupted. When the packet reaches this point
341 	 * the 'flg' variable indicates whether a checksum is needed
342 	 * or not. The algorithm is as follows, if the current packet
343 	 * is a More fragment set the count of packets to be checksummed
344 	 * after it to 3. If it't not and the count of is more than 0
345 	 * then calculate the checksum in software, if a hardware checksum
346 	 * was requested. Then decrment the count. Same algorithm applies
347 	 * to TCP.
348 	 */
349 	if (flg & CH_UDP_MF) {
350 		sge->do_udp_csum = 3;
351 	} else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) {
352 		if ((flg & CH_NO_HWCKSUM) == 0) {
353 			/*
354 			 *  Calc Checksum here.
355 			 */
356 			csum = calc_ocsum(m0,
357 			    sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
358 			csum_loc = (uint16_t *)(m0->b_rptr +
359 			    sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
360 			csum_loc += (((*(char *)csum_loc) & 0x0f) << 1);
361 
362 			sge->intr_cnt.tx_soft_cksums++;
363 			((struct udphdr *)(csum_loc))->uh_sum = csum;
364 			((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1;
365 		}
366 		sge->do_udp_csum--;
367 	} else if (flg & CH_TCP_MF) {
368 		sge->do_tcp_csum = 3;
369 	} else if (sge->do_tcp_csum != 0) {
370 		if ((flg & CH_NO_HWCKSUM) == 0) {
371 			sge->intr_cnt.tx_soft_cksums++;
372 			/*
373 			 *  Calc Checksum here.
374 			 */
375 		}
376 		sge->do_tcp_csum--;
377 	}
378 #endif	/* TX_CKSUM_FIX */
379 #ifdef TX_THREAD_RECLAIM
380 	reclaim_cnt = Q->cq_complete;
381 	if (reclaim_cnt > SGE_BATCH_THRESH) {
382 		sge->intr_cnt.tx_reclaims[qid]++;
383 		free_cmdQ_buffers(sge, Q, reclaim_cnt);
384 		Q->cq_complete = 0;
385 	}
386 #endif
387 	genbit = Q->cq_genbit;
388 	pidx = Q->cq_pidx;
389 	credits = Q->cq_credits;
390 
391 	if ((credits - 1) < count) {
392 		spin_unlock(qlock);
393 		sge->intr_cnt.cmdQ_full[qid]++;
394 		return (1);
395 	}
396 
397 	atomic_sub(count, &Q->cq_credits);
398 	Q->cq_pidx += count;
399 	if (Q->cq_pidx >= entries_n) {
400 		Q->cq_pidx -= entries_n;
401 		Q->cq_genbit ^= 1;
402 	}
403 
404 	spin_unlock(qlock);
405 
406 #ifdef SUN_KSTATS
407 	if (count > MBLK_MAX)
408 		sge->intr_cnt.tx_descs[MBLK_MAX - 1]++;
409 	else
410 		sge->intr_cnt.tx_descs[count]++;
411 #endif
412 
413 	ce = &cq[pidx];
414 	*ce = *cmp;
415 	mapping = cmp->ce_pa;
416 	j++;
417 
418 	e = &q[pidx];
419 
420 	offset = (caddr_t)e - (caddr_t)q;
421 
422 	e->Sop =  1;
423 	e->DataValid = 1;
424 	e->BufferLength = cmp->ce_len;
425 	e->AddrHigh = ((u64)mapping >> 32);
426 	e->AddrLow = ((u64)mapping & 0xffffffff);
427 
428 	--count;
429 	if (count > 0) {
430 		unsigned int i;
431 
432 		e->Eop = 0;
433 		wmb();
434 		e->GenerationBit = e->GenerationBit2 = genbit;
435 
436 		for (i = 0; i < count; i++) {
437 
438 			ce++;
439 			e++;
440 			cmp++;
441 			if (++pidx == entries_n) {
442 				pidx = 0;
443 				genbit ^= 1;
444 				/* sync from offset to end of cmdQ */
445 				(void) ddi_dma_sync(dh, (off_t)(offset),
446 				    j*sizeof (*e), DDI_DMA_SYNC_FORDEV);
447 				offset = j = 0;
448 				ce = cq;
449 				e = q;
450 			}
451 
452 			*ce = *cmp;
453 			mapping = cmp->ce_pa;
454 			j++;
455 			e->Sop = 0;
456 			e->DataValid = 1;
457 			e->BufferLength = cmp->ce_len;
458 			e->AddrHigh = ((u64)mapping >> 32);
459 			e->AddrLow = ((u64)mapping & 0xffffffff);
460 
461 			if (i < (count - 1)) {
462 				e->Eop = 0;
463 				wmb();
464 				e->GenerationBit = e->GenerationBit2 = genbit;
465 			}
466 		}
467 	}
468 
469 	ce->ce_mp = m0;
470 
471 	e->Eop = 1;
472 	wmb();
473 	e->GenerationBit = e->GenerationBit2 = genbit;
474 
475 	(void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e),
476 	    DDI_DMA_SYNC_FORDEV);
477 
478 	/*
479 	 * We always ring the doorbell for cmdQ1.  For cmdQ0, we only ring
480 	 * the doorbell if the Q is asleep. There is a natural race, where
481 	 * the hardware is going to sleep just after we checked, however,
482 	 * then the interrupt handler will detect the outstanding TX packet
483 	 * and ring the doorbell for us.
484 	 */
485 	if (qid) {
486 		doorbell_pio(sge, F_CMDQ1_ENABLE);
487 	} else {
488 		if (atomic_read(Q->cq_asleep)) {
489 			atomic_set(&Q->cq_asleep, 0);
490 /* NOT YET		doorbell_pio(sge, F_CMDQ0_ENABLE); */
491 			atomic_set(&Q->cq_pio_pidx, Q->cq_pidx);
492 		}
493 	}
494 	doorbell_pio(sge, F_CMDQ0_ENABLE);
495 
496 	return (0);
497 }
498 
499 #define	SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
500 
501 /*
502  * Disable SGE error interrupts.
503  */
504 int
505 t1_sge_intr_disable(pesge* sge)
506 {
507 	u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
508 
509 	t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
510 	t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0);
511 	return (0);
512 }
513 
514 #define	SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
515 	F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
516 
517 /*
518  * Enable SGE error interrupts.
519  */
520 int
521 t1_sge_intr_enable(pesge* sge)
522 {
523 	u32 en = SGE_INT_ENABLE;
524 	u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
525 
526 	t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
527 
528 	if (sge->obj->ch_flags & TSO_CAPABLE)
529 		en &= ~F_PACKET_TOO_BIG;
530 	t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en);
531 	return (0);
532 }
533 
534 /*
535  * Clear SGE error interrupts.
536  */
537 int
538 t1_sge_intr_clear(pesge* sge)
539 {
540 	t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK);
541 	t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff);
542 	return (0);
543 }
544 
545 #define	SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
546 
547 int
548 t1_sge_intr_error_handler(pesge *sge)
549 {
550 	peobj *obj = sge->obj;
551 	u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE);
552 
553 	if (cause & F_RESPQ_EXHAUSTED)
554 		sge->intr_cnt.respQ_empty++;
555 	if (cause & F_RESPQ_OVERFLOW) {
556 		sge->intr_cnt.respQ_overflow++;
557 		cmn_err(CE_WARN, "%s: SGE response queue overflow\n",
558 		    obj->ch_name);
559 	}
560 	if (cause & F_FL_EXHAUSTED) {
561 		sge->intr_cnt.freelistQ_empty++;
562 		freelQs_empty(sge);
563 	}
564 	if (cause & F_PACKET_TOO_BIG) {
565 		sge->intr_cnt.pkt_too_big++;
566 		cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n",
567 		    obj->ch_name);
568 	}
569 	if (cause & F_PACKET_MISMATCH) {
570 		sge->intr_cnt.pkt_mismatch++;
571 		cmn_err(CE_WARN, "%s: SGE packet mismatch\n",
572 		    obj->ch_name);
573 	}
574 	if (cause & SGE_INT_FATAL)
575 		t1_fatal_err(obj);
576 
577 	t1_write_reg_4(obj, A_SG_INT_CAUSE, cause);
578 	return (0);
579 }
580 
581 /*
582  *
583  * PARAM:   sge     - SGE instance pointer.
584  */
585 int
586 sge_data_in(pesge *sge)
587 {
588 	peobj *adapter = sge->obj;
589 	struct respQ *Q = &sge->respQ;
590 	respQ_e *e;				/* response queue entry */
591 	respQ_e *q = Q->rq_entries;		/* base response queue */
592 	uint32_t cidx = Q->rq_cidx;
593 	uint32_t genbit = Q->rq_genbit;
594 	uint32_t entries_n = Q->rq_entries_n;
595 	uint32_t credits = Q->rq_credits;
596 	uint32_t credits_thresh = Q->rq_credits_thresh;
597 	uint32_t ret = 0;
598 #ifndef TX_THREAD_RECLAIM
599 	uint32_t credits_pend[2] = {0, 0};
600 #endif
601 	uint32_t flags = 0;
602 	uint32_t flagt;
603 	ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh;
604 
605 	t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA);
606 
607 	/*
608 	 * Catch the case where an interrupt arrives
609 	 * early.
610 	 */
611 	if ((q == NULL) || (dh == NULL)) {
612 		goto check_slow_ints;
613 	}
614 
615 	/* initial response queue entry */
616 	e = &q[cidx];
617 
618 	/* pull physical memory of response queue entry into cache */
619 	(void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
620 	    sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
621 
622 	while (e->GenerationBit == genbit) {
623 		if (--credits < credits_thresh) {
624 			uint32_t n = entries_n - credits - 1;
625 			t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n);
626 			credits += n;
627 		}
628 		if (likely(e->DataValid)) {
629 			(void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid],
630 			    e->BufferLength, e->Offload);
631 			if ((e->Sop != 1) || (e->Eop != 1)) {
632 				sge->intr_cnt.rx_badEopSop++;
633 				cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d",
634 				    e->Sop, e->Eop, e->BufferLength);
635 			}
636 		}
637 		flagt = e->Qsleeping;
638 		flags |= flagt;
639 		if (flagt & F_CMDQ0_ENABLE)
640 			sge->intr_cnt.rx_cmdq0++;
641 		if (flagt & F_CMDQ1_ENABLE)
642 			sge->intr_cnt.rx_cmdq1++;
643 		if (flagt & F_FL0_ENABLE)
644 			sge->intr_cnt.rx_flq0++;
645 		if (flagt & F_FL1_ENABLE)
646 			sge->intr_cnt.rx_flq1++;
647 #ifdef TX_THREAD_RECLAIM
648 		spin_lock(&sge->cmdQ[0].cq_qlock);
649 		sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn;
650 		spin_unlock(&sge->cmdQ[0].cq_qlock);
651 		spin_lock(&sge->cmdQ[1].cq_qlock);
652 		sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn;
653 		if ((adapter->ch_blked) &&
654 		    (sge->cmdQ[0].cq_complete +
655 		    sge->cmdQ[1].cq_complete) > 16) {
656 			adapter->ch_blked = 0;
657 			ch_gld_ok(adapter);
658 		}
659 		spin_unlock(&sge->cmdQ[1].cq_qlock);
660 #else
661 		credits_pend[0] += e->Cmdq0CreditReturn;
662 		credits_pend[1] += e->Cmdq1CreditReturn;
663 #ifdef CONFIG_SMP
664 		if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) {
665 			free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
666 			credits_pend[0] = 0;
667 		}
668 		if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) {
669 			free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
670 			credits_pend[1] = 0;
671 		}
672 #endif
673 #endif
674 #ifdef HOST_PAUSE
675 		t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]);
676 #endif
677 		e++;
678 		if (unlikely(++cidx == entries_n)) {
679 			cidx = 0;
680 			genbit ^= 1;
681 			e = q;
682 		}
683 
684 		/* pull physical memory of response queue entry into cache */
685 		(void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
686 		    sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
687 
688 		ret = 1;
689 	}
690 
691 #ifndef TX_THREAD_RECLAIM
692 	if (credits_pend[0])
693 		free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
694 	if (credits_pend[1])
695 		free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
696 #endif
697 	if (flags & F_CMDQ0_ENABLE) {
698 		struct cmdQ *cmdQ = &sge->cmdQ[0];
699 		atomic_set(&cmdQ->cq_asleep, 1);
700 		if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) {
701 			doorbell_pio(sge, F_CMDQ0_ENABLE);
702 			atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx);
703 		}
704 	}
705 
706 	/* the SGE told us one of the free lists is empty */
707 	if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE)))
708 		freelQs_empty(sge);
709 
710 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
711 	if (adapter->ch_tx_overflow_mutex)
712 		mutex_enter(adapter->ch_tx_overflow_mutex);
713 	if (adapter->ch_blked &&
714 	    (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) &&
715 	    (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) {
716 		adapter->ch_blked = 0;
717 		if (adapter->ch_tx_overflow_cv)
718 			cv_broadcast(adapter->ch_tx_overflow_cv);
719 		ch_gld_ok(adapter);
720 	}
721 	if (adapter->ch_tx_overflow_mutex)
722 		mutex_exit(adapter->ch_tx_overflow_mutex);
723 #else
724 #ifndef TX_THREAD_RECLAIM
725 	if (adapter->ch_blked &&
726 	    (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) &&
727 	    (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) {
728 		adapter->ch_blked = 0;
729 		ch_gld_ok(adapter);
730 	}
731 #endif
732 #endif	/* CONFIG_CHELSIO_T1_OFFLOAD */
733 
734 	Q->rq_genbit = genbit;
735 	Q->rq_cidx = cidx;
736 	Q->rq_credits = credits;
737 
738 	t1_write_reg_4(adapter, A_SG_SLEEPING, cidx);
739 
740 check_slow_ints:
741 	/* handle non-data interrupts */
742 	if (unlikely(!ret))
743 		ret = t1_slow_intr_handler(adapter);
744 
745 	return (ret);
746 }
747 
748 /*
749  * allocate a mblk with DMA mapped mblk.
750  * When checksum offload is enabled, we start the DMA at a 2 byte offset so
751  * the IP header will be aligned. We do this for sparc only.
752  */
753 static uint64_t
754 os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh)
755 {
756 	ch_esb_t *ch_get_small_rbuf(ch_t *sa);
757 	ch_esb_t *ch_get_big_rbuf(ch_t *sa);
758 	ch_esb_t *rbp;
759 	uint32_t rxoff = sa->sge->rx_offset;
760 
761 	if (sz == SGE_SM_BUF_SZ(sa)) {
762 		/* get pre-mapped buffer */
763 		if ((rbp = ch_get_small_rbuf(sa)) == NULL) {
764 			sa->norcvbuf++;
765 			return ((uint64_t)0);
766 		}
767 
768 		*mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
769 		    SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
770 		if (*mb == NULL) {
771 			mutex_enter(&sa->ch_small_esbl);
772 			rbp->cs_next = sa->ch_small_esb_free;
773 			sa->ch_small_esb_free = rbp;
774 			mutex_exit(&sa->ch_small_esbl);
775 			return ((uint64_t)0);
776 		}
777 		*dh = rbp->cs_dh;
778 
779 		return (rbp->cs_pa + rxoff);
780 	} else {
781 		/* get pre-mapped buffer */
782 		if ((rbp = ch_get_big_rbuf(sa)) == NULL) {
783 			sa->norcvbuf++;
784 			return ((uint64_t)0);
785 		}
786 
787 		*mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
788 		    SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
789 		if (*mb == NULL) {
790 			mutex_enter(&sa->ch_big_esbl);
791 			rbp->cs_next = sa->ch_big_esb_free;
792 			sa->ch_big_esb_free = rbp;
793 			mutex_exit(&sa->ch_big_esbl);
794 			return ((uint64_t)0);
795 		}
796 		*dh = rbp->cs_dh;
797 
798 		return (rbp->cs_pa + rxoff);
799 	}
800 }
801 
802 static inline unsigned int
803 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)
804 {
805 	mblk_t *skb;
806 	peobj *adapter = sge->obj;
807 	struct freelQ_ce *cq = Q->fq_centries;
808 	struct freelQ_ce *ce = &cq[Q->fq_cidx];
809 	ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh;
810 	uint32_t cidx = Q->fq_cidx;
811 	uint32_t entries_n = Q->fq_entries_n;
812 	uint32_t sz = Q->fq_rx_buffer_size;
813 	uint32_t useit = 1;
814 	uint32_t rxoff = sge->rx_offset;
815 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
816 	uint32_t rv;
817 #endif
818 
819 	if (Q->fq_id)
820 		sge->intr_cnt.rx_flq1_cnt++;
821 	else
822 		sge->intr_cnt.rx_flq0_cnt++;
823 	/*
824 	 * If pkt size falls below threshold, then we'll copy data to
825 	 * an blk and reuse mblk.
826 	 *
827 	 * NOTE that rxoff is 2 for T1 adapters. We align the the start
828 	 * of the DMA buffer begin at rxoff offset for T1 cards instead of
829 	 * at the beginning of the buffer, thus the length of the received
830 	 * data does not include this offset. We therefore always add
831 	 * SGE_RX_OFFSET to the allocb size so we have space to provide the
832 	 * offset for the copied data.
833 	 */
834 #ifdef HOST_PAUSE
835 	/*
836 	 * If we have Host pause compiled in, then we look at the
837 	 * free list, if the pause is on and we're not in offload
838 	 * mode then we drop packets, this is designed to avoid
839 	 * overwhelming the machine. If the machine is powerfull enough
840 	 * this will not happen. The 'rx_pkt_drops' will show when
841 	 * packets are being dropped and how much.
842 	 */
843 	if ((offload == 0) && adapter->pause_on) {
844 		freelQ_e *e;
845 		/* Ditch the packet and reuse original buffer */
846 		e = &Q->fq_entries[cidx];
847 		e->GenerationBit  ^= 1;
848 		e->GenerationBit2 ^= 1;
849 		sge->intr_cnt.rx_pkt_drops++;
850 		goto rx_entry_consumed;
851 	} else if (((adapter->pause_on ||
852 	    (len <= SGE_RX_COPY_THRESHOLD)) &&
853 	    (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))))
854 #else
855 	if ((len <= SGE_RX_COPY_THRESHOLD) &&
856 	    (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))
857 #endif
858 	{
859 		freelQ_e *e;
860 		char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr;
861 
862 		/*
863 		 * pull physical memory of pkt data into cache
864 		 * Note that len does not include offset for T1.
865 		 */
866 		(void) ddi_dma_sync(dh, (off_t)(rxoff), len,
867 		    DDI_DMA_SYNC_FORKERNEL);
868 
869 		if (offload == 0) {
870 			/*
871 			 * create 2 byte offset so IP header aligned on
872 			 * 4 byte boundry
873 			 */
874 			skb_reserve(skb, SGE_RX_OFFSET);
875 			/*
876 			 * if hardware inserted 2 byte offset then need to
877 			 * start copying with extra offset
878 			 */
879 			src += sge->rx_pkt_pad;
880 		}
881 		memcpy(skb->b_rptr, src, len);
882 		useit = 0;	/* mblk copy, don't inc esballoc in use cnt */
883 
884 		/* so we can reuse original buffer */
885 		e = &Q->fq_entries[cidx];
886 		e->GenerationBit  ^= 1;
887 		e->GenerationBit2 ^= 1;
888 		sge->intr_cnt.rx_pkt_copied++;
889 	} else {
890 		/* consume buffer off the ring */
891 		skb = ce->fe_mp;
892 		ce->fe_mp = NULL;
893 
894 		/*
895 		 * if not offload (tunneled pkt), & hardward padded, then
896 		 * adjust start of pkt to point to start of data i.e.
897 		 * skip pad (2 bytes).
898 		 */
899 		if (!offload && sge->rx_pkt_pad)
900 			__skb_pull(skb, SGE_RX_OFFSET);
901 
902 		/*
903 		 * pull physical memory of pkt data into cache
904 		 * Note that len does not include offset for T1.
905 		 */
906 		(void) ddi_dma_sync(dh, (off_t)(rxoff), len,
907 		    DDI_DMA_SYNC_FORKERNEL);
908 	}
909 
910 	/* set length of data in skb */
911 	skb_put(skb, len);
912 
913 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
914 	if (likely(offload)) {
915 		if (likely(toe_running(adapter))) {
916 			/* sends pkt upstream to toe layer */
917 			if (useit) {
918 				if (sz == SGE_SM_BUF_SZ(adapter)) {
919 					atomic_add(1,
920 					&buffers_in_use[adapter->ch_sm_index]);
921 				} else {
922 					atomic_add(1,
923 					&buffers_in_use[adapter->ch_big_index]);
924 				}
925 			}
926 			if (adapter->toe_rcv)
927 				adapter->toe_rcv(adapter->ch_toeinst, skb);
928 			else
929 				freemsg(skb);
930 		} else {
931 			cmn_err(CE_WARN,
932 			    "%s: unexpected offloaded packet, cmd %u\n",
933 			    adapter->ch_name, *skb->b_rptr);
934 
935 			/* discard packet */
936 			freemsg(skb);
937 		}
938 	}
939 #else
940 	if (unlikely(offload)) {
941 		cmn_err(CE_WARN,
942 		    "%s: unexpected offloaded packet, cmd %u\n",
943 		    adapter->ch_name, *skb->b_rptr);
944 
945 		/* discard paket */
946 		freemsg(skb);
947 	}
948 #endif
949 	else {
950 		struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr;
951 		int flg = 0;
952 		uint32_t cksum;
953 
954 		/* adjust beginning of data to skip CPL header */
955 		skb_pull(skb, SZ_CPL_RX_PKT);
956 
957 		/* extract checksum from CPL header here */
958 
959 		/*
960 		 * bump count of mlbks in used by protocol stack(s)
961 		 */
962 		if (useit) {
963 			if (sz == SGE_SM_BUF_SZ(adapter)) {
964 				atomic_add(1,
965 				    &buffers_in_use[adapter->ch_sm_index]);
966 			} else {
967 				atomic_add(1,
968 				    &buffers_in_use[adapter->ch_big_index]);
969 			}
970 		}
971 
972 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
973 		/*
974 		 * let the TOE layer have a crack at the packet first.
975 		 */
976 		if (adapter->toe_tunnel) {
977 			rv = adapter->toe_tunnel(adapter->ch_toeinst, skb);
978 			/*
979 			 * The TOE may have consumed the packet.
980 			 */
981 			if (rv)
982 				goto rx_entry_consumed;
983 		}
984 #endif	/* CONFIG_CHELSIO_T1_OFFLOAD */
985 
986 		cksum = p->csum;
987 
988 		/*
989 		 * NOTE: 14+9 = size of MAC + offset to IP protocol field
990 		 */
991 		if (adapter->ch_config.cksum_enabled &&
992 		    (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) ==
993 		    ETHERTYPE_IP) &&
994 		    ((skb->b_rptr[14+9] == IPPROTO_TCP) ||
995 		    (skb->b_rptr[14+9] == IPPROTO_UDP))) {
996 			flg = 1;
997 		}
998 
999 		ch_send_up(adapter, skb, cksum, flg);
1000 	}
1001 
1002 rx_entry_consumed:
1003 
1004 	if (++cidx == entries_n)
1005 		cidx = 0;
1006 
1007 	Q->fq_cidx = cidx;
1008 
1009 	if (unlikely(--Q->fq_credits < (entries_n>>2)))
1010 		/* allocate new buffers on the free list */
1011 		alloc_freelQ_buffers(sge, Q);
1012 	return (1);
1013 }
1014 
1015 #ifdef HOST_PAUSE
1016 static void
1017 t1_sge_check_pause(pesge *sge, struct freelQ *Q)
1018 {
1019 	peobj *adapter = sge->obj;
1020 
1021 	/*
1022 	 * If the number of available credits shrinks below
1023 	 * the Pause on threshold then enable the pause and
1024 	 * try and allocate more buffers.
1025 	 * On the next pass, if there's more credits returned
1026 	 * then check that you've went above the pause
1027 	 * threshold and then disable the pause.
1028 	 */
1029 	if (Q->fq_credits < Q->fq_pause_on_thresh) {
1030 		if (do_host_pause) {
1031 			sge->intr_cnt.rx_pause_on++;
1032 			adapter->txxg_cfg1 |=
1033 			    SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1034 			(void) t1_tpi_write(adapter,
1035 			    SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1036 			    adapter->txxg_cfg1);
1037 			adapter->pause_on = 1;
1038 			adapter->pause_time = gethrtime();
1039 		}
1040 		alloc_freelQ_buffers(sge, Q);
1041 	} else if ((adapter->pause_on) &&
1042 	    (Q->fq_credits > Q->fq_pause_off_thresh)) {
1043 		hrtime_t time;
1044 		sge->intr_cnt.rx_pause_off++;
1045 		adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1046 		(void) t1_tpi_write(adapter,
1047 		    SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1048 		    adapter->txxg_cfg1);
1049 		adapter->pause_on = 0;
1050 		time = (gethrtime() - adapter->pause_time)/1000;
1051 		sge->intr_cnt.rx_pause_ms += time;
1052 		if (time > sge->intr_cnt.rx_pause_spike)
1053 			sge->intr_cnt.rx_pause_spike = (uint32_t)time;
1054 	}
1055 	sge->intr_cnt.rx_fl_credits = Q->fq_credits;
1056 }
1057 #endif	/* HOST_PAUSE */
1058 
1059 static void
1060 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q)
1061 {
1062 	uint32_t pidx = Q->fq_pidx;
1063 	struct freelQ_ce *ce = &Q->fq_centries[pidx];
1064 	freelQ_e *fq = Q->fq_entries;		/* base of freelist Q */
1065 	freelQ_e *e = &Q->fq_entries[pidx];
1066 	uint32_t sz = Q->fq_rx_buffer_size;
1067 	uint32_t rxoff = sge->rx_offset;
1068 	uint32_t credits = Q->fq_credits;
1069 	uint32_t entries_n = Q->fq_entries_n;
1070 	uint32_t genbit = Q->fq_genbit;
1071 	ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh;
1072 	ulong_t dh;
1073 	uint64_t mapping;
1074 	off_t offset = (off_t)((caddr_t)e - (caddr_t)fq);
1075 	size_t len = 0;
1076 
1077 	while (credits < entries_n) {
1078 		if (e->GenerationBit != genbit) {
1079 			mblk_t *skb;
1080 
1081 			mapping = os_freelist_buffer_alloc(sge->obj, sz,
1082 			    &skb, &dh);
1083 			if (mapping == 0) {
1084 				sge->intr_cnt.rx_flbuf_fails++;
1085 				break;
1086 			}
1087 			sge->intr_cnt.rx_flbuf_allocs++;
1088 
1089 			ce->fe_mp = skb;
1090 			ce->fe_dh = dh;
1091 
1092 			/*
1093 			 * Note that for T1, we've started the beginning of
1094 			 * of the buffer by an offset of 2 bytes. We thus
1095 			 * decrement the length to account for this.
1096 			 */
1097 			e->AddrLow = (u32)mapping;
1098 			e->AddrHigh = (u64)mapping >> 32;
1099 			e->BufferLength = sz - rxoff;
1100 			wmb();
1101 			e->GenerationBit = e->GenerationBit2 = genbit;
1102 		}
1103 
1104 		len += sizeof (*e);
1105 
1106 		ce++;
1107 		e++;
1108 		credits++;
1109 		if (++pidx == entries_n) {
1110 			/*
1111 			 * sync freelist entries to physical memory up to
1112 			 * end of the table.
1113 			 */
1114 			(void) ddi_dma_sync(th, offset, len,
1115 			    DDI_DMA_SYNC_FORDEV);
1116 			offset = 0;
1117 			len = 0;
1118 
1119 			pidx = 0;
1120 			genbit ^= 1;
1121 			ce = Q->fq_centries;
1122 			e = Q->fq_entries;
1123 		}
1124 	}
1125 
1126 	/* sync freelist entries that have been modified. */
1127 	if (len)
1128 		(void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV);
1129 
1130 	Q->fq_genbit = genbit;
1131 	Q->fq_pidx = pidx;
1132 	Q->fq_credits = credits;
1133 }
1134 
1135 static void
1136 freelQs_empty(pesge *sge)
1137 {
1138 	u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE);
1139 	u32 irqholdoff_reg;
1140 
1141 	alloc_freelQ_buffers(sge, &sge->freelQ[0]);
1142 	alloc_freelQ_buffers(sge, &sge->freelQ[1]);
1143 
1144 	if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) &&
1145 	    (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) {
1146 		irq_reg |= F_FL_EXHAUSTED;
1147 		irqholdoff_reg = sge->intrtimer[sge->currIndex];
1148 	} else {
1149 		/* Clear the F_FL_EXHAUSTED interrupts for now */
1150 		irq_reg &= ~F_FL_EXHAUSTED;
1151 		irqholdoff_reg = sge->intrtimer_nres;
1152 	}
1153 	t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg);
1154 	t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg);
1155 
1156 	/* We reenable the Qs to force an Freelist GTS interrupt later */
1157 	doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE);
1158 }
1159 
1160 /*
1161  * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
1162  * Free xmit buffers
1163  */
1164 static void
1165 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)
1166 {
1167 	mblk_t *skb;
1168 	struct cmdQ_ce *ce;
1169 	struct cmdQ_ce *cq = Q->cq_centries;
1170 	uint32_t entries_n = Q->cq_entries_n;
1171 	uint32_t cidx = Q->cq_cidx;
1172 	uint32_t i = credits_pend;
1173 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1174 	ch_t *chp = sge->obj;
1175 #endif
1176 	ce = &cq[cidx];
1177 
1178 	while (i--) {
1179 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1180 		/* if flag set, then toe buffer */
1181 		switch (ce->ce_flg & 0x7) {
1182 		case DH_DMA:
1183 			if (ce->ce_dh) {
1184 				ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1185 				ce->ce_dh = NULL;	/* may not be needed */
1186 			}
1187 			skb = ce->ce_mp;
1188 			if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1189 				freemsg(skb);
1190 			}
1191 			ce->ce_mp = NULL;
1192 			break;
1193 
1194 #if defined(__sparc)
1195 		case DH_DVMA:
1196 			if (ce->ce_dh) {
1197 				ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1198 				ce->ce_dh = NULL;	/* may not be needed */
1199 			}
1200 			skb = ce->ce_mp;
1201 			if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1202 				freemsg(skb);
1203 			}
1204 			ce->ce_mp = NULL;
1205 			break;
1206 #endif	/* __sparc */
1207 
1208 		case DH_TOE:
1209 			chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp));
1210 			ce->ce_mp = NULL;
1211 			break;
1212 		}
1213 #else	/* CONFIG_CHELSIO_T1_OFFLOAD */
1214 		if (ce->ce_dh) {
1215 			if ((ce->ce_flg & 7) == DH_DMA) {
1216 				ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1217 			}
1218 #if defined(__sparc)
1219 			else {
1220 				ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1221 			}
1222 #endif	/* __sparc */
1223 			ce->ce_dh = NULL; /* may not be needed */
1224 		}
1225 
1226 		skb = ce->ce_mp;
1227 		if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1228 			freemsg(skb);
1229 		}
1230 		ce->ce_mp = NULL;
1231 #endif	/* !CONFIG_CHELSIO_T1_OFFLOAD */
1232 
1233 		ce++;
1234 		if (++cidx == entries_n) {
1235 			cidx = 0;
1236 			ce = cq;
1237 		}
1238 	}
1239 
1240 	Q->cq_cidx = cidx;
1241 	atomic_add(credits_pend, &Q->cq_credits);
1242 }
1243 
1244 struct sge_intr_counts *
1245 sge_get_stat(pesge *sge)
1246 {
1247 	return (&sge->intr_cnt);
1248 }
1249 
1250 /*
1251  * Allocates both RX and TX resources and configures the SGE. However,
1252  * the hardware is not enabled yet.
1253  *
1254  * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
1255  * jumbo_fl is set to the index of the freelist containing the jumbo buffers.
1256  */
1257 int
1258 t1_sge_configure(pesge *sge, struct sge_params *p)
1259 {
1260 	sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET;
1261 	sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0;
1262 	/* if we're a T2 card, then we have hardware offset support */
1263 	sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0;
1264 
1265 	if (alloc_rx_resources(sge, p))
1266 		return (-ENOMEM);
1267 	if (alloc_tx_resources(sge, p)) {
1268 		free_rx_resources(sge);
1269 		return (-ENOMEM);
1270 	}
1271 	configure_sge(sge, p);
1272 
1273 	/*
1274 	 * Now that we have sized the free lists calculate the payload
1275 	 * capacity of the large buffers.  Other parts of the driver use
1276 	 * this to set the max offload coalescing size so that RX packets
1277 	 * do not overflow our large buffers.
1278 	 */
1279 	p->large_buf_capacity = jumbo_payload_capacity(sge);
1280 	return (0);
1281 }
1282 
1283 /*
1284  * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
1285  * response Q.
1286  */
1287 static int
1288 alloc_rx_resources(pesge *sge, struct sge_params *p)
1289 {
1290 	unsigned int size, i;
1291 
1292 	for (i = 0; i < SGE_FREELQ_N; i++) {
1293 		struct freelQ *Q = &sge->freelQ[i];
1294 
1295 		Q->fq_id = i;
1296 		Q->fq_genbit = 1;
1297 		Q->fq_entries_n = p->freelQ_size[i];
1298 #ifdef HOST_PAUSE
1299 		Q->fq_pause_on_thresh = flq_pause_window;
1300 		Q->fq_pause_off_thresh = Q->fq_entries_n >> 1;
1301 #endif
1302 		size = sizeof (freelQ_e) * Q->fq_entries_n;
1303 
1304 		Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1305 		    size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT);
1306 
1307 
1308 		if (!Q->fq_entries)
1309 			goto err_no_mem;
1310 		memset(Q->fq_entries, 0, size);
1311 		size = sizeof (struct freelQ_ce) * Q->fq_entries_n;
1312 		Q->fq_centries = t1_os_malloc_wait_zero(size);
1313 		if (!Q->fq_centries)
1314 			goto err_no_mem;
1315 		memset(Q->fq_centries, 0, size);
1316 	}
1317 
1318 	/*
1319 	 * Calculate the buffer sizes for the two free lists.  FL0 accommodates
1320 	 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
1321 	 * including all the sk_buff overhead.
1322 	 * For T1C FL0 and FL1 are reversed.
1323 	 */
1324 #ifdef NOTYET
1325 	sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE +
1326 	    sizeof (struct cpl_rx_data) +
1327 	    SGE_RX_OFFSET - sge->rx_pkt_pad;
1328 #else
1329 	sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size =
1330 	    sge->obj->ch_sm_buf_sz;
1331 	if (is_T2(sge->obj))
1332 		sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz;
1333 	else
1334 		sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz;
1335 #endif
1336 #ifdef NOTYET
1337 	sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) -
1338 	    SKB_DATA_ALIGN(sizeof (struct skb_shared_info));
1339 #else
1340 	sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz;
1341 	if (is_T2(sge->obj))
1342 		sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz;
1343 	else
1344 		sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz;
1345 #endif
1346 
1347 	sge->respQ.rq_genbit = 1;
1348 	sge->respQ.rq_entries_n = sge_respq_cnt;
1349 	sge->respQ.rq_credits = sge_respq_cnt;
1350 	sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2);
1351 	size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1352 
1353 	sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1354 	    size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh),
1355 	    &(sge->respQ.rq_ah), 0);
1356 
1357 	if (!sge->respQ.rq_entries)
1358 		goto err_no_mem;
1359 	memset(sge->respQ.rq_entries, 0, size);
1360 	return (0);
1361 
1362 err_no_mem:
1363 	free_rx_resources(sge);
1364 	return (1);
1365 }
1366 
1367 /*
1368  * Allocates basic TX resources, consisting of memory mapped command Qs.
1369  */
1370 static int
1371 alloc_tx_resources(pesge *sge, struct sge_params *p)
1372 {
1373 	unsigned int size, i;
1374 
1375 	for (i = 0; i < SGE_CMDQ_N; i++) {
1376 		struct cmdQ *Q = &sge->cmdQ[i];
1377 
1378 		Q->cq_genbit = 1;
1379 		Q->cq_entries_n = p->cmdQ_size[i];
1380 		atomic_set(&Q->cq_credits, Q->cq_entries_n);
1381 		atomic_set(&Q->cq_asleep, 1);
1382 
1383 		mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER,
1384 		    sge->obj->ch_icookp);
1385 
1386 		size = sizeof (cmdQ_e) * Q->cq_entries_n;
1387 		Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1388 		    size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT);
1389 
1390 		if (!Q->cq_entries)
1391 			goto err_no_mem;
1392 		memset(Q->cq_entries, 0, size);
1393 		size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1394 		Q->cq_centries = t1_os_malloc_wait_zero(size);
1395 		if (!Q->cq_centries)
1396 			goto err_no_mem;
1397 		memset(Q->cq_centries, 0, size);
1398 
1399 		/* allocate pre-mapped dma headers */
1400 		pe_dma_handle_init(sge->obj, Q->cq_entries_n);
1401 	}
1402 
1403 	return (0);
1404 
1405 err_no_mem:
1406 	free_tx_resources(sge);
1407 	return (1);
1408 }
1409 
1410 /*
1411  * Sets the interrupt latency timer when the adaptive Rx coalescing
1412  * is turned off. Do nothing when it is turned on again.
1413  *
1414  * This routine relies on the fact that the caller has already set
1415  * the adaptive policy in adapter->sge_params before calling it.
1416  */
1417 int
1418 t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p)
1419 {
1420 	if (!p->coalesce_enable) {
1421 		u32 newTimer = p->rx_coalesce_usecs *
1422 		    (board_info(sge->obj)->clock_core / 1000000);
1423 
1424 		t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer);
1425 	}
1426 	return (0);
1427 }
1428 
1429 /*
1430  * Programs the various SGE registers. However, the engine is not yet enabled,
1431  * but sge->sge_control is setup and ready to go.
1432  */
1433 static void
1434 configure_sge(pesge *sge, struct sge_params *p)
1435 {
1436 	ch_t *ap = sge->obj;
1437 	int i;
1438 
1439 	t1_write_reg_4(ap, A_SG_CONTROL, 0);
1440 
1441 	setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n,
1442 	    A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
1443 	setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n,
1444 	    A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
1445 	setup_ring_params(ap, sge->freelQ[0].fq_pa,
1446 	    sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR,
1447 	    A_SG_FL0BASEUPR, A_SG_FL0SIZE);
1448 	setup_ring_params(ap, sge->freelQ[1].fq_pa,
1449 	    sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR,
1450 	    A_SG_FL1BASEUPR, A_SG_FL1SIZE);
1451 
1452 	/* The threshold comparison uses <. */
1453 	t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) -
1454 	    SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1);
1455 	setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n,
1456 	    A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
1457 	t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n);
1458 	sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
1459 	    F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
1460 	    V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
1461 #if 1
1462 		/*
1463 		 * if the the following bit is not set, then we'll get an
1464 		 * interrupt everytime command Q 0 goes empty. Since we're
1465 		 * always ringing the doorbell, we can turn it on.
1466 		 */
1467 	    F_DISABLE_CMDQ0_GTS |
1468 #endif
1469 	    V_RX_PKT_OFFSET(sge->rx_pkt_pad);
1470 
1471 #if BYTE_ORDER == BIG_ENDIAN
1472 	sge->sge_control |= F_ENABLE_BIG_ENDIAN;
1473 #endif
1474 
1475 	/*
1476 	 * Initialize the SGE Interrupt Timer arrray:
1477 	 * intrtimer[0] = (SGE_INTRTIMER0) usec
1478 	 * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
1479 	 * intrtimer[10] = (SGE_INTRTIMER1) usec
1480 	 *
1481 	 */
1482 	sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000;
1483 	for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) {
1484 		sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i);
1485 		sge->intrtimer[i] *= sge->intrtimer[0];
1486 	}
1487 	sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] =
1488 	    sge->intrtimer[0] * SGE_INTRTIMER1;
1489 	/* Initialize resource timer */
1490 	sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] *
1491 	    SGE_INTRTIMER_NRES);
1492 	/* Finally finish initialization of intrtimer[0] */
1493 	sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0);
1494 	/* Initialize for a throughput oriented workload */
1495 	sge->currIndex = SGE_INTR_MAXBUCKETS - 1;
1496 
1497 	if (p->coalesce_enable)
1498 		t1_write_reg_4(ap, A_SG_INTRTIMER,
1499 		    sge->intrtimer[sge->currIndex]);
1500 	else
1501 		(void) t1_sge_set_coalesce_params(sge, p);
1502 }
1503 
1504 static inline void
1505 setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo,
1506     int base_reg_hi, int size_reg)
1507 {
1508 	t1_write_reg_4(adapter, base_reg_lo, (u32)addr);
1509 	t1_write_reg_4(adapter, base_reg_hi, addr >> 32);
1510 	t1_write_reg_4(adapter, size_reg, size);
1511 }
1512 
1513 /*
1514  * Frees RX resources.
1515  */
1516 static void
1517 free_rx_resources(pesge *sge)
1518 {
1519 	unsigned int size, i;
1520 
1521 	if (sge->respQ.rq_entries) {
1522 		size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1523 
1524 		pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries,
1525 		    sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah);
1526 	}
1527 
1528 	for (i = 0; i < SGE_FREELQ_N; i++) {
1529 		struct freelQ *Q = &sge->freelQ[i];
1530 
1531 		if (Q->fq_centries) {
1532 			free_freelQ_buffers(sge, Q);
1533 
1534 			t1_os_free(Q->fq_centries,
1535 			    Q->fq_entries_n * sizeof (freelQ_ce_t));
1536 		}
1537 		if (Q->fq_entries) {
1538 			size = sizeof (freelQ_e) * Q->fq_entries_n;
1539 
1540 			/* free the freelist queue */
1541 			pe_os_free_contig(sge->obj, size, Q->fq_entries,
1542 			    Q->fq_pa, Q->fq_dh, Q->fq_ah);
1543 
1544 		}
1545 	}
1546 }
1547 
1548 /*
1549  * Frees all RX buffers on the freelist Q. The caller must make sure that
1550  * the SGE is turned off before calling this function.
1551  */
1552 static void
1553 free_freelQ_buffers(pesge *sge, struct freelQ *Q)
1554 {
1555 	struct freelQ_ce *ce;
1556 	struct freelQ_ce *cq = Q->fq_centries;
1557 	uint32_t credits = Q->fq_credits;
1558 	uint32_t entries_n = Q->fq_entries_n;
1559 	uint32_t cidx = Q->fq_cidx;
1560 	uint32_t i = Q->fq_id;
1561 
1562 	ce = &cq[cidx];
1563 
1564 	credits = entries_n;
1565 	while (credits--) {
1566 		mblk_t *mp;
1567 		if ((mp = ce->fe_mp) != NULL) {
1568 			/* bump in-use count of receive buffers */
1569 			if (i != sge->jumbo_fl) {
1570 				atomic_add(1,
1571 				    &buffers_in_use[sge->obj->ch_sm_index]);
1572 			} else {
1573 				atomic_add(1,
1574 				    &buffers_in_use[sge->obj->ch_big_index]);
1575 			}
1576 
1577 			/*
1578 			 * note. freeb() callback of esb-alloced mblk will
1579 			 * cause receive buffer to be put back on sa free list.
1580 			 */
1581 			freeb(mp);
1582 			ce->fe_mp = NULL;
1583 		}
1584 
1585 		ce++;
1586 		if (++cidx == entries_n) {
1587 			cidx = 0;
1588 			ce = cq;
1589 		}
1590 	}
1591 
1592 	Q->fq_cidx = cidx;
1593 	Q->fq_credits = credits;
1594 }
1595 
1596 /*
1597  * Free TX resources.
1598  *
1599  * Assumes that SGE is stopped and all interrupts are disabled.
1600  */
1601 static void
1602 free_tx_resources(pesge *sge)
1603 {
1604 	unsigned int size;
1605 	uint32_t i;
1606 
1607 	for (i = 0; i < SGE_CMDQ_N; i++) {
1608 		struct cmdQ *Q = &sge->cmdQ[i];
1609 
1610 		if (Q->cq_centries) {
1611 			unsigned int pending = Q->cq_entries_n -
1612 			    atomic_read(Q->cq_credits);
1613 
1614 			mutex_destroy(&Q->cq_qlock);
1615 
1616 			if (pending)
1617 				free_cmdQ_buffers(sge, Q, pending);
1618 
1619 			size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1620 			t1_os_free(Q->cq_centries, size);
1621 		}
1622 
1623 		if (Q->cq_entries) {
1624 			size = sizeof (cmdQ_e) * Q->cq_entries_n;
1625 			pe_os_free_contig(sge->obj, size, Q->cq_entries,
1626 			    Q->cq_pa, Q->cq_dh, Q->cq_ah);
1627 		}
1628 	}
1629 }
1630 
1631 /*
1632  * Return the payload capacity of the jumbo free-list buffers.
1633  */
1634 static inline unsigned int jumbo_payload_capacity(pesge *sge)
1635 {
1636 	return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size -
1637 	    sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset);
1638 }
1639 
1640 /* PR2928 & PR3309 */
1641 void
1642 t1_sge_set_ptimeout(adapter_t *adapter, u32 val)
1643 {
1644 	pesge *sge = adapter->sge;
1645 
1646 	if (is_T2(adapter))
1647 		sge->ptimeout = max(val, 1);
1648 }
1649 
1650 /* PR2928 & PR3309 */
1651 u32
1652 t1_sge_get_ptimeout(adapter_t *adapter)
1653 {
1654 	pesge *sge = adapter->sge;
1655 
1656 	return (is_T2(adapter) ? sge->ptimeout : 0);
1657 }
1658 
1659 void
1660 sge_add_fake_arp(pesge *sge, void *bp)
1661 {
1662 	sge->pskb = bp;
1663 }
1664 
1665 #ifdef SUN_KSTATS
1666 static int
1667 sge_kstat_setup(pesge *sge)
1668 {
1669 	int status;
1670 	p_kstat_t ksp;
1671 	size_t ch_kstat_sz;
1672 	p_ch_kstat_t chkp;
1673 	char kstat_name[32];
1674 	int instance;
1675 	int i;
1676 
1677 	status = -1;
1678 	ch_kstat_sz = sizeof (ch_kstat_t);
1679 	instance = ddi_get_instance(sge->obj->ch_dip);
1680 	if ((ksp = kstat_create(CHNAME "_debug", instance,
1681 	    NULL, "net_debug", KSTAT_TYPE_NAMED,
1682 	    ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL)
1683 		goto sge_kstat_setup_exit;
1684 	chkp = (p_ch_kstat_t)ksp->ks_data;
1685 	kstat_named_init(&chkp->respQ_empty,		"respQ_empty",
1686 	    KSTAT_DATA_UINT32);
1687 	kstat_named_init(&chkp->respQ_overflow,		"respQ_overflow",
1688 	    KSTAT_DATA_UINT32);
1689 	kstat_named_init(&chkp->freelistQ_empty,	"freelistQ_empty",
1690 	    KSTAT_DATA_UINT32);
1691 	kstat_named_init(&chkp->pkt_too_big,		"pkt_too_big",
1692 	    KSTAT_DATA_UINT32);
1693 	kstat_named_init(&chkp->pkt_mismatch,		"pkt_mismatch",
1694 	    KSTAT_DATA_UINT32);
1695 	kstat_named_init(&chkp->cmdQ_full[0],		"cmdQ_full[0]",
1696 	    KSTAT_DATA_UINT32);
1697 	kstat_named_init(&chkp->cmdQ_full[1],		"cmdQ_full[1]",
1698 	    KSTAT_DATA_UINT32);
1699 	kstat_named_init(&chkp->tx_reclaims[0],		"tx_reclaims[0]",
1700 	    KSTAT_DATA_UINT32);
1701 	kstat_named_init(&chkp->tx_reclaims[1],		"tx_reclaims[1]",
1702 	    KSTAT_DATA_UINT32);
1703 	kstat_named_init(&chkp->tx_msg_pullups,		"tx_msg_pullups",
1704 	    KSTAT_DATA_UINT32);
1705 	kstat_named_init(&chkp->tx_hdr_pullups,		"tx_hdr_pullups",
1706 	    KSTAT_DATA_UINT32);
1707 	kstat_named_init(&chkp->tx_tcp_ip_frag,		"tx_tcp_ip_frag",
1708 	    KSTAT_DATA_UINT32);
1709 	kstat_named_init(&chkp->tx_udp_ip_frag,		"tx_udp_ip_frag",
1710 	    KSTAT_DATA_UINT32);
1711 	kstat_named_init(&chkp->tx_soft_cksums,		"tx_soft_cksums",
1712 	    KSTAT_DATA_UINT32);
1713 	kstat_named_init(&chkp->tx_need_cpl_space,	"tx_need_cpl_space",
1714 	    KSTAT_DATA_UINT32);
1715 	kstat_named_init(&chkp->tx_multi_mblks,		"tx_multi_mblks",
1716 	    KSTAT_DATA_UINT32);
1717 	kstat_named_init(&chkp->tx_no_dvma1,	"tx_num_multi_dvma_fails",
1718 	    KSTAT_DATA_UINT32);
1719 	kstat_named_init(&chkp->tx_no_dvma2,	"tx_num_single_dvma_fails",
1720 	    KSTAT_DATA_UINT32);
1721 	kstat_named_init(&chkp->tx_no_dma1,	"tx_num_multi_dma_fails",
1722 	    KSTAT_DATA_UINT32);
1723 	kstat_named_init(&chkp->tx_no_dma2,	"tx_num_single_dma_fails",
1724 	    KSTAT_DATA_UINT32);
1725 	kstat_named_init(&chkp->rx_cmdq0,		"rx_cmdq0",
1726 	    KSTAT_DATA_UINT32);
1727 	kstat_named_init(&chkp->rx_cmdq1,		"rx_cmdq1",
1728 	    KSTAT_DATA_UINT32);
1729 	kstat_named_init(&chkp->rx_flq0,		"rx_flq0",
1730 	    KSTAT_DATA_UINT32);
1731 	kstat_named_init(&chkp->rx_flq1,		"rx_flq1",
1732 	    KSTAT_DATA_UINT32);
1733 	kstat_named_init(&chkp->rx_flq0_sz,		"rx_flq0_buffer_sz",
1734 	    KSTAT_DATA_UINT32);
1735 	kstat_named_init(&chkp->rx_flq1_sz,		"rx_flq1_buffer_sz",
1736 	    KSTAT_DATA_UINT32);
1737 	kstat_named_init(&chkp->rx_pkt_drops,		"rx_pkt_drops",
1738 	    KSTAT_DATA_UINT32);
1739 	kstat_named_init(&chkp->rx_pkt_copied,		"rx_pkt_copied",
1740 	    KSTAT_DATA_UINT32);
1741 	kstat_named_init(&chkp->rx_pause_on,		"rx_pause_on",
1742 	    KSTAT_DATA_UINT32);
1743 	kstat_named_init(&chkp->rx_pause_off,		"rx_pause_off",
1744 	    KSTAT_DATA_UINT32);
1745 	kstat_named_init(&chkp->rx_pause_ms,		"rx_pause_ms",
1746 	    KSTAT_DATA_UINT32);
1747 	kstat_named_init(&chkp->rx_pause_spike,		"rx_pause_spike",
1748 	    KSTAT_DATA_UINT32);
1749 	kstat_named_init(&chkp->rx_fl_credits,		"rx_fl_credits",
1750 	    KSTAT_DATA_UINT32);
1751 	kstat_named_init(&chkp->rx_flbuf_fails,		"rx_flbuf_fails",
1752 	    KSTAT_DATA_UINT32);
1753 	kstat_named_init(&chkp->rx_flbuf_allocs,	"rx_flbuf_allocs",
1754 	    KSTAT_DATA_UINT32);
1755 	kstat_named_init(&chkp->rx_badEopSop,		"rx_badEopSop",
1756 	    KSTAT_DATA_UINT32);
1757 	kstat_named_init(&chkp->rx_flq0_cnt,		"rx_flq0_cnt",
1758 	    KSTAT_DATA_UINT32);
1759 	kstat_named_init(&chkp->rx_flq1_cnt,		"rx_flq1_cnt",
1760 	    KSTAT_DATA_UINT32);
1761 	kstat_named_init(&chkp->arp_sent,		"arp_sent",
1762 	    KSTAT_DATA_UINT32);
1763 	kstat_named_init(&chkp->tx_doorbells,		"tx_doorbells",
1764 	    KSTAT_DATA_UINT32);
1765 	kstat_named_init(&chkp->intr_doorbells,		"intr_doorbells",
1766 	    KSTAT_DATA_UINT32);
1767 	kstat_named_init(&chkp->intr1_doorbells,	"intr1_doorbells",
1768 	    KSTAT_DATA_UINT32);
1769 	kstat_named_init(&chkp->sleep_cnt,		"sleep_cnt",
1770 	    KSTAT_DATA_UINT32);
1771 	kstat_named_init(&chkp->pe_allocb_cnt,		"pe_allocb_cnt",
1772 	    KSTAT_DATA_UINT32);
1773 	for (i = 0; i < MBLK_MAX; i++) {
1774 		(void) sprintf(kstat_name, "tx_descs[%02d]", i);
1775 		kstat_named_init(&chkp->tx_descs[i],
1776 		    kstat_name, KSTAT_DATA_UINT32);
1777 	}
1778 	ksp->ks_update = sge_kstat_update;
1779 	ksp->ks_private = (void *)sge;
1780 	sge->ksp = ksp;
1781 	kstat_install(ksp);
1782 	status = 0;
1783 
1784 sge_kstat_setup_exit:
1785 	return (status);
1786 }
1787 
1788 static void
1789 sge_kstat_remove(pesge *sge)
1790 {
1791 	if (sge->ksp)
1792 		kstat_delete(sge->ksp);
1793 }
1794 
1795 static int
1796 sge_kstat_update(p_kstat_t ksp, int rw)
1797 {
1798 	pesge *sge;
1799 	p_ch_stats_t statsp;
1800 	p_ch_kstat_t chkp;
1801 	int i;
1802 
1803 	sge = (pesge *)ksp->ks_private;
1804 	statsp = (p_ch_stats_t)&sge->intr_cnt;
1805 	chkp = (p_ch_kstat_t)ksp->ks_data;
1806 	if (rw == KSTAT_WRITE) {
1807 		statsp->respQ_empty	= chkp->respQ_empty.value.ui32;
1808 		statsp->respQ_overflow	= chkp->respQ_overflow.value.ui32;
1809 		statsp->freelistQ_empty	= chkp->freelistQ_empty.value.ui32;
1810 		statsp->pkt_too_big	= chkp->pkt_too_big.value.ui32;
1811 		statsp->pkt_mismatch	= chkp->pkt_mismatch.value.ui32;
1812 		statsp->cmdQ_full[0]	= chkp->cmdQ_full[0].value.ui32;
1813 		statsp->cmdQ_full[1]	= chkp->cmdQ_full[1].value.ui32;
1814 		statsp->tx_reclaims[0]	= chkp->tx_reclaims[0].value.ui32;
1815 		statsp->tx_reclaims[1]	= chkp->tx_reclaims[1].value.ui32;
1816 		statsp->tx_msg_pullups	= chkp->tx_msg_pullups.value.ui32;
1817 		statsp->tx_hdr_pullups	= chkp->tx_hdr_pullups.value.ui32;
1818 		statsp->tx_tcp_ip_frag	= chkp->tx_tcp_ip_frag.value.ui32;
1819 		statsp->tx_udp_ip_frag	= chkp->tx_udp_ip_frag.value.ui32;
1820 		statsp->tx_soft_cksums	= chkp->tx_soft_cksums.value.ui32;
1821 		statsp->tx_need_cpl_space
1822 		    = chkp->tx_need_cpl_space.value.ui32;
1823 		statsp->tx_multi_mblks	= chkp->tx_multi_mblks.value.ui32;
1824 		statsp->tx_no_dvma1	= chkp->tx_no_dvma1.value.ui32;
1825 		statsp->tx_no_dvma2	= chkp->tx_no_dvma2.value.ui32;
1826 		statsp->tx_no_dma1	= chkp->tx_no_dma1.value.ui32;
1827 		statsp->tx_no_dma2	= chkp->tx_no_dma2.value.ui32;
1828 		statsp->rx_cmdq0	= chkp->rx_cmdq0.value.ui32;
1829 		statsp->rx_cmdq1	= chkp->rx_cmdq1.value.ui32;
1830 		statsp->rx_flq0		= chkp->rx_flq0.value.ui32;
1831 		statsp->rx_flq1		= chkp->rx_flq1.value.ui32;
1832 		statsp->rx_flq0_sz	= chkp->rx_flq0_sz.value.ui32;
1833 		statsp->rx_flq1_sz	= chkp->rx_flq1_sz.value.ui32;
1834 		statsp->rx_pkt_drops	= chkp->rx_pkt_drops.value.ui32;
1835 		statsp->rx_pkt_copied	= chkp->rx_pkt_copied.value.ui32;
1836 		statsp->rx_pause_on	= chkp->rx_pause_on.value.ui32;
1837 		statsp->rx_pause_off	= chkp->rx_pause_off.value.ui32;
1838 		statsp->rx_pause_ms	= chkp->rx_pause_ms.value.ui32;
1839 		statsp->rx_pause_spike	= chkp->rx_pause_spike.value.ui32;
1840 		statsp->rx_fl_credits	= chkp->rx_fl_credits.value.ui32;
1841 		statsp->rx_flbuf_fails	= chkp->rx_flbuf_fails.value.ui32;
1842 		statsp->rx_flbuf_allocs	= chkp->rx_flbuf_allocs.value.ui32;
1843 		statsp->rx_badEopSop	= chkp->rx_badEopSop.value.ui32;
1844 		statsp->rx_flq0_cnt	= chkp->rx_flq0_cnt.value.ui32;
1845 		statsp->rx_flq1_cnt	= chkp->rx_flq1_cnt.value.ui32;
1846 		statsp->arp_sent	= chkp->arp_sent.value.ui32;
1847 		statsp->tx_doorbells	= chkp->tx_doorbells.value.ui32;
1848 		statsp->intr_doorbells	= chkp->intr_doorbells.value.ui32;
1849 		statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32;
1850 		statsp->sleep_cnt	= chkp->sleep_cnt.value.ui32;
1851 		statsp->pe_allocb_cnt	= chkp->pe_allocb_cnt.value.ui32;
1852 		for (i = 0; i < MBLK_MAX; i++) {
1853 			statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32;
1854 		}
1855 	} else {
1856 		chkp->respQ_empty.value.ui32	= statsp->respQ_empty;
1857 		chkp->respQ_overflow.value.ui32	= statsp->respQ_overflow;
1858 		chkp->freelistQ_empty.value.ui32
1859 		    = statsp->freelistQ_empty;
1860 		chkp->pkt_too_big.value.ui32	= statsp->pkt_too_big;
1861 		chkp->pkt_mismatch.value.ui32	= statsp->pkt_mismatch;
1862 		chkp->cmdQ_full[0].value.ui32	= statsp->cmdQ_full[0];
1863 		chkp->cmdQ_full[1].value.ui32	= statsp->cmdQ_full[1];
1864 		chkp->tx_reclaims[0].value.ui32	= statsp->tx_reclaims[0];
1865 		chkp->tx_reclaims[1].value.ui32	= statsp->tx_reclaims[1];
1866 		chkp->tx_msg_pullups.value.ui32	= statsp->tx_msg_pullups;
1867 		chkp->tx_hdr_pullups.value.ui32	= statsp->tx_hdr_pullups;
1868 		chkp->tx_tcp_ip_frag.value.ui32	= statsp->tx_tcp_ip_frag;
1869 		chkp->tx_udp_ip_frag.value.ui32	= statsp->tx_udp_ip_frag;
1870 		chkp->tx_soft_cksums.value.ui32	= statsp->tx_soft_cksums;
1871 		chkp->tx_need_cpl_space.value.ui32
1872 		    = statsp->tx_need_cpl_space;
1873 		chkp->tx_multi_mblks.value.ui32	= statsp->tx_multi_mblks;
1874 		chkp->tx_no_dvma1.value.ui32	= statsp->tx_no_dvma1;
1875 		chkp->tx_no_dvma2.value.ui32	= statsp->tx_no_dvma2;
1876 		chkp->tx_no_dma1.value.ui32	= statsp->tx_no_dma1;
1877 		chkp->tx_no_dma2.value.ui32	= statsp->tx_no_dma2;
1878 		chkp->rx_cmdq0.value.ui32	= statsp->rx_cmdq0;
1879 		chkp->rx_cmdq1.value.ui32	= statsp->rx_cmdq1;
1880 		chkp->rx_flq0.value.ui32	= statsp->rx_flq0;
1881 		chkp->rx_flq1.value.ui32	= statsp->rx_flq1;
1882 		chkp->rx_flq0_sz.value.ui32	= statsp->rx_flq0_sz;
1883 		chkp->rx_flq1_sz.value.ui32	= statsp->rx_flq1_sz;
1884 		chkp->rx_pkt_drops.value.ui32	= statsp->rx_pkt_drops;
1885 		chkp->rx_pkt_copied.value.ui32	= statsp->rx_pkt_copied;
1886 		chkp->rx_pause_on.value.ui32	= statsp->rx_pause_on;
1887 		chkp->rx_pause_off.value.ui32	= statsp->rx_pause_off;
1888 		chkp->rx_pause_ms.value.ui32	= statsp->rx_pause_ms;
1889 		chkp->rx_pause_spike.value.ui32	= statsp->rx_pause_spike;
1890 		chkp->rx_fl_credits.value.ui32	= statsp->rx_fl_credits;
1891 		chkp->rx_flbuf_fails.value.ui32
1892 		    = statsp->rx_flbuf_fails;
1893 		chkp->rx_flbuf_allocs.value.ui32
1894 		    = statsp->rx_flbuf_allocs;
1895 		chkp->rx_badEopSop.value.ui32	= statsp->rx_badEopSop;
1896 		chkp->rx_flq0_cnt.value.ui32	= statsp->rx_flq0_cnt;
1897 		chkp->rx_flq1_cnt.value.ui32	= statsp->rx_flq1_cnt;
1898 		chkp->arp_sent.value.ui32	= statsp->arp_sent;
1899 		chkp->tx_doorbells.value.ui32	= statsp->tx_doorbells;
1900 		chkp->intr_doorbells.value.ui32	= statsp->intr_doorbells;
1901 		chkp->intr1_doorbells.value.ui32
1902 		    = statsp->intr1_doorbells;
1903 		chkp->sleep_cnt.value.ui32	= statsp->sleep_cnt;
1904 		chkp->pe_allocb_cnt.value.ui32	= statsp->pe_allocb_cnt;
1905 		for (i = 0; i < MBLK_MAX; i++) {
1906 			chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i];
1907 		}
1908 	}
1909 	return (0);
1910 }
1911 #endif
1912 
1913 static uint16_t
1914 calc_ocsum(mblk_t *mp, int offset)
1915 {
1916 	uint8_t *addrp;
1917 	uint32_t src;
1918 	uint32_t dst;
1919 
1920 	ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset);
1921 	uint32_t sum;
1922 	int iplen = IPH_HDR_LENGTH(ihdr);
1923 	struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen);
1924 	uchar_t *byte;
1925 	int len;
1926 
1927 	addrp = (uint8_t *)&ihdr->ipha_src;
1928 	src =  ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1929 	    ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1930 
1931 	addrp = (uint8_t *)&ihdr->ipha_dst;
1932 	dst =  ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1933 	    ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1934 
1935 	sum = (uint16_t)(src >> 16) +
1936 	    (uint16_t)(src) +
1937 	    (uint16_t)(dst >> 16) +
1938 	    (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP));
1939 
1940 	sum = (uint16_t)(sum >> 16) + (uint16_t)(sum);
1941 
1942 	if (sum > 0xffff)
1943 		sum -= 0xffff;
1944 
1945 	udpp->uh_sum = 0;
1946 	byte = mp->b_rptr + offset + iplen;
1947 	do {
1948 		len = (mp->b_wptr - byte);
1949 		sum = bcksum(byte, len, sum);
1950 		if (sum > 0xffff)
1951 			sum -= 0xffff;
1952 		mp = mp->b_cont;
1953 		if (mp)
1954 			byte = mp->b_rptr;
1955 	} while (mp);
1956 
1957 	sum = ~sum & 0xffff;
1958 
1959 	return (sum);
1960 }
1961