1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * This file is part of the Chelsio T1 Ethernet driver.
29 *
30 * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved.
31 */
32
33 #pragma ident "%Z%%M% %I% %E% SMI"
34
35 #include <sys/types.h>
36 #include <sys/param.h>
37 #include <sys/cmn_err.h>
38 #include <sys/sunddi.h>
39 #include <sys/kmem.h>
40 #include <sys/cmn_err.h>
41 #include <sys/byteorder.h>
42 #include <sys/atomic.h>
43 #include <sys/stropts.h>
44 #include <sys/stream.h>
45 #include <sys/strsubr.h>
46 #include <sys/dlpi.h>
47 #include <sys/kstat.h>
48 #include <sys/ethernet.h>
49 #include <netinet/in.h>
50 #include <netinet/udp.h>
51 #include <inet/common.h>
52 #include <inet/nd.h>
53 #include <inet/ip.h>
54 #include <inet/tcp.h>
55 #include <netinet/udp.h>
56 #include <sys/gld.h>
57 #include "ostypes.h"
58 #include "common.h"
59 #ifdef CONFIG_CHELSIO_T1_1G
60 #include "fpga_defs.h"
61 #endif
62 #include "regs.h"
63 #include "suni1x10gexp_regs.h"
64 #include "sge.h"
65 #include "espi.h"
66
67 #include "ch.h"
68
69 extern uint32_t buffers_in_use[];
70
71 uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N;
72 uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N;
73 uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N;
74 uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N;
75 uint32_t sge_respq_cnt = SGE_RESPQ_E_N;
76
77 uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N;
78 uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N;
79 uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N;
80 uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N;
81 uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N;
82
83 #ifdef HOST_PAUSE
84 uint32_t do_host_pause = 1;
85 uint32_t flq_pause_window = 64;
86 #endif
87
88 static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb,
89 ulong_t *dh);
90 void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t);
91
92 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q,
93 unsigned int len, unsigned int offload);
94 #ifdef HOST_PAUSE
95 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
96 #endif
97 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
98 static void freelQs_empty(pesge *sge);
99 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
100 static int alloc_rx_resources(pesge *sge, struct sge_params *p);
101 static int alloc_tx_resources(pesge *sge, struct sge_params *p);
102 static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size,
103 int base_reg_lo, int base_reg_hi, int size_reg);
104 static void configure_sge(pesge *sge, struct sge_params *p);
105 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
106 static void free_rx_resources(pesge *sge);
107 static void free_tx_resources(pesge *sge);
108 static inline unsigned int jumbo_payload_capacity(pesge *sge);
109 #ifdef SUN_KSTATS
110 static int sge_kstat_setup(pesge *);
111 static void sge_kstat_remove(pesge *);
112 static int sge_kstat_update(p_kstat_t, int);
113 #endif
114 static uint16_t calc_ocsum(mblk_t *, int);
115
116 /*
117 * Local routines.
118 */
119 static inline void sge_ring_doorbell(pesge *sge, u32 control_reg);
120
121 static inline void
sge_ring_doorbell(pesge * sge,u32 control_reg)122 sge_ring_doorbell(pesge *sge, u32 control_reg)
123 {
124 membar_producer();
125 t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg);
126 }
127
128 /*
129 * DESC:
130 *
131 * NOTES: Must have at least 1 command queue and 1 freelist queue.
132 *
133 */
134 pesge *
t1_sge_create(ch_t * sa,struct sge_params * p)135 t1_sge_create(ch_t *sa, struct sge_params *p)
136 {
137 pesge *sge;
138
139 sge = t1_os_malloc_wait_zero(sizeof (pesge));
140
141 if (sge == NULL)
142 goto error_no_mem;
143
144 memset(sge, 0, sizeof (*sge));
145
146 /*
147 * PR2928 & PR3309
148 * set default timeout value - 20 msec
149 * we set the initial value to 2 which gurantees at least one tick.
150 */
151 if (is_T2(sa))
152 sge->ptimeout = 1;
153
154 sge->obj = sa;
155 #ifdef SUN_KSTATS
156 if (sge_kstat_setup(sge) != 0)
157 goto t1_sge_create_fail1;
158 #endif
159 p->cmdQ_size[0] = sge_cmdq0_cnt;
160 p->cmdQ_size[1] = sge_cmdq1_cnt;
161
162 /* note that jumbo frame index is inverted for T2 */
163 if (is_T2(sa)) {
164 p->freelQ_size[1] = sge_flq0_cnt;
165 p->freelQ_size[0] = sge_flq1_cnt;
166 } else {
167 p->freelQ_size[0] = sge_flq0_cnt;
168 p->freelQ_size[1] = sge_flq1_cnt;
169 }
170
171 #if CH_DEBUG
172 /* DEBUG only */
173 cmn_err(CE_NOTE, "sge: %p\n", sge);
174 cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]);
175 cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]);
176 cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]);
177 cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ);
178 cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt);
179 #endif
180 #ifdef SUN_KSTATS
181 goto error_no_mem;
182
183 t1_sge_create_fail1:
184 t1_os_free(sge, sizeof (pesge));
185 sge = NULL;
186 #endif
187 error_no_mem:
188 return (sge);
189 }
190
191 int
t1_sge_destroy(pesge * sge)192 t1_sge_destroy(pesge* sge)
193 {
194 if (sge != NULL) {
195 free_tx_resources(sge);
196 free_rx_resources(sge);
197
198 /* PR2928 & PR3309 */
199 if ((is_T2(sge->obj)) && (sge->pskb))
200 pe_free_fake_arp(sge->pskb);
201 #ifdef SUN_KSTATS
202 sge_kstat_remove(sge);
203 #endif
204 t1_os_free(sge, sizeof (pesge));
205 }
206 return (0);
207 }
208
209 /*
210 * PR2928 & PR3309
211 * call out event from timeout
212 *
213 * there is a potential race between the timeout and the close.
214 * unless we protect the timeout, the close could occur at the
215 * same time. Then if the timeout service routine was slow or
216 * interrupted, the sge_stop() could complete with a timeoutID
217 * that has expired, thus letting another timeout occur. If the
218 * service routine was delayed still further, a detach could occur.
219 * the second time could then end up accessing memory that has been
220 * released back to the system. Bad things could then occur. We
221 * set a flag in sge_stop() to tell the service routine not to
222 * issue further timeouts. sge_stop() will block until a timeout
223 * has occured. If the command Q is full then we shouldn't put out
224 * an arp.
225 */
226
227 void
t1_espi_workaround(ch_t * adapter)228 t1_espi_workaround(ch_t *adapter)
229 {
230 pesge *sge = adapter->sge;
231 ch_t *chp = (ch_t *)sge->obj;
232 int rv = 1;
233
234 if ((chp->ch_state == PERUNNING) &&
235 atomic_read(&sge->cmdQ[0].cq_asleep)) {
236 u32 seop;
237 seop = t1_espi_get_mon(adapter, 0x930, 0);
238 if ((seop & 0xfff0fff) == 0xfff) {
239 /* after first arp */
240 if (sge->pskb)
241 rv = pe_start(adapter, (mblk_t *)sge->pskb,
242 CH_ARP);
243 if (!rv)
244 sge->intr_cnt.arp_sent++;
245 }
246 }
247 #ifdef HOST_PAUSE
248 /*
249 * If we are already in sge_data_in, then we can skip calling
250 * t1_sge_check_pause() this clock cycle. lockstat showed that
251 * we were blocking on the mutex ~ 2% of the time.
252 */
253 if (mutex_tryenter(&adapter->ch_intr)) {
254 t1_sge_check_pause(sge, &sge->freelQ[0]);
255 t1_sge_check_pause(sge, &sge->freelQ[1]);
256 mutex_exit(&adapter->ch_intr);
257 }
258 #endif
259 }
260
261 int
sge_start(pesge * sge)262 sge_start(pesge *sge)
263 {
264 t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control);
265 /* PR2928 & PR3309, also need to avoid Pause deadlock */
266 ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic,
267 (void (*)(void *))t1_espi_workaround, sge->obj);
268 ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout);
269 return (0);
270 }
271
272 /*
273 * Disables SGE queues.
274 */
275 int
sge_stop(pesge * sge)276 sge_stop(pesge *sge)
277 {
278 uint32_t status;
279 int loops;
280
281 DBGASSERT(sge);
282
283 /* PR2928 & PR3309, also need to avoid Pause deadlock */
284 t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0);
285
286 /* wait until there's no more outstanding interrupts pending */
287 loops = 0;
288 do {
289 status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE);
290 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status);
291 drv_usecwait(125);
292 loops++;
293 } while (status && (loops < 1000));
294
295 ch_stop_cyclic(&sge->espi_wa_cyclic);
296
297 return (0);
298 }
299
300 uint32_t sge_cmdq_send_fail;
301
302 int
sge_data_out(pesge * sge,int qid,mblk_t * m0,cmdQ_ce_t * cmp,int count,uint32_t flg)303 sge_data_out(pesge* sge, int qid, mblk_t *m0,
304 cmdQ_ce_t *cmp, int count, uint32_t flg)
305 {
306 struct cmdQ *Q = &sge->cmdQ[qid];
307 ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh;
308 spinlock_t *qlock = &Q->cq_qlock;
309 cmdQ_e *e;
310 cmdQ_e *q = Q->cq_entries;
311 uint32_t credits;
312 uint32_t pidx;
313 uint32_t genbit;
314 uint32_t entries_n = Q->cq_entries_n;
315 cmdQ_ce_t *ce;
316 cmdQ_ce_t *cq = Q->cq_centries;
317 dma_addr_t mapping;
318 uint32_t j = 0;
319 uint32_t offset;
320 #if defined(TX_CKSUM_FIX)
321 uint16_t csum;
322 uint16_t *csum_loc;
323 #endif
324 #ifdef TX_THREAD_RECLAIM
325 uint32_t reclaim_cnt;
326 #endif
327
328 /*
329 * We must exit if we don't have enough free command queue entries
330 * available.
331 */
332
333 spin_lock(qlock);
334
335 #if defined(TX_CKSUM_FIX)
336 /*
337 * This checksum fix will address a fragmented datagram
338 * checksum error. Which will lead to the next packet after
339 * the last packet with the More fragment bit set having its
340 * checksum corrupted. When the packet reaches this point
341 * the 'flg' variable indicates whether a checksum is needed
342 * or not. The algorithm is as follows, if the current packet
343 * is a More fragment set the count of packets to be checksummed
344 * after it to 3. If it't not and the count of is more than 0
345 * then calculate the checksum in software, if a hardware checksum
346 * was requested. Then decrment the count. Same algorithm applies
347 * to TCP.
348 */
349 if (flg & CH_UDP_MF) {
350 sge->do_udp_csum = 3;
351 } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) {
352 if ((flg & CH_NO_HWCKSUM) == 0) {
353 /*
354 * Calc Checksum here.
355 */
356 csum = calc_ocsum(m0,
357 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
358 csum_loc = (uint16_t *)(m0->b_rptr +
359 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
360 csum_loc += (((*(char *)csum_loc) & 0x0f) << 1);
361
362 sge->intr_cnt.tx_soft_cksums++;
363 ((struct udphdr *)(csum_loc))->uh_sum = csum;
364 ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1;
365 }
366 sge->do_udp_csum--;
367 } else if (flg & CH_TCP_MF) {
368 sge->do_tcp_csum = 3;
369 } else if (sge->do_tcp_csum != 0) {
370 if ((flg & CH_NO_HWCKSUM) == 0) {
371 sge->intr_cnt.tx_soft_cksums++;
372 /*
373 * Calc Checksum here.
374 */
375 }
376 sge->do_tcp_csum--;
377 }
378 #endif /* TX_CKSUM_FIX */
379 #ifdef TX_THREAD_RECLAIM
380 reclaim_cnt = Q->cq_complete;
381 if (reclaim_cnt > SGE_BATCH_THRESH) {
382 sge->intr_cnt.tx_reclaims[qid]++;
383 free_cmdQ_buffers(sge, Q, reclaim_cnt);
384 Q->cq_complete = 0;
385 }
386 #endif
387 genbit = Q->cq_genbit;
388 pidx = Q->cq_pidx;
389 credits = Q->cq_credits;
390
391 if ((credits - 1) < count) {
392 spin_unlock(qlock);
393 sge->intr_cnt.cmdQ_full[qid]++;
394 return (1);
395 }
396
397 atomic_sub(count, &Q->cq_credits);
398 Q->cq_pidx += count;
399 if (Q->cq_pidx >= entries_n) {
400 Q->cq_pidx -= entries_n;
401 Q->cq_genbit ^= 1;
402 }
403
404 spin_unlock(qlock);
405
406 #ifdef SUN_KSTATS
407 if (count > MBLK_MAX)
408 sge->intr_cnt.tx_descs[MBLK_MAX - 1]++;
409 else
410 sge->intr_cnt.tx_descs[count]++;
411 #endif
412
413 ce = &cq[pidx];
414 *ce = *cmp;
415 mapping = cmp->ce_pa;
416 j++;
417
418 e = &q[pidx];
419
420 offset = (caddr_t)e - (caddr_t)q;
421
422 e->Sop = 1;
423 e->DataValid = 1;
424 e->BufferLength = cmp->ce_len;
425 e->AddrHigh = ((u64)mapping >> 32);
426 e->AddrLow = ((u64)mapping & 0xffffffff);
427
428 --count;
429 if (count > 0) {
430 unsigned int i;
431
432 e->Eop = 0;
433 wmb();
434 e->GenerationBit = e->GenerationBit2 = genbit;
435
436 for (i = 0; i < count; i++) {
437
438 ce++;
439 e++;
440 cmp++;
441 if (++pidx == entries_n) {
442 pidx = 0;
443 genbit ^= 1;
444 /* sync from offset to end of cmdQ */
445 (void) ddi_dma_sync(dh, (off_t)(offset),
446 j*sizeof (*e), DDI_DMA_SYNC_FORDEV);
447 offset = j = 0;
448 ce = cq;
449 e = q;
450 }
451
452 *ce = *cmp;
453 mapping = cmp->ce_pa;
454 j++;
455 e->Sop = 0;
456 e->DataValid = 1;
457 e->BufferLength = cmp->ce_len;
458 e->AddrHigh = ((u64)mapping >> 32);
459 e->AddrLow = ((u64)mapping & 0xffffffff);
460
461 if (i < (count - 1)) {
462 e->Eop = 0;
463 wmb();
464 e->GenerationBit = e->GenerationBit2 = genbit;
465 }
466 }
467 }
468
469 ce->ce_mp = m0;
470
471 e->Eop = 1;
472 wmb();
473 e->GenerationBit = e->GenerationBit2 = genbit;
474
475 (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e),
476 DDI_DMA_SYNC_FORDEV);
477
478 /*
479 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
480 * the doorbell if the Q is asleep. There is a natural race, where
481 * the hardware is going to sleep just after we checked, however,
482 * then the interrupt handler will detect the outstanding TX packet
483 * and ring the doorbell for us.
484 */
485 if (qid) {
486 doorbell_pio(sge, F_CMDQ1_ENABLE);
487 } else {
488 if (atomic_read(Q->cq_asleep)) {
489 atomic_set(&Q->cq_asleep, 0);
490 /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */
491 atomic_set(&Q->cq_pio_pidx, Q->cq_pidx);
492 }
493 }
494 doorbell_pio(sge, F_CMDQ0_ENABLE);
495
496 return (0);
497 }
498
499 #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
500
501 /*
502 * Disable SGE error interrupts.
503 */
504 int
t1_sge_intr_disable(pesge * sge)505 t1_sge_intr_disable(pesge* sge)
506 {
507 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
508
509 t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
510 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0);
511 return (0);
512 }
513
514 #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
515 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
516
517 /*
518 * Enable SGE error interrupts.
519 */
520 int
t1_sge_intr_enable(pesge * sge)521 t1_sge_intr_enable(pesge* sge)
522 {
523 u32 en = SGE_INT_ENABLE;
524 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
525
526 t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
527
528 if (sge->obj->ch_flags & TSO_CAPABLE)
529 en &= ~F_PACKET_TOO_BIG;
530 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en);
531 return (0);
532 }
533
534 /*
535 * Clear SGE error interrupts.
536 */
537 int
t1_sge_intr_clear(pesge * sge)538 t1_sge_intr_clear(pesge* sge)
539 {
540 t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK);
541 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff);
542 return (0);
543 }
544
545 #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
546
547 int
t1_sge_intr_error_handler(pesge * sge)548 t1_sge_intr_error_handler(pesge *sge)
549 {
550 peobj *obj = sge->obj;
551 u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE);
552
553 if (cause & F_RESPQ_EXHAUSTED)
554 sge->intr_cnt.respQ_empty++;
555 if (cause & F_RESPQ_OVERFLOW) {
556 sge->intr_cnt.respQ_overflow++;
557 cmn_err(CE_WARN, "%s: SGE response queue overflow\n",
558 obj->ch_name);
559 }
560 if (cause & F_FL_EXHAUSTED) {
561 sge->intr_cnt.freelistQ_empty++;
562 freelQs_empty(sge);
563 }
564 if (cause & F_PACKET_TOO_BIG) {
565 sge->intr_cnt.pkt_too_big++;
566 cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n",
567 obj->ch_name);
568 }
569 if (cause & F_PACKET_MISMATCH) {
570 sge->intr_cnt.pkt_mismatch++;
571 cmn_err(CE_WARN, "%s: SGE packet mismatch\n",
572 obj->ch_name);
573 }
574 if (cause & SGE_INT_FATAL)
575 t1_fatal_err(obj);
576
577 t1_write_reg_4(obj, A_SG_INT_CAUSE, cause);
578 return (0);
579 }
580
581 /*
582 *
583 * PARAM: sge - SGE instance pointer.
584 */
585 int
sge_data_in(pesge * sge)586 sge_data_in(pesge *sge)
587 {
588 peobj *adapter = sge->obj;
589 struct respQ *Q = &sge->respQ;
590 respQ_e *e; /* response queue entry */
591 respQ_e *q = Q->rq_entries; /* base response queue */
592 uint32_t cidx = Q->rq_cidx;
593 uint32_t genbit = Q->rq_genbit;
594 uint32_t entries_n = Q->rq_entries_n;
595 uint32_t credits = Q->rq_credits;
596 uint32_t credits_thresh = Q->rq_credits_thresh;
597 uint32_t ret = 0;
598 #ifndef TX_THREAD_RECLAIM
599 uint32_t credits_pend[2] = {0, 0};
600 #endif
601 uint32_t flags = 0;
602 uint32_t flagt;
603 ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh;
604
605 t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA);
606
607 /*
608 * Catch the case where an interrupt arrives
609 * early.
610 */
611 if ((q == NULL) || (dh == NULL)) {
612 goto check_slow_ints;
613 }
614
615 /* initial response queue entry */
616 e = &q[cidx];
617
618 /* pull physical memory of response queue entry into cache */
619 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
620 sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
621
622 while (e->GenerationBit == genbit) {
623 if (--credits < credits_thresh) {
624 uint32_t n = entries_n - credits - 1;
625 t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n);
626 credits += n;
627 }
628 if (likely(e->DataValid)) {
629 (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid],
630 e->BufferLength, e->Offload);
631 if ((e->Sop != 1) || (e->Eop != 1)) {
632 sge->intr_cnt.rx_badEopSop++;
633 cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d",
634 e->Sop, e->Eop, e->BufferLength);
635 }
636 }
637 flagt = e->Qsleeping;
638 flags |= flagt;
639 if (flagt & F_CMDQ0_ENABLE)
640 sge->intr_cnt.rx_cmdq0++;
641 if (flagt & F_CMDQ1_ENABLE)
642 sge->intr_cnt.rx_cmdq1++;
643 if (flagt & F_FL0_ENABLE)
644 sge->intr_cnt.rx_flq0++;
645 if (flagt & F_FL1_ENABLE)
646 sge->intr_cnt.rx_flq1++;
647 #ifdef TX_THREAD_RECLAIM
648 spin_lock(&sge->cmdQ[0].cq_qlock);
649 sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn;
650 spin_unlock(&sge->cmdQ[0].cq_qlock);
651 spin_lock(&sge->cmdQ[1].cq_qlock);
652 sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn;
653 if ((adapter->ch_blked) &&
654 (sge->cmdQ[0].cq_complete +
655 sge->cmdQ[1].cq_complete) > 16) {
656 adapter->ch_blked = 0;
657 ch_gld_ok(adapter);
658 }
659 spin_unlock(&sge->cmdQ[1].cq_qlock);
660 #else
661 credits_pend[0] += e->Cmdq0CreditReturn;
662 credits_pend[1] += e->Cmdq1CreditReturn;
663 #ifdef CONFIG_SMP
664 if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) {
665 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
666 credits_pend[0] = 0;
667 }
668 if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) {
669 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
670 credits_pend[1] = 0;
671 }
672 #endif
673 #endif
674 #ifdef HOST_PAUSE
675 t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]);
676 #endif
677 e++;
678 if (unlikely(++cidx == entries_n)) {
679 cidx = 0;
680 genbit ^= 1;
681 e = q;
682 }
683
684 /* pull physical memory of response queue entry into cache */
685 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
686 sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
687
688 ret = 1;
689 }
690
691 #ifndef TX_THREAD_RECLAIM
692 if (credits_pend[0])
693 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
694 if (credits_pend[1])
695 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
696 #endif
697 if (flags & F_CMDQ0_ENABLE) {
698 struct cmdQ *cmdQ = &sge->cmdQ[0];
699 atomic_set(&cmdQ->cq_asleep, 1);
700 if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) {
701 doorbell_pio(sge, F_CMDQ0_ENABLE);
702 atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx);
703 }
704 }
705
706 /* the SGE told us one of the free lists is empty */
707 if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE)))
708 freelQs_empty(sge);
709
710 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
711 if (adapter->ch_tx_overflow_mutex)
712 mutex_enter(adapter->ch_tx_overflow_mutex);
713 if (adapter->ch_blked &&
714 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) &&
715 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) {
716 adapter->ch_blked = 0;
717 if (adapter->ch_tx_overflow_cv)
718 cv_broadcast(adapter->ch_tx_overflow_cv);
719 ch_gld_ok(adapter);
720 }
721 if (adapter->ch_tx_overflow_mutex)
722 mutex_exit(adapter->ch_tx_overflow_mutex);
723 #else
724 #ifndef TX_THREAD_RECLAIM
725 if (adapter->ch_blked &&
726 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) &&
727 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) {
728 adapter->ch_blked = 0;
729 ch_gld_ok(adapter);
730 }
731 #endif
732 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
733
734 Q->rq_genbit = genbit;
735 Q->rq_cidx = cidx;
736 Q->rq_credits = credits;
737
738 t1_write_reg_4(adapter, A_SG_SLEEPING, cidx);
739
740 check_slow_ints:
741 /* handle non-data interrupts */
742 if (unlikely(!ret))
743 ret = t1_slow_intr_handler(adapter);
744
745 return (ret);
746 }
747
748 /*
749 * allocate a mblk with DMA mapped mblk.
750 * When checksum offload is enabled, we start the DMA at a 2 byte offset so
751 * the IP header will be aligned. We do this for sparc only.
752 */
753 static uint64_t
os_freelist_buffer_alloc(ch_t * sa,int sz,mblk_t ** mb,ulong_t * dh)754 os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh)
755 {
756 ch_esb_t *ch_get_small_rbuf(ch_t *sa);
757 ch_esb_t *ch_get_big_rbuf(ch_t *sa);
758 ch_esb_t *rbp;
759 uint32_t rxoff = sa->sge->rx_offset;
760
761 if (sz == SGE_SM_BUF_SZ(sa)) {
762 /* get pre-mapped buffer */
763 if ((rbp = ch_get_small_rbuf(sa)) == NULL) {
764 sa->norcvbuf++;
765 return ((uint64_t)0);
766 }
767
768 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
769 SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
770 if (*mb == NULL) {
771 mutex_enter(&sa->ch_small_esbl);
772 rbp->cs_next = sa->ch_small_esb_free;
773 sa->ch_small_esb_free = rbp;
774 mutex_exit(&sa->ch_small_esbl);
775 return ((uint64_t)0);
776 }
777 *dh = rbp->cs_dh;
778
779 return (rbp->cs_pa + rxoff);
780 } else {
781 /* get pre-mapped buffer */
782 if ((rbp = ch_get_big_rbuf(sa)) == NULL) {
783 sa->norcvbuf++;
784 return ((uint64_t)0);
785 }
786
787 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
788 SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
789 if (*mb == NULL) {
790 mutex_enter(&sa->ch_big_esbl);
791 rbp->cs_next = sa->ch_big_esb_free;
792 sa->ch_big_esb_free = rbp;
793 mutex_exit(&sa->ch_big_esbl);
794 return ((uint64_t)0);
795 }
796 *dh = rbp->cs_dh;
797
798 return (rbp->cs_pa + rxoff);
799 }
800 }
801
802 static inline unsigned int
t1_sge_rx(pesge * sge,struct freelQ * Q,unsigned int len,unsigned int offload)803 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)
804 {
805 mblk_t *skb;
806 peobj *adapter = sge->obj;
807 struct freelQ_ce *cq = Q->fq_centries;
808 struct freelQ_ce *ce = &cq[Q->fq_cidx];
809 ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh;
810 uint32_t cidx = Q->fq_cidx;
811 uint32_t entries_n = Q->fq_entries_n;
812 uint32_t sz = Q->fq_rx_buffer_size;
813 uint32_t useit = 1;
814 uint32_t rxoff = sge->rx_offset;
815 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
816 uint32_t rv;
817 #endif
818
819 if (Q->fq_id)
820 sge->intr_cnt.rx_flq1_cnt++;
821 else
822 sge->intr_cnt.rx_flq0_cnt++;
823 /*
824 * If pkt size falls below threshold, then we'll copy data to
825 * an blk and reuse mblk.
826 *
827 * NOTE that rxoff is 2 for T1 adapters. We align the the start
828 * of the DMA buffer begin at rxoff offset for T1 cards instead of
829 * at the beginning of the buffer, thus the length of the received
830 * data does not include this offset. We therefore always add
831 * SGE_RX_OFFSET to the allocb size so we have space to provide the
832 * offset for the copied data.
833 */
834 #ifdef HOST_PAUSE
835 /*
836 * If we have Host pause compiled in, then we look at the
837 * free list, if the pause is on and we're not in offload
838 * mode then we drop packets, this is designed to avoid
839 * overwhelming the machine. If the machine is powerfull enough
840 * this will not happen. The 'rx_pkt_drops' will show when
841 * packets are being dropped and how much.
842 */
843 if ((offload == 0) && adapter->pause_on) {
844 freelQ_e *e;
845 /* Ditch the packet and reuse original buffer */
846 e = &Q->fq_entries[cidx];
847 e->GenerationBit ^= 1;
848 e->GenerationBit2 ^= 1;
849 sge->intr_cnt.rx_pkt_drops++;
850 goto rx_entry_consumed;
851 } else if (((adapter->pause_on ||
852 (len <= SGE_RX_COPY_THRESHOLD)) &&
853 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))))
854 #else
855 if ((len <= SGE_RX_COPY_THRESHOLD) &&
856 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))
857 #endif
858 {
859 freelQ_e *e;
860 char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr;
861
862 /*
863 * pull physical memory of pkt data into cache
864 * Note that len does not include offset for T1.
865 */
866 (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
867 DDI_DMA_SYNC_FORKERNEL);
868
869 if (offload == 0) {
870 /*
871 * create 2 byte offset so IP header aligned on
872 * 4 byte boundry
873 */
874 skb_reserve(skb, SGE_RX_OFFSET);
875 /*
876 * if hardware inserted 2 byte offset then need to
877 * start copying with extra offset
878 */
879 src += sge->rx_pkt_pad;
880 }
881 memcpy(skb->b_rptr, src, len);
882 useit = 0; /* mblk copy, don't inc esballoc in use cnt */
883
884 /* so we can reuse original buffer */
885 e = &Q->fq_entries[cidx];
886 e->GenerationBit ^= 1;
887 e->GenerationBit2 ^= 1;
888 sge->intr_cnt.rx_pkt_copied++;
889 } else {
890 /* consume buffer off the ring */
891 skb = ce->fe_mp;
892 ce->fe_mp = NULL;
893
894 /*
895 * if not offload (tunneled pkt), & hardward padded, then
896 * adjust start of pkt to point to start of data i.e.
897 * skip pad (2 bytes).
898 */
899 if (!offload && sge->rx_pkt_pad)
900 __skb_pull(skb, SGE_RX_OFFSET);
901
902 /*
903 * pull physical memory of pkt data into cache
904 * Note that len does not include offset for T1.
905 */
906 (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
907 DDI_DMA_SYNC_FORKERNEL);
908 }
909
910 /* set length of data in skb */
911 skb_put(skb, len);
912
913 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
914 if (likely(offload)) {
915 if (likely(toe_running(adapter))) {
916 /* sends pkt upstream to toe layer */
917 if (useit) {
918 if (sz == SGE_SM_BUF_SZ(adapter)) {
919 atomic_add(1,
920 &buffers_in_use[adapter->ch_sm_index]);
921 } else {
922 atomic_add(1,
923 &buffers_in_use[adapter->ch_big_index]);
924 }
925 }
926 if (adapter->toe_rcv)
927 adapter->toe_rcv(adapter->ch_toeinst, skb);
928 else
929 freemsg(skb);
930 } else {
931 cmn_err(CE_WARN,
932 "%s: unexpected offloaded packet, cmd %u\n",
933 adapter->ch_name, *skb->b_rptr);
934
935 /* discard packet */
936 freemsg(skb);
937 }
938 }
939 #else
940 if (unlikely(offload)) {
941 cmn_err(CE_WARN,
942 "%s: unexpected offloaded packet, cmd %u\n",
943 adapter->ch_name, *skb->b_rptr);
944
945 /* discard paket */
946 freemsg(skb);
947 }
948 #endif
949 else {
950 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr;
951 int flg = 0;
952 uint32_t cksum;
953
954 /* adjust beginning of data to skip CPL header */
955 skb_pull(skb, SZ_CPL_RX_PKT);
956
957 /* extract checksum from CPL header here */
958
959 /*
960 * bump count of mlbks in used by protocol stack(s)
961 */
962 if (useit) {
963 if (sz == SGE_SM_BUF_SZ(adapter)) {
964 atomic_add(1,
965 &buffers_in_use[adapter->ch_sm_index]);
966 } else {
967 atomic_add(1,
968 &buffers_in_use[adapter->ch_big_index]);
969 }
970 }
971
972 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
973 /*
974 * let the TOE layer have a crack at the packet first.
975 */
976 if (adapter->toe_tunnel) {
977 rv = adapter->toe_tunnel(adapter->ch_toeinst, skb);
978 /*
979 * The TOE may have consumed the packet.
980 */
981 if (rv)
982 goto rx_entry_consumed;
983 }
984 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
985
986 cksum = p->csum;
987
988 /*
989 * NOTE: 14+9 = size of MAC + offset to IP protocol field
990 */
991 if (adapter->ch_config.cksum_enabled &&
992 (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) ==
993 ETHERTYPE_IP) &&
994 ((skb->b_rptr[14+9] == IPPROTO_TCP) ||
995 (skb->b_rptr[14+9] == IPPROTO_UDP))) {
996 flg = 1;
997 }
998
999 ch_send_up(adapter, skb, cksum, flg);
1000 }
1001
1002 rx_entry_consumed:
1003
1004 if (++cidx == entries_n)
1005 cidx = 0;
1006
1007 Q->fq_cidx = cidx;
1008
1009 if (unlikely(--Q->fq_credits < (entries_n>>2)))
1010 /* allocate new buffers on the free list */
1011 alloc_freelQ_buffers(sge, Q);
1012 return (1);
1013 }
1014
1015 #ifdef HOST_PAUSE
1016 static void
t1_sge_check_pause(pesge * sge,struct freelQ * Q)1017 t1_sge_check_pause(pesge *sge, struct freelQ *Q)
1018 {
1019 peobj *adapter = sge->obj;
1020
1021 /*
1022 * If the number of available credits shrinks below
1023 * the Pause on threshold then enable the pause and
1024 * try and allocate more buffers.
1025 * On the next pass, if there's more credits returned
1026 * then check that you've went above the pause
1027 * threshold and then disable the pause.
1028 */
1029 if (Q->fq_credits < Q->fq_pause_on_thresh) {
1030 if (do_host_pause) {
1031 sge->intr_cnt.rx_pause_on++;
1032 adapter->txxg_cfg1 |=
1033 SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1034 (void) t1_tpi_write(adapter,
1035 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1036 adapter->txxg_cfg1);
1037 adapter->pause_on = 1;
1038 adapter->pause_time = gethrtime();
1039 }
1040 alloc_freelQ_buffers(sge, Q);
1041 } else if ((adapter->pause_on) &&
1042 (Q->fq_credits > Q->fq_pause_off_thresh)) {
1043 hrtime_t time;
1044 sge->intr_cnt.rx_pause_off++;
1045 adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1046 (void) t1_tpi_write(adapter,
1047 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1048 adapter->txxg_cfg1);
1049 adapter->pause_on = 0;
1050 time = (gethrtime() - adapter->pause_time)/1000;
1051 sge->intr_cnt.rx_pause_ms += time;
1052 if (time > sge->intr_cnt.rx_pause_spike)
1053 sge->intr_cnt.rx_pause_spike = (uint32_t)time;
1054 }
1055 sge->intr_cnt.rx_fl_credits = Q->fq_credits;
1056 }
1057 #endif /* HOST_PAUSE */
1058
1059 static void
alloc_freelQ_buffers(pesge * sge,struct freelQ * Q)1060 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q)
1061 {
1062 uint32_t pidx = Q->fq_pidx;
1063 struct freelQ_ce *ce = &Q->fq_centries[pidx];
1064 freelQ_e *fq = Q->fq_entries; /* base of freelist Q */
1065 freelQ_e *e = &Q->fq_entries[pidx];
1066 uint32_t sz = Q->fq_rx_buffer_size;
1067 uint32_t rxoff = sge->rx_offset;
1068 uint32_t credits = Q->fq_credits;
1069 uint32_t entries_n = Q->fq_entries_n;
1070 uint32_t genbit = Q->fq_genbit;
1071 ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh;
1072 ulong_t dh;
1073 uint64_t mapping;
1074 off_t offset = (off_t)((caddr_t)e - (caddr_t)fq);
1075 size_t len = 0;
1076
1077 while (credits < entries_n) {
1078 if (e->GenerationBit != genbit) {
1079 mblk_t *skb;
1080
1081 mapping = os_freelist_buffer_alloc(sge->obj, sz,
1082 &skb, &dh);
1083 if (mapping == 0) {
1084 sge->intr_cnt.rx_flbuf_fails++;
1085 break;
1086 }
1087 sge->intr_cnt.rx_flbuf_allocs++;
1088
1089 ce->fe_mp = skb;
1090 ce->fe_dh = dh;
1091
1092 /*
1093 * Note that for T1, we've started the beginning of
1094 * of the buffer by an offset of 2 bytes. We thus
1095 * decrement the length to account for this.
1096 */
1097 e->AddrLow = (u32)mapping;
1098 e->AddrHigh = (u64)mapping >> 32;
1099 e->BufferLength = sz - rxoff;
1100 wmb();
1101 e->GenerationBit = e->GenerationBit2 = genbit;
1102 }
1103
1104 len += sizeof (*e);
1105
1106 ce++;
1107 e++;
1108 credits++;
1109 if (++pidx == entries_n) {
1110 /*
1111 * sync freelist entries to physical memory up to
1112 * end of the table.
1113 */
1114 (void) ddi_dma_sync(th, offset, len,
1115 DDI_DMA_SYNC_FORDEV);
1116 offset = 0;
1117 len = 0;
1118
1119 pidx = 0;
1120 genbit ^= 1;
1121 ce = Q->fq_centries;
1122 e = Q->fq_entries;
1123 }
1124 }
1125
1126 /* sync freelist entries that have been modified. */
1127 if (len)
1128 (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV);
1129
1130 Q->fq_genbit = genbit;
1131 Q->fq_pidx = pidx;
1132 Q->fq_credits = credits;
1133 }
1134
1135 static void
freelQs_empty(pesge * sge)1136 freelQs_empty(pesge *sge)
1137 {
1138 u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE);
1139 u32 irqholdoff_reg;
1140
1141 alloc_freelQ_buffers(sge, &sge->freelQ[0]);
1142 alloc_freelQ_buffers(sge, &sge->freelQ[1]);
1143
1144 if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) &&
1145 (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) {
1146 irq_reg |= F_FL_EXHAUSTED;
1147 irqholdoff_reg = sge->intrtimer[sge->currIndex];
1148 } else {
1149 /* Clear the F_FL_EXHAUSTED interrupts for now */
1150 irq_reg &= ~F_FL_EXHAUSTED;
1151 irqholdoff_reg = sge->intrtimer_nres;
1152 }
1153 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg);
1154 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg);
1155
1156 /* We reenable the Qs to force an Freelist GTS interrupt later */
1157 doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE);
1158 }
1159
1160 /*
1161 * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
1162 * Free xmit buffers
1163 */
1164 static void
free_cmdQ_buffers(pesge * sge,struct cmdQ * Q,unsigned int credits_pend)1165 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)
1166 {
1167 mblk_t *skb;
1168 struct cmdQ_ce *ce;
1169 struct cmdQ_ce *cq = Q->cq_centries;
1170 uint32_t entries_n = Q->cq_entries_n;
1171 uint32_t cidx = Q->cq_cidx;
1172 uint32_t i = credits_pend;
1173 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1174 ch_t *chp = sge->obj;
1175 #endif
1176 ce = &cq[cidx];
1177
1178 while (i--) {
1179 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1180 /* if flag set, then toe buffer */
1181 switch (ce->ce_flg & 0x7) {
1182 case DH_DMA:
1183 if (ce->ce_dh) {
1184 ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1185 ce->ce_dh = NULL; /* may not be needed */
1186 }
1187 skb = ce->ce_mp;
1188 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1189 freemsg(skb);
1190 }
1191 ce->ce_mp = NULL;
1192 break;
1193
1194 #if defined(__sparc)
1195 case DH_DVMA:
1196 if (ce->ce_dh) {
1197 ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1198 ce->ce_dh = NULL; /* may not be needed */
1199 }
1200 skb = ce->ce_mp;
1201 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1202 freemsg(skb);
1203 }
1204 ce->ce_mp = NULL;
1205 break;
1206 #endif /* __sparc */
1207
1208 case DH_TOE:
1209 chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp));
1210 ce->ce_mp = NULL;
1211 break;
1212 }
1213 #else /* CONFIG_CHELSIO_T1_OFFLOAD */
1214 if (ce->ce_dh) {
1215 if ((ce->ce_flg & 7) == DH_DMA) {
1216 ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1217 }
1218 #if defined(__sparc)
1219 else {
1220 ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1221 }
1222 #endif /* __sparc */
1223 ce->ce_dh = NULL; /* may not be needed */
1224 }
1225
1226 skb = ce->ce_mp;
1227 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) {
1228 freemsg(skb);
1229 }
1230 ce->ce_mp = NULL;
1231 #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */
1232
1233 ce++;
1234 if (++cidx == entries_n) {
1235 cidx = 0;
1236 ce = cq;
1237 }
1238 }
1239
1240 Q->cq_cidx = cidx;
1241 atomic_add(credits_pend, &Q->cq_credits);
1242 }
1243
1244 struct sge_intr_counts *
sge_get_stat(pesge * sge)1245 sge_get_stat(pesge *sge)
1246 {
1247 return (&sge->intr_cnt);
1248 }
1249
1250 /*
1251 * Allocates both RX and TX resources and configures the SGE. However,
1252 * the hardware is not enabled yet.
1253 *
1254 * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
1255 * jumbo_fl is set to the index of the freelist containing the jumbo buffers.
1256 */
1257 int
t1_sge_configure(pesge * sge,struct sge_params * p)1258 t1_sge_configure(pesge *sge, struct sge_params *p)
1259 {
1260 sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET;
1261 sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0;
1262 /* if we're a T2 card, then we have hardware offset support */
1263 sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0;
1264
1265 if (alloc_rx_resources(sge, p))
1266 return (-ENOMEM);
1267 if (alloc_tx_resources(sge, p)) {
1268 free_rx_resources(sge);
1269 return (-ENOMEM);
1270 }
1271 configure_sge(sge, p);
1272
1273 /*
1274 * Now that we have sized the free lists calculate the payload
1275 * capacity of the large buffers. Other parts of the driver use
1276 * this to set the max offload coalescing size so that RX packets
1277 * do not overflow our large buffers.
1278 */
1279 p->large_buf_capacity = jumbo_payload_capacity(sge);
1280 return (0);
1281 }
1282
1283 /*
1284 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
1285 * response Q.
1286 */
1287 static int
alloc_rx_resources(pesge * sge,struct sge_params * p)1288 alloc_rx_resources(pesge *sge, struct sge_params *p)
1289 {
1290 unsigned int size, i;
1291
1292 for (i = 0; i < SGE_FREELQ_N; i++) {
1293 struct freelQ *Q = &sge->freelQ[i];
1294
1295 Q->fq_id = i;
1296 Q->fq_genbit = 1;
1297 Q->fq_entries_n = p->freelQ_size[i];
1298 #ifdef HOST_PAUSE
1299 Q->fq_pause_on_thresh = flq_pause_window;
1300 Q->fq_pause_off_thresh = Q->fq_entries_n >> 1;
1301 #endif
1302 size = sizeof (freelQ_e) * Q->fq_entries_n;
1303
1304 Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1305 size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT);
1306
1307
1308 if (!Q->fq_entries)
1309 goto err_no_mem;
1310 memset(Q->fq_entries, 0, size);
1311 size = sizeof (struct freelQ_ce) * Q->fq_entries_n;
1312 Q->fq_centries = t1_os_malloc_wait_zero(size);
1313 if (!Q->fq_centries)
1314 goto err_no_mem;
1315 memset(Q->fq_centries, 0, size);
1316 }
1317
1318 /*
1319 * Calculate the buffer sizes for the two free lists. FL0 accommodates
1320 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
1321 * including all the sk_buff overhead.
1322 * For T1C FL0 and FL1 are reversed.
1323 */
1324 #ifdef NOTYET
1325 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE +
1326 sizeof (struct cpl_rx_data) +
1327 SGE_RX_OFFSET - sge->rx_pkt_pad;
1328 #else
1329 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size =
1330 sge->obj->ch_sm_buf_sz;
1331 if (is_T2(sge->obj))
1332 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz;
1333 else
1334 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz;
1335 #endif
1336 #ifdef NOTYET
1337 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) -
1338 SKB_DATA_ALIGN(sizeof (struct skb_shared_info));
1339 #else
1340 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz;
1341 if (is_T2(sge->obj))
1342 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz;
1343 else
1344 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz;
1345 #endif
1346
1347 sge->respQ.rq_genbit = 1;
1348 sge->respQ.rq_entries_n = sge_respq_cnt;
1349 sge->respQ.rq_credits = sge_respq_cnt;
1350 sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2);
1351 size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1352
1353 sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1354 size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh),
1355 &(sge->respQ.rq_ah), 0);
1356
1357 if (!sge->respQ.rq_entries)
1358 goto err_no_mem;
1359 memset(sge->respQ.rq_entries, 0, size);
1360 return (0);
1361
1362 err_no_mem:
1363 free_rx_resources(sge);
1364 return (1);
1365 }
1366
1367 /*
1368 * Allocates basic TX resources, consisting of memory mapped command Qs.
1369 */
1370 static int
alloc_tx_resources(pesge * sge,struct sge_params * p)1371 alloc_tx_resources(pesge *sge, struct sge_params *p)
1372 {
1373 unsigned int size, i;
1374
1375 for (i = 0; i < SGE_CMDQ_N; i++) {
1376 struct cmdQ *Q = &sge->cmdQ[i];
1377
1378 Q->cq_genbit = 1;
1379 Q->cq_entries_n = p->cmdQ_size[i];
1380 atomic_set(&Q->cq_credits, Q->cq_entries_n);
1381 atomic_set(&Q->cq_asleep, 1);
1382
1383 mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER,
1384 sge->obj->ch_icookp);
1385
1386 size = sizeof (cmdQ_e) * Q->cq_entries_n;
1387 Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1388 size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT);
1389
1390 if (!Q->cq_entries)
1391 goto err_no_mem;
1392 memset(Q->cq_entries, 0, size);
1393 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1394 Q->cq_centries = t1_os_malloc_wait_zero(size);
1395 if (!Q->cq_centries)
1396 goto err_no_mem;
1397 memset(Q->cq_centries, 0, size);
1398
1399 /* allocate pre-mapped dma headers */
1400 pe_dma_handle_init(sge->obj, Q->cq_entries_n);
1401 }
1402
1403 return (0);
1404
1405 err_no_mem:
1406 free_tx_resources(sge);
1407 return (1);
1408 }
1409
1410 /*
1411 * Sets the interrupt latency timer when the adaptive Rx coalescing
1412 * is turned off. Do nothing when it is turned on again.
1413 *
1414 * This routine relies on the fact that the caller has already set
1415 * the adaptive policy in adapter->sge_params before calling it.
1416 */
1417 int
t1_sge_set_coalesce_params(pesge * sge,struct sge_params * p)1418 t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p)
1419 {
1420 if (!p->coalesce_enable) {
1421 u32 newTimer = p->rx_coalesce_usecs *
1422 (board_info(sge->obj)->clock_core / 1000000);
1423
1424 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer);
1425 }
1426 return (0);
1427 }
1428
1429 /*
1430 * Programs the various SGE registers. However, the engine is not yet enabled,
1431 * but sge->sge_control is setup and ready to go.
1432 */
1433 static void
configure_sge(pesge * sge,struct sge_params * p)1434 configure_sge(pesge *sge, struct sge_params *p)
1435 {
1436 ch_t *ap = sge->obj;
1437 int i;
1438
1439 t1_write_reg_4(ap, A_SG_CONTROL, 0);
1440
1441 setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n,
1442 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
1443 setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n,
1444 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
1445 setup_ring_params(ap, sge->freelQ[0].fq_pa,
1446 sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR,
1447 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
1448 setup_ring_params(ap, sge->freelQ[1].fq_pa,
1449 sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR,
1450 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
1451
1452 /* The threshold comparison uses <. */
1453 t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) -
1454 SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1);
1455 setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n,
1456 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
1457 t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n);
1458 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
1459 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
1460 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
1461 #if 1
1462 /*
1463 * if the the following bit is not set, then we'll get an
1464 * interrupt everytime command Q 0 goes empty. Since we're
1465 * always ringing the doorbell, we can turn it on.
1466 */
1467 F_DISABLE_CMDQ0_GTS |
1468 #endif
1469 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
1470
1471 #if BYTE_ORDER == BIG_ENDIAN
1472 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
1473 #endif
1474
1475 /*
1476 * Initialize the SGE Interrupt Timer arrray:
1477 * intrtimer[0] = (SGE_INTRTIMER0) usec
1478 * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
1479 * intrtimer[10] = (SGE_INTRTIMER1) usec
1480 *
1481 */
1482 sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000;
1483 for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) {
1484 sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i);
1485 sge->intrtimer[i] *= sge->intrtimer[0];
1486 }
1487 sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] =
1488 sge->intrtimer[0] * SGE_INTRTIMER1;
1489 /* Initialize resource timer */
1490 sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] *
1491 SGE_INTRTIMER_NRES);
1492 /* Finally finish initialization of intrtimer[0] */
1493 sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0);
1494 /* Initialize for a throughput oriented workload */
1495 sge->currIndex = SGE_INTR_MAXBUCKETS - 1;
1496
1497 if (p->coalesce_enable)
1498 t1_write_reg_4(ap, A_SG_INTRTIMER,
1499 sge->intrtimer[sge->currIndex]);
1500 else
1501 (void) t1_sge_set_coalesce_params(sge, p);
1502 }
1503
1504 static inline void
setup_ring_params(ch_t * adapter,u64 addr,u32 size,int base_reg_lo,int base_reg_hi,int size_reg)1505 setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo,
1506 int base_reg_hi, int size_reg)
1507 {
1508 t1_write_reg_4(adapter, base_reg_lo, (u32)addr);
1509 t1_write_reg_4(adapter, base_reg_hi, addr >> 32);
1510 t1_write_reg_4(adapter, size_reg, size);
1511 }
1512
1513 /*
1514 * Frees RX resources.
1515 */
1516 static void
free_rx_resources(pesge * sge)1517 free_rx_resources(pesge *sge)
1518 {
1519 unsigned int size, i;
1520
1521 if (sge->respQ.rq_entries) {
1522 size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1523
1524 pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries,
1525 sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah);
1526 }
1527
1528 for (i = 0; i < SGE_FREELQ_N; i++) {
1529 struct freelQ *Q = &sge->freelQ[i];
1530
1531 if (Q->fq_centries) {
1532 free_freelQ_buffers(sge, Q);
1533
1534 t1_os_free(Q->fq_centries,
1535 Q->fq_entries_n * sizeof (freelQ_ce_t));
1536 }
1537 if (Q->fq_entries) {
1538 size = sizeof (freelQ_e) * Q->fq_entries_n;
1539
1540 /* free the freelist queue */
1541 pe_os_free_contig(sge->obj, size, Q->fq_entries,
1542 Q->fq_pa, Q->fq_dh, Q->fq_ah);
1543
1544 }
1545 }
1546 }
1547
1548 /*
1549 * Frees all RX buffers on the freelist Q. The caller must make sure that
1550 * the SGE is turned off before calling this function.
1551 */
1552 static void
free_freelQ_buffers(pesge * sge,struct freelQ * Q)1553 free_freelQ_buffers(pesge *sge, struct freelQ *Q)
1554 {
1555 struct freelQ_ce *ce;
1556 struct freelQ_ce *cq = Q->fq_centries;
1557 uint32_t credits = Q->fq_credits;
1558 uint32_t entries_n = Q->fq_entries_n;
1559 uint32_t cidx = Q->fq_cidx;
1560 uint32_t i = Q->fq_id;
1561
1562 ce = &cq[cidx];
1563
1564 credits = entries_n;
1565 while (credits--) {
1566 mblk_t *mp;
1567 if ((mp = ce->fe_mp) != NULL) {
1568 /* bump in-use count of receive buffers */
1569 if (i != sge->jumbo_fl) {
1570 atomic_add(1,
1571 &buffers_in_use[sge->obj->ch_sm_index]);
1572 } else {
1573 atomic_add(1,
1574 &buffers_in_use[sge->obj->ch_big_index]);
1575 }
1576
1577 /*
1578 * note. freeb() callback of esb-alloced mblk will
1579 * cause receive buffer to be put back on sa free list.
1580 */
1581 freeb(mp);
1582 ce->fe_mp = NULL;
1583 }
1584
1585 ce++;
1586 if (++cidx == entries_n) {
1587 cidx = 0;
1588 ce = cq;
1589 }
1590 }
1591
1592 Q->fq_cidx = cidx;
1593 Q->fq_credits = credits;
1594 }
1595
1596 /*
1597 * Free TX resources.
1598 *
1599 * Assumes that SGE is stopped and all interrupts are disabled.
1600 */
1601 static void
free_tx_resources(pesge * sge)1602 free_tx_resources(pesge *sge)
1603 {
1604 unsigned int size;
1605 uint32_t i;
1606
1607 for (i = 0; i < SGE_CMDQ_N; i++) {
1608 struct cmdQ *Q = &sge->cmdQ[i];
1609
1610 if (Q->cq_centries) {
1611 unsigned int pending = Q->cq_entries_n -
1612 atomic_read(Q->cq_credits);
1613
1614 mutex_destroy(&Q->cq_qlock);
1615
1616 if (pending)
1617 free_cmdQ_buffers(sge, Q, pending);
1618
1619 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1620 t1_os_free(Q->cq_centries, size);
1621 }
1622
1623 if (Q->cq_entries) {
1624 size = sizeof (cmdQ_e) * Q->cq_entries_n;
1625 pe_os_free_contig(sge->obj, size, Q->cq_entries,
1626 Q->cq_pa, Q->cq_dh, Q->cq_ah);
1627 }
1628 }
1629 }
1630
1631 /*
1632 * Return the payload capacity of the jumbo free-list buffers.
1633 */
jumbo_payload_capacity(pesge * sge)1634 static inline unsigned int jumbo_payload_capacity(pesge *sge)
1635 {
1636 return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size -
1637 sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset);
1638 }
1639
1640 /* PR2928 & PR3309 */
1641 void
t1_sge_set_ptimeout(adapter_t * adapter,u32 val)1642 t1_sge_set_ptimeout(adapter_t *adapter, u32 val)
1643 {
1644 pesge *sge = adapter->sge;
1645
1646 if (is_T2(adapter))
1647 sge->ptimeout = max(val, 1);
1648 }
1649
1650 /* PR2928 & PR3309 */
1651 u32
t1_sge_get_ptimeout(adapter_t * adapter)1652 t1_sge_get_ptimeout(adapter_t *adapter)
1653 {
1654 pesge *sge = adapter->sge;
1655
1656 return (is_T2(adapter) ? sge->ptimeout : 0);
1657 }
1658
1659 void
sge_add_fake_arp(pesge * sge,void * bp)1660 sge_add_fake_arp(pesge *sge, void *bp)
1661 {
1662 sge->pskb = bp;
1663 }
1664
1665 #ifdef SUN_KSTATS
1666 static int
sge_kstat_setup(pesge * sge)1667 sge_kstat_setup(pesge *sge)
1668 {
1669 int status;
1670 p_kstat_t ksp;
1671 size_t ch_kstat_sz;
1672 p_ch_kstat_t chkp;
1673 char kstat_name[32];
1674 int instance;
1675 int i;
1676
1677 status = -1;
1678 ch_kstat_sz = sizeof (ch_kstat_t);
1679 instance = ddi_get_instance(sge->obj->ch_dip);
1680 if ((ksp = kstat_create(CHNAME "_debug", instance,
1681 NULL, "net_debug", KSTAT_TYPE_NAMED,
1682 ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL)
1683 goto sge_kstat_setup_exit;
1684 chkp = (p_ch_kstat_t)ksp->ks_data;
1685 kstat_named_init(&chkp->respQ_empty, "respQ_empty",
1686 KSTAT_DATA_UINT32);
1687 kstat_named_init(&chkp->respQ_overflow, "respQ_overflow",
1688 KSTAT_DATA_UINT32);
1689 kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty",
1690 KSTAT_DATA_UINT32);
1691 kstat_named_init(&chkp->pkt_too_big, "pkt_too_big",
1692 KSTAT_DATA_UINT32);
1693 kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch",
1694 KSTAT_DATA_UINT32);
1695 kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]",
1696 KSTAT_DATA_UINT32);
1697 kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]",
1698 KSTAT_DATA_UINT32);
1699 kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]",
1700 KSTAT_DATA_UINT32);
1701 kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]",
1702 KSTAT_DATA_UINT32);
1703 kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups",
1704 KSTAT_DATA_UINT32);
1705 kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups",
1706 KSTAT_DATA_UINT32);
1707 kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag",
1708 KSTAT_DATA_UINT32);
1709 kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag",
1710 KSTAT_DATA_UINT32);
1711 kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums",
1712 KSTAT_DATA_UINT32);
1713 kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space",
1714 KSTAT_DATA_UINT32);
1715 kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks",
1716 KSTAT_DATA_UINT32);
1717 kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails",
1718 KSTAT_DATA_UINT32);
1719 kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails",
1720 KSTAT_DATA_UINT32);
1721 kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails",
1722 KSTAT_DATA_UINT32);
1723 kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails",
1724 KSTAT_DATA_UINT32);
1725 kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0",
1726 KSTAT_DATA_UINT32);
1727 kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1",
1728 KSTAT_DATA_UINT32);
1729 kstat_named_init(&chkp->rx_flq0, "rx_flq0",
1730 KSTAT_DATA_UINT32);
1731 kstat_named_init(&chkp->rx_flq1, "rx_flq1",
1732 KSTAT_DATA_UINT32);
1733 kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz",
1734 KSTAT_DATA_UINT32);
1735 kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz",
1736 KSTAT_DATA_UINT32);
1737 kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops",
1738 KSTAT_DATA_UINT32);
1739 kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied",
1740 KSTAT_DATA_UINT32);
1741 kstat_named_init(&chkp->rx_pause_on, "rx_pause_on",
1742 KSTAT_DATA_UINT32);
1743 kstat_named_init(&chkp->rx_pause_off, "rx_pause_off",
1744 KSTAT_DATA_UINT32);
1745 kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms",
1746 KSTAT_DATA_UINT32);
1747 kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike",
1748 KSTAT_DATA_UINT32);
1749 kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits",
1750 KSTAT_DATA_UINT32);
1751 kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails",
1752 KSTAT_DATA_UINT32);
1753 kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs",
1754 KSTAT_DATA_UINT32);
1755 kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop",
1756 KSTAT_DATA_UINT32);
1757 kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt",
1758 KSTAT_DATA_UINT32);
1759 kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt",
1760 KSTAT_DATA_UINT32);
1761 kstat_named_init(&chkp->arp_sent, "arp_sent",
1762 KSTAT_DATA_UINT32);
1763 kstat_named_init(&chkp->tx_doorbells, "tx_doorbells",
1764 KSTAT_DATA_UINT32);
1765 kstat_named_init(&chkp->intr_doorbells, "intr_doorbells",
1766 KSTAT_DATA_UINT32);
1767 kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells",
1768 KSTAT_DATA_UINT32);
1769 kstat_named_init(&chkp->sleep_cnt, "sleep_cnt",
1770 KSTAT_DATA_UINT32);
1771 kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt",
1772 KSTAT_DATA_UINT32);
1773 for (i = 0; i < MBLK_MAX; i++) {
1774 (void) sprintf(kstat_name, "tx_descs[%02d]", i);
1775 kstat_named_init(&chkp->tx_descs[i],
1776 kstat_name, KSTAT_DATA_UINT32);
1777 }
1778 ksp->ks_update = sge_kstat_update;
1779 ksp->ks_private = (void *)sge;
1780 sge->ksp = ksp;
1781 kstat_install(ksp);
1782 status = 0;
1783
1784 sge_kstat_setup_exit:
1785 return (status);
1786 }
1787
1788 static void
sge_kstat_remove(pesge * sge)1789 sge_kstat_remove(pesge *sge)
1790 {
1791 if (sge->ksp)
1792 kstat_delete(sge->ksp);
1793 }
1794
1795 static int
sge_kstat_update(p_kstat_t ksp,int rw)1796 sge_kstat_update(p_kstat_t ksp, int rw)
1797 {
1798 pesge *sge;
1799 p_ch_stats_t statsp;
1800 p_ch_kstat_t chkp;
1801 int i;
1802
1803 sge = (pesge *)ksp->ks_private;
1804 statsp = (p_ch_stats_t)&sge->intr_cnt;
1805 chkp = (p_ch_kstat_t)ksp->ks_data;
1806 if (rw == KSTAT_WRITE) {
1807 statsp->respQ_empty = chkp->respQ_empty.value.ui32;
1808 statsp->respQ_overflow = chkp->respQ_overflow.value.ui32;
1809 statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32;
1810 statsp->pkt_too_big = chkp->pkt_too_big.value.ui32;
1811 statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32;
1812 statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32;
1813 statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32;
1814 statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32;
1815 statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32;
1816 statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32;
1817 statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32;
1818 statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32;
1819 statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32;
1820 statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32;
1821 statsp->tx_need_cpl_space
1822 = chkp->tx_need_cpl_space.value.ui32;
1823 statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32;
1824 statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32;
1825 statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32;
1826 statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32;
1827 statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32;
1828 statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32;
1829 statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32;
1830 statsp->rx_flq0 = chkp->rx_flq0.value.ui32;
1831 statsp->rx_flq1 = chkp->rx_flq1.value.ui32;
1832 statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32;
1833 statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32;
1834 statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32;
1835 statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32;
1836 statsp->rx_pause_on = chkp->rx_pause_on.value.ui32;
1837 statsp->rx_pause_off = chkp->rx_pause_off.value.ui32;
1838 statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32;
1839 statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32;
1840 statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32;
1841 statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32;
1842 statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32;
1843 statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32;
1844 statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32;
1845 statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32;
1846 statsp->arp_sent = chkp->arp_sent.value.ui32;
1847 statsp->tx_doorbells = chkp->tx_doorbells.value.ui32;
1848 statsp->intr_doorbells = chkp->intr_doorbells.value.ui32;
1849 statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32;
1850 statsp->sleep_cnt = chkp->sleep_cnt.value.ui32;
1851 statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32;
1852 for (i = 0; i < MBLK_MAX; i++) {
1853 statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32;
1854 }
1855 } else {
1856 chkp->respQ_empty.value.ui32 = statsp->respQ_empty;
1857 chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow;
1858 chkp->freelistQ_empty.value.ui32
1859 = statsp->freelistQ_empty;
1860 chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big;
1861 chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch;
1862 chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0];
1863 chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1];
1864 chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0];
1865 chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1];
1866 chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups;
1867 chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups;
1868 chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag;
1869 chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag;
1870 chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums;
1871 chkp->tx_need_cpl_space.value.ui32
1872 = statsp->tx_need_cpl_space;
1873 chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks;
1874 chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1;
1875 chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2;
1876 chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1;
1877 chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2;
1878 chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0;
1879 chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1;
1880 chkp->rx_flq0.value.ui32 = statsp->rx_flq0;
1881 chkp->rx_flq1.value.ui32 = statsp->rx_flq1;
1882 chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz;
1883 chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz;
1884 chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops;
1885 chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied;
1886 chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on;
1887 chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off;
1888 chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms;
1889 chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike;
1890 chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits;
1891 chkp->rx_flbuf_fails.value.ui32
1892 = statsp->rx_flbuf_fails;
1893 chkp->rx_flbuf_allocs.value.ui32
1894 = statsp->rx_flbuf_allocs;
1895 chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop;
1896 chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt;
1897 chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt;
1898 chkp->arp_sent.value.ui32 = statsp->arp_sent;
1899 chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells;
1900 chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells;
1901 chkp->intr1_doorbells.value.ui32
1902 = statsp->intr1_doorbells;
1903 chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt;
1904 chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt;
1905 for (i = 0; i < MBLK_MAX; i++) {
1906 chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i];
1907 }
1908 }
1909 return (0);
1910 }
1911 #endif
1912
1913 static uint16_t
calc_ocsum(mblk_t * mp,int offset)1914 calc_ocsum(mblk_t *mp, int offset)
1915 {
1916 uint8_t *addrp;
1917 uint32_t src;
1918 uint32_t dst;
1919
1920 ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset);
1921 uint32_t sum;
1922 int iplen = IPH_HDR_LENGTH(ihdr);
1923 struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen);
1924 uchar_t *byte;
1925 int len;
1926
1927 addrp = (uint8_t *)&ihdr->ipha_src;
1928 src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1929 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1930
1931 addrp = (uint8_t *)&ihdr->ipha_dst;
1932 dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1933 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1934
1935 sum = (uint16_t)(src >> 16) +
1936 (uint16_t)(src) +
1937 (uint16_t)(dst >> 16) +
1938 (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP));
1939
1940 sum = (uint16_t)(sum >> 16) + (uint16_t)(sum);
1941
1942 if (sum > 0xffff)
1943 sum -= 0xffff;
1944
1945 udpp->uh_sum = 0;
1946 byte = mp->b_rptr + offset + iplen;
1947 do {
1948 len = (mp->b_wptr - byte);
1949 sum = bcksum(byte, len, sum);
1950 if (sum > 0xffff)
1951 sum -= 0xffff;
1952 mp = mp->b_cont;
1953 if (mp)
1954 byte = mp->b_rptr;
1955 } while (mp);
1956
1957 sum = ~sum & 0xffff;
1958
1959 return (sum);
1960 }
1961