1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * This file is part of the Chelsio T1 Ethernet driver.
29 *
30 * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved.
31 */
32
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/cmn_err.h>
36 #include <sys/sunddi.h>
37 #include <sys/kmem.h>
38 #include <sys/cmn_err.h>
39 #include <sys/byteorder.h>
40 #include <sys/atomic.h>
41 #include <sys/stropts.h>
42 #include <sys/stream.h>
43 #include <sys/strsubr.h>
44 #include <sys/dlpi.h>
45 #include <sys/kstat.h>
46 #include <sys/ethernet.h>
47 #include <netinet/in.h>
48 #include <netinet/udp.h>
49 #include <inet/common.h>
50 #include <inet/nd.h>
51 #include <inet/ip.h>
52 #include <inet/tcp.h>
53 #include <netinet/udp.h>
54 #include <sys/gld.h>
55 #include "ostypes.h"
56 #include "common.h"
57 #ifdef CONFIG_CHELSIO_T1_1G
58 #include "fpga_defs.h"
59 #endif
60 #include "regs.h"
61 #include "suni1x10gexp_regs.h"
62 #include "sge.h"
63 #include "espi.h"
64
65 #include "ch.h"
66
67 extern uint32_t buffers_in_use[];
68
69 uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N;
70 uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N;
71 uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N;
72 uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N;
73 uint32_t sge_respq_cnt = SGE_RESPQ_E_N;
74
75 uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N;
76 uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N;
77 uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N;
78 uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N;
79 uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N;
80
81 #ifdef HOST_PAUSE
82 uint32_t do_host_pause = 1;
83 uint32_t flq_pause_window = 64;
84 #endif
85
86 static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb,
87 ulong_t *dh);
88 void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t);
89
90 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q,
91 unsigned int len, unsigned int offload);
92 #ifdef HOST_PAUSE
93 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q);
94 #endif
95 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q);
96 static void freelQs_empty(pesge *sge);
97 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend);
98 static int alloc_rx_resources(pesge *sge, struct sge_params *p);
99 static int alloc_tx_resources(pesge *sge, struct sge_params *p);
100 static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size,
101 int base_reg_lo, int base_reg_hi, int size_reg);
102 static void configure_sge(pesge *sge, struct sge_params *p);
103 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q);
104 static void free_rx_resources(pesge *sge);
105 static void free_tx_resources(pesge *sge);
106 static inline unsigned int jumbo_payload_capacity(pesge *sge);
107 #ifdef SUN_KSTATS
108 static int sge_kstat_setup(pesge *);
109 static void sge_kstat_remove(pesge *);
110 static int sge_kstat_update(p_kstat_t, int);
111 #endif
112 static uint16_t calc_ocsum(mblk_t *, int);
113
114 /*
115 * Local routines.
116 */
117 static inline void sge_ring_doorbell(pesge *sge, u32 control_reg);
118
119 static inline void
sge_ring_doorbell(pesge * sge,u32 control_reg)120 sge_ring_doorbell(pesge *sge, u32 control_reg)
121 {
122 membar_producer();
123 t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg);
124 }
125
126 /*
127 * DESC:
128 *
129 * NOTES: Must have at least 1 command queue and 1 freelist queue.
130 *
131 */
132 pesge *
t1_sge_create(ch_t * sa,struct sge_params * p)133 t1_sge_create(ch_t *sa, struct sge_params *p)
134 {
135 pesge *sge;
136
137 sge = t1_os_malloc_wait_zero(sizeof (pesge));
138
139 if (sge == NULL)
140 goto error_no_mem;
141
142 memset(sge, 0, sizeof (*sge));
143
144 /*
145 * PR2928 & PR3309
146 * set default timeout value - 20 msec
147 * we set the initial value to 2 which gurantees at least one tick.
148 */
149 if (is_T2(sa))
150 sge->ptimeout = 1;
151
152 sge->obj = sa;
153 #ifdef SUN_KSTATS
154 if (sge_kstat_setup(sge) != 0)
155 goto t1_sge_create_fail1;
156 #endif
157 p->cmdQ_size[0] = sge_cmdq0_cnt;
158 p->cmdQ_size[1] = sge_cmdq1_cnt;
159
160 /* note that jumbo frame index is inverted for T2 */
161 if (is_T2(sa)) {
162 p->freelQ_size[1] = sge_flq0_cnt;
163 p->freelQ_size[0] = sge_flq1_cnt;
164 } else {
165 p->freelQ_size[0] = sge_flq0_cnt;
166 p->freelQ_size[1] = sge_flq1_cnt;
167 }
168
169 #if CH_DEBUG
170 /* DEBUG only */
171 cmn_err(CE_NOTE, "sge: %p\n", sge);
172 cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]);
173 cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]);
174 cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]);
175 cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ);
176 cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt);
177 #endif
178 #ifdef SUN_KSTATS
179 goto error_no_mem;
180
181 t1_sge_create_fail1:
182 t1_os_free(sge, sizeof (pesge));
183 sge = NULL;
184 #endif
185 error_no_mem:
186 return (sge);
187 }
188
189 int
t1_sge_destroy(pesge * sge)190 t1_sge_destroy(pesge* sge)
191 {
192 if (sge != NULL) {
193 free_tx_resources(sge);
194 free_rx_resources(sge);
195
196 /* PR2928 & PR3309 */
197 if ((is_T2(sge->obj)) && (sge->pskb))
198 pe_free_fake_arp(sge->pskb);
199 #ifdef SUN_KSTATS
200 sge_kstat_remove(sge);
201 #endif
202 t1_os_free(sge, sizeof (pesge));
203 }
204 return (0);
205 }
206
207 /*
208 * PR2928 & PR3309
209 * call out event from timeout
210 *
211 * there is a potential race between the timeout and the close.
212 * unless we protect the timeout, the close could occur at the
213 * same time. Then if the timeout service routine was slow or
214 * interrupted, the sge_stop() could complete with a timeoutID
215 * that has expired, thus letting another timeout occur. If the
216 * service routine was delayed still further, a detach could occur.
217 * the second time could then end up accessing memory that has been
218 * released back to the system. Bad things could then occur. We
219 * set a flag in sge_stop() to tell the service routine not to
220 * issue further timeouts. sge_stop() will block until a timeout
221 * has occured. If the command Q is full then we shouldn't put out
222 * an arp.
223 */
224
225 void
t1_espi_workaround(ch_t * adapter)226 t1_espi_workaround(ch_t *adapter)
227 {
228 pesge *sge = adapter->sge;
229 ch_t *chp = (ch_t *)sge->obj;
230 int rv = 1;
231
232 if ((chp->ch_state == PERUNNING) &&
233 atomic_read(sge->cmdQ[0].cq_asleep)) {
234 u32 seop;
235 seop = t1_espi_get_mon(adapter, 0x930, 0);
236 if ((seop & 0xfff0fff) == 0xfff) {
237 /* after first arp */
238 if (sge->pskb) {
239 rv = pe_start(adapter, (mblk_t *)sge->pskb,
240 CH_ARP);
241 if (!rv)
242 sge->intr_cnt.arp_sent++;
243 }
244 }
245 }
246 #ifdef HOST_PAUSE
247 /*
248 * If we are already in sge_data_in, then we can skip calling
249 * t1_sge_check_pause() this clock cycle. lockstat showed that
250 * we were blocking on the mutex ~ 2% of the time.
251 */
252 if (mutex_tryenter(&adapter->ch_intr)) {
253 t1_sge_check_pause(sge, &sge->freelQ[0]);
254 t1_sge_check_pause(sge, &sge->freelQ[1]);
255 mutex_exit(&adapter->ch_intr);
256 }
257 #endif
258 }
259
260 int
sge_start(pesge * sge)261 sge_start(pesge *sge)
262 {
263 t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control);
264 /* PR2928 & PR3309, also need to avoid Pause deadlock */
265 ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic,
266 (void (*)(void *))t1_espi_workaround, sge->obj);
267 ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout);
268 return (0);
269 }
270
271 /*
272 * Disables SGE queues.
273 */
274 int
sge_stop(pesge * sge)275 sge_stop(pesge *sge)
276 {
277 uint32_t status;
278 int loops;
279
280 DBGASSERT(sge);
281
282 /* PR2928 & PR3309, also need to avoid Pause deadlock */
283 t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0);
284
285 /* wait until there's no more outstanding interrupts pending */
286 loops = 0;
287 do {
288 status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE);
289 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status);
290 drv_usecwait(125);
291 loops++;
292 } while (status && (loops < 1000));
293
294 ch_stop_cyclic(&sge->espi_wa_cyclic);
295
296 return (0);
297 }
298
299 uint32_t sge_cmdq_send_fail;
300
301 int
sge_data_out(pesge * sge,int qid,mblk_t * m0,cmdQ_ce_t * cmp,int count,uint32_t flg)302 sge_data_out(pesge* sge, int qid, mblk_t *m0,
303 cmdQ_ce_t *cmp, int count, uint32_t flg)
304 {
305 struct cmdQ *Q = &sge->cmdQ[qid];
306 ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh;
307 spinlock_t *qlock = &Q->cq_qlock;
308 cmdQ_e *e;
309 cmdQ_e *q = Q->cq_entries;
310 uint32_t credits;
311 uint32_t pidx;
312 uint32_t genbit;
313 uint32_t entries_n = Q->cq_entries_n;
314 cmdQ_ce_t *ce;
315 cmdQ_ce_t *cq = Q->cq_centries;
316 dma_addr_t mapping;
317 uint32_t j = 0;
318 uint32_t offset;
319 #if defined(TX_CKSUM_FIX)
320 uint16_t csum;
321 uint16_t *csum_loc;
322 #endif
323 #ifdef TX_THREAD_RECLAIM
324 uint32_t reclaim_cnt;
325 #endif
326
327 /*
328 * We must exit if we don't have enough free command queue entries
329 * available.
330 */
331
332 spin_lock(qlock);
333
334 #if defined(TX_CKSUM_FIX)
335 /*
336 * This checksum fix will address a fragmented datagram
337 * checksum error. Which will lead to the next packet after
338 * the last packet with the More fragment bit set having its
339 * checksum corrupted. When the packet reaches this point
340 * the 'flg' variable indicates whether a checksum is needed
341 * or not. The algorithm is as follows, if the current packet
342 * is a More fragment set the count of packets to be checksummed
343 * after it to 3. If it't not and the count of is more than 0
344 * then calculate the checksum in software, if a hardware checksum
345 * was requested. Then decrment the count. Same algorithm applies
346 * to TCP.
347 */
348 if (flg & CH_UDP_MF) {
349 sge->do_udp_csum = 3;
350 } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) {
351 if ((flg & CH_NO_HWCKSUM) == 0) {
352 /*
353 * Calc Checksum here.
354 */
355 csum = calc_ocsum(m0,
356 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
357 csum_loc = (uint16_t *)(m0->b_rptr +
358 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE);
359 csum_loc += (((*(char *)csum_loc) & 0x0f) << 1);
360
361 sge->intr_cnt.tx_soft_cksums++;
362 ((struct udphdr *)(csum_loc))->uh_sum = csum;
363 ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1;
364 }
365 sge->do_udp_csum--;
366 } else if (flg & CH_TCP_MF) {
367 sge->do_tcp_csum = 3;
368 } else if (sge->do_tcp_csum != 0) {
369 if ((flg & CH_NO_HWCKSUM) == 0) {
370 sge->intr_cnt.tx_soft_cksums++;
371 /*
372 * Calc Checksum here.
373 */
374 }
375 sge->do_tcp_csum--;
376 }
377 #endif /* TX_CKSUM_FIX */
378 #ifdef TX_THREAD_RECLAIM
379 reclaim_cnt = Q->cq_complete;
380 if (reclaim_cnt > SGE_BATCH_THRESH) {
381 sge->intr_cnt.tx_reclaims[qid]++;
382 free_cmdQ_buffers(sge, Q, reclaim_cnt);
383 Q->cq_complete = 0;
384 }
385 #endif
386 genbit = Q->cq_genbit;
387 pidx = Q->cq_pidx;
388 credits = Q->cq_credits;
389
390 if ((credits - 1) < count) {
391 spin_unlock(qlock);
392 sge->intr_cnt.cmdQ_full[qid]++;
393 return (1);
394 }
395
396 atomic_sub(count, &Q->cq_credits);
397 Q->cq_pidx += count;
398 if (Q->cq_pidx >= entries_n) {
399 Q->cq_pidx -= entries_n;
400 Q->cq_genbit ^= 1;
401 }
402
403 spin_unlock(qlock);
404
405 #ifdef SUN_KSTATS
406 if (count > MBLK_MAX)
407 sge->intr_cnt.tx_descs[MBLK_MAX - 1]++;
408 else
409 sge->intr_cnt.tx_descs[count]++;
410 #endif
411
412 ce = &cq[pidx];
413 *ce = *cmp;
414 mapping = cmp->ce_pa;
415 j++;
416
417 e = &q[pidx];
418
419 offset = (caddr_t)e - (caddr_t)q;
420
421 e->Sop = 1;
422 e->DataValid = 1;
423 e->BufferLength = cmp->ce_len;
424 e->AddrHigh = ((u64)mapping >> 32);
425 e->AddrLow = ((u64)mapping & 0xffffffff);
426
427 --count;
428 if (count > 0) {
429 unsigned int i;
430
431 e->Eop = 0;
432 wmb();
433 e->GenerationBit = e->GenerationBit2 = genbit;
434
435 for (i = 0; i < count; i++) {
436
437 ce++;
438 e++;
439 cmp++;
440 if (++pidx == entries_n) {
441 pidx = 0;
442 genbit ^= 1;
443 /* sync from offset to end of cmdQ */
444 (void) ddi_dma_sync(dh, (off_t)(offset),
445 j*sizeof (*e), DDI_DMA_SYNC_FORDEV);
446 offset = j = 0;
447 ce = cq;
448 e = q;
449 }
450
451 *ce = *cmp;
452 mapping = cmp->ce_pa;
453 j++;
454 e->Sop = 0;
455 e->DataValid = 1;
456 e->BufferLength = cmp->ce_len;
457 e->AddrHigh = ((u64)mapping >> 32);
458 e->AddrLow = ((u64)mapping & 0xffffffff);
459
460 if (i < (count - 1)) {
461 e->Eop = 0;
462 wmb();
463 e->GenerationBit = e->GenerationBit2 = genbit;
464 }
465 }
466 }
467
468 ce->ce_mp = m0;
469
470 e->Eop = 1;
471 wmb();
472 e->GenerationBit = e->GenerationBit2 = genbit;
473
474 (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e),
475 DDI_DMA_SYNC_FORDEV);
476
477 /*
478 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring
479 * the doorbell if the Q is asleep. There is a natural race, where
480 * the hardware is going to sleep just after we checked, however,
481 * then the interrupt handler will detect the outstanding TX packet
482 * and ring the doorbell for us.
483 */
484 if (qid) {
485 doorbell_pio(sge, F_CMDQ1_ENABLE);
486 } else {
487 if (atomic_read(Q->cq_asleep)) {
488 atomic_set(&Q->cq_asleep, 0);
489 /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */
490 atomic_set(&Q->cq_pio_pidx, Q->cq_pidx);
491 }
492 }
493 doorbell_pio(sge, F_CMDQ0_ENABLE);
494
495 return (0);
496 }
497
498 #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA)
499
500 /*
501 * Disable SGE error interrupts.
502 */
503 int
t1_sge_intr_disable(pesge * sge)504 t1_sge_intr_disable(pesge* sge)
505 {
506 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
507
508 t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK);
509 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0);
510 return (0);
511 }
512
513 #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \
514 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
515
516 /*
517 * Enable SGE error interrupts.
518 */
519 int
t1_sge_intr_enable(pesge * sge)520 t1_sge_intr_enable(pesge* sge)
521 {
522 u32 en = SGE_INT_ENABLE;
523 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE);
524
525 t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK);
526
527 if (sge->obj->ch_flags & TSO_CAPABLE)
528 en &= ~F_PACKET_TOO_BIG;
529 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en);
530 return (0);
531 }
532
533 /*
534 * Clear SGE error interrupts.
535 */
536 int
t1_sge_intr_clear(pesge * sge)537 t1_sge_intr_clear(pesge* sge)
538 {
539 t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK);
540 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff);
541 return (0);
542 }
543
544 #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH)
545
546 int
t1_sge_intr_error_handler(pesge * sge)547 t1_sge_intr_error_handler(pesge *sge)
548 {
549 peobj *obj = sge->obj;
550 u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE);
551
552 if (cause & F_RESPQ_EXHAUSTED)
553 sge->intr_cnt.respQ_empty++;
554 if (cause & F_RESPQ_OVERFLOW) {
555 sge->intr_cnt.respQ_overflow++;
556 cmn_err(CE_WARN, "%s: SGE response queue overflow\n",
557 obj->ch_name);
558 }
559 if (cause & F_FL_EXHAUSTED) {
560 sge->intr_cnt.freelistQ_empty++;
561 freelQs_empty(sge);
562 }
563 if (cause & F_PACKET_TOO_BIG) {
564 sge->intr_cnt.pkt_too_big++;
565 cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n",
566 obj->ch_name);
567 }
568 if (cause & F_PACKET_MISMATCH) {
569 sge->intr_cnt.pkt_mismatch++;
570 cmn_err(CE_WARN, "%s: SGE packet mismatch\n",
571 obj->ch_name);
572 }
573 if (cause & SGE_INT_FATAL)
574 t1_fatal_err(obj);
575
576 t1_write_reg_4(obj, A_SG_INT_CAUSE, cause);
577 return (0);
578 }
579
580 /*
581 *
582 * PARAM: sge - SGE instance pointer.
583 */
584 int
sge_data_in(pesge * sge)585 sge_data_in(pesge *sge)
586 {
587 peobj *adapter = sge->obj;
588 struct respQ *Q = &sge->respQ;
589 respQ_e *e; /* response queue entry */
590 respQ_e *q = Q->rq_entries; /* base response queue */
591 uint32_t cidx = Q->rq_cidx;
592 uint32_t genbit = Q->rq_genbit;
593 uint32_t entries_n = Q->rq_entries_n;
594 uint32_t credits = Q->rq_credits;
595 uint32_t credits_thresh = Q->rq_credits_thresh;
596 uint32_t ret = 0;
597 #ifndef TX_THREAD_RECLAIM
598 uint32_t credits_pend[2] = {0, 0};
599 #endif
600 uint32_t flags = 0;
601 uint32_t flagt;
602 ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh;
603
604 t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA);
605
606 /*
607 * Catch the case where an interrupt arrives
608 * early.
609 */
610 if ((q == NULL) || (dh == NULL)) {
611 goto check_slow_ints;
612 }
613
614 /* initial response queue entry */
615 e = &q[cidx];
616
617 /* pull physical memory of response queue entry into cache */
618 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
619 sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
620
621 while (e->GenerationBit == genbit) {
622 if (--credits < credits_thresh) {
623 uint32_t n = entries_n - credits - 1;
624 t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n);
625 credits += n;
626 }
627 if (likely(e->DataValid)) {
628 (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid],
629 e->BufferLength, e->Offload);
630 if ((e->Sop != 1) || (e->Eop != 1)) {
631 sge->intr_cnt.rx_badEopSop++;
632 cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d",
633 e->Sop, e->Eop, e->BufferLength);
634 }
635 }
636 flagt = e->Qsleeping;
637 flags |= flagt;
638 if (flagt & F_CMDQ0_ENABLE)
639 sge->intr_cnt.rx_cmdq0++;
640 if (flagt & F_CMDQ1_ENABLE)
641 sge->intr_cnt.rx_cmdq1++;
642 if (flagt & F_FL0_ENABLE)
643 sge->intr_cnt.rx_flq0++;
644 if (flagt & F_FL1_ENABLE)
645 sge->intr_cnt.rx_flq1++;
646 #ifdef TX_THREAD_RECLAIM
647 spin_lock(&sge->cmdQ[0].cq_qlock);
648 sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn;
649 spin_unlock(&sge->cmdQ[0].cq_qlock);
650 spin_lock(&sge->cmdQ[1].cq_qlock);
651 sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn;
652 if ((adapter->ch_blked) &&
653 (sge->cmdQ[0].cq_complete +
654 sge->cmdQ[1].cq_complete) > 16) {
655 adapter->ch_blked = 0;
656 ch_gld_ok(adapter);
657 }
658 spin_unlock(&sge->cmdQ[1].cq_qlock);
659 #else
660 credits_pend[0] += e->Cmdq0CreditReturn;
661 credits_pend[1] += e->Cmdq1CreditReturn;
662 #ifdef CONFIG_SMP
663 if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) {
664 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
665 credits_pend[0] = 0;
666 }
667 if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) {
668 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
669 credits_pend[1] = 0;
670 }
671 #endif
672 #endif
673 #ifdef HOST_PAUSE
674 t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]);
675 #endif
676 e++;
677 if (unlikely(++cidx == entries_n)) {
678 cidx = 0;
679 genbit ^= 1;
680 e = q;
681 }
682
683 /* pull physical memory of response queue entry into cache */
684 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q),
685 sizeof (*e), DDI_DMA_SYNC_FORKERNEL);
686
687 ret = 1;
688 }
689
690 #ifndef TX_THREAD_RECLAIM
691 if (credits_pend[0])
692 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]);
693 if (credits_pend[1])
694 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]);
695 #endif
696 if (flags & F_CMDQ0_ENABLE) {
697 struct cmdQ *cmdQ = &sge->cmdQ[0];
698 atomic_set(&cmdQ->cq_asleep, 1);
699 if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) {
700 doorbell_pio(sge, F_CMDQ0_ENABLE);
701 atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx);
702 }
703 }
704
705 /* the SGE told us one of the free lists is empty */
706 if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE)))
707 freelQs_empty(sge);
708
709 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
710 if (adapter->ch_tx_overflow_mutex)
711 mutex_enter(adapter->ch_tx_overflow_mutex);
712 if (adapter->ch_blked &&
713 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) &&
714 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) {
715 adapter->ch_blked = 0;
716 if (adapter->ch_tx_overflow_cv)
717 cv_broadcast(adapter->ch_tx_overflow_cv);
718 ch_gld_ok(adapter);
719 }
720 if (adapter->ch_tx_overflow_mutex)
721 mutex_exit(adapter->ch_tx_overflow_mutex);
722 #else
723 #ifndef TX_THREAD_RECLAIM
724 if (adapter->ch_blked &&
725 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) &&
726 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) {
727 adapter->ch_blked = 0;
728 ch_gld_ok(adapter);
729 }
730 #endif
731 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
732
733 Q->rq_genbit = genbit;
734 Q->rq_cidx = cidx;
735 Q->rq_credits = credits;
736
737 t1_write_reg_4(adapter, A_SG_SLEEPING, cidx);
738
739 check_slow_ints:
740 /* handle non-data interrupts */
741 if (unlikely(!ret))
742 ret = t1_slow_intr_handler(adapter);
743
744 return (ret);
745 }
746
747 /*
748 * allocate a mblk with DMA mapped mblk.
749 * When checksum offload is enabled, we start the DMA at a 2 byte offset so
750 * the IP header will be aligned. We do this for sparc only.
751 */
752 static uint64_t
os_freelist_buffer_alloc(ch_t * sa,int sz,mblk_t ** mb,ulong_t * dh)753 os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh)
754 {
755 ch_esb_t *ch_get_small_rbuf(ch_t *sa);
756 ch_esb_t *ch_get_big_rbuf(ch_t *sa);
757 ch_esb_t *rbp;
758 uint32_t rxoff = sa->sge->rx_offset;
759
760 if (sz == SGE_SM_BUF_SZ(sa)) {
761 /* get pre-mapped buffer */
762 if ((rbp = ch_get_small_rbuf(sa)) == NULL) {
763 sa->norcvbuf++;
764 return ((uint64_t)0);
765 }
766
767 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
768 SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
769 if (*mb == NULL) {
770 mutex_enter(&sa->ch_small_esbl);
771 rbp->cs_next = sa->ch_small_esb_free;
772 sa->ch_small_esb_free = rbp;
773 mutex_exit(&sa->ch_small_esbl);
774 return ((uint64_t)0);
775 }
776 *dh = rbp->cs_dh;
777
778 return (rbp->cs_pa + rxoff);
779 } else {
780 /* get pre-mapped buffer */
781 if ((rbp = ch_get_big_rbuf(sa)) == NULL) {
782 sa->norcvbuf++;
783 return ((uint64_t)0);
784 }
785
786 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff,
787 SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn);
788 if (*mb == NULL) {
789 mutex_enter(&sa->ch_big_esbl);
790 rbp->cs_next = sa->ch_big_esb_free;
791 sa->ch_big_esb_free = rbp;
792 mutex_exit(&sa->ch_big_esbl);
793 return ((uint64_t)0);
794 }
795 *dh = rbp->cs_dh;
796
797 return (rbp->cs_pa + rxoff);
798 }
799 }
800
801 static inline unsigned int
t1_sge_rx(pesge * sge,struct freelQ * Q,unsigned int len,unsigned int offload)802 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload)
803 {
804 mblk_t *skb;
805 peobj *adapter = sge->obj;
806 struct freelQ_ce *cq = Q->fq_centries;
807 struct freelQ_ce *ce = &cq[Q->fq_cidx];
808 ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh;
809 uint32_t cidx = Q->fq_cidx;
810 uint32_t entries_n = Q->fq_entries_n;
811 uint32_t sz = Q->fq_rx_buffer_size;
812 uint32_t useit = 1;
813 uint32_t rxoff = sge->rx_offset;
814 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
815 uint32_t rv;
816 #endif
817
818 if (Q->fq_id)
819 sge->intr_cnt.rx_flq1_cnt++;
820 else
821 sge->intr_cnt.rx_flq0_cnt++;
822 /*
823 * If pkt size falls below threshold, then we'll copy data to
824 * an blk and reuse mblk.
825 *
826 * NOTE that rxoff is 2 for T1 adapters. We align the the start
827 * of the DMA buffer begin at rxoff offset for T1 cards instead of
828 * at the beginning of the buffer, thus the length of the received
829 * data does not include this offset. We therefore always add
830 * SGE_RX_OFFSET to the allocb size so we have space to provide the
831 * offset for the copied data.
832 */
833 #ifdef HOST_PAUSE
834 /*
835 * If we have Host pause compiled in, then we look at the
836 * free list, if the pause is on and we're not in offload
837 * mode then we drop packets, this is designed to avoid
838 * overwhelming the machine. If the machine is powerfull enough
839 * this will not happen. The 'rx_pkt_drops' will show when
840 * packets are being dropped and how much.
841 */
842 if ((offload == 0) && adapter->pause_on) {
843 freelQ_e *e;
844 /* Ditch the packet and reuse original buffer */
845 e = &Q->fq_entries[cidx];
846 e->GenerationBit ^= 1;
847 e->GenerationBit2 ^= 1;
848 sge->intr_cnt.rx_pkt_drops++;
849 goto rx_entry_consumed;
850 } else if (((adapter->pause_on ||
851 (len <= SGE_RX_COPY_THRESHOLD)) &&
852 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))))
853 #else
854 if ((len <= SGE_RX_COPY_THRESHOLD) &&
855 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))
856 #endif
857 {
858 freelQ_e *e;
859 char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr;
860
861 /*
862 * pull physical memory of pkt data into cache
863 * Note that len does not include offset for T1.
864 */
865 (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
866 DDI_DMA_SYNC_FORKERNEL);
867
868 if (offload == 0) {
869 /*
870 * create 2 byte offset so IP header aligned on
871 * 4 byte boundry
872 */
873 skb_reserve(skb, SGE_RX_OFFSET);
874 /*
875 * if hardware inserted 2 byte offset then need to
876 * start copying with extra offset
877 */
878 src += sge->rx_pkt_pad;
879 }
880 memcpy(skb->b_rptr, src, len);
881 useit = 0; /* mblk copy, don't inc esballoc in use cnt */
882
883 /* so we can reuse original buffer */
884 e = &Q->fq_entries[cidx];
885 e->GenerationBit ^= 1;
886 e->GenerationBit2 ^= 1;
887 sge->intr_cnt.rx_pkt_copied++;
888 } else {
889 /* consume buffer off the ring */
890 skb = ce->fe_mp;
891 ce->fe_mp = NULL;
892
893 /*
894 * if not offload (tunneled pkt), & hardward padded, then
895 * adjust start of pkt to point to start of data i.e.
896 * skip pad (2 bytes).
897 */
898 if (!offload && sge->rx_pkt_pad)
899 __skb_pull(skb, SGE_RX_OFFSET);
900
901 /*
902 * pull physical memory of pkt data into cache
903 * Note that len does not include offset for T1.
904 */
905 (void) ddi_dma_sync(dh, (off_t)(rxoff), len,
906 DDI_DMA_SYNC_FORKERNEL);
907 }
908
909 /* set length of data in skb */
910 skb_put(skb, len);
911
912 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
913 if (likely(offload)) {
914 if (likely(toe_running(adapter))) {
915 /* sends pkt upstream to toe layer */
916 if (useit) {
917 uint_t index;
918 if (sz == SGE_SM_BUF_SZ(adapter))
919 index = adapter->ch_sm_index;
920 else
921 index = adapter->ch_big_index;
922 atomic_add(1, &buffers_in_use[index]);
923 }
924 if (adapter->toe_rcv)
925 adapter->toe_rcv(adapter->ch_toeinst, skb);
926 else
927 freemsg(skb);
928 } else {
929 cmn_err(CE_WARN,
930 "%s: unexpected offloaded packet, cmd %u\n",
931 adapter->ch_name, *skb->b_rptr);
932
933 /* discard packet */
934 freemsg(skb);
935 }
936 }
937 #else
938 if (unlikely(offload)) {
939 cmn_err(CE_WARN,
940 "%s: unexpected offloaded packet, cmd %u\n",
941 adapter->ch_name, *skb->b_rptr);
942
943 /* discard paket */
944 freemsg(skb);
945 }
946 #endif
947 else {
948 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr;
949 int flg = 0;
950 uint32_t cksum;
951
952 /* adjust beginning of data to skip CPL header */
953 skb_pull(skb, SZ_CPL_RX_PKT);
954
955 /* extract checksum from CPL header here */
956
957 /*
958 * bump count of mlbks in used by protocol stack(s)
959 */
960 if (useit) {
961 if (sz == SGE_SM_BUF_SZ(adapter)) {
962 atomic_add(1,
963 &buffers_in_use[adapter->ch_sm_index]);
964 } else {
965 atomic_add(1,
966 &buffers_in_use[adapter->ch_big_index]);
967 }
968 }
969
970 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
971 /*
972 * let the TOE layer have a crack at the packet first.
973 */
974 if (adapter->toe_tunnel) {
975 rv = adapter->toe_tunnel(adapter->ch_toeinst, skb);
976 /*
977 * The TOE may have consumed the packet.
978 */
979 if (rv)
980 goto rx_entry_consumed;
981 }
982 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */
983
984 cksum = p->csum;
985
986 /*
987 * NOTE: 14+9 = size of MAC + offset to IP protocol field
988 */
989 if (adapter->ch_config.cksum_enabled &&
990 (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) ==
991 ETHERTYPE_IP) &&
992 ((skb->b_rptr[14+9] == IPPROTO_TCP) ||
993 (skb->b_rptr[14+9] == IPPROTO_UDP))) {
994 flg = 1;
995 }
996
997 ch_send_up(adapter, skb, cksum, flg);
998 }
999
1000 rx_entry_consumed:
1001
1002 if (++cidx == entries_n)
1003 cidx = 0;
1004
1005 Q->fq_cidx = cidx;
1006
1007 if (unlikely(--Q->fq_credits < (entries_n>>2)))
1008 /* allocate new buffers on the free list */
1009 alloc_freelQ_buffers(sge, Q);
1010 return (1);
1011 }
1012
1013 #ifdef HOST_PAUSE
1014 static void
t1_sge_check_pause(pesge * sge,struct freelQ * Q)1015 t1_sge_check_pause(pesge *sge, struct freelQ *Q)
1016 {
1017 peobj *adapter = sge->obj;
1018
1019 /*
1020 * If the number of available credits shrinks below
1021 * the Pause on threshold then enable the pause and
1022 * try and allocate more buffers.
1023 * On the next pass, if there's more credits returned
1024 * then check that you've went above the pause
1025 * threshold and then disable the pause.
1026 */
1027 if (Q->fq_credits < Q->fq_pause_on_thresh) {
1028 if (do_host_pause) {
1029 sge->intr_cnt.rx_pause_on++;
1030 adapter->txxg_cfg1 |=
1031 SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1032 (void) t1_tpi_write(adapter,
1033 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1034 adapter->txxg_cfg1);
1035 adapter->pause_on = 1;
1036 adapter->pause_time = gethrtime();
1037 }
1038 alloc_freelQ_buffers(sge, Q);
1039 } else if ((adapter->pause_on) &&
1040 (Q->fq_credits > Q->fq_pause_off_thresh)) {
1041 hrtime_t time;
1042 sge->intr_cnt.rx_pause_off++;
1043 adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE;
1044 (void) t1_tpi_write(adapter,
1045 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2,
1046 adapter->txxg_cfg1);
1047 adapter->pause_on = 0;
1048 time = (gethrtime() - adapter->pause_time)/1000;
1049 sge->intr_cnt.rx_pause_ms += time;
1050 if (time > sge->intr_cnt.rx_pause_spike)
1051 sge->intr_cnt.rx_pause_spike = (uint32_t)time;
1052 }
1053 sge->intr_cnt.rx_fl_credits = Q->fq_credits;
1054 }
1055 #endif /* HOST_PAUSE */
1056
1057 static void
alloc_freelQ_buffers(pesge * sge,struct freelQ * Q)1058 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q)
1059 {
1060 uint32_t pidx = Q->fq_pidx;
1061 struct freelQ_ce *ce = &Q->fq_centries[pidx];
1062 freelQ_e *fq = Q->fq_entries; /* base of freelist Q */
1063 freelQ_e *e = &Q->fq_entries[pidx];
1064 uint32_t sz = Q->fq_rx_buffer_size;
1065 uint32_t rxoff = sge->rx_offset;
1066 uint32_t credits = Q->fq_credits;
1067 uint32_t entries_n = Q->fq_entries_n;
1068 uint32_t genbit = Q->fq_genbit;
1069 ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh;
1070 ulong_t dh;
1071 uint64_t mapping;
1072 off_t offset = (off_t)((caddr_t)e - (caddr_t)fq);
1073 size_t len = 0;
1074
1075 while (credits < entries_n) {
1076 if (e->GenerationBit != genbit) {
1077 mblk_t *skb;
1078
1079 mapping = os_freelist_buffer_alloc(sge->obj, sz,
1080 &skb, &dh);
1081 if (mapping == 0) {
1082 sge->intr_cnt.rx_flbuf_fails++;
1083 break;
1084 }
1085 sge->intr_cnt.rx_flbuf_allocs++;
1086
1087 ce->fe_mp = skb;
1088 ce->fe_dh = dh;
1089
1090 /*
1091 * Note that for T1, we've started the beginning of
1092 * of the buffer by an offset of 2 bytes. We thus
1093 * decrement the length to account for this.
1094 */
1095 e->AddrLow = (u32)mapping;
1096 e->AddrHigh = (u64)mapping >> 32;
1097 e->BufferLength = sz - rxoff;
1098 wmb();
1099 e->GenerationBit = e->GenerationBit2 = genbit;
1100 }
1101
1102 len += sizeof (*e);
1103
1104 ce++;
1105 e++;
1106 credits++;
1107 if (++pidx == entries_n) {
1108 /*
1109 * sync freelist entries to physical memory up to
1110 * end of the table.
1111 */
1112 (void) ddi_dma_sync(th, offset, len,
1113 DDI_DMA_SYNC_FORDEV);
1114 offset = 0;
1115 len = 0;
1116
1117 pidx = 0;
1118 genbit ^= 1;
1119 ce = Q->fq_centries;
1120 e = Q->fq_entries;
1121 }
1122 }
1123
1124 /* sync freelist entries that have been modified. */
1125 if (len)
1126 (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV);
1127
1128 Q->fq_genbit = genbit;
1129 Q->fq_pidx = pidx;
1130 Q->fq_credits = credits;
1131 }
1132
1133 static void
freelQs_empty(pesge * sge)1134 freelQs_empty(pesge *sge)
1135 {
1136 u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE);
1137 u32 irqholdoff_reg;
1138
1139 alloc_freelQ_buffers(sge, &sge->freelQ[0]);
1140 alloc_freelQ_buffers(sge, &sge->freelQ[1]);
1141
1142 if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) &&
1143 (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) {
1144 irq_reg |= F_FL_EXHAUSTED;
1145 irqholdoff_reg = sge->intrtimer[sge->currIndex];
1146 } else {
1147 /* Clear the F_FL_EXHAUSTED interrupts for now */
1148 irq_reg &= ~F_FL_EXHAUSTED;
1149 irqholdoff_reg = sge->intrtimer_nres;
1150 }
1151 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg);
1152 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg);
1153
1154 /* We reenable the Qs to force an Freelist GTS interrupt later */
1155 doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE);
1156 }
1157
1158 /*
1159 * Frees 'credits_pend' TX buffers and returns the credits to Q->credits.
1160 * Free xmit buffers
1161 */
1162 static void
free_cmdQ_buffers(pesge * sge,struct cmdQ * Q,unsigned int credits_pend)1163 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend)
1164 {
1165 mblk_t *skb;
1166 struct cmdQ_ce *ce;
1167 struct cmdQ_ce *cq = Q->cq_centries;
1168 uint32_t entries_n = Q->cq_entries_n;
1169 uint32_t cidx = Q->cq_cidx;
1170 uint32_t i = credits_pend;
1171 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1172 ch_t *chp = sge->obj;
1173 #endif
1174 ce = &cq[cidx];
1175
1176 while (i--) {
1177 #ifdef CONFIG_CHELSIO_T1_OFFLOAD
1178 /* if flag set, then toe buffer */
1179 switch (ce->ce_flg & 0x7) {
1180 case DH_DMA:
1181 if (ce->ce_dh) {
1182 ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1183 ce->ce_dh = NULL; /* may not be needed */
1184 }
1185 skb = ce->ce_mp;
1186 if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1187 freemsg(skb);
1188 }
1189 ce->ce_mp = NULL;
1190 break;
1191
1192 #if defined(__sparc)
1193 case DH_DVMA:
1194 if (ce->ce_dh) {
1195 ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1196 ce->ce_dh = NULL; /* may not be needed */
1197 }
1198 skb = ce->ce_mp;
1199 if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1200 freemsg(skb);
1201 }
1202 ce->ce_mp = NULL;
1203 break;
1204 #endif /* __sparc */
1205
1206 case DH_TOE:
1207 chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp));
1208 ce->ce_mp = NULL;
1209 break;
1210 }
1211 #else /* CONFIG_CHELSIO_T1_OFFLOAD */
1212 if (ce->ce_dh) {
1213 if ((ce->ce_flg & 7) == DH_DMA) {
1214 ch_unbind_dma_handle(sge->obj, ce->ce_dh);
1215 }
1216 #if defined(__sparc)
1217 else {
1218 ch_unbind_dvma_handle(sge->obj, ce->ce_dh);
1219 }
1220 #endif /* __sparc */
1221 ce->ce_dh = NULL; /* may not be needed */
1222 }
1223
1224 skb = ce->ce_mp;
1225 if (skb && ((ce->ce_flg & CH_ARP) == 0)) {
1226 freemsg(skb);
1227 }
1228 ce->ce_mp = NULL;
1229 #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */
1230
1231 ce++;
1232 if (++cidx == entries_n) {
1233 cidx = 0;
1234 ce = cq;
1235 }
1236 }
1237
1238 Q->cq_cidx = cidx;
1239 atomic_add(credits_pend, &Q->cq_credits);
1240 }
1241
1242 struct sge_intr_counts *
sge_get_stat(pesge * sge)1243 sge_get_stat(pesge *sge)
1244 {
1245 return (&sge->intr_cnt);
1246 }
1247
1248 /*
1249 * Allocates both RX and TX resources and configures the SGE. However,
1250 * the hardware is not enabled yet.
1251 *
1252 * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic.
1253 * jumbo_fl is set to the index of the freelist containing the jumbo buffers.
1254 */
1255 int
t1_sge_configure(pesge * sge,struct sge_params * p)1256 t1_sge_configure(pesge *sge, struct sge_params *p)
1257 {
1258 sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET;
1259 sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0;
1260 /* if we're a T2 card, then we have hardware offset support */
1261 sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0;
1262
1263 if (alloc_rx_resources(sge, p))
1264 return (-ENOMEM);
1265 if (alloc_tx_resources(sge, p)) {
1266 free_rx_resources(sge);
1267 return (-ENOMEM);
1268 }
1269 configure_sge(sge, p);
1270
1271 /*
1272 * Now that we have sized the free lists calculate the payload
1273 * capacity of the large buffers. Other parts of the driver use
1274 * this to set the max offload coalescing size so that RX packets
1275 * do not overflow our large buffers.
1276 */
1277 p->large_buf_capacity = jumbo_payload_capacity(sge);
1278 return (0);
1279 }
1280
1281 /*
1282 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a
1283 * response Q.
1284 */
1285 static int
alloc_rx_resources(pesge * sge,struct sge_params * p)1286 alloc_rx_resources(pesge *sge, struct sge_params *p)
1287 {
1288 unsigned int size, i;
1289
1290 for (i = 0; i < SGE_FREELQ_N; i++) {
1291 struct freelQ *Q = &sge->freelQ[i];
1292
1293 Q->fq_id = i;
1294 Q->fq_genbit = 1;
1295 Q->fq_entries_n = p->freelQ_size[i];
1296 #ifdef HOST_PAUSE
1297 Q->fq_pause_on_thresh = flq_pause_window;
1298 Q->fq_pause_off_thresh = Q->fq_entries_n >> 1;
1299 #endif
1300 size = sizeof (freelQ_e) * Q->fq_entries_n;
1301
1302 Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1303 size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT);
1304
1305
1306 if (!Q->fq_entries)
1307 goto err_no_mem;
1308 memset(Q->fq_entries, 0, size);
1309 size = sizeof (struct freelQ_ce) * Q->fq_entries_n;
1310 Q->fq_centries = t1_os_malloc_wait_zero(size);
1311 if (!Q->fq_centries)
1312 goto err_no_mem;
1313 memset(Q->fq_centries, 0, size);
1314 }
1315
1316 /*
1317 * Calculate the buffer sizes for the two free lists. FL0 accommodates
1318 * regular sized Ethernet frames, FL1 is sized not to exceed 16K,
1319 * including all the sk_buff overhead.
1320 * For T1C FL0 and FL1 are reversed.
1321 */
1322 #ifdef NOTYET
1323 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE +
1324 sizeof (struct cpl_rx_data) +
1325 SGE_RX_OFFSET - sge->rx_pkt_pad;
1326 #else
1327 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size =
1328 sge->obj->ch_sm_buf_sz;
1329 if (is_T2(sge->obj))
1330 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz;
1331 else
1332 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz;
1333 #endif
1334 #ifdef NOTYET
1335 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) -
1336 SKB_DATA_ALIGN(sizeof (struct skb_shared_info));
1337 #else
1338 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz;
1339 if (is_T2(sge->obj))
1340 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz;
1341 else
1342 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz;
1343 #endif
1344
1345 sge->respQ.rq_genbit = 1;
1346 sge->respQ.rq_entries_n = sge_respq_cnt;
1347 sge->respQ.rq_credits = sge_respq_cnt;
1348 sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2);
1349 size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1350
1351 sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1352 size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh),
1353 &(sge->respQ.rq_ah), 0);
1354
1355 if (!sge->respQ.rq_entries)
1356 goto err_no_mem;
1357 memset(sge->respQ.rq_entries, 0, size);
1358 return (0);
1359
1360 err_no_mem:
1361 free_rx_resources(sge);
1362 return (1);
1363 }
1364
1365 /*
1366 * Allocates basic TX resources, consisting of memory mapped command Qs.
1367 */
1368 static int
alloc_tx_resources(pesge * sge,struct sge_params * p)1369 alloc_tx_resources(pesge *sge, struct sge_params *p)
1370 {
1371 unsigned int size, i;
1372
1373 for (i = 0; i < SGE_CMDQ_N; i++) {
1374 struct cmdQ *Q = &sge->cmdQ[i];
1375
1376 Q->cq_genbit = 1;
1377 Q->cq_entries_n = p->cmdQ_size[i];
1378 atomic_set(&Q->cq_credits, Q->cq_entries_n);
1379 atomic_set(&Q->cq_asleep, 1);
1380
1381 mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER,
1382 sge->obj->ch_icookp);
1383
1384 size = sizeof (cmdQ_e) * Q->cq_entries_n;
1385 Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj,
1386 size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT);
1387
1388 if (!Q->cq_entries)
1389 goto err_no_mem;
1390 memset(Q->cq_entries, 0, size);
1391 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1392 Q->cq_centries = t1_os_malloc_wait_zero(size);
1393 if (!Q->cq_centries)
1394 goto err_no_mem;
1395 memset(Q->cq_centries, 0, size);
1396
1397 /* allocate pre-mapped dma headers */
1398 pe_dma_handle_init(sge->obj, Q->cq_entries_n);
1399 }
1400
1401 return (0);
1402
1403 err_no_mem:
1404 free_tx_resources(sge);
1405 return (1);
1406 }
1407
1408 /*
1409 * Sets the interrupt latency timer when the adaptive Rx coalescing
1410 * is turned off. Do nothing when it is turned on again.
1411 *
1412 * This routine relies on the fact that the caller has already set
1413 * the adaptive policy in adapter->sge_params before calling it.
1414 */
1415 int
t1_sge_set_coalesce_params(pesge * sge,struct sge_params * p)1416 t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p)
1417 {
1418 if (!p->coalesce_enable) {
1419 u32 newTimer = p->rx_coalesce_usecs *
1420 (board_info(sge->obj)->clock_core / 1000000);
1421
1422 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer);
1423 }
1424 return (0);
1425 }
1426
1427 /*
1428 * Programs the various SGE registers. However, the engine is not yet enabled,
1429 * but sge->sge_control is setup and ready to go.
1430 */
1431 static void
configure_sge(pesge * sge,struct sge_params * p)1432 configure_sge(pesge *sge, struct sge_params *p)
1433 {
1434 ch_t *ap = sge->obj;
1435 int i;
1436
1437 t1_write_reg_4(ap, A_SG_CONTROL, 0);
1438
1439 setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n,
1440 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE);
1441 setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n,
1442 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE);
1443 setup_ring_params(ap, sge->freelQ[0].fq_pa,
1444 sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR,
1445 A_SG_FL0BASEUPR, A_SG_FL0SIZE);
1446 setup_ring_params(ap, sge->freelQ[1].fq_pa,
1447 sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR,
1448 A_SG_FL1BASEUPR, A_SG_FL1SIZE);
1449
1450 /* The threshold comparison uses <. */
1451 t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) -
1452 SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1);
1453 setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n,
1454 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE);
1455 t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n);
1456 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE |
1457 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE |
1458 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE |
1459 #if 1
1460 /*
1461 * if the the following bit is not set, then we'll get an
1462 * interrupt everytime command Q 0 goes empty. Since we're
1463 * always ringing the doorbell, we can turn it on.
1464 */
1465 F_DISABLE_CMDQ0_GTS |
1466 #endif
1467 V_RX_PKT_OFFSET(sge->rx_pkt_pad);
1468
1469 #if BYTE_ORDER == BIG_ENDIAN
1470 sge->sge_control |= F_ENABLE_BIG_ENDIAN;
1471 #endif
1472
1473 /*
1474 * Initialize the SGE Interrupt Timer arrray:
1475 * intrtimer[0] = (SGE_INTRTIMER0) usec
1476 * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec
1477 * intrtimer[10] = (SGE_INTRTIMER1) usec
1478 *
1479 */
1480 sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000;
1481 for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) {
1482 sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i);
1483 sge->intrtimer[i] *= sge->intrtimer[0];
1484 }
1485 sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] =
1486 sge->intrtimer[0] * SGE_INTRTIMER1;
1487 /* Initialize resource timer */
1488 sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] *
1489 SGE_INTRTIMER_NRES);
1490 /* Finally finish initialization of intrtimer[0] */
1491 sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0);
1492 /* Initialize for a throughput oriented workload */
1493 sge->currIndex = SGE_INTR_MAXBUCKETS - 1;
1494
1495 if (p->coalesce_enable)
1496 t1_write_reg_4(ap, A_SG_INTRTIMER,
1497 sge->intrtimer[sge->currIndex]);
1498 else
1499 (void) t1_sge_set_coalesce_params(sge, p);
1500 }
1501
1502 static inline void
setup_ring_params(ch_t * adapter,u64 addr,u32 size,int base_reg_lo,int base_reg_hi,int size_reg)1503 setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo,
1504 int base_reg_hi, int size_reg)
1505 {
1506 t1_write_reg_4(adapter, base_reg_lo, (u32)addr);
1507 t1_write_reg_4(adapter, base_reg_hi, addr >> 32);
1508 t1_write_reg_4(adapter, size_reg, size);
1509 }
1510
1511 /*
1512 * Frees RX resources.
1513 */
1514 static void
free_rx_resources(pesge * sge)1515 free_rx_resources(pesge *sge)
1516 {
1517 unsigned int size, i;
1518
1519 if (sge->respQ.rq_entries) {
1520 size = sizeof (respQ_e) * sge->respQ.rq_entries_n;
1521
1522 pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries,
1523 sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah);
1524 }
1525
1526 for (i = 0; i < SGE_FREELQ_N; i++) {
1527 struct freelQ *Q = &sge->freelQ[i];
1528
1529 if (Q->fq_centries) {
1530 free_freelQ_buffers(sge, Q);
1531
1532 t1_os_free(Q->fq_centries,
1533 Q->fq_entries_n * sizeof (freelQ_ce_t));
1534 }
1535 if (Q->fq_entries) {
1536 size = sizeof (freelQ_e) * Q->fq_entries_n;
1537
1538 /* free the freelist queue */
1539 pe_os_free_contig(sge->obj, size, Q->fq_entries,
1540 Q->fq_pa, Q->fq_dh, Q->fq_ah);
1541
1542 }
1543 }
1544 }
1545
1546 /*
1547 * Frees all RX buffers on the freelist Q. The caller must make sure that
1548 * the SGE is turned off before calling this function.
1549 */
1550 static void
free_freelQ_buffers(pesge * sge,struct freelQ * Q)1551 free_freelQ_buffers(pesge *sge, struct freelQ *Q)
1552 {
1553 struct freelQ_ce *ce;
1554 struct freelQ_ce *cq = Q->fq_centries;
1555 uint32_t credits = Q->fq_credits;
1556 uint32_t entries_n = Q->fq_entries_n;
1557 uint32_t cidx = Q->fq_cidx;
1558 uint32_t i = Q->fq_id;
1559
1560 ce = &cq[cidx];
1561
1562 credits = entries_n;
1563 while (credits--) {
1564 mblk_t *mp;
1565 if ((mp = ce->fe_mp) != NULL) {
1566 /* bump in-use count of receive buffers */
1567 if (i != sge->jumbo_fl) {
1568 atomic_add(1,
1569 &buffers_in_use[sge->obj->ch_sm_index]);
1570 } else {
1571 atomic_add(1,
1572 &buffers_in_use[sge->obj->ch_big_index]);
1573 }
1574
1575 /*
1576 * note. freeb() callback of esb-alloced mblk will
1577 * cause receive buffer to be put back on sa free list.
1578 */
1579 freeb(mp);
1580 ce->fe_mp = NULL;
1581 }
1582
1583 ce++;
1584 if (++cidx == entries_n) {
1585 cidx = 0;
1586 ce = cq;
1587 }
1588 }
1589
1590 Q->fq_cidx = cidx;
1591 Q->fq_credits = credits;
1592 }
1593
1594 /*
1595 * Free TX resources.
1596 *
1597 * Assumes that SGE is stopped and all interrupts are disabled.
1598 */
1599 static void
free_tx_resources(pesge * sge)1600 free_tx_resources(pesge *sge)
1601 {
1602 unsigned int size;
1603 uint32_t i;
1604
1605 for (i = 0; i < SGE_CMDQ_N; i++) {
1606 struct cmdQ *Q = &sge->cmdQ[i];
1607
1608 if (Q->cq_centries) {
1609 unsigned int pending = Q->cq_entries_n -
1610 atomic_read(Q->cq_credits);
1611
1612 mutex_destroy(&Q->cq_qlock);
1613
1614 if (pending)
1615 free_cmdQ_buffers(sge, Q, pending);
1616
1617 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n;
1618 t1_os_free(Q->cq_centries, size);
1619 }
1620
1621 if (Q->cq_entries) {
1622 size = sizeof (cmdQ_e) * Q->cq_entries_n;
1623 pe_os_free_contig(sge->obj, size, Q->cq_entries,
1624 Q->cq_pa, Q->cq_dh, Q->cq_ah);
1625 }
1626 }
1627 }
1628
1629 /*
1630 * Return the payload capacity of the jumbo free-list buffers.
1631 */
jumbo_payload_capacity(pesge * sge)1632 static inline unsigned int jumbo_payload_capacity(pesge *sge)
1633 {
1634 return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size -
1635 sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset);
1636 }
1637
1638 /* PR2928 & PR3309 */
1639 void
t1_sge_set_ptimeout(adapter_t * adapter,u32 val)1640 t1_sge_set_ptimeout(adapter_t *adapter, u32 val)
1641 {
1642 pesge *sge = adapter->sge;
1643
1644 if (is_T2(adapter))
1645 sge->ptimeout = max(val, 1);
1646 }
1647
1648 /* PR2928 & PR3309 */
1649 u32
t1_sge_get_ptimeout(adapter_t * adapter)1650 t1_sge_get_ptimeout(adapter_t *adapter)
1651 {
1652 pesge *sge = adapter->sge;
1653
1654 return (is_T2(adapter) ? sge->ptimeout : 0);
1655 }
1656
1657 void
sge_add_fake_arp(pesge * sge,void * bp)1658 sge_add_fake_arp(pesge *sge, void *bp)
1659 {
1660 sge->pskb = bp;
1661 }
1662
1663 #ifdef SUN_KSTATS
1664 static int
sge_kstat_setup(pesge * sge)1665 sge_kstat_setup(pesge *sge)
1666 {
1667 int status;
1668 p_kstat_t ksp;
1669 size_t ch_kstat_sz;
1670 p_ch_kstat_t chkp;
1671 char kstat_name[32];
1672 int instance;
1673 int i;
1674
1675 status = -1;
1676 ch_kstat_sz = sizeof (ch_kstat_t);
1677 instance = ddi_get_instance(sge->obj->ch_dip);
1678 if ((ksp = kstat_create(CHNAME "_debug", instance,
1679 NULL, "net_debug", KSTAT_TYPE_NAMED,
1680 ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL)
1681 goto sge_kstat_setup_exit;
1682 chkp = (p_ch_kstat_t)ksp->ks_data;
1683 kstat_named_init(&chkp->respQ_empty, "respQ_empty",
1684 KSTAT_DATA_UINT32);
1685 kstat_named_init(&chkp->respQ_overflow, "respQ_overflow",
1686 KSTAT_DATA_UINT32);
1687 kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty",
1688 KSTAT_DATA_UINT32);
1689 kstat_named_init(&chkp->pkt_too_big, "pkt_too_big",
1690 KSTAT_DATA_UINT32);
1691 kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch",
1692 KSTAT_DATA_UINT32);
1693 kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]",
1694 KSTAT_DATA_UINT32);
1695 kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]",
1696 KSTAT_DATA_UINT32);
1697 kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]",
1698 KSTAT_DATA_UINT32);
1699 kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]",
1700 KSTAT_DATA_UINT32);
1701 kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups",
1702 KSTAT_DATA_UINT32);
1703 kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups",
1704 KSTAT_DATA_UINT32);
1705 kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag",
1706 KSTAT_DATA_UINT32);
1707 kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag",
1708 KSTAT_DATA_UINT32);
1709 kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums",
1710 KSTAT_DATA_UINT32);
1711 kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space",
1712 KSTAT_DATA_UINT32);
1713 kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks",
1714 KSTAT_DATA_UINT32);
1715 kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails",
1716 KSTAT_DATA_UINT32);
1717 kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails",
1718 KSTAT_DATA_UINT32);
1719 kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails",
1720 KSTAT_DATA_UINT32);
1721 kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails",
1722 KSTAT_DATA_UINT32);
1723 kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0",
1724 KSTAT_DATA_UINT32);
1725 kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1",
1726 KSTAT_DATA_UINT32);
1727 kstat_named_init(&chkp->rx_flq0, "rx_flq0",
1728 KSTAT_DATA_UINT32);
1729 kstat_named_init(&chkp->rx_flq1, "rx_flq1",
1730 KSTAT_DATA_UINT32);
1731 kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz",
1732 KSTAT_DATA_UINT32);
1733 kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz",
1734 KSTAT_DATA_UINT32);
1735 kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops",
1736 KSTAT_DATA_UINT32);
1737 kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied",
1738 KSTAT_DATA_UINT32);
1739 kstat_named_init(&chkp->rx_pause_on, "rx_pause_on",
1740 KSTAT_DATA_UINT32);
1741 kstat_named_init(&chkp->rx_pause_off, "rx_pause_off",
1742 KSTAT_DATA_UINT32);
1743 kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms",
1744 KSTAT_DATA_UINT32);
1745 kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike",
1746 KSTAT_DATA_UINT32);
1747 kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits",
1748 KSTAT_DATA_UINT32);
1749 kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails",
1750 KSTAT_DATA_UINT32);
1751 kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs",
1752 KSTAT_DATA_UINT32);
1753 kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop",
1754 KSTAT_DATA_UINT32);
1755 kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt",
1756 KSTAT_DATA_UINT32);
1757 kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt",
1758 KSTAT_DATA_UINT32);
1759 kstat_named_init(&chkp->arp_sent, "arp_sent",
1760 KSTAT_DATA_UINT32);
1761 kstat_named_init(&chkp->tx_doorbells, "tx_doorbells",
1762 KSTAT_DATA_UINT32);
1763 kstat_named_init(&chkp->intr_doorbells, "intr_doorbells",
1764 KSTAT_DATA_UINT32);
1765 kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells",
1766 KSTAT_DATA_UINT32);
1767 kstat_named_init(&chkp->sleep_cnt, "sleep_cnt",
1768 KSTAT_DATA_UINT32);
1769 kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt",
1770 KSTAT_DATA_UINT32);
1771 for (i = 0; i < MBLK_MAX; i++) {
1772 (void) sprintf(kstat_name, "tx_descs[%02d]", i);
1773 kstat_named_init(&chkp->tx_descs[i],
1774 kstat_name, KSTAT_DATA_UINT32);
1775 }
1776 ksp->ks_update = sge_kstat_update;
1777 ksp->ks_private = (void *)sge;
1778 sge->ksp = ksp;
1779 kstat_install(ksp);
1780 status = 0;
1781
1782 sge_kstat_setup_exit:
1783 return (status);
1784 }
1785
1786 static void
sge_kstat_remove(pesge * sge)1787 sge_kstat_remove(pesge *sge)
1788 {
1789 if (sge->ksp)
1790 kstat_delete(sge->ksp);
1791 }
1792
1793 static int
sge_kstat_update(p_kstat_t ksp,int rw)1794 sge_kstat_update(p_kstat_t ksp, int rw)
1795 {
1796 pesge *sge;
1797 p_ch_stats_t statsp;
1798 p_ch_kstat_t chkp;
1799 int i;
1800
1801 sge = (pesge *)ksp->ks_private;
1802 statsp = (p_ch_stats_t)&sge->intr_cnt;
1803 chkp = (p_ch_kstat_t)ksp->ks_data;
1804 if (rw == KSTAT_WRITE) {
1805 statsp->respQ_empty = chkp->respQ_empty.value.ui32;
1806 statsp->respQ_overflow = chkp->respQ_overflow.value.ui32;
1807 statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32;
1808 statsp->pkt_too_big = chkp->pkt_too_big.value.ui32;
1809 statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32;
1810 statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32;
1811 statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32;
1812 statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32;
1813 statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32;
1814 statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32;
1815 statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32;
1816 statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32;
1817 statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32;
1818 statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32;
1819 statsp->tx_need_cpl_space
1820 = chkp->tx_need_cpl_space.value.ui32;
1821 statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32;
1822 statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32;
1823 statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32;
1824 statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32;
1825 statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32;
1826 statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32;
1827 statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32;
1828 statsp->rx_flq0 = chkp->rx_flq0.value.ui32;
1829 statsp->rx_flq1 = chkp->rx_flq1.value.ui32;
1830 statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32;
1831 statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32;
1832 statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32;
1833 statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32;
1834 statsp->rx_pause_on = chkp->rx_pause_on.value.ui32;
1835 statsp->rx_pause_off = chkp->rx_pause_off.value.ui32;
1836 statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32;
1837 statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32;
1838 statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32;
1839 statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32;
1840 statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32;
1841 statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32;
1842 statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32;
1843 statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32;
1844 statsp->arp_sent = chkp->arp_sent.value.ui32;
1845 statsp->tx_doorbells = chkp->tx_doorbells.value.ui32;
1846 statsp->intr_doorbells = chkp->intr_doorbells.value.ui32;
1847 statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32;
1848 statsp->sleep_cnt = chkp->sleep_cnt.value.ui32;
1849 statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32;
1850 for (i = 0; i < MBLK_MAX; i++) {
1851 statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32;
1852 }
1853 } else {
1854 chkp->respQ_empty.value.ui32 = statsp->respQ_empty;
1855 chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow;
1856 chkp->freelistQ_empty.value.ui32
1857 = statsp->freelistQ_empty;
1858 chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big;
1859 chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch;
1860 chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0];
1861 chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1];
1862 chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0];
1863 chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1];
1864 chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups;
1865 chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups;
1866 chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag;
1867 chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag;
1868 chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums;
1869 chkp->tx_need_cpl_space.value.ui32
1870 = statsp->tx_need_cpl_space;
1871 chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks;
1872 chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1;
1873 chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2;
1874 chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1;
1875 chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2;
1876 chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0;
1877 chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1;
1878 chkp->rx_flq0.value.ui32 = statsp->rx_flq0;
1879 chkp->rx_flq1.value.ui32 = statsp->rx_flq1;
1880 chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz;
1881 chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz;
1882 chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops;
1883 chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied;
1884 chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on;
1885 chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off;
1886 chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms;
1887 chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike;
1888 chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits;
1889 chkp->rx_flbuf_fails.value.ui32
1890 = statsp->rx_flbuf_fails;
1891 chkp->rx_flbuf_allocs.value.ui32
1892 = statsp->rx_flbuf_allocs;
1893 chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop;
1894 chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt;
1895 chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt;
1896 chkp->arp_sent.value.ui32 = statsp->arp_sent;
1897 chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells;
1898 chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells;
1899 chkp->intr1_doorbells.value.ui32
1900 = statsp->intr1_doorbells;
1901 chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt;
1902 chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt;
1903 for (i = 0; i < MBLK_MAX; i++) {
1904 chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i];
1905 }
1906 }
1907 return (0);
1908 }
1909 #endif
1910
1911 static uint16_t
calc_ocsum(mblk_t * mp,int offset)1912 calc_ocsum(mblk_t *mp, int offset)
1913 {
1914 uint8_t *addrp;
1915 uint32_t src;
1916 uint32_t dst;
1917
1918 ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset);
1919 uint32_t sum;
1920 int iplen = IPH_HDR_LENGTH(ihdr);
1921 struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen);
1922 uchar_t *byte;
1923 int len;
1924
1925 addrp = (uint8_t *)&ihdr->ipha_src;
1926 src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1927 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1928
1929 addrp = (uint8_t *)&ihdr->ipha_dst;
1930 dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) |
1931 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]);
1932
1933 sum = (uint16_t)(src >> 16) +
1934 (uint16_t)(src) +
1935 (uint16_t)(dst >> 16) +
1936 (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP));
1937
1938 sum = (uint16_t)(sum >> 16) + (uint16_t)(sum);
1939
1940 if (sum > 0xffff)
1941 sum -= 0xffff;
1942
1943 udpp->uh_sum = 0;
1944 byte = mp->b_rptr + offset + iplen;
1945 do {
1946 len = (mp->b_wptr - byte);
1947 sum = bcksum(byte, len, sum);
1948 if (sum > 0xffff)
1949 sum -= 0xffff;
1950 mp = mp->b_cont;
1951 if (mp)
1952 byte = mp->b_rptr;
1953 } while (mp);
1954
1955 sum = ~sum & 0xffff;
1956
1957 return (sum);
1958 }
1959