1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * This file is part of the Chelsio T1 Ethernet driver. 29 * 30 * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved. 31 */ 32 33 #pragma ident "%Z%%M% %I% %E% SMI" 34 35 #include <sys/types.h> 36 #include <sys/param.h> 37 #include <sys/cmn_err.h> 38 #include <sys/sunddi.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <sys/byteorder.h> 42 #include <sys/atomic.h> 43 #include <sys/stropts.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/dlpi.h> 47 #include <sys/kstat.h> 48 #include <sys/ethernet.h> 49 #include <netinet/in.h> 50 #include <netinet/udp.h> 51 #include <inet/common.h> 52 #include <inet/nd.h> 53 #include <inet/ip.h> 54 #include <inet/tcp.h> 55 #include <netinet/udp.h> 56 #include <sys/gld.h> 57 #include "ostypes.h" 58 #include "common.h" 59 #ifdef CONFIG_CHELSIO_T1_1G 60 #include "fpga_defs.h" 61 #endif 62 #include "regs.h" 63 #include "suni1x10gexp_regs.h" 64 #include "sge.h" 65 #include "espi.h" 66 67 #include "ch.h" 68 69 extern uint32_t buffers_in_use[]; 70 71 uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N; 72 uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N; 73 uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N; 74 uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N; 75 uint32_t sge_respq_cnt = SGE_RESPQ_E_N; 76 77 uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N; 78 uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N; 79 uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N; 80 uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N; 81 uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N; 82 83 #ifdef HOST_PAUSE 84 uint32_t do_host_pause = 1; 85 uint32_t flq_pause_window = 64; 86 #endif 87 88 static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, 89 ulong_t *dh); 90 void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t); 91 92 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q, 93 unsigned int len, unsigned int offload); 94 #ifdef HOST_PAUSE 95 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q); 96 #endif 97 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q); 98 static void freelQs_empty(pesge *sge); 99 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend); 100 static int alloc_rx_resources(pesge *sge, struct sge_params *p); 101 static int alloc_tx_resources(pesge *sge, struct sge_params *p); 102 static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size, 103 int base_reg_lo, int base_reg_hi, int size_reg); 104 static void configure_sge(pesge *sge, struct sge_params *p); 105 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q); 106 static void free_rx_resources(pesge *sge); 107 static void free_tx_resources(pesge *sge); 108 static inline unsigned int jumbo_payload_capacity(pesge *sge); 109 #ifdef SUN_KSTATS 110 static int sge_kstat_setup(pesge *); 111 static void sge_kstat_remove(pesge *); 112 static int sge_kstat_update(p_kstat_t, int); 113 #endif 114 static uint16_t calc_ocsum(mblk_t *, int); 115 116 /* 117 * Local routines. 118 */ 119 static inline void sge_ring_doorbell(pesge *sge, u32 control_reg); 120 121 static inline void 122 sge_ring_doorbell(pesge *sge, u32 control_reg) 123 { 124 membar_producer(); 125 t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg); 126 } 127 128 /* 129 * DESC: 130 * 131 * NOTES: Must have at least 1 command queue and 1 freelist queue. 132 * 133 */ 134 pesge * 135 t1_sge_create(ch_t *sa, struct sge_params *p) 136 { 137 pesge *sge; 138 139 sge = t1_os_malloc_wait_zero(sizeof (pesge)); 140 141 if (sge == NULL) 142 goto error_no_mem; 143 144 memset(sge, 0, sizeof (*sge)); 145 146 /* 147 * PR2928 & PR3309 148 * set default timeout value - 20 msec 149 * we set the initial value to 2 which gurantees at least one tick. 150 */ 151 if (is_T2(sa)) 152 sge->ptimeout = 1; 153 154 sge->obj = sa; 155 #ifdef SUN_KSTATS 156 if (sge_kstat_setup(sge) != 0) 157 goto t1_sge_create_fail1; 158 #endif 159 p->cmdQ_size[0] = sge_cmdq0_cnt; 160 p->cmdQ_size[1] = sge_cmdq1_cnt; 161 162 /* note that jumbo frame index is inverted for T2 */ 163 if (is_T2(sa)) { 164 p->freelQ_size[1] = sge_flq0_cnt; 165 p->freelQ_size[0] = sge_flq1_cnt; 166 } else { 167 p->freelQ_size[0] = sge_flq0_cnt; 168 p->freelQ_size[1] = sge_flq1_cnt; 169 } 170 171 #if CH_DEBUG 172 /* DEBUG only */ 173 cmn_err(CE_NOTE, "sge: %p\n", sge); 174 cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]); 175 cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]); 176 cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]); 177 cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ); 178 cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt); 179 #endif 180 #ifdef SUN_KSTATS 181 goto error_no_mem; 182 183 t1_sge_create_fail1: 184 t1_os_free(sge, sizeof (pesge)); 185 sge = NULL; 186 #endif 187 error_no_mem: 188 return (sge); 189 } 190 191 int 192 t1_sge_destroy(pesge* sge) 193 { 194 if (sge != NULL) { 195 free_tx_resources(sge); 196 free_rx_resources(sge); 197 198 /* PR2928 & PR3309 */ 199 if ((is_T2(sge->obj)) && (sge->pskb)) 200 pe_free_fake_arp(sge->pskb); 201 #ifdef SUN_KSTATS 202 sge_kstat_remove(sge); 203 #endif 204 t1_os_free(sge, sizeof (pesge)); 205 } 206 return (0); 207 } 208 209 /* 210 * PR2928 & PR3309 211 * call out event from timeout 212 * 213 * there is a potential race between the timeout and the close. 214 * unless we protect the timeout, the close could occur at the 215 * same time. Then if the timeout service routine was slow or 216 * interrupted, the sge_stop() could complete with a timeoutID 217 * that has expired, thus letting another timeout occur. If the 218 * service routine was delayed still further, a detach could occur. 219 * the second time could then end up accessing memory that has been 220 * released back to the system. Bad things could then occur. We 221 * set a flag in sge_stop() to tell the service routine not to 222 * issue further timeouts. sge_stop() will block until a timeout 223 * has occured. If the command Q is full then we shouldn't put out 224 * an arp. 225 */ 226 227 void 228 t1_espi_workaround(ch_t *adapter) 229 { 230 pesge *sge = adapter->sge; 231 ch_t *chp = (ch_t *)sge->obj; 232 int rv = 1; 233 234 if ((chp->ch_state == PERUNNING) && 235 atomic_read(&sge->cmdQ[0].cq_asleep)) { 236 u32 seop; 237 seop = t1_espi_get_mon(adapter, 0x930, 0); 238 if ((seop & 0xfff0fff) == 0xfff) { 239 /* after first arp */ 240 if (sge->pskb) 241 rv = pe_start(adapter, (mblk_t *)sge->pskb, 242 CH_ARP); 243 if (!rv) 244 sge->intr_cnt.arp_sent++; 245 } 246 } 247 #ifdef HOST_PAUSE 248 /* 249 * If we are already in sge_data_in, then we can skip calling 250 * t1_sge_check_pause() this clock cycle. lockstat showed that 251 * we were blocking on the mutex ~ 2% of the time. 252 */ 253 if (mutex_tryenter(&adapter->ch_intr)) { 254 t1_sge_check_pause(sge, &sge->freelQ[0]); 255 t1_sge_check_pause(sge, &sge->freelQ[1]); 256 mutex_exit(&adapter->ch_intr); 257 } 258 #endif 259 } 260 261 int 262 sge_start(pesge *sge) 263 { 264 t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control); 265 /* PR2928 & PR3309, also need to avoid Pause deadlock */ 266 ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic, 267 (void (*)(void *))t1_espi_workaround, sge->obj); 268 ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout); 269 return (0); 270 } 271 272 /* 273 * Disables SGE queues. 274 */ 275 int 276 sge_stop(pesge *sge) 277 { 278 uint32_t status; 279 int loops; 280 281 DBGASSERT(sge); 282 283 /* PR2928 & PR3309, also need to avoid Pause deadlock */ 284 t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0); 285 286 /* wait until there's no more outstanding interrupts pending */ 287 loops = 0; 288 do { 289 status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE); 290 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status); 291 drv_usecwait(125); 292 loops++; 293 } while (status && (loops < 1000)); 294 295 ch_stop_cyclic(&sge->espi_wa_cyclic); 296 297 return (0); 298 } 299 300 uint32_t sge_cmdq_send_fail; 301 302 int 303 sge_data_out(pesge* sge, int qid, mblk_t *m0, 304 cmdQ_ce_t *cmp, int count, uint32_t flg) 305 { 306 struct cmdQ *Q = &sge->cmdQ[qid]; 307 ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh; 308 spinlock_t *qlock = &Q->cq_qlock; 309 cmdQ_e *e; 310 cmdQ_e *q = Q->cq_entries; 311 uint32_t credits; 312 uint32_t pidx; 313 uint32_t genbit; 314 uint32_t entries_n = Q->cq_entries_n; 315 cmdQ_ce_t *ce; 316 cmdQ_ce_t *cq = Q->cq_centries; 317 dma_addr_t mapping; 318 uint32_t j = 0; 319 uint32_t offset; 320 #if defined(TX_CKSUM_FIX) 321 uint16_t csum; 322 uint16_t *csum_loc; 323 #endif 324 #ifdef TX_THREAD_RECLAIM 325 uint32_t reclaim_cnt; 326 #endif 327 328 /* 329 * We must exit if we don't have enough free command queue entries 330 * available. 331 */ 332 333 spin_lock(qlock); 334 335 #if defined(TX_CKSUM_FIX) 336 /* 337 * This checksum fix will address a fragmented datagram 338 * checksum error. Which will lead to the next packet after 339 * the last packet with the More fragment bit set having its 340 * checksum corrupted. When the packet reaches this point 341 * the 'flg' variable indicates whether a checksum is needed 342 * or not. The algorithm is as follows, if the current packet 343 * is a More fragment set the count of packets to be checksummed 344 * after it to 3. If it't not and the count of is more than 0 345 * then calculate the checksum in software, if a hardware checksum 346 * was requested. Then decrment the count. Same algorithm applies 347 * to TCP. 348 */ 349 if (flg & CH_UDP_MF) { 350 sge->do_udp_csum = 3; 351 } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) { 352 if ((flg & CH_NO_HWCKSUM) == 0) { 353 /* 354 * Calc Checksum here. 355 */ 356 csum = calc_ocsum(m0, 357 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 358 csum_loc = (uint16_t *)(m0->b_rptr + 359 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 360 csum_loc += (((*(char *)csum_loc) & 0x0f) << 1); 361 362 sge->intr_cnt.tx_soft_cksums++; 363 ((struct udphdr *)(csum_loc))->uh_sum = csum; 364 ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1; 365 } 366 sge->do_udp_csum--; 367 } else if (flg & CH_TCP_MF) { 368 sge->do_tcp_csum = 3; 369 } else if (sge->do_tcp_csum != 0) { 370 if ((flg & CH_NO_HWCKSUM) == 0) { 371 sge->intr_cnt.tx_soft_cksums++; 372 /* 373 * Calc Checksum here. 374 */ 375 } 376 sge->do_tcp_csum--; 377 } 378 #endif /* TX_CKSUM_FIX */ 379 #ifdef TX_THREAD_RECLAIM 380 reclaim_cnt = Q->cq_complete; 381 if (reclaim_cnt > SGE_BATCH_THRESH) { 382 sge->intr_cnt.tx_reclaims[qid]++; 383 free_cmdQ_buffers(sge, Q, reclaim_cnt); 384 Q->cq_complete = 0; 385 } 386 #endif 387 genbit = Q->cq_genbit; 388 pidx = Q->cq_pidx; 389 credits = Q->cq_credits; 390 391 if ((credits - 1) < count) { 392 spin_unlock(qlock); 393 sge->intr_cnt.cmdQ_full[qid]++; 394 return (1); 395 } 396 397 atomic_sub(count, &Q->cq_credits); 398 Q->cq_pidx += count; 399 if (Q->cq_pidx >= entries_n) { 400 Q->cq_pidx -= entries_n; 401 Q->cq_genbit ^= 1; 402 } 403 404 spin_unlock(qlock); 405 406 #ifdef SUN_KSTATS 407 if (count > MBLK_MAX) 408 sge->intr_cnt.tx_descs[MBLK_MAX - 1]++; 409 else 410 sge->intr_cnt.tx_descs[count]++; 411 #endif 412 413 ce = &cq[pidx]; 414 *ce = *cmp; 415 mapping = cmp->ce_pa; 416 j++; 417 418 e = &q[pidx]; 419 420 offset = (caddr_t)e - (caddr_t)q; 421 422 e->Sop = 1; 423 e->DataValid = 1; 424 e->BufferLength = cmp->ce_len; 425 e->AddrHigh = ((u64)mapping >> 32); 426 e->AddrLow = ((u64)mapping & 0xffffffff); 427 428 --count; 429 if (count > 0) { 430 unsigned int i; 431 432 e->Eop = 0; 433 wmb(); 434 e->GenerationBit = e->GenerationBit2 = genbit; 435 436 for (i = 0; i < count; i++) { 437 438 ce++; 439 e++; 440 cmp++; 441 if (++pidx == entries_n) { 442 pidx = 0; 443 genbit ^= 1; 444 /* sync from offset to end of cmdQ */ 445 (void) ddi_dma_sync(dh, (off_t)(offset), 446 j*sizeof (*e), DDI_DMA_SYNC_FORDEV); 447 offset = j = 0; 448 ce = cq; 449 e = q; 450 } 451 452 *ce = *cmp; 453 mapping = cmp->ce_pa; 454 j++; 455 e->Sop = 0; 456 e->DataValid = 1; 457 e->BufferLength = cmp->ce_len; 458 e->AddrHigh = ((u64)mapping >> 32); 459 e->AddrLow = ((u64)mapping & 0xffffffff); 460 461 if (i < (count - 1)) { 462 e->Eop = 0; 463 wmb(); 464 e->GenerationBit = e->GenerationBit2 = genbit; 465 } 466 } 467 } 468 469 ce->ce_mp = m0; 470 471 e->Eop = 1; 472 wmb(); 473 e->GenerationBit = e->GenerationBit2 = genbit; 474 475 (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e), 476 DDI_DMA_SYNC_FORDEV); 477 478 /* 479 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring 480 * the doorbell if the Q is asleep. There is a natural race, where 481 * the hardware is going to sleep just after we checked, however, 482 * then the interrupt handler will detect the outstanding TX packet 483 * and ring the doorbell for us. 484 */ 485 if (qid) { 486 doorbell_pio(sge, F_CMDQ1_ENABLE); 487 } else { 488 if (atomic_read(Q->cq_asleep)) { 489 atomic_set(&Q->cq_asleep, 0); 490 /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */ 491 atomic_set(&Q->cq_pio_pidx, Q->cq_pidx); 492 } 493 } 494 doorbell_pio(sge, F_CMDQ0_ENABLE); 495 496 return (0); 497 } 498 499 #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA) 500 501 /* 502 * Disable SGE error interrupts. 503 */ 504 int 505 t1_sge_intr_disable(pesge* sge) 506 { 507 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 508 509 t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK); 510 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0); 511 return (0); 512 } 513 514 #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \ 515 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 516 517 /* 518 * Enable SGE error interrupts. 519 */ 520 int 521 t1_sge_intr_enable(pesge* sge) 522 { 523 u32 en = SGE_INT_ENABLE; 524 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 525 526 t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK); 527 528 if (sge->obj->ch_flags & TSO_CAPABLE) 529 en &= ~F_PACKET_TOO_BIG; 530 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en); 531 return (0); 532 } 533 534 /* 535 * Clear SGE error interrupts. 536 */ 537 int 538 t1_sge_intr_clear(pesge* sge) 539 { 540 t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK); 541 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff); 542 return (0); 543 } 544 545 #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 546 547 int 548 t1_sge_intr_error_handler(pesge *sge) 549 { 550 peobj *obj = sge->obj; 551 u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE); 552 553 if (cause & F_RESPQ_EXHAUSTED) 554 sge->intr_cnt.respQ_empty++; 555 if (cause & F_RESPQ_OVERFLOW) { 556 sge->intr_cnt.respQ_overflow++; 557 cmn_err(CE_WARN, "%s: SGE response queue overflow\n", 558 obj->ch_name); 559 } 560 if (cause & F_FL_EXHAUSTED) { 561 sge->intr_cnt.freelistQ_empty++; 562 freelQs_empty(sge); 563 } 564 if (cause & F_PACKET_TOO_BIG) { 565 sge->intr_cnt.pkt_too_big++; 566 cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n", 567 obj->ch_name); 568 } 569 if (cause & F_PACKET_MISMATCH) { 570 sge->intr_cnt.pkt_mismatch++; 571 cmn_err(CE_WARN, "%s: SGE packet mismatch\n", 572 obj->ch_name); 573 } 574 if (cause & SGE_INT_FATAL) 575 t1_fatal_err(obj); 576 577 t1_write_reg_4(obj, A_SG_INT_CAUSE, cause); 578 return (0); 579 } 580 581 /* 582 * 583 * PARAM: sge - SGE instance pointer. 584 */ 585 int 586 sge_data_in(pesge *sge) 587 { 588 peobj *adapter = sge->obj; 589 struct respQ *Q = &sge->respQ; 590 respQ_e *e; /* response queue entry */ 591 respQ_e *q = Q->rq_entries; /* base response queue */ 592 uint32_t cidx = Q->rq_cidx; 593 uint32_t genbit = Q->rq_genbit; 594 uint32_t entries_n = Q->rq_entries_n; 595 uint32_t credits = Q->rq_credits; 596 uint32_t credits_thresh = Q->rq_credits_thresh; 597 uint32_t ret = 0; 598 #ifndef TX_THREAD_RECLAIM 599 uint32_t credits_pend[2] = {0, 0}; 600 #endif 601 uint32_t flags = 0; 602 uint32_t flagt; 603 ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh; 604 605 t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA); 606 607 /* 608 * Catch the case where an interrupt arrives 609 * early. 610 */ 611 if ((q == NULL) || (dh == NULL)) { 612 goto check_slow_ints; 613 } 614 615 /* initial response queue entry */ 616 e = &q[cidx]; 617 618 /* pull physical memory of response queue entry into cache */ 619 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 620 sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 621 622 while (e->GenerationBit == genbit) { 623 if (--credits < credits_thresh) { 624 uint32_t n = entries_n - credits - 1; 625 t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n); 626 credits += n; 627 } 628 if (likely(e->DataValid)) { 629 (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid], 630 e->BufferLength, e->Offload); 631 if ((e->Sop != 1) || (e->Eop != 1)) { 632 sge->intr_cnt.rx_badEopSop++; 633 cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d", 634 e->Sop, e->Eop, e->BufferLength); 635 } 636 } 637 flagt = e->Qsleeping; 638 flags |= flagt; 639 if (flagt & F_CMDQ0_ENABLE) 640 sge->intr_cnt.rx_cmdq0++; 641 if (flagt & F_CMDQ1_ENABLE) 642 sge->intr_cnt.rx_cmdq1++; 643 if (flagt & F_FL0_ENABLE) 644 sge->intr_cnt.rx_flq0++; 645 if (flagt & F_FL1_ENABLE) 646 sge->intr_cnt.rx_flq1++; 647 #ifdef TX_THREAD_RECLAIM 648 spin_lock(&sge->cmdQ[0].cq_qlock); 649 sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn; 650 spin_unlock(&sge->cmdQ[0].cq_qlock); 651 spin_lock(&sge->cmdQ[1].cq_qlock); 652 sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn; 653 if ((adapter->ch_blked) && 654 (sge->cmdQ[0].cq_complete + 655 sge->cmdQ[1].cq_complete) > 16) { 656 adapter->ch_blked = 0; 657 ch_gld_ok(adapter); 658 } 659 spin_unlock(&sge->cmdQ[1].cq_qlock); 660 #else 661 credits_pend[0] += e->Cmdq0CreditReturn; 662 credits_pend[1] += e->Cmdq1CreditReturn; 663 #ifdef CONFIG_SMP 664 if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) { 665 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 666 credits_pend[0] = 0; 667 } 668 if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) { 669 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 670 credits_pend[1] = 0; 671 } 672 #endif 673 #endif 674 #ifdef HOST_PAUSE 675 t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]); 676 #endif 677 e++; 678 if (unlikely(++cidx == entries_n)) { 679 cidx = 0; 680 genbit ^= 1; 681 e = q; 682 } 683 684 /* pull physical memory of response queue entry into cache */ 685 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 686 sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 687 688 ret = 1; 689 } 690 691 #ifndef TX_THREAD_RECLAIM 692 if (credits_pend[0]) 693 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 694 if (credits_pend[1]) 695 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 696 #endif 697 if (flags & F_CMDQ0_ENABLE) { 698 struct cmdQ *cmdQ = &sge->cmdQ[0]; 699 atomic_set(&cmdQ->cq_asleep, 1); 700 if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) { 701 doorbell_pio(sge, F_CMDQ0_ENABLE); 702 atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx); 703 } 704 } 705 706 /* the SGE told us one of the free lists is empty */ 707 if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE))) 708 freelQs_empty(sge); 709 710 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 711 if (adapter->ch_tx_overflow_mutex) 712 mutex_enter(adapter->ch_tx_overflow_mutex); 713 if (adapter->ch_blked && 714 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) && 715 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) { 716 adapter->ch_blked = 0; 717 if (adapter->ch_tx_overflow_cv) 718 cv_broadcast(adapter->ch_tx_overflow_cv); 719 ch_gld_ok(adapter); 720 } 721 if (adapter->ch_tx_overflow_mutex) 722 mutex_exit(adapter->ch_tx_overflow_mutex); 723 #else 724 #ifndef TX_THREAD_RECLAIM 725 if (adapter->ch_blked && 726 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) && 727 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) { 728 adapter->ch_blked = 0; 729 ch_gld_ok(adapter); 730 } 731 #endif 732 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 733 734 Q->rq_genbit = genbit; 735 Q->rq_cidx = cidx; 736 Q->rq_credits = credits; 737 738 t1_write_reg_4(adapter, A_SG_SLEEPING, cidx); 739 740 check_slow_ints: 741 /* handle non-data interrupts */ 742 if (unlikely(!ret)) 743 ret = t1_slow_intr_handler(adapter); 744 745 return (ret); 746 } 747 748 /* 749 * allocate a mblk with DMA mapped mblk. 750 * When checksum offload is enabled, we start the DMA at a 2 byte offset so 751 * the IP header will be aligned. We do this for sparc only. 752 */ 753 static uint64_t 754 os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh) 755 { 756 ch_esb_t *ch_get_small_rbuf(ch_t *sa); 757 ch_esb_t *ch_get_big_rbuf(ch_t *sa); 758 ch_esb_t *rbp; 759 uint32_t rxoff = sa->sge->rx_offset; 760 761 if (sz == SGE_SM_BUF_SZ(sa)) { 762 /* get pre-mapped buffer */ 763 if ((rbp = ch_get_small_rbuf(sa)) == NULL) { 764 sa->norcvbuf++; 765 return ((uint64_t)0); 766 } 767 768 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 769 SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 770 if (*mb == NULL) { 771 mutex_enter(&sa->ch_small_esbl); 772 rbp->cs_next = sa->ch_small_esb_free; 773 sa->ch_small_esb_free = rbp; 774 mutex_exit(&sa->ch_small_esbl); 775 return ((uint64_t)0); 776 } 777 *dh = rbp->cs_dh; 778 779 return (rbp->cs_pa + rxoff); 780 } else { 781 /* get pre-mapped buffer */ 782 if ((rbp = ch_get_big_rbuf(sa)) == NULL) { 783 sa->norcvbuf++; 784 return ((uint64_t)0); 785 } 786 787 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 788 SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 789 if (*mb == NULL) { 790 mutex_enter(&sa->ch_big_esbl); 791 rbp->cs_next = sa->ch_big_esb_free; 792 sa->ch_big_esb_free = rbp; 793 mutex_exit(&sa->ch_big_esbl); 794 return ((uint64_t)0); 795 } 796 *dh = rbp->cs_dh; 797 798 return (rbp->cs_pa + rxoff); 799 } 800 } 801 802 static inline unsigned int 803 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload) 804 { 805 mblk_t *skb; 806 peobj *adapter = sge->obj; 807 struct freelQ_ce *cq = Q->fq_centries; 808 struct freelQ_ce *ce = &cq[Q->fq_cidx]; 809 ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh; 810 uint32_t cidx = Q->fq_cidx; 811 uint32_t entries_n = Q->fq_entries_n; 812 uint32_t sz = Q->fq_rx_buffer_size; 813 uint32_t useit = 1; 814 uint32_t rxoff = sge->rx_offset; 815 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 816 uint32_t rv; 817 #endif 818 819 if (Q->fq_id) 820 sge->intr_cnt.rx_flq1_cnt++; 821 else 822 sge->intr_cnt.rx_flq0_cnt++; 823 /* 824 * If pkt size falls below threshold, then we'll copy data to 825 * an blk and reuse mblk. 826 * 827 * NOTE that rxoff is 2 for T1 adapters. We align the the start 828 * of the DMA buffer begin at rxoff offset for T1 cards instead of 829 * at the beginning of the buffer, thus the length of the received 830 * data does not include this offset. We therefore always add 831 * SGE_RX_OFFSET to the allocb size so we have space to provide the 832 * offset for the copied data. 833 */ 834 #ifdef HOST_PAUSE 835 /* 836 * If we have Host pause compiled in, then we look at the 837 * free list, if the pause is on and we're not in offload 838 * mode then we drop packets, this is designed to avoid 839 * overwhelming the machine. If the machine is powerfull enough 840 * this will not happen. The 'rx_pkt_drops' will show when 841 * packets are being dropped and how much. 842 */ 843 if ((offload == 0) && adapter->pause_on) { 844 freelQ_e *e; 845 /* Ditch the packet and reuse original buffer */ 846 e = &Q->fq_entries[cidx]; 847 e->GenerationBit ^= 1; 848 e->GenerationBit2 ^= 1; 849 sge->intr_cnt.rx_pkt_drops++; 850 goto rx_entry_consumed; 851 } else if (((adapter->pause_on || 852 (len <= SGE_RX_COPY_THRESHOLD)) && 853 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))) 854 #else 855 if ((len <= SGE_RX_COPY_THRESHOLD) && 856 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))) 857 #endif 858 { 859 freelQ_e *e; 860 char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr; 861 862 /* 863 * pull physical memory of pkt data into cache 864 * Note that len does not include offset for T1. 865 */ 866 (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 867 DDI_DMA_SYNC_FORKERNEL); 868 869 if (offload == 0) { 870 /* 871 * create 2 byte offset so IP header aligned on 872 * 4 byte boundry 873 */ 874 skb_reserve(skb, SGE_RX_OFFSET); 875 /* 876 * if hardware inserted 2 byte offset then need to 877 * start copying with extra offset 878 */ 879 src += sge->rx_pkt_pad; 880 } 881 memcpy(skb->b_rptr, src, len); 882 useit = 0; /* mblk copy, don't inc esballoc in use cnt */ 883 884 /* so we can reuse original buffer */ 885 e = &Q->fq_entries[cidx]; 886 e->GenerationBit ^= 1; 887 e->GenerationBit2 ^= 1; 888 sge->intr_cnt.rx_pkt_copied++; 889 } else { 890 /* consume buffer off the ring */ 891 skb = ce->fe_mp; 892 ce->fe_mp = NULL; 893 894 /* 895 * if not offload (tunneled pkt), & hardward padded, then 896 * adjust start of pkt to point to start of data i.e. 897 * skip pad (2 bytes). 898 */ 899 if (!offload && sge->rx_pkt_pad) 900 __skb_pull(skb, SGE_RX_OFFSET); 901 902 /* 903 * pull physical memory of pkt data into cache 904 * Note that len does not include offset for T1. 905 */ 906 (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 907 DDI_DMA_SYNC_FORKERNEL); 908 } 909 910 /* set length of data in skb */ 911 skb_put(skb, len); 912 913 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 914 if (likely(offload)) { 915 if (likely(toe_running(adapter))) { 916 /* sends pkt upstream to toe layer */ 917 if (useit) { 918 if (sz == SGE_SM_BUF_SZ(adapter)) { 919 atomic_add(1, 920 &buffers_in_use[adapter->ch_sm_index]); 921 } else { 922 atomic_add(1, 923 &buffers_in_use[adapter->ch_big_index]); 924 } 925 } 926 if (adapter->toe_rcv) 927 adapter->toe_rcv(adapter->ch_toeinst, skb); 928 else 929 freemsg(skb); 930 } else { 931 cmn_err(CE_WARN, 932 "%s: unexpected offloaded packet, cmd %u\n", 933 adapter->ch_name, *skb->b_rptr); 934 935 /* discard packet */ 936 freemsg(skb); 937 } 938 } 939 #else 940 if (unlikely(offload)) { 941 cmn_err(CE_WARN, 942 "%s: unexpected offloaded packet, cmd %u\n", 943 adapter->ch_name, *skb->b_rptr); 944 945 /* discard paket */ 946 freemsg(skb); 947 } 948 #endif 949 else { 950 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr; 951 int flg = 0; 952 uint32_t cksum; 953 954 /* adjust beginning of data to skip CPL header */ 955 skb_pull(skb, SZ_CPL_RX_PKT); 956 957 /* extract checksum from CPL header here */ 958 959 /* 960 * bump count of mlbks in used by protocol stack(s) 961 */ 962 if (useit) { 963 if (sz == SGE_SM_BUF_SZ(adapter)) { 964 atomic_add(1, 965 &buffers_in_use[adapter->ch_sm_index]); 966 } else { 967 atomic_add(1, 968 &buffers_in_use[adapter->ch_big_index]); 969 } 970 } 971 972 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 973 /* 974 * let the TOE layer have a crack at the packet first. 975 */ 976 if (adapter->toe_tunnel) { 977 rv = adapter->toe_tunnel(adapter->ch_toeinst, skb); 978 /* 979 * The TOE may have consumed the packet. 980 */ 981 if (rv) 982 goto rx_entry_consumed; 983 } 984 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 985 986 cksum = p->csum; 987 988 /* 989 * NOTE: 14+9 = size of MAC + offset to IP protocol field 990 */ 991 if (adapter->ch_config.cksum_enabled && 992 (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) == 993 ETHERTYPE_IP) && 994 ((skb->b_rptr[14+9] == IPPROTO_TCP) || 995 (skb->b_rptr[14+9] == IPPROTO_UDP))) { 996 flg = 1; 997 } 998 999 ch_send_up(adapter, skb, cksum, flg); 1000 } 1001 1002 rx_entry_consumed: 1003 1004 if (++cidx == entries_n) 1005 cidx = 0; 1006 1007 Q->fq_cidx = cidx; 1008 1009 if (unlikely(--Q->fq_credits < (entries_n>>2))) 1010 /* allocate new buffers on the free list */ 1011 alloc_freelQ_buffers(sge, Q); 1012 return (1); 1013 } 1014 1015 #ifdef HOST_PAUSE 1016 static void 1017 t1_sge_check_pause(pesge *sge, struct freelQ *Q) 1018 { 1019 peobj *adapter = sge->obj; 1020 1021 /* 1022 * If the number of available credits shrinks below 1023 * the Pause on threshold then enable the pause and 1024 * try and allocate more buffers. 1025 * On the next pass, if there's more credits returned 1026 * then check that you've went above the pause 1027 * threshold and then disable the pause. 1028 */ 1029 if (Q->fq_credits < Q->fq_pause_on_thresh) { 1030 if (do_host_pause) { 1031 sge->intr_cnt.rx_pause_on++; 1032 adapter->txxg_cfg1 |= 1033 SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1034 (void) t1_tpi_write(adapter, 1035 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1036 adapter->txxg_cfg1); 1037 adapter->pause_on = 1; 1038 adapter->pause_time = gethrtime(); 1039 } 1040 alloc_freelQ_buffers(sge, Q); 1041 } else if ((adapter->pause_on) && 1042 (Q->fq_credits > Q->fq_pause_off_thresh)) { 1043 hrtime_t time; 1044 sge->intr_cnt.rx_pause_off++; 1045 adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1046 (void) t1_tpi_write(adapter, 1047 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1048 adapter->txxg_cfg1); 1049 adapter->pause_on = 0; 1050 time = (gethrtime() - adapter->pause_time)/1000; 1051 sge->intr_cnt.rx_pause_ms += time; 1052 if (time > sge->intr_cnt.rx_pause_spike) 1053 sge->intr_cnt.rx_pause_spike = (uint32_t)time; 1054 } 1055 sge->intr_cnt.rx_fl_credits = Q->fq_credits; 1056 } 1057 #endif /* HOST_PAUSE */ 1058 1059 static void 1060 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q) 1061 { 1062 uint32_t pidx = Q->fq_pidx; 1063 struct freelQ_ce *ce = &Q->fq_centries[pidx]; 1064 freelQ_e *fq = Q->fq_entries; /* base of freelist Q */ 1065 freelQ_e *e = &Q->fq_entries[pidx]; 1066 uint32_t sz = Q->fq_rx_buffer_size; 1067 uint32_t rxoff = sge->rx_offset; 1068 uint32_t credits = Q->fq_credits; 1069 uint32_t entries_n = Q->fq_entries_n; 1070 uint32_t genbit = Q->fq_genbit; 1071 ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh; 1072 ulong_t dh; 1073 uint64_t mapping; 1074 off_t offset = (off_t)((caddr_t)e - (caddr_t)fq); 1075 size_t len = 0; 1076 1077 while (credits < entries_n) { 1078 if (e->GenerationBit != genbit) { 1079 mblk_t *skb; 1080 1081 mapping = os_freelist_buffer_alloc(sge->obj, sz, 1082 &skb, &dh); 1083 if (mapping == 0) { 1084 sge->intr_cnt.rx_flbuf_fails++; 1085 break; 1086 } 1087 sge->intr_cnt.rx_flbuf_allocs++; 1088 1089 ce->fe_mp = skb; 1090 ce->fe_dh = dh; 1091 1092 /* 1093 * Note that for T1, we've started the beginning of 1094 * of the buffer by an offset of 2 bytes. We thus 1095 * decrement the length to account for this. 1096 */ 1097 e->AddrLow = (u32)mapping; 1098 e->AddrHigh = (u64)mapping >> 32; 1099 e->BufferLength = sz - rxoff; 1100 wmb(); 1101 e->GenerationBit = e->GenerationBit2 = genbit; 1102 } 1103 1104 len += sizeof (*e); 1105 1106 ce++; 1107 e++; 1108 credits++; 1109 if (++pidx == entries_n) { 1110 /* 1111 * sync freelist entries to physical memory up to 1112 * end of the table. 1113 */ 1114 (void) ddi_dma_sync(th, offset, len, 1115 DDI_DMA_SYNC_FORDEV); 1116 offset = 0; 1117 len = 0; 1118 1119 pidx = 0; 1120 genbit ^= 1; 1121 ce = Q->fq_centries; 1122 e = Q->fq_entries; 1123 } 1124 } 1125 1126 /* sync freelist entries that have been modified. */ 1127 if (len) 1128 (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV); 1129 1130 Q->fq_genbit = genbit; 1131 Q->fq_pidx = pidx; 1132 Q->fq_credits = credits; 1133 } 1134 1135 static void 1136 freelQs_empty(pesge *sge) 1137 { 1138 u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE); 1139 u32 irqholdoff_reg; 1140 1141 alloc_freelQ_buffers(sge, &sge->freelQ[0]); 1142 alloc_freelQ_buffers(sge, &sge->freelQ[1]); 1143 1144 if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) && 1145 (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) { 1146 irq_reg |= F_FL_EXHAUSTED; 1147 irqholdoff_reg = sge->intrtimer[sge->currIndex]; 1148 } else { 1149 /* Clear the F_FL_EXHAUSTED interrupts for now */ 1150 irq_reg &= ~F_FL_EXHAUSTED; 1151 irqholdoff_reg = sge->intrtimer_nres; 1152 } 1153 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg); 1154 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg); 1155 1156 /* We reenable the Qs to force an Freelist GTS interrupt later */ 1157 doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE); 1158 } 1159 1160 /* 1161 * Frees 'credits_pend' TX buffers and returns the credits to Q->credits. 1162 * Free xmit buffers 1163 */ 1164 static void 1165 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend) 1166 { 1167 mblk_t *skb; 1168 struct cmdQ_ce *ce; 1169 struct cmdQ_ce *cq = Q->cq_centries; 1170 uint32_t entries_n = Q->cq_entries_n; 1171 uint32_t cidx = Q->cq_cidx; 1172 uint32_t i = credits_pend; 1173 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1174 ch_t *chp = sge->obj; 1175 #endif 1176 ce = &cq[cidx]; 1177 1178 while (i--) { 1179 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1180 /* if flag set, then toe buffer */ 1181 switch (ce->ce_flg & 0x7) { 1182 case DH_DMA: 1183 if (ce->ce_dh) { 1184 ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1185 ce->ce_dh = NULL; /* may not be needed */ 1186 } 1187 skb = ce->ce_mp; 1188 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1189 freemsg(skb); 1190 } 1191 ce->ce_mp = NULL; 1192 break; 1193 1194 #if defined(__sparc) 1195 case DH_DVMA: 1196 if (ce->ce_dh) { 1197 ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1198 ce->ce_dh = NULL; /* may not be needed */ 1199 } 1200 skb = ce->ce_mp; 1201 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1202 freemsg(skb); 1203 } 1204 ce->ce_mp = NULL; 1205 break; 1206 #endif /* __sparc */ 1207 1208 case DH_TOE: 1209 chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp)); 1210 ce->ce_mp = NULL; 1211 break; 1212 } 1213 #else /* CONFIG_CHELSIO_T1_OFFLOAD */ 1214 if (ce->ce_dh) { 1215 if ((ce->ce_flg & 7) == DH_DMA) { 1216 ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1217 } 1218 #if defined(__sparc) 1219 else { 1220 ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1221 } 1222 #endif /* __sparc */ 1223 ce->ce_dh = NULL; /* may not be needed */ 1224 } 1225 1226 skb = ce->ce_mp; 1227 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1228 freemsg(skb); 1229 } 1230 ce->ce_mp = NULL; 1231 #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */ 1232 1233 ce++; 1234 if (++cidx == entries_n) { 1235 cidx = 0; 1236 ce = cq; 1237 } 1238 } 1239 1240 Q->cq_cidx = cidx; 1241 atomic_add(credits_pend, &Q->cq_credits); 1242 } 1243 1244 struct sge_intr_counts * 1245 sge_get_stat(pesge *sge) 1246 { 1247 return (&sge->intr_cnt); 1248 } 1249 1250 /* 1251 * Allocates both RX and TX resources and configures the SGE. However, 1252 * the hardware is not enabled yet. 1253 * 1254 * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic. 1255 * jumbo_fl is set to the index of the freelist containing the jumbo buffers. 1256 */ 1257 int 1258 t1_sge_configure(pesge *sge, struct sge_params *p) 1259 { 1260 sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET; 1261 sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0; 1262 /* if we're a T2 card, then we have hardware offset support */ 1263 sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0; 1264 1265 if (alloc_rx_resources(sge, p)) 1266 return (-ENOMEM); 1267 if (alloc_tx_resources(sge, p)) { 1268 free_rx_resources(sge); 1269 return (-ENOMEM); 1270 } 1271 configure_sge(sge, p); 1272 1273 /* 1274 * Now that we have sized the free lists calculate the payload 1275 * capacity of the large buffers. Other parts of the driver use 1276 * this to set the max offload coalescing size so that RX packets 1277 * do not overflow our large buffers. 1278 */ 1279 p->large_buf_capacity = jumbo_payload_capacity(sge); 1280 return (0); 1281 } 1282 1283 /* 1284 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a 1285 * response Q. 1286 */ 1287 static int 1288 alloc_rx_resources(pesge *sge, struct sge_params *p) 1289 { 1290 unsigned int size, i; 1291 1292 for (i = 0; i < SGE_FREELQ_N; i++) { 1293 struct freelQ *Q = &sge->freelQ[i]; 1294 1295 Q->fq_id = i; 1296 Q->fq_genbit = 1; 1297 Q->fq_entries_n = p->freelQ_size[i]; 1298 #ifdef HOST_PAUSE 1299 Q->fq_pause_on_thresh = flq_pause_window; 1300 Q->fq_pause_off_thresh = Q->fq_entries_n >> 1; 1301 #endif 1302 size = sizeof (freelQ_e) * Q->fq_entries_n; 1303 1304 Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1305 size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT); 1306 1307 1308 if (!Q->fq_entries) 1309 goto err_no_mem; 1310 memset(Q->fq_entries, 0, size); 1311 size = sizeof (struct freelQ_ce) * Q->fq_entries_n; 1312 Q->fq_centries = t1_os_malloc_wait_zero(size); 1313 if (!Q->fq_centries) 1314 goto err_no_mem; 1315 memset(Q->fq_centries, 0, size); 1316 } 1317 1318 /* 1319 * Calculate the buffer sizes for the two free lists. FL0 accommodates 1320 * regular sized Ethernet frames, FL1 is sized not to exceed 16K, 1321 * including all the sk_buff overhead. 1322 * For T1C FL0 and FL1 are reversed. 1323 */ 1324 #ifdef NOTYET 1325 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE + 1326 sizeof (struct cpl_rx_data) + 1327 SGE_RX_OFFSET - sge->rx_pkt_pad; 1328 #else 1329 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = 1330 sge->obj->ch_sm_buf_sz; 1331 if (is_T2(sge->obj)) 1332 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz; 1333 else 1334 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz; 1335 #endif 1336 #ifdef NOTYET 1337 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) - 1338 SKB_DATA_ALIGN(sizeof (struct skb_shared_info)); 1339 #else 1340 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz; 1341 if (is_T2(sge->obj)) 1342 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz; 1343 else 1344 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz; 1345 #endif 1346 1347 sge->respQ.rq_genbit = 1; 1348 sge->respQ.rq_entries_n = sge_respq_cnt; 1349 sge->respQ.rq_credits = sge_respq_cnt; 1350 sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2); 1351 size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1352 1353 sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1354 size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh), 1355 &(sge->respQ.rq_ah), 0); 1356 1357 if (!sge->respQ.rq_entries) 1358 goto err_no_mem; 1359 memset(sge->respQ.rq_entries, 0, size); 1360 return (0); 1361 1362 err_no_mem: 1363 free_rx_resources(sge); 1364 return (1); 1365 } 1366 1367 /* 1368 * Allocates basic TX resources, consisting of memory mapped command Qs. 1369 */ 1370 static int 1371 alloc_tx_resources(pesge *sge, struct sge_params *p) 1372 { 1373 unsigned int size, i; 1374 1375 for (i = 0; i < SGE_CMDQ_N; i++) { 1376 struct cmdQ *Q = &sge->cmdQ[i]; 1377 1378 Q->cq_genbit = 1; 1379 Q->cq_entries_n = p->cmdQ_size[i]; 1380 atomic_set(&Q->cq_credits, Q->cq_entries_n); 1381 atomic_set(&Q->cq_asleep, 1); 1382 1383 mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER, 1384 sge->obj->ch_icookp); 1385 1386 size = sizeof (cmdQ_e) * Q->cq_entries_n; 1387 Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1388 size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT); 1389 1390 if (!Q->cq_entries) 1391 goto err_no_mem; 1392 memset(Q->cq_entries, 0, size); 1393 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1394 Q->cq_centries = t1_os_malloc_wait_zero(size); 1395 if (!Q->cq_centries) 1396 goto err_no_mem; 1397 memset(Q->cq_centries, 0, size); 1398 1399 /* allocate pre-mapped dma headers */ 1400 pe_dma_handle_init(sge->obj, Q->cq_entries_n); 1401 } 1402 1403 return (0); 1404 1405 err_no_mem: 1406 free_tx_resources(sge); 1407 return (1); 1408 } 1409 1410 /* 1411 * Sets the interrupt latency timer when the adaptive Rx coalescing 1412 * is turned off. Do nothing when it is turned on again. 1413 * 1414 * This routine relies on the fact that the caller has already set 1415 * the adaptive policy in adapter->sge_params before calling it. 1416 */ 1417 int 1418 t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p) 1419 { 1420 if (!p->coalesce_enable) { 1421 u32 newTimer = p->rx_coalesce_usecs * 1422 (board_info(sge->obj)->clock_core / 1000000); 1423 1424 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer); 1425 } 1426 return (0); 1427 } 1428 1429 /* 1430 * Programs the various SGE registers. However, the engine is not yet enabled, 1431 * but sge->sge_control is setup and ready to go. 1432 */ 1433 static void 1434 configure_sge(pesge *sge, struct sge_params *p) 1435 { 1436 ch_t *ap = sge->obj; 1437 int i; 1438 1439 t1_write_reg_4(ap, A_SG_CONTROL, 0); 1440 1441 setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n, 1442 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE); 1443 setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n, 1444 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE); 1445 setup_ring_params(ap, sge->freelQ[0].fq_pa, 1446 sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR, 1447 A_SG_FL0BASEUPR, A_SG_FL0SIZE); 1448 setup_ring_params(ap, sge->freelQ[1].fq_pa, 1449 sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR, 1450 A_SG_FL1BASEUPR, A_SG_FL1SIZE); 1451 1452 /* The threshold comparison uses <. */ 1453 t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) - 1454 SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1); 1455 setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n, 1456 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE); 1457 t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n); 1458 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE | 1459 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE | 1460 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE | 1461 #if 1 1462 /* 1463 * if the the following bit is not set, then we'll get an 1464 * interrupt everytime command Q 0 goes empty. Since we're 1465 * always ringing the doorbell, we can turn it on. 1466 */ 1467 F_DISABLE_CMDQ0_GTS | 1468 #endif 1469 V_RX_PKT_OFFSET(sge->rx_pkt_pad); 1470 1471 #if BYTE_ORDER == BIG_ENDIAN 1472 sge->sge_control |= F_ENABLE_BIG_ENDIAN; 1473 #endif 1474 1475 /* 1476 * Initialize the SGE Interrupt Timer arrray: 1477 * intrtimer[0] = (SGE_INTRTIMER0) usec 1478 * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec 1479 * intrtimer[10] = (SGE_INTRTIMER1) usec 1480 * 1481 */ 1482 sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000; 1483 for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) { 1484 sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i); 1485 sge->intrtimer[i] *= sge->intrtimer[0]; 1486 } 1487 sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] = 1488 sge->intrtimer[0] * SGE_INTRTIMER1; 1489 /* Initialize resource timer */ 1490 sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] * 1491 SGE_INTRTIMER_NRES); 1492 /* Finally finish initialization of intrtimer[0] */ 1493 sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0); 1494 /* Initialize for a throughput oriented workload */ 1495 sge->currIndex = SGE_INTR_MAXBUCKETS - 1; 1496 1497 if (p->coalesce_enable) 1498 t1_write_reg_4(ap, A_SG_INTRTIMER, 1499 sge->intrtimer[sge->currIndex]); 1500 else 1501 (void) t1_sge_set_coalesce_params(sge, p); 1502 } 1503 1504 static inline void 1505 setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo, 1506 int base_reg_hi, int size_reg) 1507 { 1508 t1_write_reg_4(adapter, base_reg_lo, (u32)addr); 1509 t1_write_reg_4(adapter, base_reg_hi, addr >> 32); 1510 t1_write_reg_4(adapter, size_reg, size); 1511 } 1512 1513 /* 1514 * Frees RX resources. 1515 */ 1516 static void 1517 free_rx_resources(pesge *sge) 1518 { 1519 unsigned int size, i; 1520 1521 if (sge->respQ.rq_entries) { 1522 size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1523 1524 pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries, 1525 sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah); 1526 } 1527 1528 for (i = 0; i < SGE_FREELQ_N; i++) { 1529 struct freelQ *Q = &sge->freelQ[i]; 1530 1531 if (Q->fq_centries) { 1532 free_freelQ_buffers(sge, Q); 1533 1534 t1_os_free(Q->fq_centries, 1535 Q->fq_entries_n * sizeof (freelQ_ce_t)); 1536 } 1537 if (Q->fq_entries) { 1538 size = sizeof (freelQ_e) * Q->fq_entries_n; 1539 1540 /* free the freelist queue */ 1541 pe_os_free_contig(sge->obj, size, Q->fq_entries, 1542 Q->fq_pa, Q->fq_dh, Q->fq_ah); 1543 1544 } 1545 } 1546 } 1547 1548 /* 1549 * Frees all RX buffers on the freelist Q. The caller must make sure that 1550 * the SGE is turned off before calling this function. 1551 */ 1552 static void 1553 free_freelQ_buffers(pesge *sge, struct freelQ *Q) 1554 { 1555 struct freelQ_ce *ce; 1556 struct freelQ_ce *cq = Q->fq_centries; 1557 uint32_t credits = Q->fq_credits; 1558 uint32_t entries_n = Q->fq_entries_n; 1559 uint32_t cidx = Q->fq_cidx; 1560 uint32_t i = Q->fq_id; 1561 1562 ce = &cq[cidx]; 1563 1564 credits = entries_n; 1565 while (credits--) { 1566 mblk_t *mp; 1567 if ((mp = ce->fe_mp) != NULL) { 1568 /* bump in-use count of receive buffers */ 1569 if (i != sge->jumbo_fl) { 1570 atomic_add(1, 1571 &buffers_in_use[sge->obj->ch_sm_index]); 1572 } else { 1573 atomic_add(1, 1574 &buffers_in_use[sge->obj->ch_big_index]); 1575 } 1576 1577 /* 1578 * note. freeb() callback of esb-alloced mblk will 1579 * cause receive buffer to be put back on sa free list. 1580 */ 1581 freeb(mp); 1582 ce->fe_mp = NULL; 1583 } 1584 1585 ce++; 1586 if (++cidx == entries_n) { 1587 cidx = 0; 1588 ce = cq; 1589 } 1590 } 1591 1592 Q->fq_cidx = cidx; 1593 Q->fq_credits = credits; 1594 } 1595 1596 /* 1597 * Free TX resources. 1598 * 1599 * Assumes that SGE is stopped and all interrupts are disabled. 1600 */ 1601 static void 1602 free_tx_resources(pesge *sge) 1603 { 1604 unsigned int size; 1605 uint32_t i; 1606 1607 for (i = 0; i < SGE_CMDQ_N; i++) { 1608 struct cmdQ *Q = &sge->cmdQ[i]; 1609 1610 if (Q->cq_centries) { 1611 unsigned int pending = Q->cq_entries_n - 1612 atomic_read(Q->cq_credits); 1613 1614 mutex_destroy(&Q->cq_qlock); 1615 1616 if (pending) 1617 free_cmdQ_buffers(sge, Q, pending); 1618 1619 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1620 t1_os_free(Q->cq_centries, size); 1621 } 1622 1623 if (Q->cq_entries) { 1624 size = sizeof (cmdQ_e) * Q->cq_entries_n; 1625 pe_os_free_contig(sge->obj, size, Q->cq_entries, 1626 Q->cq_pa, Q->cq_dh, Q->cq_ah); 1627 } 1628 } 1629 } 1630 1631 /* 1632 * Return the payload capacity of the jumbo free-list buffers. 1633 */ 1634 static inline unsigned int jumbo_payload_capacity(pesge *sge) 1635 { 1636 return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size - 1637 sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset); 1638 } 1639 1640 /* PR2928 & PR3309 */ 1641 void 1642 t1_sge_set_ptimeout(adapter_t *adapter, u32 val) 1643 { 1644 pesge *sge = adapter->sge; 1645 1646 if (is_T2(adapter)) 1647 sge->ptimeout = max(val, 1); 1648 } 1649 1650 /* PR2928 & PR3309 */ 1651 u32 1652 t1_sge_get_ptimeout(adapter_t *adapter) 1653 { 1654 pesge *sge = adapter->sge; 1655 1656 return (is_T2(adapter) ? sge->ptimeout : 0); 1657 } 1658 1659 void 1660 sge_add_fake_arp(pesge *sge, void *bp) 1661 { 1662 sge->pskb = bp; 1663 } 1664 1665 #ifdef SUN_KSTATS 1666 static int 1667 sge_kstat_setup(pesge *sge) 1668 { 1669 int status; 1670 p_kstat_t ksp; 1671 size_t ch_kstat_sz; 1672 p_ch_kstat_t chkp; 1673 char kstat_name[32]; 1674 int instance; 1675 int i; 1676 1677 status = -1; 1678 ch_kstat_sz = sizeof (ch_kstat_t); 1679 instance = ddi_get_instance(sge->obj->ch_dip); 1680 if ((ksp = kstat_create(CHNAME "_debug", instance, 1681 NULL, "net_debug", KSTAT_TYPE_NAMED, 1682 ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL) 1683 goto sge_kstat_setup_exit; 1684 chkp = (p_ch_kstat_t)ksp->ks_data; 1685 kstat_named_init(&chkp->respQ_empty, "respQ_empty", 1686 KSTAT_DATA_UINT32); 1687 kstat_named_init(&chkp->respQ_overflow, "respQ_overflow", 1688 KSTAT_DATA_UINT32); 1689 kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty", 1690 KSTAT_DATA_UINT32); 1691 kstat_named_init(&chkp->pkt_too_big, "pkt_too_big", 1692 KSTAT_DATA_UINT32); 1693 kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch", 1694 KSTAT_DATA_UINT32); 1695 kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]", 1696 KSTAT_DATA_UINT32); 1697 kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]", 1698 KSTAT_DATA_UINT32); 1699 kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]", 1700 KSTAT_DATA_UINT32); 1701 kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]", 1702 KSTAT_DATA_UINT32); 1703 kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups", 1704 KSTAT_DATA_UINT32); 1705 kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups", 1706 KSTAT_DATA_UINT32); 1707 kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag", 1708 KSTAT_DATA_UINT32); 1709 kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag", 1710 KSTAT_DATA_UINT32); 1711 kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums", 1712 KSTAT_DATA_UINT32); 1713 kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space", 1714 KSTAT_DATA_UINT32); 1715 kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks", 1716 KSTAT_DATA_UINT32); 1717 kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails", 1718 KSTAT_DATA_UINT32); 1719 kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails", 1720 KSTAT_DATA_UINT32); 1721 kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails", 1722 KSTAT_DATA_UINT32); 1723 kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails", 1724 KSTAT_DATA_UINT32); 1725 kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0", 1726 KSTAT_DATA_UINT32); 1727 kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1", 1728 KSTAT_DATA_UINT32); 1729 kstat_named_init(&chkp->rx_flq0, "rx_flq0", 1730 KSTAT_DATA_UINT32); 1731 kstat_named_init(&chkp->rx_flq1, "rx_flq1", 1732 KSTAT_DATA_UINT32); 1733 kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz", 1734 KSTAT_DATA_UINT32); 1735 kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz", 1736 KSTAT_DATA_UINT32); 1737 kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops", 1738 KSTAT_DATA_UINT32); 1739 kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied", 1740 KSTAT_DATA_UINT32); 1741 kstat_named_init(&chkp->rx_pause_on, "rx_pause_on", 1742 KSTAT_DATA_UINT32); 1743 kstat_named_init(&chkp->rx_pause_off, "rx_pause_off", 1744 KSTAT_DATA_UINT32); 1745 kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms", 1746 KSTAT_DATA_UINT32); 1747 kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike", 1748 KSTAT_DATA_UINT32); 1749 kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits", 1750 KSTAT_DATA_UINT32); 1751 kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails", 1752 KSTAT_DATA_UINT32); 1753 kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs", 1754 KSTAT_DATA_UINT32); 1755 kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop", 1756 KSTAT_DATA_UINT32); 1757 kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt", 1758 KSTAT_DATA_UINT32); 1759 kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt", 1760 KSTAT_DATA_UINT32); 1761 kstat_named_init(&chkp->arp_sent, "arp_sent", 1762 KSTAT_DATA_UINT32); 1763 kstat_named_init(&chkp->tx_doorbells, "tx_doorbells", 1764 KSTAT_DATA_UINT32); 1765 kstat_named_init(&chkp->intr_doorbells, "intr_doorbells", 1766 KSTAT_DATA_UINT32); 1767 kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells", 1768 KSTAT_DATA_UINT32); 1769 kstat_named_init(&chkp->sleep_cnt, "sleep_cnt", 1770 KSTAT_DATA_UINT32); 1771 kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt", 1772 KSTAT_DATA_UINT32); 1773 for (i = 0; i < MBLK_MAX; i++) { 1774 (void) sprintf(kstat_name, "tx_descs[%02d]", i); 1775 kstat_named_init(&chkp->tx_descs[i], 1776 kstat_name, KSTAT_DATA_UINT32); 1777 } 1778 ksp->ks_update = sge_kstat_update; 1779 ksp->ks_private = (void *)sge; 1780 sge->ksp = ksp; 1781 kstat_install(ksp); 1782 status = 0; 1783 1784 sge_kstat_setup_exit: 1785 return (status); 1786 } 1787 1788 static void 1789 sge_kstat_remove(pesge *sge) 1790 { 1791 if (sge->ksp) 1792 kstat_delete(sge->ksp); 1793 } 1794 1795 static int 1796 sge_kstat_update(p_kstat_t ksp, int rw) 1797 { 1798 pesge *sge; 1799 p_ch_stats_t statsp; 1800 p_ch_kstat_t chkp; 1801 int i; 1802 1803 sge = (pesge *)ksp->ks_private; 1804 statsp = (p_ch_stats_t)&sge->intr_cnt; 1805 chkp = (p_ch_kstat_t)ksp->ks_data; 1806 if (rw == KSTAT_WRITE) { 1807 statsp->respQ_empty = chkp->respQ_empty.value.ui32; 1808 statsp->respQ_overflow = chkp->respQ_overflow.value.ui32; 1809 statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32; 1810 statsp->pkt_too_big = chkp->pkt_too_big.value.ui32; 1811 statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32; 1812 statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32; 1813 statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32; 1814 statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32; 1815 statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32; 1816 statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32; 1817 statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32; 1818 statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32; 1819 statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32; 1820 statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32; 1821 statsp->tx_need_cpl_space 1822 = chkp->tx_need_cpl_space.value.ui32; 1823 statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32; 1824 statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32; 1825 statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32; 1826 statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32; 1827 statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32; 1828 statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32; 1829 statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32; 1830 statsp->rx_flq0 = chkp->rx_flq0.value.ui32; 1831 statsp->rx_flq1 = chkp->rx_flq1.value.ui32; 1832 statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32; 1833 statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32; 1834 statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32; 1835 statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32; 1836 statsp->rx_pause_on = chkp->rx_pause_on.value.ui32; 1837 statsp->rx_pause_off = chkp->rx_pause_off.value.ui32; 1838 statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32; 1839 statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32; 1840 statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32; 1841 statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32; 1842 statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32; 1843 statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32; 1844 statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32; 1845 statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32; 1846 statsp->arp_sent = chkp->arp_sent.value.ui32; 1847 statsp->tx_doorbells = chkp->tx_doorbells.value.ui32; 1848 statsp->intr_doorbells = chkp->intr_doorbells.value.ui32; 1849 statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32; 1850 statsp->sleep_cnt = chkp->sleep_cnt.value.ui32; 1851 statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32; 1852 for (i = 0; i < MBLK_MAX; i++) { 1853 statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32; 1854 } 1855 } else { 1856 chkp->respQ_empty.value.ui32 = statsp->respQ_empty; 1857 chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow; 1858 chkp->freelistQ_empty.value.ui32 1859 = statsp->freelistQ_empty; 1860 chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big; 1861 chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch; 1862 chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0]; 1863 chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1]; 1864 chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0]; 1865 chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1]; 1866 chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups; 1867 chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups; 1868 chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag; 1869 chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag; 1870 chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums; 1871 chkp->tx_need_cpl_space.value.ui32 1872 = statsp->tx_need_cpl_space; 1873 chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks; 1874 chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1; 1875 chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2; 1876 chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1; 1877 chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2; 1878 chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0; 1879 chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1; 1880 chkp->rx_flq0.value.ui32 = statsp->rx_flq0; 1881 chkp->rx_flq1.value.ui32 = statsp->rx_flq1; 1882 chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz; 1883 chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz; 1884 chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops; 1885 chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied; 1886 chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on; 1887 chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off; 1888 chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms; 1889 chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike; 1890 chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits; 1891 chkp->rx_flbuf_fails.value.ui32 1892 = statsp->rx_flbuf_fails; 1893 chkp->rx_flbuf_allocs.value.ui32 1894 = statsp->rx_flbuf_allocs; 1895 chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop; 1896 chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt; 1897 chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt; 1898 chkp->arp_sent.value.ui32 = statsp->arp_sent; 1899 chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells; 1900 chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells; 1901 chkp->intr1_doorbells.value.ui32 1902 = statsp->intr1_doorbells; 1903 chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt; 1904 chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt; 1905 for (i = 0; i < MBLK_MAX; i++) { 1906 chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i]; 1907 } 1908 } 1909 return (0); 1910 } 1911 #endif 1912 1913 static uint16_t 1914 calc_ocsum(mblk_t *mp, int offset) 1915 { 1916 uint8_t *addrp; 1917 uint32_t src; 1918 uint32_t dst; 1919 1920 ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset); 1921 uint32_t sum; 1922 int iplen = IPH_HDR_LENGTH(ihdr); 1923 struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen); 1924 uchar_t *byte; 1925 int len; 1926 1927 addrp = (uint8_t *)&ihdr->ipha_src; 1928 src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1929 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1930 1931 addrp = (uint8_t *)&ihdr->ipha_dst; 1932 dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1933 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1934 1935 sum = (uint16_t)(src >> 16) + 1936 (uint16_t)(src) + 1937 (uint16_t)(dst >> 16) + 1938 (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP)); 1939 1940 sum = (uint16_t)(sum >> 16) + (uint16_t)(sum); 1941 1942 if (sum > 0xffff) 1943 sum -= 0xffff; 1944 1945 udpp->uh_sum = 0; 1946 byte = mp->b_rptr + offset + iplen; 1947 do { 1948 len = (mp->b_wptr - byte); 1949 sum = bcksum(byte, len, sum); 1950 if (sum > 0xffff) 1951 sum -= 0xffff; 1952 mp = mp->b_cont; 1953 if (mp) 1954 byte = mp->b_rptr; 1955 } while (mp); 1956 1957 sum = ~sum & 0xffff; 1958 1959 return (sum); 1960 } 1961