1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * This file is part of the Chelsio T1 Ethernet driver. 29 * 30 * Copyright (C) 2003-2005 Chelsio Communications. All rights reserved. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/param.h> 35 #include <sys/cmn_err.h> 36 #include <sys/sunddi.h> 37 #include <sys/kmem.h> 38 #include <sys/cmn_err.h> 39 #include <sys/byteorder.h> 40 #include <sys/atomic.h> 41 #include <sys/stropts.h> 42 #include <sys/stream.h> 43 #include <sys/strsubr.h> 44 #include <sys/dlpi.h> 45 #include <sys/kstat.h> 46 #include <sys/ethernet.h> 47 #include <netinet/in.h> 48 #include <netinet/udp.h> 49 #include <inet/common.h> 50 #include <inet/nd.h> 51 #include <inet/ip.h> 52 #include <inet/tcp.h> 53 #include <netinet/udp.h> 54 #include <sys/gld.h> 55 #include "ostypes.h" 56 #include "common.h" 57 #ifdef CONFIG_CHELSIO_T1_1G 58 #include "fpga_defs.h" 59 #endif 60 #include "regs.h" 61 #include "suni1x10gexp_regs.h" 62 #include "sge.h" 63 #include "espi.h" 64 65 #include "ch.h" 66 67 extern uint32_t buffers_in_use[]; 68 69 uint32_t sge_cmdq0_cnt = SGE_CMDQ0_E_N; 70 uint32_t sge_cmdq1_cnt = SGE_CMDQ1_E_N; 71 uint32_t sge_flq0_cnt = SGE_FREELQ0_E_N; 72 uint32_t sge_flq1_cnt = SGE_FREELQ1_E_N; 73 uint32_t sge_respq_cnt = SGE_RESPQ_E_N; 74 75 uint32_t sge_cmdq0_cnt_orig = SGE_CMDQ0_E_N; 76 uint32_t sge_cmdq1_cnt_orig = SGE_CMDQ1_E_N; 77 uint32_t sge_flq0_cnt_orig = SGE_FREELQ0_E_N; 78 uint32_t sge_flq1_cnt_orig = SGE_FREELQ1_E_N; 79 uint32_t sge_respq_cnt_orig = SGE_RESPQ_E_N; 80 81 #ifdef HOST_PAUSE 82 uint32_t do_host_pause = 1; 83 uint32_t flq_pause_window = 64; 84 #endif 85 86 static uint64_t os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, 87 ulong_t *dh); 88 void pe_os_free_contig(ch_t *, size_t, void *, uint64_t, ulong_t, ulong_t); 89 90 static inline uint32_t t1_sge_rx(pesge *sge, freelQ_t *Q, 91 unsigned int len, unsigned int offload); 92 #ifdef HOST_PAUSE 93 static void t1_sge_check_pause(pesge *sge, struct freelQ *Q); 94 #endif 95 static void alloc_freelQ_buffers(pesge *sge, struct freelQ *Q); 96 static void freelQs_empty(pesge *sge); 97 static void free_cmdQ_buffers(pesge *sge, cmdQ_t *Q, uint32_t credits_pend); 98 static int alloc_rx_resources(pesge *sge, struct sge_params *p); 99 static int alloc_tx_resources(pesge *sge, struct sge_params *p); 100 static inline void setup_ring_params(ch_t *adapter, u64 addr, u32 size, 101 int base_reg_lo, int base_reg_hi, int size_reg); 102 static void configure_sge(pesge *sge, struct sge_params *p); 103 static void free_freelQ_buffers(pesge *sge, struct freelQ *Q); 104 static void free_rx_resources(pesge *sge); 105 static void free_tx_resources(pesge *sge); 106 static inline unsigned int jumbo_payload_capacity(pesge *sge); 107 #ifdef SUN_KSTATS 108 static int sge_kstat_setup(pesge *); 109 static void sge_kstat_remove(pesge *); 110 static int sge_kstat_update(p_kstat_t, int); 111 #endif 112 static uint16_t calc_ocsum(mblk_t *, int); 113 114 /* 115 * Local routines. 116 */ 117 static inline void sge_ring_doorbell(pesge *sge, u32 control_reg); 118 119 static inline void 120 sge_ring_doorbell(pesge *sge, u32 control_reg) 121 { 122 membar_producer(); 123 t1_write_reg_4(sge->obj, A_SG_DOORBELL, control_reg); 124 } 125 126 /* 127 * DESC: 128 * 129 * NOTES: Must have at least 1 command queue and 1 freelist queue. 130 * 131 */ 132 pesge * 133 t1_sge_create(ch_t *sa, struct sge_params *p) 134 { 135 pesge *sge; 136 137 sge = t1_os_malloc_wait_zero(sizeof (pesge)); 138 139 if (sge == NULL) 140 goto error_no_mem; 141 142 memset(sge, 0, sizeof (*sge)); 143 144 /* 145 * PR2928 & PR3309 146 * set default timeout value - 20 msec 147 * we set the initial value to 2 which gurantees at least one tick. 148 */ 149 if (is_T2(sa)) 150 sge->ptimeout = 1; 151 152 sge->obj = sa; 153 #ifdef SUN_KSTATS 154 if (sge_kstat_setup(sge) != 0) 155 goto t1_sge_create_fail1; 156 #endif 157 p->cmdQ_size[0] = sge_cmdq0_cnt; 158 p->cmdQ_size[1] = sge_cmdq1_cnt; 159 160 /* note that jumbo frame index is inverted for T2 */ 161 if (is_T2(sa)) { 162 p->freelQ_size[1] = sge_flq0_cnt; 163 p->freelQ_size[0] = sge_flq1_cnt; 164 } else { 165 p->freelQ_size[0] = sge_flq0_cnt; 166 p->freelQ_size[1] = sge_flq1_cnt; 167 } 168 169 #if CH_DEBUG 170 /* DEBUG only */ 171 cmn_err(CE_NOTE, "sge: %p\n", sge); 172 cmn_err(CE_NOTE, "&sge->cmdQ[0]: %p\n", &sge->cmdQ[0]); 173 cmn_err(CE_NOTE, "&sge->freelQ[0]: %p\n", &sge->freelQ[0]); 174 cmn_err(CE_NOTE, "&sge->freelQ[1]: %p\n", &sge->freelQ[1]); 175 cmn_err(CE_NOTE, "&sge->respQ: %p\n", &sge->respQ); 176 cmn_err(CE_NOTE, "&sge->intr_cnt: %p\n", &sge->intr_cnt); 177 #endif 178 #ifdef SUN_KSTATS 179 goto error_no_mem; 180 181 t1_sge_create_fail1: 182 t1_os_free(sge, sizeof (pesge)); 183 sge = NULL; 184 #endif 185 error_no_mem: 186 return (sge); 187 } 188 189 int 190 t1_sge_destroy(pesge* sge) 191 { 192 if (sge != NULL) { 193 free_tx_resources(sge); 194 free_rx_resources(sge); 195 196 /* PR2928 & PR3309 */ 197 if ((is_T2(sge->obj)) && (sge->pskb)) 198 pe_free_fake_arp(sge->pskb); 199 #ifdef SUN_KSTATS 200 sge_kstat_remove(sge); 201 #endif 202 t1_os_free(sge, sizeof (pesge)); 203 } 204 return (0); 205 } 206 207 /* 208 * PR2928 & PR3309 209 * call out event from timeout 210 * 211 * there is a potential race between the timeout and the close. 212 * unless we protect the timeout, the close could occur at the 213 * same time. Then if the timeout service routine was slow or 214 * interrupted, the sge_stop() could complete with a timeoutID 215 * that has expired, thus letting another timeout occur. If the 216 * service routine was delayed still further, a detach could occur. 217 * the second time could then end up accessing memory that has been 218 * released back to the system. Bad things could then occur. We 219 * set a flag in sge_stop() to tell the service routine not to 220 * issue further timeouts. sge_stop() will block until a timeout 221 * has occured. If the command Q is full then we shouldn't put out 222 * an arp. 223 */ 224 225 void 226 t1_espi_workaround(ch_t *adapter) 227 { 228 pesge *sge = adapter->sge; 229 ch_t *chp = (ch_t *)sge->obj; 230 int rv = 1; 231 232 if ((chp->ch_state == PERUNNING) && 233 atomic_read(&sge->cmdQ[0].cq_asleep)) { 234 u32 seop; 235 seop = t1_espi_get_mon(adapter, 0x930, 0); 236 if ((seop & 0xfff0fff) == 0xfff) { 237 /* after first arp */ 238 if (sge->pskb) { 239 rv = pe_start(adapter, (mblk_t *)sge->pskb, 240 CH_ARP); 241 if (!rv) 242 sge->intr_cnt.arp_sent++; 243 } 244 } 245 } 246 #ifdef HOST_PAUSE 247 /* 248 * If we are already in sge_data_in, then we can skip calling 249 * t1_sge_check_pause() this clock cycle. lockstat showed that 250 * we were blocking on the mutex ~ 2% of the time. 251 */ 252 if (mutex_tryenter(&adapter->ch_intr)) { 253 t1_sge_check_pause(sge, &sge->freelQ[0]); 254 t1_sge_check_pause(sge, &sge->freelQ[1]); 255 mutex_exit(&adapter->ch_intr); 256 } 257 #endif 258 } 259 260 int 261 sge_start(pesge *sge) 262 { 263 t1_write_reg_4(sge->obj, A_SG_CONTROL, sge->sge_control); 264 /* PR2928 & PR3309, also need to avoid Pause deadlock */ 265 ch_init_cyclic(sge->obj, &sge->espi_wa_cyclic, 266 (void (*)(void *))t1_espi_workaround, sge->obj); 267 ch_start_cyclic(&sge->espi_wa_cyclic, sge->ptimeout); 268 return (0); 269 } 270 271 /* 272 * Disables SGE queues. 273 */ 274 int 275 sge_stop(pesge *sge) 276 { 277 uint32_t status; 278 int loops; 279 280 DBGASSERT(sge); 281 282 /* PR2928 & PR3309, also need to avoid Pause deadlock */ 283 t1_write_reg_4(sge->obj, A_SG_CONTROL, 0x0); 284 285 /* wait until there's no more outstanding interrupts pending */ 286 loops = 0; 287 do { 288 status = t1_read_reg_4(sge->obj, A_SG_INT_CAUSE); 289 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, status); 290 drv_usecwait(125); 291 loops++; 292 } while (status && (loops < 1000)); 293 294 ch_stop_cyclic(&sge->espi_wa_cyclic); 295 296 return (0); 297 } 298 299 uint32_t sge_cmdq_send_fail; 300 301 int 302 sge_data_out(pesge* sge, int qid, mblk_t *m0, 303 cmdQ_ce_t *cmp, int count, uint32_t flg) 304 { 305 struct cmdQ *Q = &sge->cmdQ[qid]; 306 ddi_dma_handle_t dh = (ddi_dma_handle_t)sge->cmdQ[qid].cq_dh; 307 spinlock_t *qlock = &Q->cq_qlock; 308 cmdQ_e *e; 309 cmdQ_e *q = Q->cq_entries; 310 uint32_t credits; 311 uint32_t pidx; 312 uint32_t genbit; 313 uint32_t entries_n = Q->cq_entries_n; 314 cmdQ_ce_t *ce; 315 cmdQ_ce_t *cq = Q->cq_centries; 316 dma_addr_t mapping; 317 uint32_t j = 0; 318 uint32_t offset; 319 #if defined(TX_CKSUM_FIX) 320 uint16_t csum; 321 uint16_t *csum_loc; 322 #endif 323 #ifdef TX_THREAD_RECLAIM 324 uint32_t reclaim_cnt; 325 #endif 326 327 /* 328 * We must exit if we don't have enough free command queue entries 329 * available. 330 */ 331 332 spin_lock(qlock); 333 334 #if defined(TX_CKSUM_FIX) 335 /* 336 * This checksum fix will address a fragmented datagram 337 * checksum error. Which will lead to the next packet after 338 * the last packet with the More fragment bit set having its 339 * checksum corrupted. When the packet reaches this point 340 * the 'flg' variable indicates whether a checksum is needed 341 * or not. The algorithm is as follows, if the current packet 342 * is a More fragment set the count of packets to be checksummed 343 * after it to 3. If it't not and the count of is more than 0 344 * then calculate the checksum in software, if a hardware checksum 345 * was requested. Then decrment the count. Same algorithm applies 346 * to TCP. 347 */ 348 if (flg & CH_UDP_MF) { 349 sge->do_udp_csum = 3; 350 } else if ((flg & CH_UDP) && (sge->do_udp_csum != 0)) { 351 if ((flg & CH_NO_HWCKSUM) == 0) { 352 /* 353 * Calc Checksum here. 354 */ 355 csum = calc_ocsum(m0, 356 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 357 csum_loc = (uint16_t *)(m0->b_rptr + 358 sizeof (struct ether_header) + CPL_FORMAT_0_SIZE); 359 csum_loc += (((*(char *)csum_loc) & 0x0f) << 1); 360 361 sge->intr_cnt.tx_soft_cksums++; 362 ((struct udphdr *)(csum_loc))->uh_sum = csum; 363 ((struct cpl_tx_pkt *)m0->b_rptr)->l4_csum_dis = 1; 364 } 365 sge->do_udp_csum--; 366 } else if (flg & CH_TCP_MF) { 367 sge->do_tcp_csum = 3; 368 } else if (sge->do_tcp_csum != 0) { 369 if ((flg & CH_NO_HWCKSUM) == 0) { 370 sge->intr_cnt.tx_soft_cksums++; 371 /* 372 * Calc Checksum here. 373 */ 374 } 375 sge->do_tcp_csum--; 376 } 377 #endif /* TX_CKSUM_FIX */ 378 #ifdef TX_THREAD_RECLAIM 379 reclaim_cnt = Q->cq_complete; 380 if (reclaim_cnt > SGE_BATCH_THRESH) { 381 sge->intr_cnt.tx_reclaims[qid]++; 382 free_cmdQ_buffers(sge, Q, reclaim_cnt); 383 Q->cq_complete = 0; 384 } 385 #endif 386 genbit = Q->cq_genbit; 387 pidx = Q->cq_pidx; 388 credits = Q->cq_credits; 389 390 if ((credits - 1) < count) { 391 spin_unlock(qlock); 392 sge->intr_cnt.cmdQ_full[qid]++; 393 return (1); 394 } 395 396 atomic_sub(count, &Q->cq_credits); 397 Q->cq_pidx += count; 398 if (Q->cq_pidx >= entries_n) { 399 Q->cq_pidx -= entries_n; 400 Q->cq_genbit ^= 1; 401 } 402 403 spin_unlock(qlock); 404 405 #ifdef SUN_KSTATS 406 if (count > MBLK_MAX) 407 sge->intr_cnt.tx_descs[MBLK_MAX - 1]++; 408 else 409 sge->intr_cnt.tx_descs[count]++; 410 #endif 411 412 ce = &cq[pidx]; 413 *ce = *cmp; 414 mapping = cmp->ce_pa; 415 j++; 416 417 e = &q[pidx]; 418 419 offset = (caddr_t)e - (caddr_t)q; 420 421 e->Sop = 1; 422 e->DataValid = 1; 423 e->BufferLength = cmp->ce_len; 424 e->AddrHigh = ((u64)mapping >> 32); 425 e->AddrLow = ((u64)mapping & 0xffffffff); 426 427 --count; 428 if (count > 0) { 429 unsigned int i; 430 431 e->Eop = 0; 432 wmb(); 433 e->GenerationBit = e->GenerationBit2 = genbit; 434 435 for (i = 0; i < count; i++) { 436 437 ce++; 438 e++; 439 cmp++; 440 if (++pidx == entries_n) { 441 pidx = 0; 442 genbit ^= 1; 443 /* sync from offset to end of cmdQ */ 444 (void) ddi_dma_sync(dh, (off_t)(offset), 445 j*sizeof (*e), DDI_DMA_SYNC_FORDEV); 446 offset = j = 0; 447 ce = cq; 448 e = q; 449 } 450 451 *ce = *cmp; 452 mapping = cmp->ce_pa; 453 j++; 454 e->Sop = 0; 455 e->DataValid = 1; 456 e->BufferLength = cmp->ce_len; 457 e->AddrHigh = ((u64)mapping >> 32); 458 e->AddrLow = ((u64)mapping & 0xffffffff); 459 460 if (i < (count - 1)) { 461 e->Eop = 0; 462 wmb(); 463 e->GenerationBit = e->GenerationBit2 = genbit; 464 } 465 } 466 } 467 468 ce->ce_mp = m0; 469 470 e->Eop = 1; 471 wmb(); 472 e->GenerationBit = e->GenerationBit2 = genbit; 473 474 (void) ddi_dma_sync(dh, (off_t)(offset), j*sizeof (*e), 475 DDI_DMA_SYNC_FORDEV); 476 477 /* 478 * We always ring the doorbell for cmdQ1. For cmdQ0, we only ring 479 * the doorbell if the Q is asleep. There is a natural race, where 480 * the hardware is going to sleep just after we checked, however, 481 * then the interrupt handler will detect the outstanding TX packet 482 * and ring the doorbell for us. 483 */ 484 if (qid) { 485 doorbell_pio(sge, F_CMDQ1_ENABLE); 486 } else { 487 if (atomic_read(Q->cq_asleep)) { 488 atomic_set(&Q->cq_asleep, 0); 489 /* NOT YET doorbell_pio(sge, F_CMDQ0_ENABLE); */ 490 atomic_set(&Q->cq_pio_pidx, Q->cq_pidx); 491 } 492 } 493 doorbell_pio(sge, F_CMDQ0_ENABLE); 494 495 return (0); 496 } 497 498 #define SGE_PL_INTR_MASK (F_PL_INTR_SGE_ERR | F_PL_INTR_SGE_DATA) 499 500 /* 501 * Disable SGE error interrupts. 502 */ 503 int 504 t1_sge_intr_disable(pesge* sge) 505 { 506 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 507 508 t1_write_reg_4(sge->obj, A_PL_ENABLE, val & ~SGE_PL_INTR_MASK); 509 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, 0); 510 return (0); 511 } 512 513 #define SGE_INT_ENABLE (F_RESPQ_EXHAUSTED | F_RESPQ_OVERFLOW | \ 514 F_FL_EXHAUSTED | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 515 516 /* 517 * Enable SGE error interrupts. 518 */ 519 int 520 t1_sge_intr_enable(pesge* sge) 521 { 522 u32 en = SGE_INT_ENABLE; 523 u32 val = t1_read_reg_4(sge->obj, A_PL_ENABLE); 524 525 t1_write_reg_4(sge->obj, A_PL_ENABLE, val | SGE_PL_INTR_MASK); 526 527 if (sge->obj->ch_flags & TSO_CAPABLE) 528 en &= ~F_PACKET_TOO_BIG; 529 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, en); 530 return (0); 531 } 532 533 /* 534 * Clear SGE error interrupts. 535 */ 536 int 537 t1_sge_intr_clear(pesge* sge) 538 { 539 t1_write_reg_4(sge->obj, A_PL_CAUSE, SGE_PL_INTR_MASK); 540 t1_write_reg_4(sge->obj, A_SG_INT_CAUSE, 0xffffffff); 541 return (0); 542 } 543 544 #define SGE_INT_FATAL (F_RESPQ_OVERFLOW | F_PACKET_TOO_BIG | F_PACKET_MISMATCH) 545 546 int 547 t1_sge_intr_error_handler(pesge *sge) 548 { 549 peobj *obj = sge->obj; 550 u32 cause = t1_read_reg_4(obj, A_SG_INT_CAUSE); 551 552 if (cause & F_RESPQ_EXHAUSTED) 553 sge->intr_cnt.respQ_empty++; 554 if (cause & F_RESPQ_OVERFLOW) { 555 sge->intr_cnt.respQ_overflow++; 556 cmn_err(CE_WARN, "%s: SGE response queue overflow\n", 557 obj->ch_name); 558 } 559 if (cause & F_FL_EXHAUSTED) { 560 sge->intr_cnt.freelistQ_empty++; 561 freelQs_empty(sge); 562 } 563 if (cause & F_PACKET_TOO_BIG) { 564 sge->intr_cnt.pkt_too_big++; 565 cmn_err(CE_WARN, "%s: SGE max packet size exceeded\n", 566 obj->ch_name); 567 } 568 if (cause & F_PACKET_MISMATCH) { 569 sge->intr_cnt.pkt_mismatch++; 570 cmn_err(CE_WARN, "%s: SGE packet mismatch\n", 571 obj->ch_name); 572 } 573 if (cause & SGE_INT_FATAL) 574 t1_fatal_err(obj); 575 576 t1_write_reg_4(obj, A_SG_INT_CAUSE, cause); 577 return (0); 578 } 579 580 /* 581 * 582 * PARAM: sge - SGE instance pointer. 583 */ 584 int 585 sge_data_in(pesge *sge) 586 { 587 peobj *adapter = sge->obj; 588 struct respQ *Q = &sge->respQ; 589 respQ_e *e; /* response queue entry */ 590 respQ_e *q = Q->rq_entries; /* base response queue */ 591 uint32_t cidx = Q->rq_cidx; 592 uint32_t genbit = Q->rq_genbit; 593 uint32_t entries_n = Q->rq_entries_n; 594 uint32_t credits = Q->rq_credits; 595 uint32_t credits_thresh = Q->rq_credits_thresh; 596 uint32_t ret = 0; 597 #ifndef TX_THREAD_RECLAIM 598 uint32_t credits_pend[2] = {0, 0}; 599 #endif 600 uint32_t flags = 0; 601 uint32_t flagt; 602 ddi_dma_handle_t dh = (ddi_dma_handle_t)Q->rq_dh; 603 604 t1_write_reg_4(adapter, A_PL_CAUSE, F_PL_INTR_SGE_DATA); 605 606 /* 607 * Catch the case where an interrupt arrives 608 * early. 609 */ 610 if ((q == NULL) || (dh == NULL)) { 611 goto check_slow_ints; 612 } 613 614 /* initial response queue entry */ 615 e = &q[cidx]; 616 617 /* pull physical memory of response queue entry into cache */ 618 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 619 sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 620 621 while (e->GenerationBit == genbit) { 622 if (--credits < credits_thresh) { 623 uint32_t n = entries_n - credits - 1; 624 t1_write_reg_4(adapter, A_SG_RSPQUEUECREDIT, n); 625 credits += n; 626 } 627 if (likely(e->DataValid)) { 628 (void) t1_sge_rx(sge, &sge->freelQ[e->FreelistQid], 629 e->BufferLength, e->Offload); 630 if ((e->Sop != 1) || (e->Eop != 1)) { 631 sge->intr_cnt.rx_badEopSop++; 632 cmn_err(CE_WARN, "bad Sop %d or Eop %d: %d", 633 e->Sop, e->Eop, e->BufferLength); 634 } 635 } 636 flagt = e->Qsleeping; 637 flags |= flagt; 638 if (flagt & F_CMDQ0_ENABLE) 639 sge->intr_cnt.rx_cmdq0++; 640 if (flagt & F_CMDQ1_ENABLE) 641 sge->intr_cnt.rx_cmdq1++; 642 if (flagt & F_FL0_ENABLE) 643 sge->intr_cnt.rx_flq0++; 644 if (flagt & F_FL1_ENABLE) 645 sge->intr_cnt.rx_flq1++; 646 #ifdef TX_THREAD_RECLAIM 647 spin_lock(&sge->cmdQ[0].cq_qlock); 648 sge->cmdQ[0].cq_complete += e->Cmdq0CreditReturn; 649 spin_unlock(&sge->cmdQ[0].cq_qlock); 650 spin_lock(&sge->cmdQ[1].cq_qlock); 651 sge->cmdQ[1].cq_complete += e->Cmdq1CreditReturn; 652 if ((adapter->ch_blked) && 653 (sge->cmdQ[0].cq_complete + 654 sge->cmdQ[1].cq_complete) > 16) { 655 adapter->ch_blked = 0; 656 ch_gld_ok(adapter); 657 } 658 spin_unlock(&sge->cmdQ[1].cq_qlock); 659 #else 660 credits_pend[0] += e->Cmdq0CreditReturn; 661 credits_pend[1] += e->Cmdq1CreditReturn; 662 #ifdef CONFIG_SMP 663 if (unlikely(credits_pend[0] > SGE_BATCH_THRESH)) { 664 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 665 credits_pend[0] = 0; 666 } 667 if (unlikely(credits_pend[1] > SGE_BATCH_THRESH)) { 668 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 669 credits_pend[1] = 0; 670 } 671 #endif 672 #endif 673 #ifdef HOST_PAUSE 674 t1_sge_check_pause(sge, &sge->freelQ[e->FreelistQid]); 675 #endif 676 e++; 677 if (unlikely(++cidx == entries_n)) { 678 cidx = 0; 679 genbit ^= 1; 680 e = q; 681 } 682 683 /* pull physical memory of response queue entry into cache */ 684 (void) ddi_dma_sync(dh, (off_t)((caddr_t)e - (caddr_t)q), 685 sizeof (*e), DDI_DMA_SYNC_FORKERNEL); 686 687 ret = 1; 688 } 689 690 #ifndef TX_THREAD_RECLAIM 691 if (credits_pend[0]) 692 free_cmdQ_buffers(sge, &sge->cmdQ[0], credits_pend[0]); 693 if (credits_pend[1]) 694 free_cmdQ_buffers(sge, &sge->cmdQ[1], credits_pend[1]); 695 #endif 696 if (flags & F_CMDQ0_ENABLE) { 697 struct cmdQ *cmdQ = &sge->cmdQ[0]; 698 atomic_set(&cmdQ->cq_asleep, 1); 699 if (atomic_read(cmdQ->cq_pio_pidx) != cmdQ->cq_pidx) { 700 doorbell_pio(sge, F_CMDQ0_ENABLE); 701 atomic_set(&cmdQ->cq_pio_pidx, cmdQ->cq_pidx); 702 } 703 } 704 705 /* the SGE told us one of the free lists is empty */ 706 if (unlikely(flags & (F_FL0_ENABLE | F_FL1_ENABLE))) 707 freelQs_empty(sge); 708 709 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 710 if (adapter->ch_tx_overflow_mutex) 711 mutex_enter(adapter->ch_tx_overflow_mutex); 712 if (adapter->ch_blked && 713 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>2)) && 714 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>2))) { 715 adapter->ch_blked = 0; 716 if (adapter->ch_tx_overflow_cv) 717 cv_broadcast(adapter->ch_tx_overflow_cv); 718 ch_gld_ok(adapter); 719 } 720 if (adapter->ch_tx_overflow_mutex) 721 mutex_exit(adapter->ch_tx_overflow_mutex); 722 #else 723 #ifndef TX_THREAD_RECLAIM 724 if (adapter->ch_blked && 725 (sge->cmdQ[0].cq_credits > (sge->cmdQ[0].cq_entries_n>>1)) && 726 (sge->cmdQ[1].cq_credits > (sge->cmdQ[1].cq_entries_n>>1))) { 727 adapter->ch_blked = 0; 728 ch_gld_ok(adapter); 729 } 730 #endif 731 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 732 733 Q->rq_genbit = genbit; 734 Q->rq_cidx = cidx; 735 Q->rq_credits = credits; 736 737 t1_write_reg_4(adapter, A_SG_SLEEPING, cidx); 738 739 check_slow_ints: 740 /* handle non-data interrupts */ 741 if (unlikely(!ret)) 742 ret = t1_slow_intr_handler(adapter); 743 744 return (ret); 745 } 746 747 /* 748 * allocate a mblk with DMA mapped mblk. 749 * When checksum offload is enabled, we start the DMA at a 2 byte offset so 750 * the IP header will be aligned. We do this for sparc only. 751 */ 752 static uint64_t 753 os_freelist_buffer_alloc(ch_t *sa, int sz, mblk_t **mb, ulong_t *dh) 754 { 755 ch_esb_t *ch_get_small_rbuf(ch_t *sa); 756 ch_esb_t *ch_get_big_rbuf(ch_t *sa); 757 ch_esb_t *rbp; 758 uint32_t rxoff = sa->sge->rx_offset; 759 760 if (sz == SGE_SM_BUF_SZ(sa)) { 761 /* get pre-mapped buffer */ 762 if ((rbp = ch_get_small_rbuf(sa)) == NULL) { 763 sa->norcvbuf++; 764 return ((uint64_t)0); 765 } 766 767 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 768 SGE_SM_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 769 if (*mb == NULL) { 770 mutex_enter(&sa->ch_small_esbl); 771 rbp->cs_next = sa->ch_small_esb_free; 772 sa->ch_small_esb_free = rbp; 773 mutex_exit(&sa->ch_small_esbl); 774 return ((uint64_t)0); 775 } 776 *dh = rbp->cs_dh; 777 778 return (rbp->cs_pa + rxoff); 779 } else { 780 /* get pre-mapped buffer */ 781 if ((rbp = ch_get_big_rbuf(sa)) == NULL) { 782 sa->norcvbuf++; 783 return ((uint64_t)0); 784 } 785 786 *mb = desballoc((unsigned char *)rbp->cs_buf + rxoff, 787 SGE_BG_BUF_SZ(sa)-rxoff, BPRI_MED, &rbp->cs_frtn); 788 if (*mb == NULL) { 789 mutex_enter(&sa->ch_big_esbl); 790 rbp->cs_next = sa->ch_big_esb_free; 791 sa->ch_big_esb_free = rbp; 792 mutex_exit(&sa->ch_big_esbl); 793 return ((uint64_t)0); 794 } 795 *dh = rbp->cs_dh; 796 797 return (rbp->cs_pa + rxoff); 798 } 799 } 800 801 static inline unsigned int 802 t1_sge_rx(pesge *sge, struct freelQ *Q, unsigned int len, unsigned int offload) 803 { 804 mblk_t *skb; 805 peobj *adapter = sge->obj; 806 struct freelQ_ce *cq = Q->fq_centries; 807 struct freelQ_ce *ce = &cq[Q->fq_cidx]; 808 ddi_dma_handle_t dh = (ddi_dma_handle_t)ce->fe_dh; 809 uint32_t cidx = Q->fq_cidx; 810 uint32_t entries_n = Q->fq_entries_n; 811 uint32_t sz = Q->fq_rx_buffer_size; 812 uint32_t useit = 1; 813 uint32_t rxoff = sge->rx_offset; 814 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 815 uint32_t rv; 816 #endif 817 818 if (Q->fq_id) 819 sge->intr_cnt.rx_flq1_cnt++; 820 else 821 sge->intr_cnt.rx_flq0_cnt++; 822 /* 823 * If pkt size falls below threshold, then we'll copy data to 824 * an blk and reuse mblk. 825 * 826 * NOTE that rxoff is 2 for T1 adapters. We align the the start 827 * of the DMA buffer begin at rxoff offset for T1 cards instead of 828 * at the beginning of the buffer, thus the length of the received 829 * data does not include this offset. We therefore always add 830 * SGE_RX_OFFSET to the allocb size so we have space to provide the 831 * offset for the copied data. 832 */ 833 #ifdef HOST_PAUSE 834 /* 835 * If we have Host pause compiled in, then we look at the 836 * free list, if the pause is on and we're not in offload 837 * mode then we drop packets, this is designed to avoid 838 * overwhelming the machine. If the machine is powerfull enough 839 * this will not happen. The 'rx_pkt_drops' will show when 840 * packets are being dropped and how much. 841 */ 842 if ((offload == 0) && adapter->pause_on) { 843 freelQ_e *e; 844 /* Ditch the packet and reuse original buffer */ 845 e = &Q->fq_entries[cidx]; 846 e->GenerationBit ^= 1; 847 e->GenerationBit2 ^= 1; 848 sge->intr_cnt.rx_pkt_drops++; 849 goto rx_entry_consumed; 850 } else if (((adapter->pause_on || 851 (len <= SGE_RX_COPY_THRESHOLD)) && 852 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI)))) 853 #else 854 if ((len <= SGE_RX_COPY_THRESHOLD) && 855 (skb = allocb(len + SGE_RX_OFFSET, BPRI_HI))) 856 #endif 857 { 858 freelQ_e *e; 859 char *src = (char *)((mblk_t *)ce->fe_mp)->b_rptr; 860 861 /* 862 * pull physical memory of pkt data into cache 863 * Note that len does not include offset for T1. 864 */ 865 (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 866 DDI_DMA_SYNC_FORKERNEL); 867 868 if (offload == 0) { 869 /* 870 * create 2 byte offset so IP header aligned on 871 * 4 byte boundry 872 */ 873 skb_reserve(skb, SGE_RX_OFFSET); 874 /* 875 * if hardware inserted 2 byte offset then need to 876 * start copying with extra offset 877 */ 878 src += sge->rx_pkt_pad; 879 } 880 memcpy(skb->b_rptr, src, len); 881 useit = 0; /* mblk copy, don't inc esballoc in use cnt */ 882 883 /* so we can reuse original buffer */ 884 e = &Q->fq_entries[cidx]; 885 e->GenerationBit ^= 1; 886 e->GenerationBit2 ^= 1; 887 sge->intr_cnt.rx_pkt_copied++; 888 } else { 889 /* consume buffer off the ring */ 890 skb = ce->fe_mp; 891 ce->fe_mp = NULL; 892 893 /* 894 * if not offload (tunneled pkt), & hardward padded, then 895 * adjust start of pkt to point to start of data i.e. 896 * skip pad (2 bytes). 897 */ 898 if (!offload && sge->rx_pkt_pad) 899 __skb_pull(skb, SGE_RX_OFFSET); 900 901 /* 902 * pull physical memory of pkt data into cache 903 * Note that len does not include offset for T1. 904 */ 905 (void) ddi_dma_sync(dh, (off_t)(rxoff), len, 906 DDI_DMA_SYNC_FORKERNEL); 907 } 908 909 /* set length of data in skb */ 910 skb_put(skb, len); 911 912 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 913 if (likely(offload)) { 914 if (likely(toe_running(adapter))) { 915 /* sends pkt upstream to toe layer */ 916 if (useit) { 917 uint_t index; 918 if (sz == SGE_SM_BUF_SZ(adapter)) 919 index = adapter->ch_sm_index; 920 else 921 index = adapter->ch_big_index; 922 atomic_add(1, &buffers_in_use[index]); 923 } 924 if (adapter->toe_rcv) 925 adapter->toe_rcv(adapter->ch_toeinst, skb); 926 else 927 freemsg(skb); 928 } else { 929 cmn_err(CE_WARN, 930 "%s: unexpected offloaded packet, cmd %u\n", 931 adapter->ch_name, *skb->b_rptr); 932 933 /* discard packet */ 934 freemsg(skb); 935 } 936 } 937 #else 938 if (unlikely(offload)) { 939 cmn_err(CE_WARN, 940 "%s: unexpected offloaded packet, cmd %u\n", 941 adapter->ch_name, *skb->b_rptr); 942 943 /* discard paket */ 944 freemsg(skb); 945 } 946 #endif 947 else { 948 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)skb->b_rptr; 949 int flg = 0; 950 uint32_t cksum; 951 952 /* adjust beginning of data to skip CPL header */ 953 skb_pull(skb, SZ_CPL_RX_PKT); 954 955 /* extract checksum from CPL header here */ 956 957 /* 958 * bump count of mlbks in used by protocol stack(s) 959 */ 960 if (useit) { 961 if (sz == SGE_SM_BUF_SZ(adapter)) { 962 atomic_add(1, 963 &buffers_in_use[adapter->ch_sm_index]); 964 } else { 965 atomic_add(1, 966 &buffers_in_use[adapter->ch_big_index]); 967 } 968 } 969 970 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 971 /* 972 * let the TOE layer have a crack at the packet first. 973 */ 974 if (adapter->toe_tunnel) { 975 rv = adapter->toe_tunnel(adapter->ch_toeinst, skb); 976 /* 977 * The TOE may have consumed the packet. 978 */ 979 if (rv) 980 goto rx_entry_consumed; 981 } 982 #endif /* CONFIG_CHELSIO_T1_OFFLOAD */ 983 984 cksum = p->csum; 985 986 /* 987 * NOTE: 14+9 = size of MAC + offset to IP protocol field 988 */ 989 if (adapter->ch_config.cksum_enabled && 990 (ntohs(((struct ether_header *)skb->b_rptr)->ether_type) == 991 ETHERTYPE_IP) && 992 ((skb->b_rptr[14+9] == IPPROTO_TCP) || 993 (skb->b_rptr[14+9] == IPPROTO_UDP))) { 994 flg = 1; 995 } 996 997 ch_send_up(adapter, skb, cksum, flg); 998 } 999 1000 rx_entry_consumed: 1001 1002 if (++cidx == entries_n) 1003 cidx = 0; 1004 1005 Q->fq_cidx = cidx; 1006 1007 if (unlikely(--Q->fq_credits < (entries_n>>2))) 1008 /* allocate new buffers on the free list */ 1009 alloc_freelQ_buffers(sge, Q); 1010 return (1); 1011 } 1012 1013 #ifdef HOST_PAUSE 1014 static void 1015 t1_sge_check_pause(pesge *sge, struct freelQ *Q) 1016 { 1017 peobj *adapter = sge->obj; 1018 1019 /* 1020 * If the number of available credits shrinks below 1021 * the Pause on threshold then enable the pause and 1022 * try and allocate more buffers. 1023 * On the next pass, if there's more credits returned 1024 * then check that you've went above the pause 1025 * threshold and then disable the pause. 1026 */ 1027 if (Q->fq_credits < Q->fq_pause_on_thresh) { 1028 if (do_host_pause) { 1029 sge->intr_cnt.rx_pause_on++; 1030 adapter->txxg_cfg1 |= 1031 SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1032 (void) t1_tpi_write(adapter, 1033 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1034 adapter->txxg_cfg1); 1035 adapter->pause_on = 1; 1036 adapter->pause_time = gethrtime(); 1037 } 1038 alloc_freelQ_buffers(sge, Q); 1039 } else if ((adapter->pause_on) && 1040 (Q->fq_credits > Q->fq_pause_off_thresh)) { 1041 hrtime_t time; 1042 sge->intr_cnt.rx_pause_off++; 1043 adapter->txxg_cfg1 &= ~SUNI1x10GEXP_BITMSK_TXXG_HOSTPAUSE; 1044 (void) t1_tpi_write(adapter, 1045 SUNI1x10GEXP_REG_TXXG_CONFIG_1 << 2, 1046 adapter->txxg_cfg1); 1047 adapter->pause_on = 0; 1048 time = (gethrtime() - adapter->pause_time)/1000; 1049 sge->intr_cnt.rx_pause_ms += time; 1050 if (time > sge->intr_cnt.rx_pause_spike) 1051 sge->intr_cnt.rx_pause_spike = (uint32_t)time; 1052 } 1053 sge->intr_cnt.rx_fl_credits = Q->fq_credits; 1054 } 1055 #endif /* HOST_PAUSE */ 1056 1057 static void 1058 alloc_freelQ_buffers(pesge *sge, struct freelQ *Q) 1059 { 1060 uint32_t pidx = Q->fq_pidx; 1061 struct freelQ_ce *ce = &Q->fq_centries[pidx]; 1062 freelQ_e *fq = Q->fq_entries; /* base of freelist Q */ 1063 freelQ_e *e = &Q->fq_entries[pidx]; 1064 uint32_t sz = Q->fq_rx_buffer_size; 1065 uint32_t rxoff = sge->rx_offset; 1066 uint32_t credits = Q->fq_credits; 1067 uint32_t entries_n = Q->fq_entries_n; 1068 uint32_t genbit = Q->fq_genbit; 1069 ddi_dma_handle_t th = (ddi_dma_handle_t)Q->fq_dh; 1070 ulong_t dh; 1071 uint64_t mapping; 1072 off_t offset = (off_t)((caddr_t)e - (caddr_t)fq); 1073 size_t len = 0; 1074 1075 while (credits < entries_n) { 1076 if (e->GenerationBit != genbit) { 1077 mblk_t *skb; 1078 1079 mapping = os_freelist_buffer_alloc(sge->obj, sz, 1080 &skb, &dh); 1081 if (mapping == 0) { 1082 sge->intr_cnt.rx_flbuf_fails++; 1083 break; 1084 } 1085 sge->intr_cnt.rx_flbuf_allocs++; 1086 1087 ce->fe_mp = skb; 1088 ce->fe_dh = dh; 1089 1090 /* 1091 * Note that for T1, we've started the beginning of 1092 * of the buffer by an offset of 2 bytes. We thus 1093 * decrement the length to account for this. 1094 */ 1095 e->AddrLow = (u32)mapping; 1096 e->AddrHigh = (u64)mapping >> 32; 1097 e->BufferLength = sz - rxoff; 1098 wmb(); 1099 e->GenerationBit = e->GenerationBit2 = genbit; 1100 } 1101 1102 len += sizeof (*e); 1103 1104 ce++; 1105 e++; 1106 credits++; 1107 if (++pidx == entries_n) { 1108 /* 1109 * sync freelist entries to physical memory up to 1110 * end of the table. 1111 */ 1112 (void) ddi_dma_sync(th, offset, len, 1113 DDI_DMA_SYNC_FORDEV); 1114 offset = 0; 1115 len = 0; 1116 1117 pidx = 0; 1118 genbit ^= 1; 1119 ce = Q->fq_centries; 1120 e = Q->fq_entries; 1121 } 1122 } 1123 1124 /* sync freelist entries that have been modified. */ 1125 if (len) 1126 (void) ddi_dma_sync(th, offset, len, DDI_DMA_SYNC_FORDEV); 1127 1128 Q->fq_genbit = genbit; 1129 Q->fq_pidx = pidx; 1130 Q->fq_credits = credits; 1131 } 1132 1133 static void 1134 freelQs_empty(pesge *sge) 1135 { 1136 u32 irq_reg = t1_read_reg_4(sge->obj, A_SG_INT_ENABLE); 1137 u32 irqholdoff_reg; 1138 1139 alloc_freelQ_buffers(sge, &sge->freelQ[0]); 1140 alloc_freelQ_buffers(sge, &sge->freelQ[1]); 1141 1142 if ((sge->freelQ[0].fq_credits > sge->freelQ[0].fq_entries_n >> 2) && 1143 (sge->freelQ[1].fq_credits > sge->freelQ[1].fq_entries_n >> 2)) { 1144 irq_reg |= F_FL_EXHAUSTED; 1145 irqholdoff_reg = sge->intrtimer[sge->currIndex]; 1146 } else { 1147 /* Clear the F_FL_EXHAUSTED interrupts for now */ 1148 irq_reg &= ~F_FL_EXHAUSTED; 1149 irqholdoff_reg = sge->intrtimer_nres; 1150 } 1151 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, irqholdoff_reg); 1152 t1_write_reg_4(sge->obj, A_SG_INT_ENABLE, irq_reg); 1153 1154 /* We reenable the Qs to force an Freelist GTS interrupt later */ 1155 doorbell_pio(sge, F_FL0_ENABLE | F_FL1_ENABLE); 1156 } 1157 1158 /* 1159 * Frees 'credits_pend' TX buffers and returns the credits to Q->credits. 1160 * Free xmit buffers 1161 */ 1162 static void 1163 free_cmdQ_buffers(pesge *sge, struct cmdQ *Q, unsigned int credits_pend) 1164 { 1165 mblk_t *skb; 1166 struct cmdQ_ce *ce; 1167 struct cmdQ_ce *cq = Q->cq_centries; 1168 uint32_t entries_n = Q->cq_entries_n; 1169 uint32_t cidx = Q->cq_cidx; 1170 uint32_t i = credits_pend; 1171 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1172 ch_t *chp = sge->obj; 1173 #endif 1174 ce = &cq[cidx]; 1175 1176 while (i--) { 1177 #ifdef CONFIG_CHELSIO_T1_OFFLOAD 1178 /* if flag set, then toe buffer */ 1179 switch (ce->ce_flg & 0x7) { 1180 case DH_DMA: 1181 if (ce->ce_dh) { 1182 ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1183 ce->ce_dh = NULL; /* may not be needed */ 1184 } 1185 skb = ce->ce_mp; 1186 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1187 freemsg(skb); 1188 } 1189 ce->ce_mp = NULL; 1190 break; 1191 1192 #if defined(__sparc) 1193 case DH_DVMA: 1194 if (ce->ce_dh) { 1195 ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1196 ce->ce_dh = NULL; /* may not be needed */ 1197 } 1198 skb = ce->ce_mp; 1199 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1200 freemsg(skb); 1201 } 1202 ce->ce_mp = NULL; 1203 break; 1204 #endif /* __sparc */ 1205 1206 case DH_TOE: 1207 chp->toe_free(chp->ch_toeinst, (tbuf_t *)(ce->ce_mp)); 1208 ce->ce_mp = NULL; 1209 break; 1210 } 1211 #else /* CONFIG_CHELSIO_T1_OFFLOAD */ 1212 if (ce->ce_dh) { 1213 if ((ce->ce_flg & 7) == DH_DMA) { 1214 ch_unbind_dma_handle(sge->obj, ce->ce_dh); 1215 } 1216 #if defined(__sparc) 1217 else { 1218 ch_unbind_dvma_handle(sge->obj, ce->ce_dh); 1219 } 1220 #endif /* __sparc */ 1221 ce->ce_dh = NULL; /* may not be needed */ 1222 } 1223 1224 skb = ce->ce_mp; 1225 if (skb && ((ce->ce_flg & CH_ARP) == NULL)) { 1226 freemsg(skb); 1227 } 1228 ce->ce_mp = NULL; 1229 #endif /* !CONFIG_CHELSIO_T1_OFFLOAD */ 1230 1231 ce++; 1232 if (++cidx == entries_n) { 1233 cidx = 0; 1234 ce = cq; 1235 } 1236 } 1237 1238 Q->cq_cidx = cidx; 1239 atomic_add(credits_pend, &Q->cq_credits); 1240 } 1241 1242 struct sge_intr_counts * 1243 sge_get_stat(pesge *sge) 1244 { 1245 return (&sge->intr_cnt); 1246 } 1247 1248 /* 1249 * Allocates both RX and TX resources and configures the SGE. However, 1250 * the hardware is not enabled yet. 1251 * 1252 * rx_pkt_pad is set, if the hardware supports aligning non-offload traffic. 1253 * jumbo_fl is set to the index of the freelist containing the jumbo buffers. 1254 */ 1255 int 1256 t1_sge_configure(pesge *sge, struct sge_params *p) 1257 { 1258 sge->rx_pkt_pad = t1_is_T1B(sge->obj) ? 0 : SGE_RX_OFFSET; 1259 sge->jumbo_fl = t1_is_T1B(sge->obj) ? 1 : 0; 1260 /* if we're a T2 card, then we have hardware offset support */ 1261 sge->rx_offset = t1_is_T1B(sge->obj) ? SGE_RX_OFFSET: 0; 1262 1263 if (alloc_rx_resources(sge, p)) 1264 return (-ENOMEM); 1265 if (alloc_tx_resources(sge, p)) { 1266 free_rx_resources(sge); 1267 return (-ENOMEM); 1268 } 1269 configure_sge(sge, p); 1270 1271 /* 1272 * Now that we have sized the free lists calculate the payload 1273 * capacity of the large buffers. Other parts of the driver use 1274 * this to set the max offload coalescing size so that RX packets 1275 * do not overflow our large buffers. 1276 */ 1277 p->large_buf_capacity = jumbo_payload_capacity(sge); 1278 return (0); 1279 } 1280 1281 /* 1282 * Allocates basic RX resources, consisting of memory mapped freelist Qs and a 1283 * response Q. 1284 */ 1285 static int 1286 alloc_rx_resources(pesge *sge, struct sge_params *p) 1287 { 1288 unsigned int size, i; 1289 1290 for (i = 0; i < SGE_FREELQ_N; i++) { 1291 struct freelQ *Q = &sge->freelQ[i]; 1292 1293 Q->fq_id = i; 1294 Q->fq_genbit = 1; 1295 Q->fq_entries_n = p->freelQ_size[i]; 1296 #ifdef HOST_PAUSE 1297 Q->fq_pause_on_thresh = flq_pause_window; 1298 Q->fq_pause_off_thresh = Q->fq_entries_n >> 1; 1299 #endif 1300 size = sizeof (freelQ_e) * Q->fq_entries_n; 1301 1302 Q->fq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1303 size, &Q->fq_pa, &Q->fq_dh, &Q->fq_ah, DMA_OUT); 1304 1305 1306 if (!Q->fq_entries) 1307 goto err_no_mem; 1308 memset(Q->fq_entries, 0, size); 1309 size = sizeof (struct freelQ_ce) * Q->fq_entries_n; 1310 Q->fq_centries = t1_os_malloc_wait_zero(size); 1311 if (!Q->fq_centries) 1312 goto err_no_mem; 1313 memset(Q->fq_centries, 0, size); 1314 } 1315 1316 /* 1317 * Calculate the buffer sizes for the two free lists. FL0 accommodates 1318 * regular sized Ethernet frames, FL1 is sized not to exceed 16K, 1319 * including all the sk_buff overhead. 1320 * For T1C FL0 and FL1 are reversed. 1321 */ 1322 #ifdef NOTYET 1323 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = SGE_RX_SM_BUF_SIZE + 1324 sizeof (struct cpl_rx_data) + 1325 SGE_RX_OFFSET - sge->rx_pkt_pad; 1326 #else 1327 sge->freelQ[1 ^ sge->jumbo_fl].fq_rx_buffer_size = 1328 sge->obj->ch_sm_buf_sz; 1329 if (is_T2(sge->obj)) 1330 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_sm_buf_sz; 1331 else 1332 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_sm_buf_sz; 1333 #endif 1334 #ifdef NOTYET 1335 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = (16 * 1024) - 1336 SKB_DATA_ALIGN(sizeof (struct skb_shared_info)); 1337 #else 1338 sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size = sge->obj->ch_bg_buf_sz; 1339 if (is_T2(sge->obj)) 1340 sge->intr_cnt.rx_flq0_sz = sge->obj->ch_bg_buf_sz; 1341 else 1342 sge->intr_cnt.rx_flq1_sz = sge->obj->ch_bg_buf_sz; 1343 #endif 1344 1345 sge->respQ.rq_genbit = 1; 1346 sge->respQ.rq_entries_n = sge_respq_cnt; 1347 sge->respQ.rq_credits = sge_respq_cnt; 1348 sge->respQ.rq_credits_thresh = sge_respq_cnt - (sge_respq_cnt >> 2); 1349 size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1350 1351 sge->respQ.rq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1352 size, &(sge->respQ.rq_pa), &(sge->respQ.rq_dh), 1353 &(sge->respQ.rq_ah), 0); 1354 1355 if (!sge->respQ.rq_entries) 1356 goto err_no_mem; 1357 memset(sge->respQ.rq_entries, 0, size); 1358 return (0); 1359 1360 err_no_mem: 1361 free_rx_resources(sge); 1362 return (1); 1363 } 1364 1365 /* 1366 * Allocates basic TX resources, consisting of memory mapped command Qs. 1367 */ 1368 static int 1369 alloc_tx_resources(pesge *sge, struct sge_params *p) 1370 { 1371 unsigned int size, i; 1372 1373 for (i = 0; i < SGE_CMDQ_N; i++) { 1374 struct cmdQ *Q = &sge->cmdQ[i]; 1375 1376 Q->cq_genbit = 1; 1377 Q->cq_entries_n = p->cmdQ_size[i]; 1378 atomic_set(&Q->cq_credits, Q->cq_entries_n); 1379 atomic_set(&Q->cq_asleep, 1); 1380 1381 mutex_init(&Q->cq_qlock, NULL, MUTEX_DRIVER, 1382 sge->obj->ch_icookp); 1383 1384 size = sizeof (cmdQ_e) * Q->cq_entries_n; 1385 Q->cq_entries = pe_os_malloc_contig_wait_zero(sge->obj, 1386 size, &Q->cq_pa, &Q->cq_dh, &Q->cq_ah, DMA_OUT); 1387 1388 if (!Q->cq_entries) 1389 goto err_no_mem; 1390 memset(Q->cq_entries, 0, size); 1391 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1392 Q->cq_centries = t1_os_malloc_wait_zero(size); 1393 if (!Q->cq_centries) 1394 goto err_no_mem; 1395 memset(Q->cq_centries, 0, size); 1396 1397 /* allocate pre-mapped dma headers */ 1398 pe_dma_handle_init(sge->obj, Q->cq_entries_n); 1399 } 1400 1401 return (0); 1402 1403 err_no_mem: 1404 free_tx_resources(sge); 1405 return (1); 1406 } 1407 1408 /* 1409 * Sets the interrupt latency timer when the adaptive Rx coalescing 1410 * is turned off. Do nothing when it is turned on again. 1411 * 1412 * This routine relies on the fact that the caller has already set 1413 * the adaptive policy in adapter->sge_params before calling it. 1414 */ 1415 int 1416 t1_sge_set_coalesce_params(pesge *sge, struct sge_params *p) 1417 { 1418 if (!p->coalesce_enable) { 1419 u32 newTimer = p->rx_coalesce_usecs * 1420 (board_info(sge->obj)->clock_core / 1000000); 1421 1422 t1_write_reg_4(sge->obj, A_SG_INTRTIMER, newTimer); 1423 } 1424 return (0); 1425 } 1426 1427 /* 1428 * Programs the various SGE registers. However, the engine is not yet enabled, 1429 * but sge->sge_control is setup and ready to go. 1430 */ 1431 static void 1432 configure_sge(pesge *sge, struct sge_params *p) 1433 { 1434 ch_t *ap = sge->obj; 1435 int i; 1436 1437 t1_write_reg_4(ap, A_SG_CONTROL, 0); 1438 1439 setup_ring_params(ap, sge->cmdQ[0].cq_pa, sge->cmdQ[0].cq_entries_n, 1440 A_SG_CMD0BASELWR, A_SG_CMD0BASEUPR, A_SG_CMD0SIZE); 1441 setup_ring_params(ap, sge->cmdQ[1].cq_pa, sge->cmdQ[1].cq_entries_n, 1442 A_SG_CMD1BASELWR, A_SG_CMD1BASEUPR, A_SG_CMD1SIZE); 1443 setup_ring_params(ap, sge->freelQ[0].fq_pa, 1444 sge->freelQ[0].fq_entries_n, A_SG_FL0BASELWR, 1445 A_SG_FL0BASEUPR, A_SG_FL0SIZE); 1446 setup_ring_params(ap, sge->freelQ[1].fq_pa, 1447 sge->freelQ[1].fq_entries_n, A_SG_FL1BASELWR, 1448 A_SG_FL1BASEUPR, A_SG_FL1SIZE); 1449 1450 /* The threshold comparison uses <. */ 1451 t1_write_reg_4(ap, A_SG_FLTHRESHOLD, SGE_RX_SM_BUF_SIZE(ap) - 1452 SZ_CPL_RX_PKT - sge->rx_pkt_pad - sge->rx_offset + 1); 1453 setup_ring_params(ap, sge->respQ.rq_pa, sge->respQ.rq_entries_n, 1454 A_SG_RSPBASELWR, A_SG_RSPBASEUPR, A_SG_RSPSIZE); 1455 t1_write_reg_4(ap, A_SG_RSPQUEUECREDIT, (u32)sge->respQ.rq_entries_n); 1456 sge->sge_control = F_CMDQ0_ENABLE | F_CMDQ1_ENABLE | F_FL0_ENABLE | 1457 F_FL1_ENABLE | F_CPL_ENABLE | F_RESPONSE_QUEUE_ENABLE | 1458 V_CMDQ_PRIORITY(2) | F_DISABLE_CMDQ1_GTS | F_ISCSI_COALESCE | 1459 #if 1 1460 /* 1461 * if the the following bit is not set, then we'll get an 1462 * interrupt everytime command Q 0 goes empty. Since we're 1463 * always ringing the doorbell, we can turn it on. 1464 */ 1465 F_DISABLE_CMDQ0_GTS | 1466 #endif 1467 V_RX_PKT_OFFSET(sge->rx_pkt_pad); 1468 1469 #if BYTE_ORDER == BIG_ENDIAN 1470 sge->sge_control |= F_ENABLE_BIG_ENDIAN; 1471 #endif 1472 1473 /* 1474 * Initialize the SGE Interrupt Timer arrray: 1475 * intrtimer[0] = (SGE_INTRTIMER0) usec 1476 * intrtimer[0<i<10] = (SGE_INTRTIMER0 + 2*i) usec 1477 * intrtimer[10] = (SGE_INTRTIMER1) usec 1478 * 1479 */ 1480 sge->intrtimer[0] = board_info(sge->obj)->clock_core / 1000000; 1481 for (i = 1; i < SGE_INTR_MAXBUCKETS - 1; ++i) { 1482 sge->intrtimer[i] = SGE_INTRTIMER0 + (2 * i); 1483 sge->intrtimer[i] *= sge->intrtimer[0]; 1484 } 1485 sge->intrtimer[SGE_INTR_MAXBUCKETS - 1] = 1486 sge->intrtimer[0] * SGE_INTRTIMER1; 1487 /* Initialize resource timer */ 1488 sge->intrtimer_nres = (uint32_t)(sge->intrtimer[0] * 1489 SGE_INTRTIMER_NRES); 1490 /* Finally finish initialization of intrtimer[0] */ 1491 sge->intrtimer[0] = (uint32_t)(sge->intrtimer[0] * SGE_INTRTIMER0); 1492 /* Initialize for a throughput oriented workload */ 1493 sge->currIndex = SGE_INTR_MAXBUCKETS - 1; 1494 1495 if (p->coalesce_enable) 1496 t1_write_reg_4(ap, A_SG_INTRTIMER, 1497 sge->intrtimer[sge->currIndex]); 1498 else 1499 (void) t1_sge_set_coalesce_params(sge, p); 1500 } 1501 1502 static inline void 1503 setup_ring_params(ch_t *adapter, u64 addr, u32 size, int base_reg_lo, 1504 int base_reg_hi, int size_reg) 1505 { 1506 t1_write_reg_4(adapter, base_reg_lo, (u32)addr); 1507 t1_write_reg_4(adapter, base_reg_hi, addr >> 32); 1508 t1_write_reg_4(adapter, size_reg, size); 1509 } 1510 1511 /* 1512 * Frees RX resources. 1513 */ 1514 static void 1515 free_rx_resources(pesge *sge) 1516 { 1517 unsigned int size, i; 1518 1519 if (sge->respQ.rq_entries) { 1520 size = sizeof (respQ_e) * sge->respQ.rq_entries_n; 1521 1522 pe_os_free_contig(sge->obj, size, sge->respQ.rq_entries, 1523 sge->respQ.rq_pa, sge->respQ.rq_dh, sge->respQ.rq_ah); 1524 } 1525 1526 for (i = 0; i < SGE_FREELQ_N; i++) { 1527 struct freelQ *Q = &sge->freelQ[i]; 1528 1529 if (Q->fq_centries) { 1530 free_freelQ_buffers(sge, Q); 1531 1532 t1_os_free(Q->fq_centries, 1533 Q->fq_entries_n * sizeof (freelQ_ce_t)); 1534 } 1535 if (Q->fq_entries) { 1536 size = sizeof (freelQ_e) * Q->fq_entries_n; 1537 1538 /* free the freelist queue */ 1539 pe_os_free_contig(sge->obj, size, Q->fq_entries, 1540 Q->fq_pa, Q->fq_dh, Q->fq_ah); 1541 1542 } 1543 } 1544 } 1545 1546 /* 1547 * Frees all RX buffers on the freelist Q. The caller must make sure that 1548 * the SGE is turned off before calling this function. 1549 */ 1550 static void 1551 free_freelQ_buffers(pesge *sge, struct freelQ *Q) 1552 { 1553 struct freelQ_ce *ce; 1554 struct freelQ_ce *cq = Q->fq_centries; 1555 uint32_t credits = Q->fq_credits; 1556 uint32_t entries_n = Q->fq_entries_n; 1557 uint32_t cidx = Q->fq_cidx; 1558 uint32_t i = Q->fq_id; 1559 1560 ce = &cq[cidx]; 1561 1562 credits = entries_n; 1563 while (credits--) { 1564 mblk_t *mp; 1565 if ((mp = ce->fe_mp) != NULL) { 1566 /* bump in-use count of receive buffers */ 1567 if (i != sge->jumbo_fl) { 1568 atomic_add(1, 1569 &buffers_in_use[sge->obj->ch_sm_index]); 1570 } else { 1571 atomic_add(1, 1572 &buffers_in_use[sge->obj->ch_big_index]); 1573 } 1574 1575 /* 1576 * note. freeb() callback of esb-alloced mblk will 1577 * cause receive buffer to be put back on sa free list. 1578 */ 1579 freeb(mp); 1580 ce->fe_mp = NULL; 1581 } 1582 1583 ce++; 1584 if (++cidx == entries_n) { 1585 cidx = 0; 1586 ce = cq; 1587 } 1588 } 1589 1590 Q->fq_cidx = cidx; 1591 Q->fq_credits = credits; 1592 } 1593 1594 /* 1595 * Free TX resources. 1596 * 1597 * Assumes that SGE is stopped and all interrupts are disabled. 1598 */ 1599 static void 1600 free_tx_resources(pesge *sge) 1601 { 1602 unsigned int size; 1603 uint32_t i; 1604 1605 for (i = 0; i < SGE_CMDQ_N; i++) { 1606 struct cmdQ *Q = &sge->cmdQ[i]; 1607 1608 if (Q->cq_centries) { 1609 unsigned int pending = Q->cq_entries_n - 1610 atomic_read(Q->cq_credits); 1611 1612 mutex_destroy(&Q->cq_qlock); 1613 1614 if (pending) 1615 free_cmdQ_buffers(sge, Q, pending); 1616 1617 size = sizeof (struct cmdQ_ce) * Q->cq_entries_n; 1618 t1_os_free(Q->cq_centries, size); 1619 } 1620 1621 if (Q->cq_entries) { 1622 size = sizeof (cmdQ_e) * Q->cq_entries_n; 1623 pe_os_free_contig(sge->obj, size, Q->cq_entries, 1624 Q->cq_pa, Q->cq_dh, Q->cq_ah); 1625 } 1626 } 1627 } 1628 1629 /* 1630 * Return the payload capacity of the jumbo free-list buffers. 1631 */ 1632 static inline unsigned int jumbo_payload_capacity(pesge *sge) 1633 { 1634 return (sge->freelQ[sge->jumbo_fl].fq_rx_buffer_size - 1635 sizeof (struct cpl_rx_data) - sge->rx_pkt_pad - sge->rx_offset); 1636 } 1637 1638 /* PR2928 & PR3309 */ 1639 void 1640 t1_sge_set_ptimeout(adapter_t *adapter, u32 val) 1641 { 1642 pesge *sge = adapter->sge; 1643 1644 if (is_T2(adapter)) 1645 sge->ptimeout = max(val, 1); 1646 } 1647 1648 /* PR2928 & PR3309 */ 1649 u32 1650 t1_sge_get_ptimeout(adapter_t *adapter) 1651 { 1652 pesge *sge = adapter->sge; 1653 1654 return (is_T2(adapter) ? sge->ptimeout : 0); 1655 } 1656 1657 void 1658 sge_add_fake_arp(pesge *sge, void *bp) 1659 { 1660 sge->pskb = bp; 1661 } 1662 1663 #ifdef SUN_KSTATS 1664 static int 1665 sge_kstat_setup(pesge *sge) 1666 { 1667 int status; 1668 p_kstat_t ksp; 1669 size_t ch_kstat_sz; 1670 p_ch_kstat_t chkp; 1671 char kstat_name[32]; 1672 int instance; 1673 int i; 1674 1675 status = -1; 1676 ch_kstat_sz = sizeof (ch_kstat_t); 1677 instance = ddi_get_instance(sge->obj->ch_dip); 1678 if ((ksp = kstat_create(CHNAME "_debug", instance, 1679 NULL, "net_debug", KSTAT_TYPE_NAMED, 1680 ch_kstat_sz / sizeof (kstat_named_t), 0)) == NULL) 1681 goto sge_kstat_setup_exit; 1682 chkp = (p_ch_kstat_t)ksp->ks_data; 1683 kstat_named_init(&chkp->respQ_empty, "respQ_empty", 1684 KSTAT_DATA_UINT32); 1685 kstat_named_init(&chkp->respQ_overflow, "respQ_overflow", 1686 KSTAT_DATA_UINT32); 1687 kstat_named_init(&chkp->freelistQ_empty, "freelistQ_empty", 1688 KSTAT_DATA_UINT32); 1689 kstat_named_init(&chkp->pkt_too_big, "pkt_too_big", 1690 KSTAT_DATA_UINT32); 1691 kstat_named_init(&chkp->pkt_mismatch, "pkt_mismatch", 1692 KSTAT_DATA_UINT32); 1693 kstat_named_init(&chkp->cmdQ_full[0], "cmdQ_full[0]", 1694 KSTAT_DATA_UINT32); 1695 kstat_named_init(&chkp->cmdQ_full[1], "cmdQ_full[1]", 1696 KSTAT_DATA_UINT32); 1697 kstat_named_init(&chkp->tx_reclaims[0], "tx_reclaims[0]", 1698 KSTAT_DATA_UINT32); 1699 kstat_named_init(&chkp->tx_reclaims[1], "tx_reclaims[1]", 1700 KSTAT_DATA_UINT32); 1701 kstat_named_init(&chkp->tx_msg_pullups, "tx_msg_pullups", 1702 KSTAT_DATA_UINT32); 1703 kstat_named_init(&chkp->tx_hdr_pullups, "tx_hdr_pullups", 1704 KSTAT_DATA_UINT32); 1705 kstat_named_init(&chkp->tx_tcp_ip_frag, "tx_tcp_ip_frag", 1706 KSTAT_DATA_UINT32); 1707 kstat_named_init(&chkp->tx_udp_ip_frag, "tx_udp_ip_frag", 1708 KSTAT_DATA_UINT32); 1709 kstat_named_init(&chkp->tx_soft_cksums, "tx_soft_cksums", 1710 KSTAT_DATA_UINT32); 1711 kstat_named_init(&chkp->tx_need_cpl_space, "tx_need_cpl_space", 1712 KSTAT_DATA_UINT32); 1713 kstat_named_init(&chkp->tx_multi_mblks, "tx_multi_mblks", 1714 KSTAT_DATA_UINT32); 1715 kstat_named_init(&chkp->tx_no_dvma1, "tx_num_multi_dvma_fails", 1716 KSTAT_DATA_UINT32); 1717 kstat_named_init(&chkp->tx_no_dvma2, "tx_num_single_dvma_fails", 1718 KSTAT_DATA_UINT32); 1719 kstat_named_init(&chkp->tx_no_dma1, "tx_num_multi_dma_fails", 1720 KSTAT_DATA_UINT32); 1721 kstat_named_init(&chkp->tx_no_dma2, "tx_num_single_dma_fails", 1722 KSTAT_DATA_UINT32); 1723 kstat_named_init(&chkp->rx_cmdq0, "rx_cmdq0", 1724 KSTAT_DATA_UINT32); 1725 kstat_named_init(&chkp->rx_cmdq1, "rx_cmdq1", 1726 KSTAT_DATA_UINT32); 1727 kstat_named_init(&chkp->rx_flq0, "rx_flq0", 1728 KSTAT_DATA_UINT32); 1729 kstat_named_init(&chkp->rx_flq1, "rx_flq1", 1730 KSTAT_DATA_UINT32); 1731 kstat_named_init(&chkp->rx_flq0_sz, "rx_flq0_buffer_sz", 1732 KSTAT_DATA_UINT32); 1733 kstat_named_init(&chkp->rx_flq1_sz, "rx_flq1_buffer_sz", 1734 KSTAT_DATA_UINT32); 1735 kstat_named_init(&chkp->rx_pkt_drops, "rx_pkt_drops", 1736 KSTAT_DATA_UINT32); 1737 kstat_named_init(&chkp->rx_pkt_copied, "rx_pkt_copied", 1738 KSTAT_DATA_UINT32); 1739 kstat_named_init(&chkp->rx_pause_on, "rx_pause_on", 1740 KSTAT_DATA_UINT32); 1741 kstat_named_init(&chkp->rx_pause_off, "rx_pause_off", 1742 KSTAT_DATA_UINT32); 1743 kstat_named_init(&chkp->rx_pause_ms, "rx_pause_ms", 1744 KSTAT_DATA_UINT32); 1745 kstat_named_init(&chkp->rx_pause_spike, "rx_pause_spike", 1746 KSTAT_DATA_UINT32); 1747 kstat_named_init(&chkp->rx_fl_credits, "rx_fl_credits", 1748 KSTAT_DATA_UINT32); 1749 kstat_named_init(&chkp->rx_flbuf_fails, "rx_flbuf_fails", 1750 KSTAT_DATA_UINT32); 1751 kstat_named_init(&chkp->rx_flbuf_allocs, "rx_flbuf_allocs", 1752 KSTAT_DATA_UINT32); 1753 kstat_named_init(&chkp->rx_badEopSop, "rx_badEopSop", 1754 KSTAT_DATA_UINT32); 1755 kstat_named_init(&chkp->rx_flq0_cnt, "rx_flq0_cnt", 1756 KSTAT_DATA_UINT32); 1757 kstat_named_init(&chkp->rx_flq1_cnt, "rx_flq1_cnt", 1758 KSTAT_DATA_UINT32); 1759 kstat_named_init(&chkp->arp_sent, "arp_sent", 1760 KSTAT_DATA_UINT32); 1761 kstat_named_init(&chkp->tx_doorbells, "tx_doorbells", 1762 KSTAT_DATA_UINT32); 1763 kstat_named_init(&chkp->intr_doorbells, "intr_doorbells", 1764 KSTAT_DATA_UINT32); 1765 kstat_named_init(&chkp->intr1_doorbells, "intr1_doorbells", 1766 KSTAT_DATA_UINT32); 1767 kstat_named_init(&chkp->sleep_cnt, "sleep_cnt", 1768 KSTAT_DATA_UINT32); 1769 kstat_named_init(&chkp->pe_allocb_cnt, "pe_allocb_cnt", 1770 KSTAT_DATA_UINT32); 1771 for (i = 0; i < MBLK_MAX; i++) { 1772 (void) sprintf(kstat_name, "tx_descs[%02d]", i); 1773 kstat_named_init(&chkp->tx_descs[i], 1774 kstat_name, KSTAT_DATA_UINT32); 1775 } 1776 ksp->ks_update = sge_kstat_update; 1777 ksp->ks_private = (void *)sge; 1778 sge->ksp = ksp; 1779 kstat_install(ksp); 1780 status = 0; 1781 1782 sge_kstat_setup_exit: 1783 return (status); 1784 } 1785 1786 static void 1787 sge_kstat_remove(pesge *sge) 1788 { 1789 if (sge->ksp) 1790 kstat_delete(sge->ksp); 1791 } 1792 1793 static int 1794 sge_kstat_update(p_kstat_t ksp, int rw) 1795 { 1796 pesge *sge; 1797 p_ch_stats_t statsp; 1798 p_ch_kstat_t chkp; 1799 int i; 1800 1801 sge = (pesge *)ksp->ks_private; 1802 statsp = (p_ch_stats_t)&sge->intr_cnt; 1803 chkp = (p_ch_kstat_t)ksp->ks_data; 1804 if (rw == KSTAT_WRITE) { 1805 statsp->respQ_empty = chkp->respQ_empty.value.ui32; 1806 statsp->respQ_overflow = chkp->respQ_overflow.value.ui32; 1807 statsp->freelistQ_empty = chkp->freelistQ_empty.value.ui32; 1808 statsp->pkt_too_big = chkp->pkt_too_big.value.ui32; 1809 statsp->pkt_mismatch = chkp->pkt_mismatch.value.ui32; 1810 statsp->cmdQ_full[0] = chkp->cmdQ_full[0].value.ui32; 1811 statsp->cmdQ_full[1] = chkp->cmdQ_full[1].value.ui32; 1812 statsp->tx_reclaims[0] = chkp->tx_reclaims[0].value.ui32; 1813 statsp->tx_reclaims[1] = chkp->tx_reclaims[1].value.ui32; 1814 statsp->tx_msg_pullups = chkp->tx_msg_pullups.value.ui32; 1815 statsp->tx_hdr_pullups = chkp->tx_hdr_pullups.value.ui32; 1816 statsp->tx_tcp_ip_frag = chkp->tx_tcp_ip_frag.value.ui32; 1817 statsp->tx_udp_ip_frag = chkp->tx_udp_ip_frag.value.ui32; 1818 statsp->tx_soft_cksums = chkp->tx_soft_cksums.value.ui32; 1819 statsp->tx_need_cpl_space 1820 = chkp->tx_need_cpl_space.value.ui32; 1821 statsp->tx_multi_mblks = chkp->tx_multi_mblks.value.ui32; 1822 statsp->tx_no_dvma1 = chkp->tx_no_dvma1.value.ui32; 1823 statsp->tx_no_dvma2 = chkp->tx_no_dvma2.value.ui32; 1824 statsp->tx_no_dma1 = chkp->tx_no_dma1.value.ui32; 1825 statsp->tx_no_dma2 = chkp->tx_no_dma2.value.ui32; 1826 statsp->rx_cmdq0 = chkp->rx_cmdq0.value.ui32; 1827 statsp->rx_cmdq1 = chkp->rx_cmdq1.value.ui32; 1828 statsp->rx_flq0 = chkp->rx_flq0.value.ui32; 1829 statsp->rx_flq1 = chkp->rx_flq1.value.ui32; 1830 statsp->rx_flq0_sz = chkp->rx_flq0_sz.value.ui32; 1831 statsp->rx_flq1_sz = chkp->rx_flq1_sz.value.ui32; 1832 statsp->rx_pkt_drops = chkp->rx_pkt_drops.value.ui32; 1833 statsp->rx_pkt_copied = chkp->rx_pkt_copied.value.ui32; 1834 statsp->rx_pause_on = chkp->rx_pause_on.value.ui32; 1835 statsp->rx_pause_off = chkp->rx_pause_off.value.ui32; 1836 statsp->rx_pause_ms = chkp->rx_pause_ms.value.ui32; 1837 statsp->rx_pause_spike = chkp->rx_pause_spike.value.ui32; 1838 statsp->rx_fl_credits = chkp->rx_fl_credits.value.ui32; 1839 statsp->rx_flbuf_fails = chkp->rx_flbuf_fails.value.ui32; 1840 statsp->rx_flbuf_allocs = chkp->rx_flbuf_allocs.value.ui32; 1841 statsp->rx_badEopSop = chkp->rx_badEopSop.value.ui32; 1842 statsp->rx_flq0_cnt = chkp->rx_flq0_cnt.value.ui32; 1843 statsp->rx_flq1_cnt = chkp->rx_flq1_cnt.value.ui32; 1844 statsp->arp_sent = chkp->arp_sent.value.ui32; 1845 statsp->tx_doorbells = chkp->tx_doorbells.value.ui32; 1846 statsp->intr_doorbells = chkp->intr_doorbells.value.ui32; 1847 statsp->intr1_doorbells = chkp->intr1_doorbells.value.ui32; 1848 statsp->sleep_cnt = chkp->sleep_cnt.value.ui32; 1849 statsp->pe_allocb_cnt = chkp->pe_allocb_cnt.value.ui32; 1850 for (i = 0; i < MBLK_MAX; i++) { 1851 statsp->tx_descs[i] = chkp->tx_descs[i].value.ui32; 1852 } 1853 } else { 1854 chkp->respQ_empty.value.ui32 = statsp->respQ_empty; 1855 chkp->respQ_overflow.value.ui32 = statsp->respQ_overflow; 1856 chkp->freelistQ_empty.value.ui32 1857 = statsp->freelistQ_empty; 1858 chkp->pkt_too_big.value.ui32 = statsp->pkt_too_big; 1859 chkp->pkt_mismatch.value.ui32 = statsp->pkt_mismatch; 1860 chkp->cmdQ_full[0].value.ui32 = statsp->cmdQ_full[0]; 1861 chkp->cmdQ_full[1].value.ui32 = statsp->cmdQ_full[1]; 1862 chkp->tx_reclaims[0].value.ui32 = statsp->tx_reclaims[0]; 1863 chkp->tx_reclaims[1].value.ui32 = statsp->tx_reclaims[1]; 1864 chkp->tx_msg_pullups.value.ui32 = statsp->tx_msg_pullups; 1865 chkp->tx_hdr_pullups.value.ui32 = statsp->tx_hdr_pullups; 1866 chkp->tx_tcp_ip_frag.value.ui32 = statsp->tx_tcp_ip_frag; 1867 chkp->tx_udp_ip_frag.value.ui32 = statsp->tx_udp_ip_frag; 1868 chkp->tx_soft_cksums.value.ui32 = statsp->tx_soft_cksums; 1869 chkp->tx_need_cpl_space.value.ui32 1870 = statsp->tx_need_cpl_space; 1871 chkp->tx_multi_mblks.value.ui32 = statsp->tx_multi_mblks; 1872 chkp->tx_no_dvma1.value.ui32 = statsp->tx_no_dvma1; 1873 chkp->tx_no_dvma2.value.ui32 = statsp->tx_no_dvma2; 1874 chkp->tx_no_dma1.value.ui32 = statsp->tx_no_dma1; 1875 chkp->tx_no_dma2.value.ui32 = statsp->tx_no_dma2; 1876 chkp->rx_cmdq0.value.ui32 = statsp->rx_cmdq0; 1877 chkp->rx_cmdq1.value.ui32 = statsp->rx_cmdq1; 1878 chkp->rx_flq0.value.ui32 = statsp->rx_flq0; 1879 chkp->rx_flq1.value.ui32 = statsp->rx_flq1; 1880 chkp->rx_flq0_sz.value.ui32 = statsp->rx_flq0_sz; 1881 chkp->rx_flq1_sz.value.ui32 = statsp->rx_flq1_sz; 1882 chkp->rx_pkt_drops.value.ui32 = statsp->rx_pkt_drops; 1883 chkp->rx_pkt_copied.value.ui32 = statsp->rx_pkt_copied; 1884 chkp->rx_pause_on.value.ui32 = statsp->rx_pause_on; 1885 chkp->rx_pause_off.value.ui32 = statsp->rx_pause_off; 1886 chkp->rx_pause_ms.value.ui32 = statsp->rx_pause_ms; 1887 chkp->rx_pause_spike.value.ui32 = statsp->rx_pause_spike; 1888 chkp->rx_fl_credits.value.ui32 = statsp->rx_fl_credits; 1889 chkp->rx_flbuf_fails.value.ui32 1890 = statsp->rx_flbuf_fails; 1891 chkp->rx_flbuf_allocs.value.ui32 1892 = statsp->rx_flbuf_allocs; 1893 chkp->rx_badEopSop.value.ui32 = statsp->rx_badEopSop; 1894 chkp->rx_flq0_cnt.value.ui32 = statsp->rx_flq0_cnt; 1895 chkp->rx_flq1_cnt.value.ui32 = statsp->rx_flq1_cnt; 1896 chkp->arp_sent.value.ui32 = statsp->arp_sent; 1897 chkp->tx_doorbells.value.ui32 = statsp->tx_doorbells; 1898 chkp->intr_doorbells.value.ui32 = statsp->intr_doorbells; 1899 chkp->intr1_doorbells.value.ui32 1900 = statsp->intr1_doorbells; 1901 chkp->sleep_cnt.value.ui32 = statsp->sleep_cnt; 1902 chkp->pe_allocb_cnt.value.ui32 = statsp->pe_allocb_cnt; 1903 for (i = 0; i < MBLK_MAX; i++) { 1904 chkp->tx_descs[i].value.ui32 = statsp->tx_descs[i]; 1905 } 1906 } 1907 return (0); 1908 } 1909 #endif 1910 1911 static uint16_t 1912 calc_ocsum(mblk_t *mp, int offset) 1913 { 1914 uint8_t *addrp; 1915 uint32_t src; 1916 uint32_t dst; 1917 1918 ipha_t *ihdr = (ipha_t *)(mp->b_rptr + offset); 1919 uint32_t sum; 1920 int iplen = IPH_HDR_LENGTH(ihdr); 1921 struct udphdr *udpp = (struct udphdr *)(mp->b_rptr + offset + iplen); 1922 uchar_t *byte; 1923 int len; 1924 1925 addrp = (uint8_t *)&ihdr->ipha_src; 1926 src = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1927 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1928 1929 addrp = (uint8_t *)&ihdr->ipha_dst; 1930 dst = ((uint32_t)(addrp[0]) << 24) | ((uint32_t)(addrp[1]) << 16) | 1931 ((uint32_t)(addrp[2]) << 8) | (uint32_t)(addrp[3]); 1932 1933 sum = (uint16_t)(src >> 16) + 1934 (uint16_t)(src) + 1935 (uint16_t)(dst >> 16) + 1936 (uint16_t)(dst) + (udpp->uh_ulen + htons(IPPROTO_UDP)); 1937 1938 sum = (uint16_t)(sum >> 16) + (uint16_t)(sum); 1939 1940 if (sum > 0xffff) 1941 sum -= 0xffff; 1942 1943 udpp->uh_sum = 0; 1944 byte = mp->b_rptr + offset + iplen; 1945 do { 1946 len = (mp->b_wptr - byte); 1947 sum = bcksum(byte, len, sum); 1948 if (sum > 0xffff) 1949 sum -= 0xffff; 1950 mp = mp->b_cont; 1951 if (mp) 1952 byte = mp->b_rptr; 1953 } while (mp); 1954 1955 sum = ~sum & 0xffff; 1956 1957 return (sum); 1958 } 1959