1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include "nge.h" 28 29 #undef NGE_DBG 30 #define NGE_DBG NGE_DBG_RECV 31 32 #define RXD_END 0x20000000 33 #define RXD_ERR 0x40000000 34 #define RXD_OWN 0x80000000 35 #define RXD_CSUM_MSK 0x1C000000 36 #define RXD_BCNT_MSK 0x00003FFF 37 38 #define RXD_CK8G_NO_HSUM 0x0 39 #define RXD_CK8G_TCP_SUM_ERR 0x04000000 40 #define RXD_CK8G_UDP_SUM_ERR 0x08000000 41 #define RXD_CK8G_IP_HSUM_ERR 0x0C000000 42 #define RXD_CK8G_IP_HSUM 0x10000000 43 #define RXD_CK8G_TCP_SUM 0x14000000 44 #define RXD_CK8G_UDP_SUM 0x18000000 45 #define RXD_CK8G_RESV 0x1C000000 46 47 extern ddi_device_acc_attr_t nge_data_accattr; 48 49 /* 50 * Callback code invoked from STREAMs when the recv data buffer is free for 51 * recycling. 52 * 53 * The following table describes function behaviour: 54 * 55 * | mac stopped | mac running 56 * --------------------------------------------------- 57 * buffer delivered | free buffer | recycle buffer 58 * buffer not delivered | do nothing | recycle buffer (*) 59 * 60 * Note (*): 61 * Recycle buffer only if mac state did not change during execution of 62 * function. Otherwise if mac state changed, set buffer delivered & re-enter 63 * function by calling freemsg(). 64 */ 65 66 void 67 nge_recv_recycle(caddr_t arg) 68 { 69 boolean_t val; 70 boolean_t valid; 71 nge_t *ngep; 72 dma_area_t *bufp; 73 buff_ring_t *brp; 74 nge_sw_statistics_t *sw_stp; 75 76 bufp = (dma_area_t *)arg; 77 ngep = (nge_t *)bufp->private; 78 brp = ngep->buff; 79 sw_stp = &ngep->statistics.sw_statistics; 80 81 /* 82 * Free the buffer directly if the buffer was allocated 83 * previously or mac was stopped. 84 */ 85 if (bufp->signature != brp->buf_sign) { 86 if (bufp->rx_delivered == B_TRUE) { 87 nge_free_dma_mem(bufp); 88 kmem_free(bufp, sizeof (dma_area_t)); 89 val = nge_atomic_decrease(&brp->rx_hold, 1); 90 ASSERT(val == B_TRUE); 91 } 92 return; 93 } 94 95 /* 96 * recycle the data buffer again and fill them in free ring 97 */ 98 bufp->rx_recycle.free_func = nge_recv_recycle; 99 bufp->rx_recycle.free_arg = (caddr_t)bufp; 100 101 bufp->mp = desballoc(DMA_VPTR(*bufp), 102 ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle); 103 104 if (bufp->mp == NULL) { 105 sw_stp->mp_alloc_err++; 106 sw_stp->recy_free++; 107 nge_free_dma_mem(bufp); 108 kmem_free(bufp, sizeof (dma_area_t)); 109 val = nge_atomic_decrease(&brp->rx_hold, 1); 110 ASSERT(val == B_TRUE); 111 } else { 112 113 mutex_enter(brp->recycle_lock); 114 if (bufp->signature != brp->buf_sign) 115 valid = B_TRUE; 116 else 117 valid = B_FALSE; 118 bufp->rx_delivered = valid; 119 if (bufp->rx_delivered == B_FALSE) { 120 bufp->next = brp->recycle_list; 121 brp->recycle_list = bufp; 122 } 123 mutex_exit(brp->recycle_lock); 124 if (valid == B_TRUE) 125 /* call nge_rx_recycle again to free it */ 126 freemsg(bufp->mp); 127 else { 128 val = nge_atomic_decrease(&brp->rx_hold, 1); 129 ASSERT(val == B_TRUE); 130 } 131 } 132 } 133 134 /* 135 * Checking the rx's BDs (one or more) to receive 136 * one complete packet. 137 * start_index: the start indexer of BDs for one packet. 138 * end_index: the end indexer of BDs for one packet. 139 */ 140 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len); 141 #pragma inline(nge_recv_packet) 142 143 static mblk_t * 144 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len) 145 { 146 uint8_t *rptr; 147 uint32_t minsize; 148 uint32_t maxsize; 149 mblk_t *mp; 150 buff_ring_t *brp; 151 sw_rx_sbd_t *srbdp; 152 dma_area_t *bufp; 153 nge_sw_statistics_t *sw_stp; 154 void *hw_bd_p; 155 156 brp = ngep->buff; 157 minsize = ETHERMIN; 158 maxsize = ngep->max_sdu; 159 sw_stp = &ngep->statistics.sw_statistics; 160 mp = NULL; 161 162 srbdp = &brp->sw_rbds[start_index]; 163 DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL); 164 hw_bd_p = DMA_VPTR(srbdp->desc); 165 166 /* 167 * First check the free_list, if it is NULL, 168 * make the recycle_list be free_list. 169 */ 170 if (brp->free_list == NULL) { 171 mutex_enter(brp->recycle_lock); 172 brp->free_list = brp->recycle_list; 173 brp->recycle_list = NULL; 174 mutex_exit(brp->recycle_lock); 175 } 176 bufp = brp->free_list; 177 /* If it's not a qualified packet, delete it */ 178 if (len > maxsize || len < minsize) { 179 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 180 srbdp->bufp->alength); 181 srbdp->flags = CONTROLER_OWN; 182 return (NULL); 183 } 184 185 /* 186 * If receive packet size is smaller than RX bcopy threshold, 187 * or there is no available buffer in free_list or recycle list, 188 * we use bcopy directly. 189 */ 190 if (len <= ngep->param_rxbcopy_threshold || bufp == NULL) 191 brp->rx_bcopy = B_TRUE; 192 else 193 brp->rx_bcopy = B_FALSE; 194 195 if (brp->rx_bcopy) { 196 mp = allocb(len + NGE_HEADROOM, 0); 197 if (mp == NULL) { 198 sw_stp->mp_alloc_err++; 199 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 200 srbdp->bufp->alength); 201 srbdp->flags = CONTROLER_OWN; 202 return (NULL); 203 } 204 rptr = DMA_VPTR(*srbdp->bufp); 205 mp->b_rptr = mp->b_rptr + NGE_HEADROOM; 206 bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len); 207 mp->b_wptr = mp->b_rptr + len; 208 } else { 209 mp = srbdp->bufp->mp; 210 /* 211 * Make sure the packet *contents* 4-byte aligned 212 */ 213 mp->b_rptr += NGE_HEADROOM; 214 mp->b_wptr = mp->b_rptr + len; 215 mp->b_next = mp->b_cont = NULL; 216 srbdp->bufp->rx_delivered = B_TRUE; 217 srbdp->bufp = NULL; 218 nge_atomic_increase(&brp->rx_hold, 1); 219 220 /* Fill the buffer from free_list */ 221 srbdp->bufp = bufp; 222 brp->free_list = bufp->next; 223 bufp->next = NULL; 224 } 225 226 /* replenish the buffer for hardware descriptor */ 227 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 228 srbdp->bufp->alength); 229 srbdp->flags = CONTROLER_OWN; 230 sw_stp->rbytes += len; 231 sw_stp->recv_count++; 232 233 return (mp); 234 } 235 236 237 #define RX_HW_ERR 0x01 238 #define RX_SUM_NO 0x02 239 #define RX_SUM_ERR 0x04 240 241 /* 242 * Statistic the rx's error 243 * and generate a log msg for these. 244 * Note: 245 * RXE, Parity Error, Symbo error, CRC error 246 * have been recored by nvidia's hardware 247 * statistics part (nge_statistics). So it is uncessary to record them by 248 * driver in this place. 249 */ 250 static uint32_t 251 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags); 252 #pragma inline(nge_rxsta_handle) 253 254 static uint32_t 255 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags) 256 { 257 uint32_t errors; 258 uint32_t err_flag; 259 nge_sw_statistics_t *sw_stp; 260 261 err_flag = 0; 262 sw_stp = &ngep->statistics.sw_statistics; 263 264 if ((RXD_END & stflag) == 0) 265 return (RX_HW_ERR); 266 267 errors = stflag & RXD_CSUM_MSK; 268 switch (errors) { 269 default: 270 break; 271 272 case RXD_CK8G_TCP_SUM: 273 case RXD_CK8G_UDP_SUM: 274 *pflags |= HCK_FULLCKSUM; 275 *pflags |= HCK_IPV4_HDRCKSUM; 276 *pflags |= HCK_FULLCKSUM_OK; 277 break; 278 279 case RXD_CK8G_TCP_SUM_ERR: 280 case RXD_CK8G_UDP_SUM_ERR: 281 sw_stp->tcp_hwsum_err++; 282 *pflags |= HCK_IPV4_HDRCKSUM; 283 break; 284 285 case RXD_CK8G_IP_HSUM: 286 *pflags |= HCK_IPV4_HDRCKSUM; 287 break; 288 289 case RXD_CK8G_NO_HSUM: 290 err_flag |= RX_SUM_NO; 291 break; 292 293 case RXD_CK8G_IP_HSUM_ERR: 294 sw_stp->ip_hwsum_err++; 295 err_flag |= RX_SUM_ERR; 296 break; 297 } 298 299 if ((stflag & RXD_ERR) != 0) { 300 301 err_flag |= RX_HW_ERR; 302 NGE_DEBUG(("Receive desc error, status: 0x%x", stflag)); 303 } 304 305 return (err_flag); 306 } 307 308 static mblk_t * 309 nge_recv_ring(nge_t *ngep) 310 { 311 uint32_t stflag; 312 uint32_t flag_err; 313 uint32_t sum_flags; 314 size_t len; 315 uint64_t end_index; 316 uint64_t sync_start; 317 mblk_t *mp; 318 mblk_t **tail; 319 mblk_t *head; 320 recv_ring_t *rrp; 321 buff_ring_t *brp; 322 sw_rx_sbd_t *srbdp; 323 void * hw_bd_p; 324 nge_mode_cntl mode_cntl; 325 326 mp = NULL; 327 head = NULL; 328 tail = &head; 329 rrp = ngep->recv; 330 brp = ngep->buff; 331 332 end_index = sync_start = rrp->prod_index; 333 /* Sync the descriptor for kernel */ 334 if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) { 335 (void) ddi_dma_sync(rrp->desc.dma_hdl, 336 sync_start * ngep->desc_attr.rxd_size, 337 ngep->param_recv_max_packet * ngep->desc_attr.rxd_size, 338 DDI_DMA_SYNC_FORKERNEL); 339 } else { 340 (void) ddi_dma_sync(rrp->desc.dma_hdl, 341 sync_start * ngep->desc_attr.rxd_size, 342 0, 343 DDI_DMA_SYNC_FORKERNEL); 344 (void) ddi_dma_sync(rrp->desc.dma_hdl, 345 0, 346 (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) * 347 ngep->desc_attr.rxd_size, 348 DDI_DMA_SYNC_FORKERNEL); 349 } 350 351 /* 352 * Looking through the rx's ring to find the good packets 353 * and try to receive more and more packets in rx's ring 354 */ 355 for (;;) { 356 sum_flags = 0; 357 flag_err = 0; 358 end_index = rrp->prod_index; 359 srbdp = &brp->sw_rbds[end_index]; 360 hw_bd_p = DMA_VPTR(srbdp->desc); 361 stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len); 362 /* 363 * If there is no packet in receving ring 364 * break the loop 365 */ 366 if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags) 367 break; 368 369 ngep->recv_count++; 370 flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags); 371 if ((flag_err & RX_HW_ERR) == 0) { 372 srbdp->flags = NGE_END_PACKET; 373 mp = nge_recv_packet(ngep, end_index, len); 374 } else { 375 /* Hardware error, re-use the buffer */ 376 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 377 srbdp->bufp->alength); 378 srbdp->flags = CONTROLER_OWN; 379 } 380 if (mp != NULL) { 381 if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) { 382 (void) hcksum_assoc(mp, NULL, NULL, 383 0, 0, 0, 0, sum_flags, 0); 384 } 385 *tail = mp; 386 tail = &mp->b_next; 387 mp = NULL; 388 } 389 rrp->prod_index = NEXT(end_index, rrp->desc.nslots); 390 if (ngep->recv_count >= ngep->param_recv_max_packet) 391 break; 392 } 393 394 /* Sync the descriptors for device */ 395 if (sync_start + ngep->recv_count <= ngep->rx_desc) { 396 (void) ddi_dma_sync(rrp->desc.dma_hdl, 397 sync_start * ngep->desc_attr.rxd_size, 398 ngep->recv_count * ngep->desc_attr.rxd_size, 399 DDI_DMA_SYNC_FORDEV); 400 } else { 401 (void) ddi_dma_sync(rrp->desc.dma_hdl, 402 sync_start * ngep->desc_attr.rxd_size, 403 0, 404 DDI_DMA_SYNC_FORDEV); 405 (void) ddi_dma_sync(rrp->desc.dma_hdl, 406 0, 407 (ngep->recv_count + sync_start - ngep->rx_desc) * 408 ngep->desc_attr.rxd_size, 409 DDI_DMA_SYNC_FORDEV); 410 } 411 mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL); 412 mode_cntl.mode_bits.rxdm = NGE_SET; 413 mode_cntl.mode_bits.tx_rcom_en = NGE_SET; 414 nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val); 415 416 return (head); 417 } 418 419 void 420 nge_receive(nge_t *ngep) 421 { 422 mblk_t *mp; 423 recv_ring_t *rrp; 424 rrp = ngep->recv; 425 426 mp = nge_recv_ring(ngep); 427 mutex_exit(ngep->genlock); 428 if (mp != NULL) 429 mac_rx(ngep->mh, rrp->handle, mp); 430 mutex_enter(ngep->genlock); 431 } 432 433 void 434 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 435 { 436 uint64_t dmac_addr; 437 hot_rx_bd * hw_bd_p; 438 439 hw_bd_p = (hot_rx_bd *)hwd; 440 dmac_addr = cookie->dmac_laddress + NGE_HEADROOM; 441 442 hw_bd_p->cntl_status.cntl_val = 0; 443 444 hw_bd_p->host_buf_addr_hi = dmac_addr >> 32; 445 hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr; 446 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 447 448 membar_producer(); 449 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 450 } 451 452 void 453 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 454 { 455 sum_rx_bd * hw_bd_p; 456 457 hw_bd_p = hwd; 458 459 hw_bd_p->cntl_status.cntl_val = 0; 460 461 hw_bd_p->host_buf_addr = 462 (uint32_t)(cookie->dmac_address + NGE_HEADROOM); 463 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 464 465 membar_producer(); 466 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 467 } 468 469 uint32_t 470 nge_hot_rxd_check(const void *hwd, size_t *len) 471 { 472 uint32_t err_flag; 473 const hot_rx_bd * hrbdp; 474 475 hrbdp = hwd; 476 477 err_flag = hrbdp->cntl_status.cntl_val & ~RXD_BCNT_MSK; 478 *len = hrbdp->cntl_status.status_bits_legacy.bcnt; 479 480 return (err_flag); 481 } 482 483 uint32_t 484 nge_sum_rxd_check(const void *hwd, size_t *len) 485 { 486 uint32_t err_flag; 487 const sum_rx_bd * hrbdp; 488 489 hrbdp = hwd; 490 491 err_flag = hrbdp->cntl_status.cntl_val & ~RXD_BCNT_MSK; 492 *len = hrbdp->cntl_status.status_bits.bcnt; 493 494 return (err_flag); 495 } 496