1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #include "nge.h" 28 29 #undef NGE_DBG 30 #define NGE_DBG NGE_DBG_RECV 31 32 #define RXD_END 0x20000000 33 #define RXD_ERR 0x40000000 34 #define RXD_OWN 0x80000000 35 #define RXD_CSUM_MSK 0x1C000000 36 #define RXD_BCNT_MSK 0x00003FFF 37 38 #define RXD_CK8G_NO_HSUM 0x0 39 #define RXD_CK8G_TCP_SUM_ERR 0x04000000 40 #define RXD_CK8G_UDP_SUM_ERR 0x08000000 41 #define RXD_CK8G_IP_HSUM_ERR 0x0C000000 42 #define RXD_CK8G_IP_HSUM 0x10000000 43 #define RXD_CK8G_TCP_SUM 0x14000000 44 #define RXD_CK8G_UDP_SUM 0x18000000 45 #define RXD_CK8G_RESV 0x1C000000 46 47 extern ddi_device_acc_attr_t nge_data_accattr; 48 49 /* 50 * Callback code invoked from STREAMs when the recv data buffer is free for 51 * recycling. 52 * 53 * The following table describes function behaviour: 54 * 55 * | mac stopped | mac running 56 * --------------------------------------------------- 57 * buffer delivered | free buffer | recycle buffer 58 * buffer not delivered | do nothing | recycle buffer (*) 59 * 60 * Note (*): 61 * Recycle buffer only if mac state did not change during execution of 62 * function. Otherwise if mac state changed, set buffer delivered & re-enter 63 * function by calling freemsg(). 64 */ 65 66 void 67 nge_recv_recycle(caddr_t arg) 68 { 69 boolean_t val; 70 boolean_t valid; 71 nge_t *ngep; 72 dma_area_t *bufp; 73 buff_ring_t *brp; 74 nge_sw_statistics_t *sw_stp; 75 76 bufp = (dma_area_t *)arg; 77 ngep = (nge_t *)bufp->private; 78 brp = ngep->buff; 79 sw_stp = &ngep->statistics.sw_statistics; 80 81 /* 82 * Free the buffer directly if the buffer was allocated 83 * previously or mac was stopped. 84 */ 85 if (bufp->signature != brp->buf_sign) { 86 if (bufp->rx_delivered == B_TRUE) { 87 nge_free_dma_mem(bufp); 88 kmem_free(bufp, sizeof (dma_area_t)); 89 val = nge_atomic_decrease(&brp->rx_hold, 1); 90 ASSERT(val == B_TRUE); 91 } 92 return; 93 } 94 95 /* 96 * recycle the data buffer again and fill them in free ring 97 */ 98 bufp->rx_recycle.free_func = nge_recv_recycle; 99 bufp->rx_recycle.free_arg = (caddr_t)bufp; 100 101 bufp->mp = desballoc(DMA_VPTR(*bufp), 102 ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle); 103 104 if (bufp->mp == NULL) { 105 sw_stp->mp_alloc_err++; 106 sw_stp->recy_free++; 107 nge_free_dma_mem(bufp); 108 kmem_free(bufp, sizeof (dma_area_t)); 109 val = nge_atomic_decrease(&brp->rx_hold, 1); 110 ASSERT(val == B_TRUE); 111 } else { 112 113 mutex_enter(brp->recycle_lock); 114 if (bufp->signature != brp->buf_sign) 115 valid = B_TRUE; 116 else 117 valid = B_FALSE; 118 bufp->rx_delivered = valid; 119 if (bufp->rx_delivered == B_FALSE) { 120 bufp->next = brp->recycle_list; 121 brp->recycle_list = bufp; 122 } 123 mutex_exit(brp->recycle_lock); 124 if (valid == B_TRUE) 125 /* call nge_rx_recycle again to free it */ 126 freemsg(bufp->mp); 127 else { 128 val = nge_atomic_decrease(&brp->rx_hold, 1); 129 ASSERT(val == B_TRUE); 130 } 131 } 132 } 133 134 /* 135 * Checking the rx's BDs (one or more) to receive 136 * one complete packet. 137 * start_index: the start indexer of BDs for one packet. 138 * end_index: the end indexer of BDs for one packet. 139 */ 140 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len); 141 #pragma inline(nge_recv_packet) 142 143 static mblk_t * 144 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len) 145 { 146 uint8_t *rptr; 147 uint32_t minsize; 148 uint32_t maxsize; 149 mblk_t *mp; 150 buff_ring_t *brp; 151 sw_rx_sbd_t *srbdp; 152 dma_area_t *bufp; 153 nge_sw_statistics_t *sw_stp; 154 void *hw_bd_p; 155 156 brp = ngep->buff; 157 minsize = ETHERMIN; 158 maxsize = ngep->max_sdu; 159 sw_stp = &ngep->statistics.sw_statistics; 160 mp = NULL; 161 162 srbdp = &brp->sw_rbds[start_index]; 163 DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL); 164 hw_bd_p = DMA_VPTR(srbdp->desc); 165 166 /* 167 * First check the free_list, if it is NULL, 168 * make the recycle_list be free_list. 169 */ 170 if (brp->free_list == NULL) { 171 mutex_enter(brp->recycle_lock); 172 brp->free_list = brp->recycle_list; 173 brp->recycle_list = NULL; 174 mutex_exit(brp->recycle_lock); 175 } 176 bufp = brp->free_list; 177 /* If it's not a qualified packet, delete it */ 178 if (len > maxsize || len < minsize) { 179 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 180 srbdp->bufp->alength); 181 srbdp->flags = CONTROLER_OWN; 182 return (NULL); 183 } 184 185 /* 186 * If receive packet size is smaller than RX bcopy threshold, 187 * or there is no available buffer in free_list or recycle list, 188 * we use bcopy directly. 189 */ 190 if (len <= ngep->param_rxbcopy_threshold || bufp == NULL) 191 brp->rx_bcopy = B_TRUE; 192 else 193 brp->rx_bcopy = B_FALSE; 194 195 if (brp->rx_bcopy) { 196 mp = allocb(len + NGE_HEADROOM, 0); 197 if (mp == NULL) { 198 sw_stp->mp_alloc_err++; 199 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 200 srbdp->bufp->alength); 201 srbdp->flags = CONTROLER_OWN; 202 return (NULL); 203 } 204 rptr = DMA_VPTR(*srbdp->bufp); 205 mp->b_rptr = mp->b_rptr + NGE_HEADROOM; 206 bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len); 207 mp->b_wptr = mp->b_rptr + len; 208 } else { 209 mp = srbdp->bufp->mp; 210 /* 211 * Make sure the packet *contents* 4-byte aligned 212 */ 213 mp->b_rptr += NGE_HEADROOM; 214 mp->b_wptr = mp->b_rptr + len; 215 mp->b_next = mp->b_cont = NULL; 216 srbdp->bufp->rx_delivered = B_TRUE; 217 srbdp->bufp = NULL; 218 nge_atomic_increase(&brp->rx_hold, 1); 219 220 /* Fill the buffer from free_list */ 221 srbdp->bufp = bufp; 222 brp->free_list = bufp->next; 223 bufp->next = NULL; 224 } 225 226 /* replenish the buffer for hardware descriptor */ 227 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 228 srbdp->bufp->alength); 229 srbdp->flags = CONTROLER_OWN; 230 sw_stp->rbytes += len; 231 sw_stp->recv_count++; 232 233 return (mp); 234 } 235 236 237 #define RX_HW_ERR 0x01 238 #define RX_SUM_NO 0x02 239 #define RX_SUM_ERR 0x04 240 241 /* 242 * Statistic the rx's error 243 * and generate a log msg for these. 244 * Note: 245 * RXE, Parity Error, Symbo error, CRC error 246 * have been recored by nvidia's hardware 247 * statistics part (nge_statistics). So it is uncessary to record them by 248 * driver in this place. 249 */ 250 static uint32_t 251 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags); 252 #pragma inline(nge_rxsta_handle) 253 254 static uint32_t 255 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags) 256 { 257 uint32_t errors; 258 uint32_t err_flag; 259 nge_sw_statistics_t *sw_stp; 260 261 err_flag = 0; 262 sw_stp = &ngep->statistics.sw_statistics; 263 264 if ((RXD_END & stflag) == 0) 265 return (RX_HW_ERR); 266 267 errors = stflag & RXD_CSUM_MSK; 268 switch (errors) { 269 default: 270 break; 271 272 case RXD_CK8G_TCP_SUM: 273 case RXD_CK8G_UDP_SUM: 274 *pflags |= HCK_IPV4_HDRCKSUM_OK; 275 *pflags |= HCK_FULLCKSUM_OK; 276 break; 277 278 case RXD_CK8G_TCP_SUM_ERR: 279 case RXD_CK8G_UDP_SUM_ERR: 280 sw_stp->tcp_hwsum_err++; 281 *pflags |= HCK_IPV4_HDRCKSUM_OK; 282 break; 283 284 case RXD_CK8G_IP_HSUM: 285 *pflags |= HCK_IPV4_HDRCKSUM_OK; 286 break; 287 288 case RXD_CK8G_NO_HSUM: 289 err_flag |= RX_SUM_NO; 290 break; 291 292 case RXD_CK8G_IP_HSUM_ERR: 293 sw_stp->ip_hwsum_err++; 294 err_flag |= RX_SUM_ERR; 295 break; 296 } 297 298 if ((stflag & RXD_ERR) != 0) { 299 300 err_flag |= RX_HW_ERR; 301 NGE_DEBUG(("Receive desc error, status: 0x%x", stflag)); 302 } 303 304 return (err_flag); 305 } 306 307 static mblk_t * 308 nge_recv_ring(nge_t *ngep) 309 { 310 uint32_t stflag; 311 uint32_t flag_err; 312 uint32_t sum_flags; 313 size_t len; 314 uint64_t end_index; 315 uint64_t sync_start; 316 mblk_t *mp; 317 mblk_t **tail; 318 mblk_t *head; 319 recv_ring_t *rrp; 320 buff_ring_t *brp; 321 sw_rx_sbd_t *srbdp; 322 void * hw_bd_p; 323 nge_mode_cntl mode_cntl; 324 325 mp = NULL; 326 head = NULL; 327 tail = &head; 328 rrp = ngep->recv; 329 brp = ngep->buff; 330 331 end_index = sync_start = rrp->prod_index; 332 /* Sync the descriptor for kernel */ 333 if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) { 334 (void) ddi_dma_sync(rrp->desc.dma_hdl, 335 sync_start * ngep->desc_attr.rxd_size, 336 ngep->param_recv_max_packet * ngep->desc_attr.rxd_size, 337 DDI_DMA_SYNC_FORKERNEL); 338 } else { 339 (void) ddi_dma_sync(rrp->desc.dma_hdl, 340 sync_start * ngep->desc_attr.rxd_size, 341 0, 342 DDI_DMA_SYNC_FORKERNEL); 343 (void) ddi_dma_sync(rrp->desc.dma_hdl, 344 0, 345 (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) * 346 ngep->desc_attr.rxd_size, 347 DDI_DMA_SYNC_FORKERNEL); 348 } 349 350 /* 351 * Looking through the rx's ring to find the good packets 352 * and try to receive more and more packets in rx's ring 353 */ 354 for (;;) { 355 sum_flags = 0; 356 flag_err = 0; 357 end_index = rrp->prod_index; 358 srbdp = &brp->sw_rbds[end_index]; 359 hw_bd_p = DMA_VPTR(srbdp->desc); 360 stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len); 361 /* 362 * If there is no packet in receving ring 363 * break the loop 364 */ 365 if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags) 366 break; 367 368 ngep->recv_count++; 369 flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags); 370 if ((flag_err & RX_HW_ERR) == 0) { 371 srbdp->flags = NGE_END_PACKET; 372 mp = nge_recv_packet(ngep, end_index, len); 373 } else { 374 /* Hardware error, re-use the buffer */ 375 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 376 srbdp->bufp->alength); 377 srbdp->flags = CONTROLER_OWN; 378 } 379 if (mp != NULL) { 380 if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) { 381 mac_hcksum_set(mp, 0, 0, 0, 0, sum_flags); 382 } 383 *tail = mp; 384 tail = &mp->b_next; 385 mp = NULL; 386 } 387 rrp->prod_index = NEXT(end_index, rrp->desc.nslots); 388 if (ngep->recv_count >= ngep->param_recv_max_packet) 389 break; 390 } 391 392 /* Sync the descriptors for device */ 393 if (sync_start + ngep->recv_count <= ngep->rx_desc) { 394 (void) ddi_dma_sync(rrp->desc.dma_hdl, 395 sync_start * ngep->desc_attr.rxd_size, 396 ngep->recv_count * ngep->desc_attr.rxd_size, 397 DDI_DMA_SYNC_FORDEV); 398 } else { 399 (void) ddi_dma_sync(rrp->desc.dma_hdl, 400 sync_start * ngep->desc_attr.rxd_size, 401 0, 402 DDI_DMA_SYNC_FORDEV); 403 (void) ddi_dma_sync(rrp->desc.dma_hdl, 404 0, 405 (ngep->recv_count + sync_start - ngep->rx_desc) * 406 ngep->desc_attr.rxd_size, 407 DDI_DMA_SYNC_FORDEV); 408 } 409 mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL); 410 mode_cntl.mode_bits.rxdm = NGE_SET; 411 mode_cntl.mode_bits.tx_rcom_en = NGE_SET; 412 nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val); 413 414 return (head); 415 } 416 417 void 418 nge_receive(nge_t *ngep) 419 { 420 mblk_t *mp; 421 recv_ring_t *rrp; 422 rrp = ngep->recv; 423 424 mp = nge_recv_ring(ngep); 425 mutex_exit(ngep->genlock); 426 if (mp != NULL) 427 mac_rx(ngep->mh, rrp->handle, mp); 428 mutex_enter(ngep->genlock); 429 } 430 431 void 432 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 433 { 434 uint64_t dmac_addr; 435 hot_rx_bd * hw_bd_p; 436 437 hw_bd_p = (hot_rx_bd *)hwd; 438 dmac_addr = cookie->dmac_laddress + NGE_HEADROOM; 439 440 hw_bd_p->cntl_status.cntl_val = 0; 441 442 hw_bd_p->host_buf_addr_hi = dmac_addr >> 32; 443 hw_bd_p->host_buf_addr_lo = (uint32_t)dmac_addr; 444 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 445 446 membar_producer(); 447 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 448 } 449 450 void 451 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 452 { 453 sum_rx_bd * hw_bd_p; 454 455 hw_bd_p = hwd; 456 457 hw_bd_p->cntl_status.cntl_val = 0; 458 459 hw_bd_p->host_buf_addr = 460 (uint32_t)(cookie->dmac_address + NGE_HEADROOM); 461 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 462 463 membar_producer(); 464 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 465 } 466 467 uint32_t 468 nge_hot_rxd_check(const void *hwd, size_t *len) 469 { 470 uint32_t err_flag; 471 const hot_rx_bd * hrbdp; 472 473 hrbdp = hwd; 474 err_flag = hrbdp->cntl_status.cntl_val; 475 *len = err_flag & RXD_BCNT_MSK; 476 return (err_flag); 477 } 478 479 uint32_t 480 nge_sum_rxd_check(const void *hwd, size_t *len) 481 { 482 uint32_t err_flag; 483 const sum_rx_bd * hrbdp; 484 485 hrbdp = hwd; 486 487 err_flag = hrbdp->cntl_status.cntl_val; 488 *len = err_flag & RXD_BCNT_MSK; 489 return (err_flag); 490 } 491