1 /* 2 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 3 * Use is subject to license terms. 4 */ 5 6 /* 7 * This file may contain confidential information of Nvidia 8 * and should not be distributed in source form without approval 9 * from Sun Legal. 10 */ 11 12 #pragma ident "%Z%%M% %I% %E% SMI" 13 14 #include "nge.h" 15 16 #undef NGE_DBG 17 #define NGE_DBG NGE_DBG_RECV 18 19 #define RXD_END 0x20000000 20 #define RXD_ERR 0x40000000 21 #define RXD_OWN 0x80000000 22 #define RXD_CSUM_MSK 0x1C000000 23 #define RXD_BCNT_MSK 0x00003FFF 24 25 #define RXD_CK8G_NO_HSUM 0x0 26 #define RXD_CK8G_TCP_SUM_ERR 0x04000000 27 #define RXD_CK8G_UDP_SUM_ERR 0x08000000 28 #define RXD_CK8G_IP_HSUM_ERR 0x0C000000 29 #define RXD_CK8G_IP_HSUM 0x10000000 30 #define RXD_CK8G_TCP_SUM 0x14000000 31 #define RXD_CK8G_UDP_SUM 0x18000000 32 #define RXD_CK8G_RESV 0x1C000000 33 34 extern ddi_device_acc_attr_t nge_data_accattr; 35 36 /* 37 * Callback code invoked from STREAMs when the recv data buffer is free 38 * for recycling. 39 */ 40 41 void 42 nge_recv_recycle(caddr_t arg) 43 { 44 boolean_t val; 45 boolean_t valid; 46 nge_t *ngep; 47 dma_area_t *bufp; 48 buff_ring_t *brp; 49 nge_sw_statistics_t *sw_stp; 50 51 bufp = (dma_area_t *)arg; 52 ngep = (nge_t *)bufp->private; 53 brp = ngep->buff; 54 sw_stp = &ngep->statistics.sw_statistics; 55 56 /* 57 * Free the buffer directly if the buffer was allocated 58 * previously or mac was stopped. 59 */ 60 if (bufp->signature != brp->buf_sign) { 61 if (bufp->rx_delivered == B_TRUE) { 62 nge_free_dma_mem(bufp); 63 kmem_free(bufp, sizeof (dma_area_t)); 64 val = nge_atomic_decrease(&brp->rx_hold, 1); 65 ASSERT(val == B_TRUE); 66 } 67 return; 68 } 69 70 /* 71 * recycle the data buffer again and fill them in free ring 72 */ 73 bufp->rx_recycle.free_func = nge_recv_recycle; 74 bufp->rx_recycle.free_arg = (caddr_t)bufp; 75 76 bufp->mp = desballoc(DMA_VPTR(*bufp), 77 ngep->buf_size + NGE_HEADROOM, 0, &bufp->rx_recycle); 78 79 if (bufp->mp == NULL) { 80 sw_stp->mp_alloc_err++; 81 sw_stp->recy_free++; 82 nge_free_dma_mem(bufp); 83 kmem_free(bufp, sizeof (dma_area_t)); 84 val = nge_atomic_decrease(&brp->rx_hold, 1); 85 ASSERT(val == B_TRUE); 86 } else { 87 88 mutex_enter(brp->recycle_lock); 89 if (bufp->signature != brp->buf_sign) 90 valid = B_TRUE; 91 else 92 valid = B_FALSE; 93 bufp->rx_delivered = valid; 94 if (bufp->rx_delivered == B_FALSE) { 95 bufp->next = brp->recycle_list; 96 brp->recycle_list = bufp; 97 } 98 mutex_exit(brp->recycle_lock); 99 if (valid == B_TRUE) 100 /* call nge_rx_recycle again to free it */ 101 freemsg(bufp->mp); 102 else { 103 val = nge_atomic_decrease(&brp->rx_hold, 1); 104 ASSERT(val == B_TRUE); 105 } 106 } 107 } 108 109 /* 110 * Checking the rx's BDs (one or more) to receive 111 * one complete packet. 112 * start_index: the start indexer of BDs for one packet. 113 * end_index: the end indexer of BDs for one packet. 114 */ 115 static mblk_t *nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len); 116 #pragma inline(nge_recv_packet) 117 118 static mblk_t * 119 nge_recv_packet(nge_t *ngep, uint32_t start_index, size_t len) 120 { 121 uint8_t *rptr; 122 uint32_t minsize; 123 uint32_t maxsize; 124 mblk_t *mp; 125 buff_ring_t *brp; 126 sw_rx_sbd_t *srbdp; 127 dma_area_t *bufp; 128 nge_sw_statistics_t *sw_stp; 129 void *hw_bd_p; 130 131 brp = ngep->buff; 132 minsize = ETHERMIN; 133 maxsize = ngep->max_sdu; 134 sw_stp = &ngep->statistics.sw_statistics; 135 mp = NULL; 136 137 srbdp = &brp->sw_rbds[start_index]; 138 DMA_SYNC(*srbdp->bufp, DDI_DMA_SYNC_FORKERNEL); 139 hw_bd_p = DMA_VPTR(srbdp->desc); 140 141 /* 142 * First check the free_list, if it is NULL, 143 * make the recycle_list be free_list. 144 */ 145 if (brp->free_list == NULL) { 146 mutex_enter(brp->recycle_lock); 147 brp->free_list = brp->recycle_list; 148 brp->recycle_list = NULL; 149 mutex_exit(brp->recycle_lock); 150 } 151 bufp = brp->free_list; 152 /* If it's not a qualified packet, delete it */ 153 if (len > maxsize || len < minsize) { 154 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 155 srbdp->bufp->alength); 156 srbdp->flags = CONTROLER_OWN; 157 return (NULL); 158 } 159 160 /* 161 * If receive packet size is smaller than RX bcopy threshold, 162 * or there is no available buffer in free_list or recycle list, 163 * we use bcopy directly. 164 */ 165 if (len <= ngep->param_rxbcopy_threshold || bufp == NULL) 166 brp->rx_bcopy = B_TRUE; 167 else 168 brp->rx_bcopy = B_FALSE; 169 170 if (brp->rx_bcopy) { 171 mp = allocb(len + NGE_HEADROOM, 0); 172 if (mp == NULL) { 173 sw_stp->mp_alloc_err++; 174 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 175 srbdp->bufp->alength); 176 srbdp->flags = CONTROLER_OWN; 177 return (NULL); 178 } 179 rptr = DMA_VPTR(*srbdp->bufp); 180 mp->b_rptr = mp->b_rptr + NGE_HEADROOM; 181 bcopy(rptr + NGE_HEADROOM, mp->b_rptr, len); 182 mp->b_wptr = mp->b_rptr + len; 183 } else { 184 mp = srbdp->bufp->mp; 185 /* 186 * Make sure the packet *contents* 4-byte aligned 187 */ 188 mp->b_rptr += NGE_HEADROOM; 189 mp->b_wptr = mp->b_rptr + len; 190 mp->b_next = mp->b_cont = NULL; 191 srbdp->bufp->rx_delivered = B_TRUE; 192 srbdp->bufp = NULL; 193 nge_atomic_increase(&brp->rx_hold, 1); 194 195 /* Fill the buffer from free_list */ 196 srbdp->bufp = bufp; 197 brp->free_list = bufp->next; 198 bufp->next = NULL; 199 } 200 201 /* replenish the buffer for hardware descriptor */ 202 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 203 srbdp->bufp->alength); 204 srbdp->flags = CONTROLER_OWN; 205 sw_stp->rbytes += len; 206 sw_stp->recv_count++; 207 208 return (mp); 209 } 210 211 212 #define RX_HW_ERR 0x01 213 #define RX_SUM_NO 0x02 214 #define RX_SUM_ERR 0x04 215 216 /* 217 * Statistic the rx's error 218 * and generate a log msg for these. 219 * Note: 220 * RXE, Parity Error, Symbo error, CRC error 221 * have been recored by nvidia's hardware 222 * statistics part (nge_statistics). So it is uncessary to record them by 223 * driver in this place. 224 */ 225 static uint32_t 226 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags); 227 #pragma inline(nge_rxsta_handle) 228 229 static uint32_t 230 nge_rxsta_handle(nge_t *ngep, uint32_t stflag, uint32_t *pflags) 231 { 232 uint32_t errors; 233 uint32_t err_flag; 234 nge_sw_statistics_t *sw_stp; 235 236 err_flag = 0; 237 sw_stp = &ngep->statistics.sw_statistics; 238 239 if ((RXD_END & stflag) == 0) 240 return (RX_HW_ERR); 241 242 errors = stflag & RXD_CSUM_MSK; 243 switch (errors) { 244 default: 245 break; 246 247 case RXD_CK8G_TCP_SUM: 248 case RXD_CK8G_UDP_SUM: 249 *pflags |= HCK_FULLCKSUM; 250 *pflags |= HCK_IPV4_HDRCKSUM; 251 *pflags |= HCK_FULLCKSUM_OK; 252 break; 253 254 case RXD_CK8G_TCP_SUM_ERR: 255 case RXD_CK8G_UDP_SUM_ERR: 256 sw_stp->tcp_hwsum_err++; 257 *pflags |= HCK_IPV4_HDRCKSUM; 258 break; 259 260 case RXD_CK8G_IP_HSUM: 261 *pflags |= HCK_IPV4_HDRCKSUM; 262 break; 263 264 case RXD_CK8G_NO_HSUM: 265 err_flag |= RX_SUM_NO; 266 break; 267 268 case RXD_CK8G_IP_HSUM_ERR: 269 sw_stp->ip_hwsum_err++; 270 err_flag |= RX_SUM_ERR; 271 break; 272 } 273 274 if ((stflag & RXD_ERR) != 0) { 275 276 err_flag |= RX_HW_ERR; 277 NGE_DEBUG(("Receive desc error, status: 0x%x", stflag)); 278 } 279 280 return (err_flag); 281 } 282 283 static mblk_t * 284 nge_recv_ring(nge_t *ngep) 285 { 286 uint32_t stflag; 287 uint32_t flag_err; 288 uint32_t sum_flags; 289 uint32_t count; 290 size_t len; 291 uint64_t end_index; 292 uint64_t sync_start; 293 mblk_t *mp; 294 mblk_t **tail; 295 mblk_t *head; 296 recv_ring_t *rrp; 297 buff_ring_t *brp; 298 sw_rx_sbd_t *srbdp; 299 void * hw_bd_p; 300 nge_mode_cntl mode_cntl; 301 302 mp = NULL; 303 head = NULL; 304 count = 0; 305 tail = &head; 306 rrp = ngep->recv; 307 brp = ngep->buff; 308 309 end_index = sync_start = rrp->prod_index; 310 /* Sync the descriptor for kernel */ 311 if (sync_start + ngep->param_recv_max_packet <= ngep->rx_desc) { 312 (void) ddi_dma_sync(rrp->desc.dma_hdl, 313 sync_start * ngep->desc_attr.rxd_size, 314 ngep->param_recv_max_packet * ngep->desc_attr.rxd_size, 315 DDI_DMA_SYNC_FORKERNEL); 316 } else { 317 (void) ddi_dma_sync(rrp->desc.dma_hdl, 318 sync_start * ngep->desc_attr.rxd_size, 319 0, 320 DDI_DMA_SYNC_FORKERNEL); 321 (void) ddi_dma_sync(rrp->desc.dma_hdl, 322 0, 323 (ngep->param_recv_max_packet + sync_start - ngep->rx_desc) * 324 ngep->desc_attr.rxd_size, 325 DDI_DMA_SYNC_FORKERNEL); 326 } 327 328 /* 329 * Looking through the rx's ring to find the good packets 330 * and try to receive more and more packets in rx's ring 331 */ 332 for (;;) { 333 sum_flags = 0; 334 flag_err = 0; 335 end_index = rrp->prod_index; 336 srbdp = &brp->sw_rbds[end_index]; 337 hw_bd_p = DMA_VPTR(srbdp->desc); 338 stflag = ngep->desc_attr.rxd_check(hw_bd_p, &len); 339 /* 340 * If there is no packet in receving ring 341 * break the loop 342 */ 343 if ((stflag & RXD_OWN) != 0 || HOST_OWN == srbdp->flags) 344 break; 345 346 ngep->recv_count++; 347 flag_err = nge_rxsta_handle(ngep, stflag, &sum_flags); 348 if ((flag_err & RX_HW_ERR) == 0) { 349 srbdp->flags = NGE_END_PACKET; 350 mp = nge_recv_packet(ngep, end_index, len); 351 } else { 352 /* Hardware error, re-use the buffer */ 353 ngep->desc_attr.rxd_fill(hw_bd_p, &srbdp->bufp->cookie, 354 srbdp->bufp->alength); 355 srbdp->flags = CONTROLER_OWN; 356 } 357 count++; 358 if (mp != NULL) { 359 if (!(flag_err & (RX_SUM_NO | RX_SUM_ERR))) { 360 (void) hcksum_assoc(mp, NULL, NULL, 361 0, 0, 0, 0, sum_flags, 0); 362 } 363 *tail = mp; 364 tail = &mp->b_next; 365 mp = NULL; 366 } 367 rrp->prod_index = NEXT(end_index, rrp->desc.nslots); 368 if (count > ngep->param_recv_max_packet) 369 break; 370 } 371 372 /* Sync the descriptors for device */ 373 if (sync_start + count <= ngep->rx_desc) { 374 (void) ddi_dma_sync(rrp->desc.dma_hdl, 375 sync_start * ngep->desc_attr.rxd_size, 376 count * ngep->desc_attr.rxd_size, 377 DDI_DMA_SYNC_FORDEV); 378 } else { 379 (void) ddi_dma_sync(rrp->desc.dma_hdl, 380 sync_start * ngep->desc_attr.rxd_size, 381 0, 382 DDI_DMA_SYNC_FORDEV); 383 (void) ddi_dma_sync(rrp->desc.dma_hdl, 384 0, 385 (count + sync_start - ngep->rx_desc) * 386 ngep->desc_attr.rxd_size, 387 DDI_DMA_SYNC_FORDEV); 388 } 389 mode_cntl.mode_val = nge_reg_get32(ngep, NGE_MODE_CNTL); 390 mode_cntl.mode_bits.rxdm = NGE_SET; 391 mode_cntl.mode_bits.tx_rcom_en = NGE_SET; 392 nge_reg_put32(ngep, NGE_MODE_CNTL, mode_cntl.mode_val); 393 394 return (head); 395 } 396 397 void 398 nge_receive(nge_t *ngep) 399 { 400 mblk_t *mp; 401 recv_ring_t *rrp; 402 rrp = ngep->recv; 403 404 mp = nge_recv_ring(ngep); 405 mutex_exit(ngep->genlock); 406 if (mp != NULL) 407 mac_rx(ngep->mh, rrp->handle, mp); 408 mutex_enter(ngep->genlock); 409 } 410 411 void 412 nge_hot_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 413 { 414 uint64_t dmac_addr; 415 hot_rx_bd * hw_bd_p; 416 417 hw_bd_p = (hot_rx_bd *)hwd; 418 dmac_addr = cookie->dmac_laddress + NGE_HEADROOM; 419 420 hw_bd_p->cntl_status.cntl_val = 0; 421 422 hw_bd_p->host_buf_addr_hi = dmac_addr >> 32; 423 hw_bd_p->host_buf_addr_lo = dmac_addr; 424 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 425 426 membar_producer(); 427 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 428 } 429 430 void 431 nge_sum_rxd_fill(void *hwd, const ddi_dma_cookie_t *cookie, size_t len) 432 { 433 uint64_t dmac_addr; 434 sum_rx_bd * hw_bd_p; 435 436 hw_bd_p = hwd; 437 dmac_addr = cookie->dmac_address + NGE_HEADROOM; 438 439 hw_bd_p->cntl_status.cntl_val = 0; 440 441 hw_bd_p->host_buf_addr = dmac_addr; 442 hw_bd_p->cntl_status.control_bits.bcnt = len - 1; 443 444 membar_producer(); 445 hw_bd_p->cntl_status.control_bits.own = NGE_SET; 446 } 447 448 uint32_t 449 nge_hot_rxd_check(const void *hwd, size_t *len) 450 { 451 uint32_t err_flag; 452 const hot_rx_bd * hrbdp; 453 454 hrbdp = hwd; 455 456 err_flag = hrbdp->cntl_status.cntl_val & ~RXD_BCNT_MSK; 457 *len = hrbdp->cntl_status.status_bits_legacy.bcnt; 458 459 return (err_flag); 460 } 461 462 uint32_t 463 nge_sum_rxd_check(const void *hwd, size_t *len) 464 { 465 uint32_t err_flag; 466 const sum_rx_bd * hrbdp; 467 468 hrbdp = hwd; 469 470 err_flag = hrbdp->cntl_status.cntl_val & ~RXD_BCNT_MSK; 471 *len = hrbdp->cntl_status.status_bits.bcnt; 472 473 return (err_flag); 474 } 475