Lines Matching refs:ring
287 igc_rx_ring_t *ring = buf->irb_ring; in igc_rx_recycle() local
321 mutex_enter(&ring->irr_free_lock); in igc_rx_recycle()
322 ring->irr_free_list[ring->irr_nfree] = buf; in igc_rx_recycle()
323 ring->irr_nfree++; in igc_rx_recycle()
325 igc_t *igc = ring->irr_igc; in igc_rx_recycle()
326 ASSERT3U(ring->irr_nfree, <=, igc->igc_rx_nfree); in igc_rx_recycle()
328 cv_signal(&ring->irr_free_cv); in igc_rx_recycle()
329 mutex_exit(&ring->irr_free_lock); in igc_rx_recycle()
333 igc_rx_bufs_free(igc_t *igc, igc_rx_ring_t *ring) in igc_rx_bufs_free() argument
336 igc_rx_buffer_t *buf = &ring->irr_arena[i]; in igc_rx_bufs_free()
346 igc_rx_bufs_alloc(igc_t *igc, igc_rx_ring_t *ring) in igc_rx_bufs_alloc() argument
349 igc_rx_buffer_t *buf = &ring->irr_arena[i]; in igc_rx_bufs_alloc()
354 buf->irb_ring = ring; in igc_rx_bufs_alloc()
359 "ring %u buffer %u", ring->irr_idx, i); in igc_rx_bufs_alloc()
377 ring->irr_work_list[i] = buf; in igc_rx_bufs_alloc()
379 ring->irr_free_list[ring->irr_nfree] = buf; in igc_rx_bufs_alloc()
380 ring->irr_nfree++; in igc_rx_bufs_alloc()
391 igc_rx_ring_t *ring = &igc->igc_rx_rings[i]; in igc_rx_data_free() local
393 if (ring->irr_arena != NULL) { in igc_rx_data_free()
394 igc_rx_bufs_free(igc, ring); in igc_rx_data_free()
395 kmem_free(ring->irr_arena, sizeof (igc_rx_buffer_t) * in igc_rx_data_free()
397 ring->irr_arena = NULL; in igc_rx_data_free()
400 if (ring->irr_free_list != NULL) { in igc_rx_data_free()
401 kmem_free(ring->irr_free_list, igc->igc_rx_nfree * in igc_rx_data_free()
403 ring->irr_free_list = NULL; in igc_rx_data_free()
406 if (ring->irr_work_list != NULL) { in igc_rx_data_free()
407 kmem_free(ring->irr_work_list, igc->igc_rx_ndesc * in igc_rx_data_free()
409 ring->irr_work_list = NULL; in igc_rx_data_free()
412 if (ring->irr_ring != NULL) { in igc_rx_data_free()
413 igc_dma_free(&ring->irr_desc_dma); in igc_rx_data_free()
414 ring->irr_ring = NULL; in igc_rx_data_free()
415 ring->irr_next = 0; in igc_rx_data_free()
424 igc_rx_ring_t *ring = &igc->igc_rx_rings[i]; in igc_rx_data_alloc() local
431 if (!igc_dma_alloc(igc, &ring->irr_desc_dma, &desc_attr, in igc_rx_data_alloc()
437 ring->irr_ring = (void *)ring->irr_desc_dma.idb_va; in igc_rx_data_alloc()
439 ring->irr_work_list = kmem_zalloc(sizeof (igc_rx_buffer_t *) * in igc_rx_data_alloc()
441 if (ring->irr_work_list == NULL) { in igc_rx_data_alloc()
447 ring->irr_free_list = kmem_zalloc(sizeof (igc_rx_buffer_t *) * in igc_rx_data_alloc()
449 if (ring->irr_free_list == NULL) { in igc_rx_data_alloc()
456 ring->irr_arena = kmem_zalloc(sizeof (igc_rx_buffer_t) * in igc_rx_data_alloc()
458 if (ring->irr_arena == NULL) { in igc_rx_data_alloc()
464 if (!igc_rx_bufs_alloc(igc, ring)) { in igc_rx_data_alloc()
481 igc_rx_ring_desc_write(igc_rx_ring_t *ring, uint32_t idx) in igc_rx_ring_desc_write() argument
485 igc_dma_buffer_t *irb = &ring->irr_work_list[idx]->irb_dma; in igc_rx_ring_desc_write()
489 ring->irr_ring[idx].read.pkt_addr = LE_64(addr); in igc_rx_ring_desc_write()
490 ring->irr_ring[idx].read.hdr_addr = LE_64(0); in igc_rx_ring_desc_write()
502 igc_rx_ring_hw_init(igc_t *igc, igc_rx_ring_t *ring) in igc_rx_ring_hw_init() argument
508 igc_rx_ring_desc_write(ring, i); in igc_rx_ring_hw_init()
510 IGC_DMA_SYNC(&ring->irr_desc_dma, DDI_DMA_SYNC_FORDEV); in igc_rx_ring_hw_init()
515 desc = ddi_dma_cookie_one(ring->irr_desc_dma.idb_hdl); in igc_rx_ring_hw_init()
518 igc_write32(igc, IGC_RDBAH(ring->irr_idx), high); in igc_rx_ring_hw_init()
519 igc_write32(igc, IGC_RDBAL(ring->irr_idx), low); in igc_rx_ring_hw_init()
525 igc_write32(igc, IGC_RDLEN(ring->irr_idx), val); in igc_rx_ring_hw_init()
532 igc_write32(igc, IGC_SRRCTL(ring->irr_idx), val); in igc_rx_ring_hw_init()
538 val = igc_read32(igc, IGC_RXDCTL(ring->irr_idx)); in igc_rx_ring_hw_init()
544 igc_write32(igc, IGC_RXDCTL(ring->irr_idx), val); in igc_rx_ring_hw_init()
648 igc_rx_copy(igc_rx_ring_t *ring, uint32_t idx, uint32_t len) in igc_rx_copy() argument
650 const igc_rx_buffer_t *buf = ring->irr_work_list[idx]; in igc_rx_copy()
656 ring->irr_stat.irs_copy_nomem.value.ui64++; in igc_rx_copy()
663 ring->irr_stat.irs_ncopy.value.ui64++; in igc_rx_copy()
668 igc_rx_bind(igc_rx_ring_t *ring, uint32_t idx, uint32_t len) in igc_rx_bind() argument
670 igc_rx_buffer_t *buf = ring->irr_work_list[idx]; in igc_rx_bind()
673 ASSERT(MUTEX_HELD(&ring->irr_lock)); in igc_rx_bind()
679 mutex_enter(&ring->irr_free_lock); in igc_rx_bind()
680 if (ring->irr_nfree == 0) { in igc_rx_bind()
681 ring->irr_stat.irs_bind_nobuf.value.ui64++; in igc_rx_bind()
682 mutex_exit(&ring->irr_free_lock); in igc_rx_bind()
685 ring->irr_nfree--; in igc_rx_bind()
686 sub = ring->irr_free_list[ring->irr_nfree]; in igc_rx_bind()
687 mutex_exit(&ring->irr_free_lock); in igc_rx_bind()
700 ring->irr_stat.irs_bind_nomp.value.ui64++; in igc_rx_bind()
701 mutex_enter(&ring->irr_free_lock); in igc_rx_bind()
702 ring->irr_free_list[ring->irr_nfree] = sub; in igc_rx_bind()
703 ring->irr_nfree++; in igc_rx_bind()
704 mutex_exit(&ring->irr_free_lock); in igc_rx_bind()
714 ring->irr_work_list[idx] = sub; in igc_rx_bind()
715 ring->irr_stat.irs_nbind.value.ui64++; in igc_rx_bind()
731 igc_rx_hcksum(igc_rx_ring_t *ring, mblk_t *mp, uint32_t status) in igc_rx_hcksum() argument
738 ring->irr_stat.irs_ixsm.value.ui64++; in igc_rx_hcksum()
743 ring->irr_stat.irs_l4cksum_err.value.ui64++; in igc_rx_hcksum()
749 ring->irr_stat.irs_l3cksum_err.value.ui64++; in igc_rx_hcksum()
755 ring->irr_stat.irs_hcksum_hit.value.ui64++; in igc_rx_hcksum()
758 ring->irr_stat.irs_hcksum_miss.value.ui64++; in igc_rx_hcksum()
763 igc_ring_rx(igc_rx_ring_t *ring, int poll_bytes) in igc_ring_rx() argument
768 igc_t *igc = ring->irr_igc; in igc_ring_rx()
771 ASSERT(MUTEX_HELD(&ring->irr_lock)); in igc_ring_rx()
772 IGC_DMA_SYNC(&ring->irr_desc_dma, DDI_DMA_SYNC_FORKERNEL); in igc_ring_rx()
778 cur_head = ring->irr_next; in igc_ring_rx()
779 cur_desc = &ring->irr_ring[cur_head]; in igc_ring_rx()
796 ring->irr_stat.irs_desc_error.value.ui64++; in igc_ring_rx()
811 mp = igc_rx_bind(ring, cur_head, cur_length); in igc_ring_rx()
815 mp = igc_rx_copy(ring, cur_head, cur_length); in igc_ring_rx()
819 igc_rx_hcksum(ring, mp, cur_status); in igc_ring_rx()
829 igc_rx_ring_desc_write(ring, cur_head); in igc_ring_rx()
835 cur_desc = &ring->irr_ring[cur_head]; in igc_ring_rx()
861 IGC_DMA_SYNC(&ring->irr_desc_dma, DDI_DMA_SYNC_FORDEV); in igc_ring_rx()
862 ring->irr_next = cur_head; in igc_ring_rx()
864 igc_write32(igc, IGC_RDT(ring->irr_idx), tail); in igc_ring_rx()
866 ring->irr_stat.irs_rbytes.value.ui64 += rx_bytes; in igc_ring_rx()
867 ring->irr_stat.irs_ipackets.value.ui64 += rx_frames; in igc_ring_rx()
890 igc_rx_ring_t *ring = &igc->igc_rx_rings[i]; in igc_rx_drain() local
892 mutex_enter(&ring->irr_free_lock); in igc_rx_drain()
893 while (ring->irr_nfree < igc->igc_rx_nfree) { in igc_rx_drain()
894 cv_wait(&ring->irr_free_cv, &ring->irr_free_lock); in igc_rx_drain()
896 mutex_exit(&ring->irr_free_lock); in igc_rx_drain()
901 igc_tx_bufs_free(igc_t *igc, igc_tx_ring_t *ring) in igc_tx_bufs_free() argument
904 igc_tx_buffer_t *buf = &ring->itr_arena[i]; in igc_tx_bufs_free()
926 igc_tx_bufs_alloc(igc_t *igc, igc_tx_ring_t *ring) in igc_tx_bufs_alloc() argument
929 igc_tx_buffer_t *buf = &ring->itr_arena[i]; in igc_tx_bufs_alloc()
937 "ring %u buffer %u", ring->itr_idx, i); in igc_tx_bufs_alloc()
946 "ring %u TX DMA handle %u: %d", ring->itr_idx, i, in igc_tx_bufs_alloc()
951 list_insert_tail(&ring->itr_free_list, buf); in igc_tx_bufs_alloc()
961 igc_tx_ring_t *ring = &igc->igc_tx_rings[i]; in igc_tx_data_free() local
967 while (list_remove_head(&ring->itr_free_list) != NULL) in igc_tx_data_free()
970 if (ring->itr_arena != NULL) { in igc_tx_data_free()
971 igc_tx_bufs_free(igc, ring); in igc_tx_data_free()
972 kmem_free(ring->itr_arena, sizeof (igc_tx_buffer_t) * in igc_tx_data_free()
974 ring->itr_arena = NULL; in igc_tx_data_free()
977 list_destroy(&ring->itr_free_list); in igc_tx_data_free()
979 if (ring->itr_work_list != NULL) { in igc_tx_data_free()
980 kmem_free(ring->itr_work_list, igc->igc_tx_ndesc * in igc_tx_data_free()
982 ring->itr_work_list = NULL; in igc_tx_data_free()
985 if (ring->itr_ring != NULL) { in igc_tx_data_free()
986 igc_dma_free(&ring->itr_desc_dma); in igc_tx_data_free()
987 ring->itr_ring = NULL; in igc_tx_data_free()
988 ring->itr_ring_head = 0; in igc_tx_data_free()
989 ring->itr_ring_tail = 0; in igc_tx_data_free()
990 ring->itr_ring_free = 0; in igc_tx_data_free()
999 igc_tx_ring_t *ring = &igc->igc_tx_rings[i]; in igc_tx_data_alloc() local
1006 if (!igc_dma_alloc(igc, &ring->itr_desc_dma, &desc_attr, in igc_tx_data_alloc()
1012 ring->itr_ring = (void *)ring->itr_desc_dma.idb_va; in igc_tx_data_alloc()
1014 ring->itr_work_list = kmem_zalloc(sizeof (igc_tx_buffer_t *) * in igc_tx_data_alloc()
1016 if (ring->itr_work_list == NULL) { in igc_tx_data_alloc()
1022 list_create(&ring->itr_free_list, sizeof (igc_tx_buffer_t), in igc_tx_data_alloc()
1025 ring->itr_arena = kmem_zalloc(sizeof (igc_tx_buffer_t) * in igc_tx_data_alloc()
1027 if (ring->itr_arena == NULL) { in igc_tx_data_alloc()
1033 if (!igc_tx_bufs_alloc(igc, ring)) { in igc_tx_data_alloc()
1046 igc_tx_ring_hw_init(igc_t *igc, igc_tx_ring_t *ring) in igc_tx_ring_hw_init() argument
1054 desc = ddi_dma_cookie_one(ring->itr_desc_dma.idb_hdl); in igc_tx_ring_hw_init()
1057 igc_write32(igc, IGC_TDBAH(ring->itr_idx), high); in igc_tx_ring_hw_init()
1058 igc_write32(igc, IGC_TDBAL(ring->itr_idx), low); in igc_tx_ring_hw_init()
1064 igc_write32(igc, IGC_TDLEN(ring->itr_idx), val); in igc_tx_ring_hw_init()
1071 igc_write32(igc, IGC_TDH(ring->itr_idx), 0); in igc_tx_ring_hw_init()
1072 igc_write32(igc, IGC_TDT(ring->itr_idx), 0); in igc_tx_ring_hw_init()
1073 ring->itr_ring_head = 0; in igc_tx_ring_hw_init()
1074 ring->itr_ring_tail = 0; in igc_tx_ring_hw_init()
1075 ring->itr_ring_free = igc->igc_tx_ndesc; in igc_tx_ring_hw_init()
1142 igc_tx_recycle(igc_t *igc, igc_tx_ring_t *ring) in igc_tx_recycle() argument
1156 mutex_enter(&ring->itr_lock); in igc_tx_recycle()
1157 if (ring->itr_recycle) { in igc_tx_recycle()
1158 mutex_exit(&ring->itr_lock); in igc_tx_recycle()
1161 ring->itr_recycle = true; in igc_tx_recycle()
1162 head = ring->itr_ring_head; in igc_tx_recycle()
1163 tail = ring->itr_ring_tail; in igc_tx_recycle()
1164 mutex_exit(&ring->itr_lock); in igc_tx_recycle()
1169 IGC_DMA_SYNC(&ring->itr_desc_dma, DDI_DMA_SYNC_FORKERNEL); in igc_tx_recycle()
1182 igc_tx_buffer_t *check_buf = ring->itr_work_list[head]; in igc_tx_recycle()
1188 status = LE_32(ring->itr_ring[last_desc].wb.status); in igc_tx_recycle()
1203 bzero(&ring->itr_ring[desc], in igc_tx_recycle()
1206 buf = ring->itr_work_list[desc]; in igc_tx_recycle()
1209 ring->itr_work_list[desc] = NULL; in igc_tx_recycle()
1222 mutex_enter(&ring->itr_lock); in igc_tx_recycle()
1223 ring->itr_ring_head = head; in igc_tx_recycle()
1224 ring->itr_ring_free += ndesc; in igc_tx_recycle()
1225 list_move_tail(&ring->itr_free_list, &to_free); in igc_tx_recycle()
1226 if (ring->itr_mac_blocked && ring->itr_ring_free > in igc_tx_recycle()
1228 ring->itr_mac_blocked = false; in igc_tx_recycle()
1231 ring->itr_recycle = false; in igc_tx_recycle()
1232 mutex_exit(&ring->itr_lock); in igc_tx_recycle()
1235 mac_tx_ring_update(igc->igc_mac_hdl, ring->itr_rh); in igc_tx_recycle()
1243 igc_tx_buffer_alloc(igc_tx_ring_t *ring) in igc_tx_buffer_alloc() argument
1246 mutex_enter(&ring->itr_lock); in igc_tx_buffer_alloc()
1247 buf = list_remove_head(&ring->itr_free_list); in igc_tx_buffer_alloc()
1249 ring->itr_stat.its_no_tx_bufs.value.ui64++; in igc_tx_buffer_alloc()
1251 mutex_exit(&ring->itr_lock); in igc_tx_buffer_alloc()
1260 igc_tx_ring_bind(igc_tx_ring_t *ring, mblk_t *mp, igc_tx_state_t *tx) in igc_tx_ring_bind() argument
1267 buf = igc_tx_buffer_alloc(ring); in igc_tx_ring_bind()
1279 ring->itr_stat.its_tx_bind_fail.value.ui64++; in igc_tx_ring_bind()
1280 mutex_enter(&ring->itr_lock); in igc_tx_ring_bind()
1281 list_insert_tail(&ring->itr_free_list, buf); in igc_tx_ring_bind()
1282 mutex_exit(&ring->itr_lock); in igc_tx_ring_bind()
1297 ring->itr_stat.its_tx_bind.value.ui64++; in igc_tx_ring_bind()
1306 igc_tx_ring_copy(igc_tx_ring_t *ring, mblk_t *mp, igc_tx_state_t *tx) in igc_tx_ring_copy() argument
1328 tx->itx_cur_buf = igc_tx_buffer_alloc(ring); in igc_tx_ring_copy()
1355 ring->itr_stat.its_tx_copy.value.ui64++; in igc_tx_ring_copy()
1366 igc_tx_ring_context_changed(igc_tx_ring_t *ring, igc_tx_state_t *tx) in igc_tx_ring_context_changed() argument
1369 igc_tx_context_data_t *data = &ring->itr_tx_ctx; in igc_tx_ring_context_changed()
1419 igc_tx_ring_write_buf_descs(igc_t *igc, igc_tx_ring_t *ring, in igc_tx_ring_write_buf_descs() argument
1427 ASSERT(MUTEX_HELD(&ring->itr_lock)); in igc_tx_ring_write_buf_descs()
1430 for (uint_t i = 0; i < nc; i++, ring->itr_ring_tail = in igc_tx_ring_write_buf_descs()
1431 igc_next_desc(ring->itr_ring_tail, 1, igc->igc_tx_ndesc)) { in igc_tx_ring_write_buf_descs()
1443 desc = &ring->itr_ring[ring->itr_ring_tail]; in igc_tx_ring_write_buf_descs()
1453 ring->itr_work_list[ring->itr_ring_tail] = buf; in igc_tx_ring_write_buf_descs()
1465 igc_tx_ring_write_descs(igc_t *igc, igc_tx_ring_t *ring, mblk_t *mp, in igc_tx_ring_write_descs() argument
1481 mutex_enter(&ring->itr_lock); in igc_tx_ring_write_descs()
1482 if (tx->itx_ndescs + igc->igc_tx_gap > ring->itr_ring_free) { in igc_tx_ring_write_descs()
1486 mutex_exit(&ring->itr_lock); in igc_tx_ring_write_descs()
1487 igc_tx_recycle(igc, ring); in igc_tx_ring_write_descs()
1488 mutex_enter(&ring->itr_lock); in igc_tx_ring_write_descs()
1489 if (tx->itx_ndescs + igc->igc_tx_gap > ring->itr_ring_free) { in igc_tx_ring_write_descs()
1490 mutex_exit(&ring->itr_lock); in igc_tx_ring_write_descs()
1501 if (do_ctx && !igc_tx_ring_context_changed(ring, tx)) { in igc_tx_ring_write_descs()
1506 ring->itr_ring_free -= tx->itx_ndescs; in igc_tx_ring_write_descs()
1507 ctx_desc = ring->itr_ring_tail; in igc_tx_ring_write_descs()
1546 ctx = (void *)&ring->itr_ring[ctx_desc]; in igc_tx_ring_write_descs()
1551 ring->itr_ring_tail = igc_next_desc(ring->itr_ring_tail, 1, in igc_tx_ring_write_descs()
1554 ring, igc_tx_state_t *, tx, in igc_tx_ring_write_descs()
1558 first_desc = ring->itr_ring_tail; in igc_tx_ring_write_descs()
1561 igc_tx_ring_write_buf_descs(igc, ring, buf); in igc_tx_ring_write_descs()
1570 last_desc = igc_prev_desc(ring->itr_ring_tail, 1, igc->igc_tx_ndesc); in igc_tx_ring_write_descs()
1572 ring->itr_ring[last_desc].read.cmd_type_len |= LE_32(flags); in igc_tx_ring_write_descs()
1606 ring->itr_ring[first_desc].read.cmd_type_len |= LE_32(flags); in igc_tx_ring_write_descs()
1607 ring->itr_ring[first_desc].read.olinfo_status |= LE_32(status); in igc_tx_ring_write_descs()
1608 ring->itr_work_list[first_desc]->itb_mp = mp; in igc_tx_ring_write_descs()
1609 ring->itr_work_list[first_desc]->itb_first = true; in igc_tx_ring_write_descs()
1610 ring->itr_work_list[first_desc]->itb_last_desc = last_desc; in igc_tx_ring_write_descs()
1618 ring->itr_work_list[ctx_desc] = ring->itr_work_list[first_desc]; in igc_tx_ring_write_descs()
1619 ring->itr_work_list[first_desc] = NULL; in igc_tx_ring_write_descs()
1622 ring->itr_stat.its_obytes.value.ui64 += tx->itx_meoi.meoi_len; in igc_tx_ring_write_descs()
1623 ring->itr_stat.its_opackets.value.ui64++; in igc_tx_ring_write_descs()
1625 IGC_DMA_SYNC(&ring->itr_desc_dma, DDI_DMA_SYNC_FORDEV); in igc_tx_ring_write_descs()
1626 igc_write32(igc, IGC_TDT(ring->itr_idx), ring->itr_ring_tail); in igc_tx_ring_write_descs()
1627 mutex_exit(&ring->itr_lock); in igc_tx_ring_write_descs()
1663 igc_tx_ring_t *ring = arg; in igc_ring_tx() local
1664 igc_t *igc = ring->itr_igc; in igc_ring_tx()
1683 ring->itr_stat.its_bad_meo.value.ui64++; in igc_ring_tx()
1692 if (ring->itr_ring_free < igc->igc_tx_recycle_thresh) { in igc_ring_tx()
1693 igc_tx_recycle(igc, ring); in igc_ring_tx()
1696 mutex_enter(&ring->itr_lock); in igc_ring_tx()
1697 if (ring->itr_ring_free < igc->igc_tx_notify_thresh) { in igc_ring_tx()
1698 ring->itr_stat.its_ring_full.value.ui64++; in igc_ring_tx()
1699 ring->itr_mac_blocked = true; in igc_ring_tx()
1700 mutex_exit(&ring->itr_lock); in igc_ring_tx()
1703 mutex_exit(&ring->itr_lock); in igc_ring_tx()
1727 igc_tx_ring_bind(ring, cur_mp, &tx)) { in igc_ring_tx()
1731 if (!igc_tx_ring_copy(ring, cur_mp, &tx)) in igc_ring_tx()
1735 if (!igc_tx_ring_write_descs(igc, ring, mp, &tx)) { in igc_ring_tx()
1751 mutex_enter(&ring->itr_lock); in igc_ring_tx()
1752 list_move_tail(&ring->itr_free_list, &tx.itx_bufs); in igc_ring_tx()
1753 ring->itr_mac_blocked = true; in igc_ring_tx()
1754 mutex_exit(&ring->itr_lock); in igc_ring_tx()