Lines Matching refs:xnfp
352 gref_get(xnf_t *xnfp) in gref_get() argument
356 mutex_enter(&xnfp->xnf_gref_lock); in gref_get()
359 gref = gnttab_claim_grant_reference(&xnfp->xnf_gref_head); in gref_get()
362 (gnttab_alloc_grant_references(16, &xnfp->xnf_gref_head) == 0)); in gref_get()
364 mutex_exit(&xnfp->xnf_gref_lock); in gref_get()
367 xnfp->xnf_stat_gref_failure++; in gref_get()
369 atomic_inc_64(&xnfp->xnf_stat_gref_outstanding); in gref_get()
370 if (xnfp->xnf_stat_gref_outstanding > xnfp->xnf_stat_gref_peak) in gref_get()
371 xnfp->xnf_stat_gref_peak = in gref_get()
372 xnfp->xnf_stat_gref_outstanding; in gref_get()
382 gref_put(xnf_t *xnfp, grant_ref_t gref) in gref_put() argument
386 mutex_enter(&xnfp->xnf_gref_lock); in gref_put()
387 gnttab_release_grant_reference(&xnfp->xnf_gref_head, gref); in gref_put()
388 mutex_exit(&xnfp->xnf_gref_lock); in gref_put()
390 atomic_dec_64(&xnfp->xnf_stat_gref_outstanding); in gref_put()
397 txid_get(xnf_t *xnfp) in txid_get() argument
401 ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); in txid_get()
403 if (xnfp->xnf_tx_pkt_id_head == INVALID_TX_ID) in txid_get()
406 ASSERT(TX_ID_VALID(xnfp->xnf_tx_pkt_id_head)); in txid_get()
408 tidp = TX_ID_TO_TXID(xnfp, xnfp->xnf_tx_pkt_id_head); in txid_get()
409 xnfp->xnf_tx_pkt_id_head = tidp->next; in txid_get()
421 txid_put(xnf_t *xnfp, xnf_txid_t *tidp) in txid_put() argument
423 ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); in txid_put()
428 tidp->next = xnfp->xnf_tx_pkt_id_head; in txid_put()
429 xnfp->xnf_tx_pkt_id_head = tidp->id; in txid_put()
440 tx_slots_get(xnf_t *xnfp, int wanted, boolean_t wait) in tx_slots_get() argument
445 ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); in tx_slots_get()
449 slotsfree = RING_FREE_REQUESTS(&xnfp->xnf_tx_ring); in tx_slots_get()
452 slotsfree = xnf_tx_clean_ring(xnfp); in tx_slots_get()
460 cv_broadcast(&xnfp->xnf_cv_tx_slots); in tx_slots_get()
468 cv_wait(&xnfp->xnf_cv_tx_slots, &xnfp->xnf_txlock); in tx_slots_get()
471 ASSERT(slotsfree <= RING_SIZE(&(xnfp->xnf_tx_ring))); in tx_slots_get()
477 xnf_setup_rings(xnf_t *xnfp) in xnf_setup_rings() argument
486 oeid = xvdi_get_oeid(xnfp->xnf_devinfo); in xnf_setup_rings()
487 xsd = xvdi_get_xsd(xnfp->xnf_devinfo); in xnf_setup_rings()
489 if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF) in xnf_setup_rings()
490 gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0); in xnf_setup_rings()
493 xnf_btop(pa_to_ma(xnfp->xnf_tx_ring_phys_addr)), 0); in xnf_setup_rings()
499 xnfp->xnf_tx_ring_ref = (grant_ref_t)err; in xnf_setup_rings()
501 if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF) in xnf_setup_rings()
502 gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0); in xnf_setup_rings()
505 xnf_btop(pa_to_ma(xnfp->xnf_rx_ring_phys_addr)), 0); in xnf_setup_rings()
511 xnfp->xnf_rx_ring_ref = (grant_ref_t)err; in xnf_setup_rings()
513 mutex_enter(&xnfp->xnf_txlock); in xnf_setup_rings()
519 xnfp->xnf_tx_pkt_id_head = INVALID_TX_ID; /* I.e. emtpy list. */ in xnf_setup_rings()
520 for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0]; in xnf_setup_rings()
530 txid_put(xnfp, tidp); in xnf_setup_rings()
545 gref_put(xnfp, txp->tx_txreq.gref); in xnf_setup_rings()
549 xnf_buf_put(xnfp, txp->tx_bdesc, B_TRUE); in xnf_setup_rings()
553 txid_put(xnfp, tidp); in xnf_setup_rings()
554 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_setup_rings()
561 cv_broadcast(&xnfp->xnf_cv_multicast); in xnf_setup_rings()
579 SHARED_RING_INIT(xnfp->xnf_tx_ring.sring); in xnf_setup_rings()
581 FRONT_RING_INIT(&xnfp->xnf_tx_ring, in xnf_setup_rings()
582 xnfp->xnf_tx_ring.sring, PAGESIZE); in xnf_setup_rings()
584 mutex_exit(&xnfp->xnf_txlock); in xnf_setup_rings()
586 mutex_enter(&xnfp->xnf_rxlock); in xnf_setup_rings()
592 for (i = 0, bdescp = &xnfp->xnf_rx_pkt_info[0]; in xnf_setup_rings()
596 xnf_buf_put(xnfp, *bdescp, B_FALSE); in xnf_setup_rings()
602 SHARED_RING_INIT(xnfp->xnf_rx_ring.sring); in xnf_setup_rings()
604 FRONT_RING_INIT(&xnfp->xnf_rx_ring, in xnf_setup_rings()
605 xnfp->xnf_rx_ring.sring, PAGESIZE); in xnf_setup_rings()
613 bdesc = xnf_buf_get(xnfp, KM_SLEEP, B_FALSE); in xnf_setup_rings()
615 xnf_rxbuf_hang(xnfp, bdesc); in xnf_setup_rings()
619 RING_PUSH_REQUESTS(&xnfp->xnf_rx_ring); in xnf_setup_rings()
621 mutex_exit(&xnfp->xnf_rxlock); in xnf_setup_rings()
626 if (xnfp->xnf_tx_ring_ref != INVALID_GRANT_REF) in xnf_setup_rings()
627 gnttab_end_foreign_access(xnfp->xnf_tx_ring_ref, 0, 0); in xnf_setup_rings()
628 xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF; in xnf_setup_rings()
630 if (xnfp->xnf_rx_ring_ref != INVALID_GRANT_REF) in xnf_setup_rings()
631 gnttab_end_foreign_access(xnfp->xnf_rx_ring_ref, 0, 0); in xnf_setup_rings()
632 xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF; in xnf_setup_rings()
642 xnf_be_connect(xnf_t *xnfp) in xnf_be_connect() argument
650 ASSERT(!xnfp->xnf_connected); in xnf_be_connect()
652 xsd = xvdi_get_xsd(xnfp->xnf_devinfo); in xnf_be_connect()
653 xsname = xvdi_get_xsname(xnfp->xnf_devinfo); in xnf_be_connect()
655 err = xnf_setup_rings(xnfp); in xnf_be_connect()
670 xnfp->xnf_tx_ring_ref); in xnf_be_connect()
677 xnfp->xnf_rx_ring_ref); in xnf_be_connect()
684 xnfp->xnf_evtchn); in xnf_be_connect()
702 if (xnfp->xnf_be_mcast_control) { in xnf_be_connect()
711 err = xvdi_switch_state(xnfp->xnf_devinfo, xbt, XenbusStateConnected); in xnf_be_connect()
735 xnf_read_config(xnf_t *xnfp) in xnf_read_config() argument
739 char *oename = xvdi_get_oename(xnfp->xnf_devinfo); in xnf_read_config()
749 ddi_driver_name(xnfp->xnf_devinfo), in xnf_read_config()
750 ddi_get_instance(xnfp->xnf_devinfo)); in xnf_read_config()
753 if (ether_aton(mac, xnfp->xnf_mac_addr) != ETHERADDRL) { in xnf_read_config()
755 xenbus_dev_error(xvdi_get_xsd(xnfp->xnf_devinfo), ENOENT, in xnf_read_config()
756 "parsing %s/mac", xvdi_get_xsname(xnfp->xnf_devinfo)); in xnf_read_config()
769 xnfp->xnf_be_rx_copy = (be_cap != 0); in xnf_read_config()
780 xnfp->xnf_be_mcast_control = (be_cap != 0) && xnf_multicast_control; in xnf_read_config()
790 xnf_t *xnfp; in xnf_attach() local
802 xnfp = ddi_get_driver_private(devinfo); in xnf_attach()
803 xnfp->xnf_gen++; in xnf_attach()
807 xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo); in xnf_attach()
809 ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr, in xnf_attach()
810 xnfp); in xnf_attach()
813 (caddr_t)xnfp); in xnf_attach()
830 xnfp = kmem_zalloc(sizeof (*xnfp), KM_SLEEP); in xnf_attach()
833 macp->m_driver = xnfp; in xnf_attach()
834 xnfp->xnf_devinfo = devinfo; in xnf_attach()
837 macp->m_src_addr = xnfp->xnf_mac_addr; in xnf_attach()
842 xnfp->xnf_running = B_FALSE; in xnf_attach()
843 xnfp->xnf_connected = B_FALSE; in xnf_attach()
844 xnfp->xnf_be_rx_copy = B_FALSE; in xnf_attach()
845 xnfp->xnf_be_mcast_control = B_FALSE; in xnf_attach()
846 xnfp->xnf_need_sched = B_FALSE; in xnf_attach()
848 xnfp->xnf_rx_head = NULL; in xnf_attach()
849 xnfp->xnf_rx_tail = NULL; in xnf_attach()
850 xnfp->xnf_rx_new_buffers_posted = B_FALSE; in xnf_attach()
864 if (ddi_get_iblock_cookie(devinfo, 0, &xnfp->xnf_icookie) in xnf_attach()
868 mutex_init(&xnfp->xnf_txlock, in xnf_attach()
869 NULL, MUTEX_DRIVER, xnfp->xnf_icookie); in xnf_attach()
870 mutex_init(&xnfp->xnf_rxlock, in xnf_attach()
871 NULL, MUTEX_DRIVER, xnfp->xnf_icookie); in xnf_attach()
872 mutex_init(&xnfp->xnf_schedlock, in xnf_attach()
873 NULL, MUTEX_DRIVER, xnfp->xnf_icookie); in xnf_attach()
874 mutex_init(&xnfp->xnf_gref_lock, in xnf_attach()
875 NULL, MUTEX_DRIVER, xnfp->xnf_icookie); in xnf_attach()
877 cv_init(&xnfp->xnf_cv_state, NULL, CV_DEFAULT, NULL); in xnf_attach()
878 cv_init(&xnfp->xnf_cv_multicast, NULL, CV_DEFAULT, NULL); in xnf_attach()
879 cv_init(&xnfp->xnf_cv_tx_slots, NULL, CV_DEFAULT, NULL); in xnf_attach()
883 xnfp->xnf_buf_cache = kmem_cache_create(cachename, in xnf_attach()
886 NULL, xnfp, NULL, 0); in xnf_attach()
887 if (xnfp->xnf_buf_cache == NULL) in xnf_attach()
892 xnfp->xnf_tx_buf_cache = kmem_cache_create(cachename, in xnf_attach()
895 NULL, xnfp, NULL, 0); in xnf_attach()
896 if (xnfp->xnf_tx_buf_cache == NULL) in xnf_attach()
899 xnfp->xnf_gref_head = INVALID_GRANT_REF; in xnf_attach()
901 if (xnf_alloc_dma_resources(xnfp) == DDI_FAILURE) { in xnf_attach()
904 ddi_get_instance(xnfp->xnf_devinfo)); in xnf_attach()
908 xnfp->xnf_rx_ring.sring->rsp_event = in xnf_attach()
909 xnfp->xnf_tx_ring.sring->rsp_event = 1; in xnf_attach()
911 xnfp->xnf_tx_ring_ref = INVALID_GRANT_REF; in xnf_attach()
912 xnfp->xnf_rx_ring_ref = INVALID_GRANT_REF; in xnf_attach()
915 ddi_set_driver_private(devinfo, xnfp); in xnf_attach()
917 if (!xnf_kstat_init(xnfp)) in xnf_attach()
925 xnfp->xnf_evtchn = xvdi_get_evtchn(devinfo); in xnf_attach()
927 ec_bind_evtchn_to_handler(xnfp->xnf_evtchn, IPL_VIF, xnf_intr, xnfp); in xnf_attach()
929 (void) ddi_add_intr(devinfo, 0, NULL, NULL, xnf_intr, (caddr_t)xnfp); in xnf_attach()
932 err = mac_register(macp, &xnfp->xnf_mh); in xnf_attach()
956 xnf_debug_instance = xnfp; in xnf_attach()
962 (void) mac_unregister(xnfp->xnf_mh); in xnf_attach()
966 ec_unbind_evtchn(xnfp->xnf_evtchn); in xnf_attach()
969 ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); in xnf_attach()
971 xnfp->xnf_evtchn = INVALID_EVTCHN; in xnf_attach()
972 kstat_delete(xnfp->xnf_kstat_aux); in xnf_attach()
975 xnf_release_dma_resources(xnfp); in xnf_attach()
978 kmem_cache_destroy(xnfp->xnf_tx_buf_cache); in xnf_attach()
981 kmem_cache_destroy(xnfp->xnf_buf_cache); in xnf_attach()
984 cv_destroy(&xnfp->xnf_cv_tx_slots); in xnf_attach()
985 cv_destroy(&xnfp->xnf_cv_multicast); in xnf_attach()
986 cv_destroy(&xnfp->xnf_cv_state); in xnf_attach()
988 mutex_destroy(&xnfp->xnf_gref_lock); in xnf_attach()
989 mutex_destroy(&xnfp->xnf_schedlock); in xnf_attach()
990 mutex_destroy(&xnfp->xnf_rxlock); in xnf_attach()
991 mutex_destroy(&xnfp->xnf_txlock); in xnf_attach()
994 kmem_free(xnfp, sizeof (*xnfp)); in xnf_attach()
1005 xnf_t *xnfp; /* Our private device info */ in xnf_detach() local
1012 xnfp = ddi_get_driver_private(devinfo); in xnf_detach()
1017 ec_unbind_evtchn(xnfp->xnf_evtchn); in xnf_detach()
1020 ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); in xnf_detach()
1025 mutex_enter(&xnfp->xnf_rxlock); in xnf_detach()
1026 mutex_enter(&xnfp->xnf_txlock); in xnf_detach()
1028 xnfp->xnf_evtchn = INVALID_EVTCHN; in xnf_detach()
1029 xnfp->xnf_connected = B_FALSE; in xnf_detach()
1030 mutex_exit(&xnfp->xnf_txlock); in xnf_detach()
1031 mutex_exit(&xnfp->xnf_rxlock); in xnf_detach()
1034 mac_link_update(xnfp->xnf_mh, LINK_STATE_DOWN); in xnf_detach()
1044 if (xnfp->xnf_connected) in xnf_detach()
1050 if (xnfp->xnf_stat_buf_allocated > 0) in xnf_detach()
1053 if (mac_unregister(xnfp->xnf_mh) != 0) in xnf_detach()
1056 kstat_delete(xnfp->xnf_kstat_aux); in xnf_detach()
1059 xnf_stop(xnfp); in xnf_detach()
1065 ec_unbind_evtchn(xnfp->xnf_evtchn); in xnf_detach()
1068 ddi_remove_intr(devinfo, 0, xnfp->xnf_icookie); in xnf_detach()
1072 xnf_release_mblks(xnfp); in xnf_detach()
1075 xnf_release_dma_resources(xnfp); in xnf_detach()
1077 cv_destroy(&xnfp->xnf_cv_tx_slots); in xnf_detach()
1078 cv_destroy(&xnfp->xnf_cv_multicast); in xnf_detach()
1079 cv_destroy(&xnfp->xnf_cv_state); in xnf_detach()
1081 kmem_cache_destroy(xnfp->xnf_tx_buf_cache); in xnf_detach()
1082 kmem_cache_destroy(xnfp->xnf_buf_cache); in xnf_detach()
1084 mutex_destroy(&xnfp->xnf_gref_lock); in xnf_detach()
1085 mutex_destroy(&xnfp->xnf_schedlock); in xnf_detach()
1086 mutex_destroy(&xnfp->xnf_rxlock); in xnf_detach()
1087 mutex_destroy(&xnfp->xnf_txlock); in xnf_detach()
1089 kmem_free(xnfp, sizeof (*xnfp)); in xnf_detach()
1117 xnf_t *xnfp = arg; in xnf_set_multicast() local
1130 if (!xnfp->xnf_be_mcast_control) in xnf_set_multicast()
1133 txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, KM_SLEEP); in xnf_set_multicast()
1135 mutex_enter(&xnfp->xnf_txlock); in xnf_set_multicast()
1146 if (!xnfp->xnf_connected) { in xnf_set_multicast()
1147 mutex_exit(&xnfp->xnf_txlock); in xnf_set_multicast()
1159 n_slots = tx_slots_get(xnfp, 2, B_TRUE); in xnf_set_multicast()
1162 slot = xnfp->xnf_tx_ring.req_prod_pvt; in xnf_set_multicast()
1163 tidp = txid_get(xnfp); in xnf_set_multicast()
1169 txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot); in xnf_set_multicast()
1171 RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot + 1); in xnf_set_multicast()
1186 xnfp->xnf_tx_ring.req_prod_pvt = slot + 2; in xnf_set_multicast()
1188 mutex_enter(&xnfp->xnf_schedlock); in xnf_set_multicast()
1189 xnfp->xnf_pending_multicast++; in xnf_set_multicast()
1190 mutex_exit(&xnfp->xnf_schedlock); in xnf_set_multicast()
1193 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring, in xnf_set_multicast()
1196 ec_notify_via_evtchn(xnfp->xnf_evtchn); in xnf_set_multicast()
1199 cv_wait(&xnfp->xnf_cv_multicast, in xnf_set_multicast()
1200 &xnfp->xnf_txlock); in xnf_set_multicast()
1204 mutex_enter(&xnfp->xnf_schedlock); in xnf_set_multicast()
1205 xnfp->xnf_pending_multicast--; in xnf_set_multicast()
1206 mutex_exit(&xnfp->xnf_schedlock); in xnf_set_multicast()
1210 txid_put(xnfp, tidp); in xnf_set_multicast()
1212 mutex_exit(&xnfp->xnf_txlock); in xnf_set_multicast()
1214 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_set_multicast()
1240 xnf_tx_clean_ring(xnf_t *xnfp) in xnf_tx_clean_ring() argument
1244 ASSERT(MUTEX_HELD(&xnfp->xnf_txlock)); in xnf_tx_clean_ring()
1247 while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_tx_ring)) { in xnf_tx_clean_ring()
1250 cons = xnfp->xnf_tx_ring.rsp_cons; in xnf_tx_clean_ring()
1251 prod = xnfp->xnf_tx_ring.sring->rsp_prod; in xnf_tx_clean_ring()
1263 trp = RING_GET_RESPONSE(&xnfp->xnf_tx_ring, i); in xnf_tx_clean_ring()
1266 tidp = TX_ID_TO_TXID(xnfp, trp->id); in xnf_tx_clean_ring()
1286 gref_put(xnfp, txp->tx_txreq.gref); in xnf_tx_clean_ring()
1290 xnf_buf_put(xnfp, txp->tx_bdesc, in xnf_tx_clean_ring()
1295 txid_put(xnfp, tidp); in xnf_tx_clean_ring()
1296 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_tx_clean_ring()
1303 cv_broadcast(&xnfp->xnf_cv_multicast); in xnf_tx_clean_ring()
1321 xnfp->xnf_tx_ring.rsp_cons = prod; in xnf_tx_clean_ring()
1326 RING_FINAL_CHECK_FOR_RESPONSES(&xnfp->xnf_tx_ring, work_to_do); in xnf_tx_clean_ring()
1330 return (RING_FREE_REQUESTS(&xnfp->xnf_tx_ring)); in xnf_tx_clean_ring()
1339 xnf_tx_pullup(xnf_t *xnfp, mblk_t *mp) in xnf_tx_pullup() argument
1344 bd = xnf_buf_get(xnfp, KM_SLEEP, B_TRUE); in xnf_tx_pullup()
1360 xnfp->xnf_stat_tx_pullup++; in xnf_tx_pullup()
1441 tx_push_packets(xnf_t *xnfp, xnf_txbuf_t *txp) in tx_push_packets() argument
1447 mutex_enter(&xnfp->xnf_txlock); in tx_push_packets()
1449 ASSERT(xnfp->xnf_running); in tx_push_packets()
1454 while (!xnfp->xnf_connected) in tx_push_packets()
1455 cv_wait(&xnfp->xnf_cv_state, &xnfp->xnf_txlock); in tx_push_packets()
1457 slots_free = tx_slots_get(xnfp, 1, B_FALSE); in tx_push_packets()
1460 slot = xnfp->xnf_tx_ring.req_prod_pvt; in tx_push_packets()
1466 tidp = txid_get(xnfp); in tx_push_packets()
1469 txrp = RING_GET_REQUEST(&xnfp->xnf_tx_ring, slot); in tx_push_packets()
1477 xnfp->xnf_stat_opackets++; in tx_push_packets()
1478 xnfp->xnf_stat_obytes += txp->tx_txreq.size; in tx_push_packets()
1486 xnfp->xnf_tx_ring.req_prod_pvt = slot; in tx_push_packets()
1492 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_tx_ring, in tx_push_packets()
1495 ec_notify_via_evtchn(xnfp->xnf_evtchn); in tx_push_packets()
1497 mutex_exit(&xnfp->xnf_txlock); in tx_push_packets()
1508 xnf_t *xnfp = arg; in xnf_send() local
1514 oeid = xvdi_get_oeid(xnfp->xnf_devinfo); in xnf_send()
1539 xnfp->xnf_stat_tx_pagebndry++; in xnf_send()
1553 ddi_get_instance(xnfp->xnf_devinfo), length); in xnf_send()
1558 txp = kmem_cache_alloc(xnfp->xnf_tx_buf_cache, KM_SLEEP); in xnf_send()
1567 txp->tx_bdesc = xnf_tx_pullup(xnfp, mp); in xnf_send()
1569 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_send()
1595 ddi_get_instance(xnfp->xnf_devinfo), in xnf_send()
1598 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_send()
1607 txp->tx_txreq.gref = gref_get(xnfp); in xnf_send()
1611 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_send()
1637 xnfp->xnf_stat_tx_cksum_deferred++; in xnf_send()
1687 head = tx_push_packets(xnfp, head); in xnf_send()
1708 gref_put(xnfp, loop->tx_txreq.gref); in xnf_send()
1712 xnf_buf_put(xnfp, loop->tx_bdesc, B_TRUE); in xnf_send()
1720 kmem_cache_free(xnfp->xnf_tx_buf_cache, loop); in xnf_send()
1743 mutex_enter(&xnfp->xnf_schedlock); in xnf_send()
1744 xnfp->xnf_need_sched = B_TRUE; in xnf_send()
1745 mutex_exit(&xnfp->xnf_schedlock); in xnf_send()
1747 xnfp->xnf_stat_tx_defer++; in xnf_send()
1760 xnf_t *xnfp = (xnf_t *)arg; in xnf_intr() local
1764 mutex_enter(&xnfp->xnf_rxlock); in xnf_intr()
1769 if (!xnfp->xnf_connected) { in xnf_intr()
1770 mutex_exit(&xnfp->xnf_rxlock); in xnf_intr()
1771 xnfp->xnf_stat_unclaimed_interrupts++; in xnf_intr()
1782 xnf_rx_collect(xnfp); in xnf_intr()
1787 xnfp->xnf_rx_ring.sring->rsp_event = in xnf_intr()
1788 xnfp->xnf_rx_ring.rsp_cons + 1; in xnf_intr()
1791 } while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_rx_ring)); in xnf_intr()
1793 if (xnfp->xnf_rx_new_buffers_posted) { in xnf_intr()
1801 RING_PUSH_REQUESTS_AND_CHECK_NOTIFY(&xnfp->xnf_rx_ring, notify); in xnf_intr()
1803 ec_notify_via_evtchn(xnfp->xnf_evtchn); in xnf_intr()
1804 xnfp->xnf_rx_new_buffers_posted = B_FALSE; in xnf_intr()
1807 mp = xnfp->xnf_rx_head; in xnf_intr()
1808 xnfp->xnf_rx_head = xnfp->xnf_rx_tail = NULL; in xnf_intr()
1810 xnfp->xnf_stat_interrupts++; in xnf_intr()
1811 mutex_exit(&xnfp->xnf_rxlock); in xnf_intr()
1814 mac_rx(xnfp->xnf_mh, NULL, mp); in xnf_intr()
1828 mutex_enter(&xnfp->xnf_schedlock); in xnf_intr()
1829 need_sched = xnfp->xnf_need_sched; in xnf_intr()
1830 clean_ring = need_sched || (xnfp->xnf_pending_multicast > 0); in xnf_intr()
1831 mutex_exit(&xnfp->xnf_schedlock); in xnf_intr()
1836 mutex_enter(&xnfp->xnf_txlock); in xnf_intr()
1837 free_slots = tx_slots_get(xnfp, 0, B_FALSE); in xnf_intr()
1840 mutex_enter(&xnfp->xnf_schedlock); in xnf_intr()
1841 xnfp->xnf_need_sched = B_FALSE; in xnf_intr()
1842 mutex_exit(&xnfp->xnf_schedlock); in xnf_intr()
1844 mac_tx_update(xnfp->xnf_mh); in xnf_intr()
1846 mutex_exit(&xnfp->xnf_txlock); in xnf_intr()
1858 xnf_t *xnfp = arg; in xnf_start() local
1863 ddi_get_instance(xnfp->xnf_devinfo), (void *)xnfp); in xnf_start()
1866 mutex_enter(&xnfp->xnf_rxlock); in xnf_start()
1867 mutex_enter(&xnfp->xnf_txlock); in xnf_start()
1870 xnfp->xnf_running = B_TRUE; in xnf_start()
1872 mutex_exit(&xnfp->xnf_txlock); in xnf_start()
1873 mutex_exit(&xnfp->xnf_rxlock); in xnf_start()
1882 xnf_t *xnfp = arg; in xnf_stop() local
1887 ddi_get_instance(xnfp->xnf_devinfo), (void *)xnfp); in xnf_stop()
1890 mutex_enter(&xnfp->xnf_rxlock); in xnf_stop()
1891 mutex_enter(&xnfp->xnf_txlock); in xnf_stop()
1893 xnfp->xnf_running = B_FALSE; in xnf_stop()
1895 mutex_exit(&xnfp->xnf_txlock); in xnf_stop()
1896 mutex_exit(&xnfp->xnf_rxlock); in xnf_stop()
1903 xnf_rxbuf_hang(xnf_t *xnfp, xnf_buf_t *bdesc) in xnf_rxbuf_hang() argument
1908 ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock)); in xnf_rxbuf_hang()
1910 reqp = RING_GET_REQUEST(&xnfp->xnf_rx_ring, in xnf_rxbuf_hang()
1911 xnfp->xnf_rx_ring.req_prod_pvt); in xnf_rxbuf_hang()
1912 hang_ix = (RING_IDX) (reqp - RING_GET_REQUEST(&xnfp->xnf_rx_ring, 0)); in xnf_rxbuf_hang()
1913 ASSERT(xnfp->xnf_rx_pkt_info[hang_ix] == NULL); in xnf_rxbuf_hang()
1918 xnfp->xnf_rx_pkt_info[hang_ix] = bdesc; in xnf_rxbuf_hang()
1919 xnfp->xnf_rx_ring.req_prod_pvt++; in xnf_rxbuf_hang()
1921 xnfp->xnf_rx_new_buffers_posted = B_TRUE; in xnf_rxbuf_hang()
1929 xnf_rx_collect(xnf_t *xnfp) in xnf_rx_collect() argument
1933 ASSERT(MUTEX_HELD(&xnfp->xnf_rxlock)); in xnf_rx_collect()
1946 while (RING_HAS_UNCONSUMED_RESPONSES(&xnfp->xnf_rx_ring)) { in xnf_rx_collect()
1956 rxpkt = RING_GET_RESPONSE(&xnfp->xnf_rx_ring, in xnf_rx_collect()
1957 xnfp->xnf_rx_ring.rsp_cons); in xnf_rx_collect()
1967 bdesc = xnfp->xnf_rx_pkt_info[rxpkt->id]; in xnf_rx_collect()
1972 xnfp->xnf_rx_pkt_info[rxpkt->id] = NULL; in xnf_rx_collect()
1979 if (!xnfp->xnf_running) { in xnf_rx_collect()
1985 xnfp->xnf_stat_drop++; in xnf_rx_collect()
1993 xnfp->xnf_stat_errrx++; in xnf_rx_collect()
1997 xnfp->xnf_stat_runt++; in xnf_rx_collect()
2000 xnfp->xnf_stat_mac_rcv_error++; in xnf_rx_collect()
2003 xnfp->xnf_stat_norxbuf++; in xnf_rx_collect()
2010 xvdi_get_oeid(xnfp->xnf_devinfo)); in xnf_rx_collect()
2015 off, len, xvdi_get_oeid(xnfp->xnf_devinfo)); in xnf_rx_collect()
2034 nbuf = xnf_buf_get(xnfp, KM_NOSLEEP, B_FALSE); in xnf_rx_collect()
2046 xnfp->xnf_stat_rx_desballoc_fail++; in xnf_rx_collect()
2047 xnfp->xnf_stat_norxbuf++; in xnf_rx_collect()
2049 xnf_buf_put(xnfp, nbuf, B_FALSE); in xnf_rx_collect()
2064 gref_put(xnfp, bdesc->grant_ref); in xnf_rx_collect()
2081 xnfp->xnf_stat_rx_allocb_fail++; in xnf_rx_collect()
2082 xnfp->xnf_stat_norxbuf++; in xnf_rx_collect()
2093 xnf_rxbuf_hang(xnfp, bdesc); in xnf_rx_collect()
2110 xnfp->xnf_stat_rx_cksum_no_need++; in xnf_rx_collect()
2125 xnfp->xnf_stat_ipackets++; in xnf_rx_collect()
2126 xnfp->xnf_stat_rbytes += len; in xnf_rx_collect()
2129 xnfp->xnf_rx_ring.rsp_cons++; in xnf_rx_collect()
2138 if (xnfp->xnf_rx_head == NULL) { in xnf_rx_collect()
2139 ASSERT(xnfp->xnf_rx_tail == NULL); in xnf_rx_collect()
2141 xnfp->xnf_rx_head = head; in xnf_rx_collect()
2143 ASSERT(xnfp->xnf_rx_tail != NULL); in xnf_rx_collect()
2145 xnfp->xnf_rx_tail->b_next = head; in xnf_rx_collect()
2147 xnfp->xnf_rx_tail = tail; in xnf_rx_collect()
2155 xnf_alloc_dma_resources(xnf_t *xnfp) in xnf_alloc_dma_resources() argument
2157 dev_info_t *devinfo = xnfp->xnf_devinfo; in xnf_alloc_dma_resources()
2171 DDI_DMA_SLEEP, 0, &xnfp->xnf_tx_ring_dma_handle) != DDI_SUCCESS) in xnf_alloc_dma_resources()
2174 if (ddi_dma_mem_alloc(xnfp->xnf_tx_ring_dma_handle, in xnf_alloc_dma_resources()
2177 &xnfp->xnf_tx_ring_dma_acchandle) != DDI_SUCCESS) { in xnf_alloc_dma_resources()
2178 ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); in xnf_alloc_dma_resources()
2179 xnfp->xnf_tx_ring_dma_handle = NULL; in xnf_alloc_dma_resources()
2183 if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_tx_ring_dma_handle, NULL, in xnf_alloc_dma_resources()
2186 ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle); in xnf_alloc_dma_resources()
2187 ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); in xnf_alloc_dma_resources()
2188 xnfp->xnf_tx_ring_dma_handle = NULL; in xnf_alloc_dma_resources()
2189 xnfp->xnf_tx_ring_dma_acchandle = NULL; in xnf_alloc_dma_resources()
2201 FRONT_RING_INIT(&xnfp->xnf_tx_ring, (netif_tx_sring_t *)rptr, PAGESIZE); in xnf_alloc_dma_resources()
2202 xnfp->xnf_tx_ring_phys_addr = dma_cookie.dmac_laddress; in xnf_alloc_dma_resources()
2208 DDI_DMA_SLEEP, 0, &xnfp->xnf_rx_ring_dma_handle) != DDI_SUCCESS) in xnf_alloc_dma_resources()
2211 if (ddi_dma_mem_alloc(xnfp->xnf_rx_ring_dma_handle, in xnf_alloc_dma_resources()
2214 &xnfp->xnf_rx_ring_dma_acchandle) != DDI_SUCCESS) { in xnf_alloc_dma_resources()
2215 ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); in xnf_alloc_dma_resources()
2216 xnfp->xnf_rx_ring_dma_handle = NULL; in xnf_alloc_dma_resources()
2220 if ((rc = ddi_dma_addr_bind_handle(xnfp->xnf_rx_ring_dma_handle, NULL, in xnf_alloc_dma_resources()
2223 ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle); in xnf_alloc_dma_resources()
2224 ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); in xnf_alloc_dma_resources()
2225 xnfp->xnf_rx_ring_dma_handle = NULL; in xnf_alloc_dma_resources()
2226 xnfp->xnf_rx_ring_dma_acchandle = NULL; in xnf_alloc_dma_resources()
2238 FRONT_RING_INIT(&xnfp->xnf_rx_ring, (netif_rx_sring_t *)rptr, PAGESIZE); in xnf_alloc_dma_resources()
2239 xnfp->xnf_rx_ring_phys_addr = dma_cookie.dmac_laddress; in xnf_alloc_dma_resources()
2245 ddi_get_instance(xnfp->xnf_devinfo)); in xnf_alloc_dma_resources()
2247 xnf_release_dma_resources(xnfp); in xnf_alloc_dma_resources()
2255 xnf_release_dma_resources(xnf_t *xnfp) in xnf_release_dma_resources() argument
2263 mutex_enter(&xnfp->xnf_rxlock); in xnf_release_dma_resources()
2267 if ((bp = xnfp->xnf_rx_pkt_info[i]) == NULL) in xnf_release_dma_resources()
2269 xnfp->xnf_rx_pkt_info[i] = NULL; in xnf_release_dma_resources()
2270 xnf_buf_put(xnfp, bp, B_FALSE); in xnf_release_dma_resources()
2272 mutex_exit(&xnfp->xnf_rxlock); in xnf_release_dma_resources()
2275 if (xnfp->xnf_rx_ring_dma_acchandle != NULL) { in xnf_release_dma_resources()
2276 (void) ddi_dma_unbind_handle(xnfp->xnf_rx_ring_dma_handle); in xnf_release_dma_resources()
2277 ddi_dma_mem_free(&xnfp->xnf_rx_ring_dma_acchandle); in xnf_release_dma_resources()
2278 ddi_dma_free_handle(&xnfp->xnf_rx_ring_dma_handle); in xnf_release_dma_resources()
2279 xnfp->xnf_rx_ring_dma_acchandle = NULL; in xnf_release_dma_resources()
2282 if (xnfp->xnf_tx_ring_dma_acchandle != NULL) { in xnf_release_dma_resources()
2283 (void) ddi_dma_unbind_handle(xnfp->xnf_tx_ring_dma_handle); in xnf_release_dma_resources()
2284 ddi_dma_mem_free(&xnfp->xnf_tx_ring_dma_acchandle); in xnf_release_dma_resources()
2285 ddi_dma_free_handle(&xnfp->xnf_tx_ring_dma_handle); in xnf_release_dma_resources()
2286 xnfp->xnf_tx_ring_dma_acchandle = NULL; in xnf_release_dma_resources()
2295 xnf_release_mblks(xnf_t *xnfp) in xnf_release_mblks() argument
2300 for (i = 0, tidp = &xnfp->xnf_tx_pkt_id[0]; in xnf_release_mblks()
2309 txid_put(xnfp, tidp); in xnf_release_mblks()
2310 kmem_cache_free(xnfp->xnf_tx_buf_cache, txp); in xnf_release_mblks()
2320 xnf_t *xnfp = arg; in xnf_buf_constructor() local
2329 if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &buf_dma_attr, in xnf_buf_constructor()
2348 bdesc->xnfp = xnfp; in xnf_buf_constructor()
2353 bdesc->gen = xnfp->xnf_gen; in xnf_buf_constructor()
2355 atomic_inc_64(&xnfp->xnf_stat_buf_allocated); in xnf_buf_constructor()
2375 xnf_t *xnfp = arg; in xnf_buf_destructor() local
2381 atomic_dec_64(&xnfp->xnf_stat_buf_allocated); in xnf_buf_destructor()
2385 xnf_buf_get(xnf_t *xnfp, int flags, boolean_t readonly) in xnf_buf_get() argument
2394 gref = gref_get(xnfp); in xnf_buf_get()
2398 bufp = kmem_cache_alloc(xnfp->xnf_buf_cache, flags); in xnf_buf_get()
2400 gref_put(xnfp, gref); in xnf_buf_get()
2408 if (bufp->gen != xnfp->xnf_gen) in xnf_buf_get()
2412 xvdi_get_oeid(bufp->xnfp->xnf_devinfo), in xnf_buf_get()
2415 atomic_inc_64(&xnfp->xnf_stat_buf_outstanding); in xnf_buf_get()
2421 xnf_buf_put(xnf_t *xnfp, xnf_buf_t *bufp, boolean_t readonly) in xnf_buf_put() argument
2426 gref_put(xnfp, bufp->grant_ref); in xnf_buf_put()
2430 kmem_cache_free(xnfp->xnf_buf_cache, bufp); in xnf_buf_put()
2432 atomic_dec_64(&xnfp->xnf_stat_buf_outstanding); in xnf_buf_put()
2442 bdesc->gen = bdesc->xnfp->xnf_gen; in xnf_buf_refresh()
2452 xnf_t *xnfp = bdesc->xnfp; in xnf_buf_recycle() local
2454 xnf_buf_put(xnfp, bdesc, B_TRUE); in xnf_buf_recycle()
2462 xnf_t *xnfp = arg; in xnf_tx_buf_constructor() local
2467 if (ddi_dma_alloc_handle(xnfp->xnf_devinfo, &buf_dma_attr, in xnf_tx_buf_constructor()
2508 xnf_t *xnfp; in xnf_kstat_aux_update() local
2514 xnfp = ksp->ks_private; in xnf_kstat_aux_update()
2521 (knp++)->value.ui64 = xnfp->xnf_stat_tx_cksum_deferred; in xnf_kstat_aux_update()
2522 (knp++)->value.ui64 = xnfp->xnf_stat_rx_cksum_no_need; in xnf_kstat_aux_update()
2524 (knp++)->value.ui64 = xnfp->xnf_stat_interrupts; in xnf_kstat_aux_update()
2525 (knp++)->value.ui64 = xnfp->xnf_stat_unclaimed_interrupts; in xnf_kstat_aux_update()
2526 (knp++)->value.ui64 = xnfp->xnf_stat_tx_pullup; in xnf_kstat_aux_update()
2527 (knp++)->value.ui64 = xnfp->xnf_stat_tx_pagebndry; in xnf_kstat_aux_update()
2528 (knp++)->value.ui64 = xnfp->xnf_stat_tx_attempt; in xnf_kstat_aux_update()
2530 (knp++)->value.ui64 = xnfp->xnf_stat_buf_allocated; in xnf_kstat_aux_update()
2531 (knp++)->value.ui64 = xnfp->xnf_stat_buf_outstanding; in xnf_kstat_aux_update()
2532 (knp++)->value.ui64 = xnfp->xnf_stat_gref_outstanding; in xnf_kstat_aux_update()
2533 (knp++)->value.ui64 = xnfp->xnf_stat_gref_failure; in xnf_kstat_aux_update()
2534 (knp++)->value.ui64 = xnfp->xnf_stat_gref_peak; in xnf_kstat_aux_update()
2535 (knp++)->value.ui64 = xnfp->xnf_stat_rx_allocb_fail; in xnf_kstat_aux_update()
2536 (knp++)->value.ui64 = xnfp->xnf_stat_rx_desballoc_fail; in xnf_kstat_aux_update()
2542 xnf_kstat_init(xnf_t *xnfp) in xnf_kstat_init() argument
2552 if ((xnfp->xnf_kstat_aux = kstat_create("xnf", in xnf_kstat_init()
2553 ddi_get_instance(xnfp->xnf_devinfo), in xnf_kstat_init()
2558 xnfp->xnf_kstat_aux->ks_private = xnfp; in xnf_kstat_init()
2559 xnfp->xnf_kstat_aux->ks_update = xnf_kstat_aux_update; in xnf_kstat_init()
2561 knp = xnfp->xnf_kstat_aux->ks_data; in xnf_kstat_init()
2570 kstat_install(xnfp->xnf_kstat_aux); in xnf_kstat_init()
2578 xnf_t *xnfp = arg; in xnf_stat() local
2580 mutex_enter(&xnfp->xnf_rxlock); in xnf_stat()
2581 mutex_enter(&xnfp->xnf_txlock); in xnf_stat()
2585 *val = xnfp->xnf_stat_##r; \ in xnf_stat()
2590 *val = xnfp->xnf_stat_##r; \ in xnf_stat()
2617 mutex_exit(&xnfp->xnf_txlock); in xnf_stat()
2618 mutex_exit(&xnfp->xnf_rxlock); in xnf_stat()
2626 mutex_exit(&xnfp->xnf_txlock); in xnf_stat()
2627 mutex_exit(&xnfp->xnf_rxlock); in xnf_stat()
2676 xnf_t *xnfp = ddi_get_driver_private(dip); in oe_state_change() local
2679 ASSERT(xnfp != NULL); in oe_state_change()
2692 xnf_read_config(xnfp); in oe_state_change()
2694 if (!xnfp->xnf_be_rx_copy) { in oe_state_change()
2698 (void) xvdi_switch_state(xnfp->xnf_devinfo, in oe_state_change()
2706 xnf_be_connect(xnfp); in oe_state_change()
2711 mac_unicst_update(xnfp->xnf_mh, xnfp->xnf_mac_addr); in oe_state_change()
2716 mutex_enter(&xnfp->xnf_rxlock); in oe_state_change()
2717 mutex_enter(&xnfp->xnf_txlock); in oe_state_change()
2719 xnfp->xnf_connected = B_TRUE; in oe_state_change()
2724 cv_broadcast(&xnfp->xnf_cv_state); in oe_state_change()
2726 mutex_exit(&xnfp->xnf_txlock); in oe_state_change()
2727 mutex_exit(&xnfp->xnf_rxlock); in oe_state_change()
2733 ec_notify_via_evtchn(xnfp->xnf_evtchn); in oe_state_change()
2741 (void) xnf_intr((caddr_t)xnfp); in oe_state_change()
2746 mac_link_update(xnfp->xnf_mh, LINK_STATE_UP); in oe_state_change()
2752 mac_multicast_refresh(xnfp->xnf_mh, NULL, xnfp, B_TRUE); in oe_state_change()