122 static void xpvtap_drv_fini(xpvtap_state_t *state);
125 typedef void (*xpvtap_rs_cleanup_t)(xpvtap_state_t *state, uint_t rs);
134 static int xpvtap_segmf_register(xpvtap_state_t *state);
137 static int xpvtap_user_init(xpvtap_state_t *state);
138 static void xpvtap_user_fini(xpvtap_state_t *state);
139 static int xpvtap_user_ring_init(xpvtap_state_t *state);
140 static void xpvtap_user_ring_fini(xpvtap_state_t *state);
141 static int xpvtap_user_thread_init(xpvtap_state_t *state);
142 static void xpvtap_user_thread_fini(xpvtap_state_t *state);
144 static void xpvtap_user_thread_stop(xpvtap_state_t *state);
149 static int xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,
151 static int xpvtap_user_request_push(xpvtap_state_t *state,
153 static int xpvtap_user_response_get(xpvtap_state_t *state,
155 static void xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid);
217 	xpvtap_state_t *state;  in xpvtap_attach()  local
235 	state = xpvtap_drv_init(instance);  in xpvtap_attach()
236 	if (state == NULL) {  in xpvtap_attach()
239 	state->bt_dip = dip;  in xpvtap_attach()
242 	args.ar_dip = state->bt_dip;  in xpvtap_attach()
244 	args.ar_intr_arg = (caddr_t)state;  in xpvtap_attach()
246 	args.ar_ringup_arg = (caddr_t)state;  in xpvtap_attach()
248 	args.ar_ringdown_arg = (caddr_t)state;  in xpvtap_attach()
249 	e = blk_ring_init(&args, &state->bt_guest_ring);  in xpvtap_attach()
267 	blk_ring_fini(&state->bt_guest_ring);  in xpvtap_attach()
269 	xpvtap_drv_fini(state);  in xpvtap_attach()
280 	xpvtap_state_t *state;  in xpvtap_detach()  local
285 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_detach()
286 	if (state == NULL) {  in xpvtap_detach()
299 	xpvtap_user_thread_stop(state);  in xpvtap_detach()
300 	blk_ring_fini(&state->bt_guest_ring);  in xpvtap_detach()
301 	xpvtap_drv_fini(state);  in xpvtap_detach()
315 	xpvtap_state_t *state;  in xpvtap_getinfo()  local
326 		state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_getinfo()
327 		if (state == NULL) {  in xpvtap_getinfo()
330 		*result = (void *)state->bt_dip;  in xpvtap_getinfo()
355 	xpvtap_state_t *state;  in xpvtap_open()  local
364 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_open()
365 	if (state == NULL) {  in xpvtap_open()
370 	mutex_enter(&state->bt_open.bo_mutex);  in xpvtap_open()
371 	if (state->bt_open.bo_opened) {  in xpvtap_open()
372 		mutex_exit(&state->bt_open.bo_mutex);  in xpvtap_open()
375 	state->bt_open.bo_opened = B_TRUE;  in xpvtap_open()
376 	mutex_exit(&state->bt_open.bo_mutex);  in xpvtap_open()
382 	state->bt_map.um_as = curproc->p_as;  in xpvtap_open()
395 	xpvtap_state_t *state;  in xpvtap_close()  local
400 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_close()
401 	if (state == NULL) {  in xpvtap_close()
409 	mutex_enter(&state->bt_thread.ut_mutex);  in xpvtap_close()
410 	state->bt_thread.ut_wake = B_TRUE;  in xpvtap_close()
411 	state->bt_thread.ut_exit = B_TRUE;  in xpvtap_close()
412 	cv_signal(&state->bt_thread.ut_wake_cv);  in xpvtap_close()
413 	if (!state->bt_thread.ut_exit_done) {  in xpvtap_close()
414 		cv_wait(&state->bt_thread.ut_exit_done_cv,  in xpvtap_close()
415 		    &state->bt_thread.ut_mutex);  in xpvtap_close()
417 	ASSERT(state->bt_thread.ut_exit_done);  in xpvtap_close()
418 	mutex_exit(&state->bt_thread.ut_mutex);  in xpvtap_close()
420 	state->bt_map.um_as = NULL;  in xpvtap_close()
421 	state->bt_map.um_guest_pages = NULL;  in xpvtap_close()
429 	mutex_enter(&state->bt_open.bo_mutex);  in xpvtap_close()
430 	ASSERT(state->bt_open.bo_opened);  in xpvtap_close()
431 	state->bt_open.bo_opened = B_FALSE;  in xpvtap_close()
432 	cv_signal(&state->bt_open.bo_exit_cv);  in xpvtap_close()
433 	mutex_exit(&state->bt_open.bo_mutex);  in xpvtap_close()
447 	xpvtap_state_t *state;  in xpvtap_ioctl()  local
460 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_ioctl()
461 	if (state == NULL) {  in xpvtap_ioctl()
471 		mutex_enter(&state->bt_thread.ut_mutex);  in xpvtap_ioctl()
472 		state->bt_thread.ut_wake = B_TRUE;  in xpvtap_ioctl()
473 		cv_signal(&state->bt_thread.ut_wake_cv);  in xpvtap_ioctl()
474 		mutex_exit(&state->bt_thread.ut_mutex);  in xpvtap_ioctl()
496 	xpvtap_state_t *state;  in xpvtap_segmap()  local
506 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_segmap()
507 	if (state == NULL) {  in xpvtap_segmap()
542 	ASSERT(asp == state->bt_map.um_as);  in xpvtap_segmap()
568 	state->bt_map.um_guest_pages = (caddr_t)*addrp;  in xpvtap_segmap()
569 	state->bt_map.um_guest_size = (size_t)len;  in xpvtap_segmap()
572 	e = as_add_callback(asp, xpvtap_segmf_unregister, state,  in xpvtap_segmap()
580 	mutex_enter(&state->bt_thread.ut_mutex);  in xpvtap_segmap()
581 	state->bt_thread.ut_wake = B_TRUE;  in xpvtap_segmap()
582 	cv_signal(&state->bt_thread.ut_wake_cv);  in xpvtap_segmap()
583 	mutex_exit(&state->bt_thread.ut_mutex);  in xpvtap_segmap()
598 	xpvtap_state_t *state;  in xpvtap_devmap()  local
604 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_devmap()
605 	if (state == NULL) {  in xpvtap_devmap()
623 	usring = &state->bt_user_ring;  in xpvtap_devmap()
624 	e = devmap_umem_setup(dhp, state->bt_dip, NULL, usring->ur_cookie, 0,  in xpvtap_devmap()
645 	xpvtap_state_t *state;  in xpvtap_chpoll()  local
653 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_chpoll()
654 	if (state == NULL) {  in xpvtap_chpoll()
667 	usring = &state->bt_user_ring;  in xpvtap_chpoll()
685 			*phpp = &state->bt_pollhead;  in xpvtap_chpoll()
699 	xpvtap_state_t *state;  in xpvtap_drv_init()  local
707 	state = ddi_get_soft_state(xpvtap_statep, instance);  in xpvtap_drv_init()
708 	if (state == NULL) {  in xpvtap_drv_init()
712 	state->bt_instance = instance;  in xpvtap_drv_init()
713 	mutex_init(&state->bt_open.bo_mutex, NULL, MUTEX_DRIVER, NULL);  in xpvtap_drv_init()
714 	cv_init(&state->bt_open.bo_exit_cv, NULL, CV_DRIVER, NULL);  in xpvtap_drv_init()
715 	state->bt_open.bo_opened = B_FALSE;  in xpvtap_drv_init()
716 	state->bt_map.um_registered = B_FALSE;  in xpvtap_drv_init()
719 	e = xpvtap_user_init(state);  in xpvtap_drv_init()
724 	return (state);  in xpvtap_drv_init()
727 	cv_destroy(&state->bt_open.bo_exit_cv);  in xpvtap_drv_init()
728 	mutex_destroy(&state->bt_open.bo_mutex);  in xpvtap_drv_init()
739 xpvtap_drv_fini(xpvtap_state_t *state)  in xpvtap_drv_fini()  argument
741 	xpvtap_user_fini(state);  in xpvtap_drv_fini()
742 	cv_destroy(&state->bt_open.bo_exit_cv);  in xpvtap_drv_fini()
743 	mutex_destroy(&state->bt_open.bo_mutex);  in xpvtap_drv_fini()
744 	(void) ddi_soft_state_free(xpvtap_statep, state->bt_instance);  in xpvtap_drv_fini()
755 	xpvtap_state_t *state;  in xpvtap_intr()  local
758 	state = (xpvtap_state_t *)arg;  in xpvtap_intr()
761 	mutex_enter(&state->bt_thread.ut_mutex);  in xpvtap_intr()
762 	state->bt_thread.ut_wake = B_TRUE;  in xpvtap_intr()
763 	cv_signal(&state->bt_thread.ut_wake_cv);  in xpvtap_intr()
764 	mutex_exit(&state->bt_thread.ut_mutex);  in xpvtap_intr()
774 xpvtap_segmf_register(xpvtap_state_t *state)  in xpvtap_segmf_register()  argument
784 	as = state->bt_map.um_as;  in xpvtap_segmf_register()
785 	pgcnt = btopr(state->bt_map.um_guest_size);  in xpvtap_segmf_register()
786 	uaddr = state->bt_map.um_guest_pages;  in xpvtap_segmf_register()
794 	seg = as_findseg(as, state->bt_map.um_guest_pages, 0);  in xpvtap_segmf_register()
795 	if ((seg == NULL) || ((uaddr + state->bt_map.um_guest_size) >  in xpvtap_segmf_register()
816 	state->bt_map.um_registered = B_TRUE;  in xpvtap_segmf_register()
832 	xpvtap_state_t *state;  in xpvtap_segmf_unregister()  local
838 	state = (xpvtap_state_t *)arg;  in xpvtap_segmf_unregister()
839 	if (!state->bt_map.um_registered) {  in xpvtap_segmf_unregister()
845 	pgcnt = btopr(state->bt_map.um_guest_size);  in xpvtap_segmf_unregister()
846 	uaddr = state->bt_map.um_guest_pages;  in xpvtap_segmf_unregister()
849 	xpvtap_rs_flush(state->bt_map.um_rs, xpvtap_user_request_unmap, state);  in xpvtap_segmf_unregister()
864 	state->bt_map.um_registered = B_FALSE;  in xpvtap_segmf_unregister()
872 xpvtap_user_init(xpvtap_state_t *state)  in xpvtap_user_init()  argument
878 	map = &state->bt_map;  in xpvtap_user_init()
881 	e = xpvtap_user_ring_init(state);  in xpvtap_user_init()
908 	e = xpvtap_user_thread_init(state);  in xpvtap_user_init()
919 	xpvtap_user_ring_fini(state);  in xpvtap_user_init()
928 xpvtap_user_ring_init(xpvtap_state_t *state)  in xpvtap_user_ring_init()  argument
933 	usring = &state->bt_user_ring;  in xpvtap_user_ring_init()
950 xpvtap_user_thread_init(xpvtap_state_t *state)  in xpvtap_user_thread_init()  argument
956 	thread = &state->bt_thread;  in xpvtap_user_thread_init()
966 	(void) sprintf(taskqname, "xvptap_%d", state->bt_instance);  in xpvtap_user_thread_init()
967 	thread->ut_taskq = ddi_taskq_create(state->bt_dip, taskqname, 1,  in xpvtap_user_thread_init()
993 	xpvtap_state_t *state;  in xpvtap_user_thread_start()  local
997 	state = (xpvtap_state_t *)arg;  in xpvtap_user_thread_start()
998 	thread = &state->bt_thread;  in xpvtap_user_thread_start()
1002 	e = ddi_taskq_dispatch(thread->ut_taskq, xpvtap_user_thread, state,  in xpvtap_user_thread_start()
1015 xpvtap_user_thread_stop(xpvtap_state_t *state)  in xpvtap_user_thread_stop()  argument
1018 	mutex_enter(&state->bt_thread.ut_mutex);  in xpvtap_user_thread_stop()
1019 	state->bt_thread.ut_wake = B_TRUE;  in xpvtap_user_thread_stop()
1020 	state->bt_thread.ut_exit = B_TRUE;  in xpvtap_user_thread_stop()
1021 	cv_signal(&state->bt_thread.ut_wake_cv);  in xpvtap_user_thread_stop()
1022 	if (!state->bt_thread.ut_exit_done) {  in xpvtap_user_thread_stop()
1023 		cv_wait(&state->bt_thread.ut_exit_done_cv,  in xpvtap_user_thread_stop()
1024 		    &state->bt_thread.ut_mutex);  in xpvtap_user_thread_stop()
1026 	mutex_exit(&state->bt_thread.ut_mutex);  in xpvtap_user_thread_stop()
1027 	ASSERT(state->bt_thread.ut_exit_done);  in xpvtap_user_thread_stop()
1035 xpvtap_user_fini(xpvtap_state_t *state)  in xpvtap_user_fini()  argument
1040 	map = &state->bt_map;  in xpvtap_user_fini()
1042 	xpvtap_user_thread_fini(state);  in xpvtap_user_fini()
1046 	xpvtap_user_ring_fini(state);  in xpvtap_user_fini()
1054 xpvtap_user_ring_fini(xpvtap_state_t *state)  in xpvtap_user_ring_fini()  argument
1056 	ddi_umem_free(state->bt_user_ring.ur_cookie);  in xpvtap_user_ring_fini()
1064 xpvtap_user_thread_fini(xpvtap_state_t *state)  in xpvtap_user_thread_fini()  argument
1066 	ddi_taskq_destroy(state->bt_thread.ut_taskq);  in xpvtap_user_thread_fini()
1067 	cv_destroy(&state->bt_thread.ut_exit_done_cv);  in xpvtap_user_thread_fini()
1068 	cv_destroy(&state->bt_thread.ut_wake_cv);  in xpvtap_user_thread_fini()
1069 	mutex_destroy(&state->bt_thread.ut_mutex);  in xpvtap_user_thread_fini()
1081 	xpvtap_state_t *state;  in xpvtap_user_thread()  local
1088 	state = (xpvtap_state_t *)arg;  in xpvtap_user_thread()
1089 	thread = &state->bt_thread;  in xpvtap_user_thread()
1096 		cv_signal(&state->bt_thread.ut_exit_done_cv);  in xpvtap_user_thread()
1130 		e = blk_ring_request_get(state->bt_guest_ring, &req);  in xpvtap_user_thread()
1136 		e = xpvtap_user_request_map(state, &req, &uid);  in xpvtap_user_thread()
1143 			blk_ring_request_requeue(state->bt_guest_ring);  in xpvtap_user_thread()
1148 		e = xpvtap_user_request_push(state, &req, uid);  in xpvtap_user_thread()
1153 			blk_ring_response_put(state->bt_guest_ring, &resp);  in xpvtap_user_thread()
1163 		b = xpvtap_user_response_get(state, &resp, &uid);  in xpvtap_user_thread()
1172 		xpvtap_user_request_unmap(state, uid);  in xpvtap_user_thread()
1175 		blk_ring_response_put(state->bt_guest_ring, &resp);  in xpvtap_user_thread()
1186 xpvtap_user_request_map(xpvtap_state_t *state, blkif_request_t *req,  in xpvtap_user_request_map()  argument
1199 	domid = xvdi_get_oeid(state->bt_dip);  in xpvtap_user_request_map()
1201 	as = state->bt_map.um_as;  in xpvtap_user_request_map()
1202 	if ((as == NULL) || (state->bt_map.um_guest_pages == NULL)) {  in xpvtap_user_request_map()
1207 	if (!state->bt_map.um_registered) {  in xpvtap_user_request_map()
1209 		e = xpvtap_segmf_register(state);  in xpvtap_user_request_map()
1216 	e = xpvtap_rs_alloc(state->bt_map.um_rs, uid);  in xpvtap_user_request_map()
1229 	uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, *uid);  in xpvtap_user_request_map()
1232 	seg = as_findseg(as, state->bt_map.um_guest_pages, 0);  in xpvtap_user_request_map()
1262 xpvtap_user_request_push(xpvtap_state_t *state, blkif_request_t *req,  in xpvtap_user_request_push()  argument
1271 	uring = &state->bt_user_ring.ur_ring;  in xpvtap_user_request_push()
1272 	map = &state->bt_map;  in xpvtap_user_request_push()
1288 	pollwakeup(&state->bt_pollhead, POLLIN | POLLRDNORM);  in xpvtap_user_request_push()
1295 xpvtap_user_request_unmap(xpvtap_state_t *state, uint_t uid)  in xpvtap_user_request_unmap()  argument
1304 	as = state->bt_map.um_as;  in xpvtap_user_request_unmap()
1310 	req = &state->bt_map.um_outstanding_reqs[uid];  in xpvtap_user_request_unmap()
1316 		uaddr = XPVTAP_GREF_REQADDR(state->bt_map.um_guest_pages, uid);  in xpvtap_user_request_unmap()
1318 		seg = as_findseg(as, state->bt_map.um_guest_pages, 0);  in xpvtap_user_request_unmap()
1322 			xpvtap_rs_free(state->bt_map.um_rs, uid);  in xpvtap_user_request_unmap()
1335 	xpvtap_rs_free(state->bt_map.um_rs, uid);  in xpvtap_user_request_unmap()
1340 xpvtap_user_response_get(xpvtap_state_t *state, blkif_response_t *resp,  in xpvtap_user_response_get()  argument
1347 	uring = &state->bt_user_ring.ur_ring;  in xpvtap_user_response_get()
1365 	resp->id = state->bt_map.um_outstanding_reqs[*uid].id;  in xpvtap_user_response_get()
1376 	xpvtap_state_t *state;  in xpvtap_user_app_stop()  local
1379 	state = (xpvtap_state_t *)arg;  in xpvtap_user_app_stop()
1385 	mutex_enter(&state->bt_open.bo_mutex);  in xpvtap_user_app_stop()
1386 	if (state->bt_open.bo_opened) {  in xpvtap_user_app_stop()
1387 		rc = cv_reltimedwait(&state->bt_open.bo_exit_cv,  in xpvtap_user_app_stop()
1388 		    &state->bt_open.bo_mutex, drv_usectohz(10000000),  in xpvtap_user_app_stop()
1395 	mutex_exit(&state->bt_open.bo_mutex);  in xpvtap_user_app_stop()