1 /*-
67 #include <xen/xen-os.h>
74 MALLOC_DEFINE(M_GNTDEV, "gntdev", "Xen grant-table user-space device");
108 	return (f1->file_offset - f2->file_offset);  in offset_cmp()
122  * Get offset into the file which will be used while mmapping the
129 	struct file_offset_struct *offset, *offset_tmp;  in get_file_offset()  local
133 	mtx_lock(&priv_user->user_data_lock);  in get_file_offset()
134 	RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset,  in get_file_offset()
136 		if (offset->count >= count) {  in get_file_offset()
137 			offset->count -= count;  in get_file_offset()
138 			*file_offset = offset->file_offset + offset->count *  in get_file_offset()
140 			if (offset->count == 0) {  in get_file_offset()
142 				    &priv_user->file_offset, offset);  in get_file_offset()
143 				free(offset, M_GNTDEV);  in get_file_offset()
145 			mtx_unlock(&priv_user->user_data_lock);  in get_file_offset()
149 	mtx_unlock(&priv_user->user_data_lock);  in get_file_offset()
158 	struct file_offset_struct *offset, *offset_nxt, *offset_prv;  in put_file_offset()  local
160 	offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);  in put_file_offset()
161 	offset->file_offset = file_offset;  in put_file_offset()
162 	offset->count = count;  in put_file_offset()
164 	mtx_lock(&priv_user->user_data_lock);  in put_file_offset()
165 	RB_INSERT(file_offset_head, &priv_user->file_offset, offset);  in put_file_offset()
166 	offset_nxt = RB_NEXT(file_offset_head, &priv_user->file_offset, offset);  in put_file_offset()
167 	offset_prv = RB_PREV(file_offset_head, &priv_user->file_offset, offset);  in put_file_offset()
169 	    offset_nxt->file_offset == offset->file_offset + offset->count *  in put_file_offset()
171 		offset->count += offset_nxt->count;  in put_file_offset()
172 		RB_REMOVE(file_offset_head, &priv_user->file_offset,  in put_file_offset()
177 	    offset->file_offset == offset_prv->file_offset + offset_prv->count *  in put_file_offset()
179 		offset_prv->count += offset->count;  in put_file_offset()
180 		RB_REMOVE(file_offset_head, &priv_user->file_offset, offset);  in put_file_offset()
181 		free(offset, M_GNTDEV);  in put_file_offset()
183 	mtx_unlock(&priv_user->user_data_lock);  in put_file_offset()
189 static int	gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset,
225 static void	notify(struct notify_data *notify, vm_page_t page);
227 /*-------------------- Grant Allocation Methods  -----------------------------*/
236 	vm_page_t		page;  member
243 	return (g1->file_index - g2->file_index);  in gref_cmp()
249  * Traverse over the device-list of to-be-deleted grants allocated, and
260 	mtx_lock(&cleanup_data->to_kill_grefs_mtx);  in gref_list_dtor()
261 	STAILQ_SWAP(&cleanup_data->to_kill_grefs, &tmp_grefs, gntdev_gref);  in gref_list_dtor()
262 	mtx_unlock(&cleanup_data->to_kill_grefs_mtx);  in gref_list_dtor()
266 		if (gref->page && gref->page->object == NULL) {  in gref_list_dtor()
267 			if (gref->notify) {  in gref_list_dtor()
268 				notify(gref->notify, gref->page);  in gref_list_dtor()
270 			if (gref->gref_id != GRANT_REF_INVALID) {  in gref_list_dtor()
271 				if (gnttab_query_foreign_access(gref->gref_id))  in gref_list_dtor()
273 				if (gnttab_end_foreign_access_ref(gref->gref_id)  in gref_list_dtor()
276 				gnttab_free_grant_reference(gref->gref_id);  in gref_list_dtor()
278 			vm_page_unwire_noq(gref->page);  in gref_list_dtor()
279 			vm_page_free(gref->page);  in gref_list_dtor()
280 			gref->page = NULL;  in gref_list_dtor()
282 		if (gref->page == NULL) {  in gref_list_dtor()
288 			if (gref->notify)  in gref_list_dtor()
289 				free(gref->notify, M_GNTDEV);  in gref_list_dtor()
297 		mtx_lock(&cleanup_data->to_kill_grefs_mtx);  in gref_list_dtor()
298 		STAILQ_CONCAT(&cleanup_data->to_kill_grefs, &tmp_grefs);  in gref_list_dtor()
299 		mtx_unlock(&cleanup_data->to_kill_grefs_mtx);  in gref_list_dtor()
305  * program by file-offset (index).
315 	mtx_lock(&priv_user->user_data_lock);  in gntdev_find_grefs()
316 	gref_start = RB_FIND(gref_tree_head, &priv_user->gref_tree, &find_gref);  in gntdev_find_grefs()
318 	    RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) {  in gntdev_find_grefs()
319 		if (index != gref->file_index)  in gntdev_find_grefs()
322 		count--;  in gntdev_find_grefs()
324 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_find_grefs()
347 	readonly = !(arg->flags & GNTDEV_ALLOC_FLAG_WRITABLE);  in gntdev_alloc_gref()
356 	/* Get file offset for this request. */  in gntdev_alloc_gref()
357 	error = get_file_offset(priv_user, arg->count, &file_offset);  in gntdev_alloc_gref()
362 	grefs = malloc(sizeof(*grefs) * arg->count, M_GNTDEV, M_WAITOK);  in gntdev_alloc_gref()
364 	for (i = 0; i < arg->count; i++) {  in gntdev_alloc_gref()
368 		grefs[i].page = vm_page_alloc_noobj(VM_ALLOC_WIRED |  in gntdev_alloc_gref()
370 		if (grefs[i].page == NULL) {  in gntdev_alloc_gref()
371 			log(LOG_ERR, "Page allocation failed.");  in gntdev_alloc_gref()
375 		grefs[i].page->valid = VM_PAGE_BITS_ALL;  in gntdev_alloc_gref()
377 		error = gnttab_grant_foreign_access(arg->domid,  in gntdev_alloc_gref()
378 			(VM_PAGE_TO_PHYS(grefs[i].page) >> PAGE_SHIFT),  in gntdev_alloc_gref()
387 	arg->index = file_offset;  in gntdev_alloc_gref()
388 	for (i = 0; error == 0 && i < arg->count; i++) {  in gntdev_alloc_gref()
389 		if (suword32(&arg->gref_ids[i], grefs[i].gref_id) != 0)  in gntdev_alloc_gref()
395 		 * If target domain maps the gref (by guessing the gref-id),  in gntdev_alloc_gref()
397 		 * page in place so as to not leak our memory to that domain.  in gntdev_alloc_gref()
401 		for (i = 0; i < arg->count; i++)  in gntdev_alloc_gref()
412 	mtx_lock(&priv_user->user_data_lock);  in gntdev_alloc_gref()
413 	for (i = 0; i < arg->count; i++)  in gntdev_alloc_gref()
414 		RB_INSERT(gref_tree_head, &priv_user->gref_tree, &grefs[i]);  in gntdev_alloc_gref()
415 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_alloc_gref()
424  * to the to-be-deleted grants global device-list.
438 	gref = gntdev_find_grefs(priv_user, arg->index, arg->count);  in gntdev_dealloc_gref()
440 		log(LOG_ERR, "Can't find requested grant-refs.");  in gntdev_dealloc_gref()
445 	count = arg->count;  in gntdev_dealloc_gref()
446 	mtx_lock(&priv_user->user_data_lock);  in gntdev_dealloc_gref()
449 		gref_tmp = RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref);  in gntdev_dealloc_gref()
450 		RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref);  in gntdev_dealloc_gref()
453 		count--;  in gntdev_dealloc_gref()
456 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_dealloc_gref()
459 	put_file_offset(priv_user, arg->count, arg->index);  in gntdev_dealloc_gref()
464 /*-------------------- Grant Mapping Methods  --------------------------------*/
488 	return (g1->file_index - g2->file_index);  in gmap_cmp()
494  * Traverse over the device-list of to-be-deleted grant mappings, and if
505 	mtx_lock(&cleanup_data->to_kill_gmaps_mtx);  in gmap_list_dtor()
506 	STAILQ_SWAP(&cleanup_data->to_kill_gmaps, &tmp_gmaps, gntdev_gmap);  in gmap_list_dtor()
507 	mtx_unlock(&cleanup_data->to_kill_gmaps_mtx);  in gmap_list_dtor()
511 		if (gmap->map == NULL) {  in gmap_list_dtor()
518 			if (gmap->notify)  in gmap_list_dtor()
519 				free(gmap->notify, M_GNTDEV);  in gmap_list_dtor()
520 			free(gmap->grant_map_ops, M_GNTDEV);  in gmap_list_dtor()
528 		mtx_lock(&cleanup_data->to_kill_gmaps_mtx);  in gmap_list_dtor()
529 		STAILQ_CONCAT(&cleanup_data->to_kill_gmaps, &tmp_gmaps);  in gmap_list_dtor()
530 		mtx_unlock(&cleanup_data->to_kill_gmaps_mtx);  in gmap_list_dtor()
535  * Find mapped grants for a given userspace program, by file-offset (index)
536  * and count, as supplied during the map-ioctl.
546 	mtx_lock(&priv_user->user_data_lock);  in gntdev_find_gmap()
547 	gmap = RB_FIND(gmap_tree_head, &priv_user->gmap_tree, &find_gmap);  in gntdev_find_gmap()
548 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_find_gmap()
550 	if (gmap != NULL && gmap->count == count)  in gntdev_find_gmap()
559  * it have been removed, and the unmap-ioctl has been performed.
568 	unmap_ops = malloc(sizeof(struct gnttab_unmap_grant_ref) * gmap->count,  in notify_unmap_cleanup()
573 	for (i = 0; i < gmap->count; i++) {  in notify_unmap_cleanup()
574 		if (gmap->grant_map_ops[i].handle != -1) {  in notify_unmap_cleanup()
575 			unmap_ops[count].handle = gmap->grant_map_ops[i].handle;  in notify_unmap_cleanup()
577 				gmap->grant_map_ops[i].host_addr;  in notify_unmap_cleanup()
584 	if (count > 0 && gmap->notify) {  in notify_unmap_cleanup()
585 		vm_page_t page;  in notify_unmap_cleanup()  local
588 		page_offset = gmap->notify->index - gmap->file_index;  in notify_unmap_cleanup()
589 		page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + page_offset);  in notify_unmap_cleanup()
590 		notify(gmap->notify, page);  in notify_unmap_cleanup()
594 	cdev_mgtdev_pager_free_pages(gmap->map->mem);  in notify_unmap_cleanup()
600 	for (i = 0; i < gmap->count; i++) {  in notify_unmap_cleanup()
601 		gmap->grant_map_ops[i].handle = -1;  in notify_unmap_cleanup()
602 		gmap->grant_map_ops[i].host_addr = 0;  in notify_unmap_cleanup()
605 	if (gmap->map) {  in notify_unmap_cleanup()
606 		error = xenmem_free(gntdev_dev, gmap->map->pseudo_phys_res_id,  in notify_unmap_cleanup()
607 		    gmap->map->pseudo_phys_res);  in notify_unmap_cleanup()
611 		free(gmap->map, M_GNTDEV);  in notify_unmap_cleanup()
612 		gmap->map = NULL;  in notify_unmap_cleanup()
639 	gmap->count = arg->count;  in gntdev_map_grant_ref()
640 	gmap->grant_map_ops =  in gntdev_map_grant_ref()
641 	    malloc(sizeof(struct gnttab_map_grant_ref) * arg->count,  in gntdev_map_grant_ref()
644 	for (i = 0; i < arg->count; i++) {  in gntdev_map_grant_ref()
647 		error = copyin(&arg->refs[i], &ref, sizeof(ref));  in gntdev_map_grant_ref()
649 			free(gmap->grant_map_ops, M_GNTDEV);  in gntdev_map_grant_ref()
653 		gmap->grant_map_ops[i].dom = ref.domid;  in gntdev_map_grant_ref()
654 		gmap->grant_map_ops[i].ref = ref.ref;  in gntdev_map_grant_ref()
655 		gmap->grant_map_ops[i].handle = -1;  in gntdev_map_grant_ref()
656 		gmap->grant_map_ops[i].flags = GNTMAP_host_map;  in gntdev_map_grant_ref()
659 	error = get_file_offset(priv_user, arg->count, &gmap->file_index);  in gntdev_map_grant_ref()
661 		free(gmap->grant_map_ops, M_GNTDEV);  in gntdev_map_grant_ref()
666 	mtx_lock(&priv_user->user_data_lock);  in gntdev_map_grant_ref()
667 	RB_INSERT(gmap_tree_head, &priv_user->gmap_tree, gmap);  in gntdev_map_grant_ref()
668 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_map_grant_ref()
670 	arg->index = gmap->file_index;  in gntdev_map_grant_ref()
678  * to the global device-list of mappings to be deleted. A reference to
693 	gmap = gntdev_find_gmap(priv_user, arg->index, arg->count);  in gntdev_unmap_grant_ref()
695 		log(LOG_ERR, "Can't find requested grant-map.");  in gntdev_unmap_grant_ref()
699 	mtx_lock(&priv_user->user_data_lock);  in gntdev_unmap_grant_ref()
701 	RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap);  in gntdev_unmap_grant_ref()
704 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_unmap_grant_ref()
706 	if (gmap->map)  in gntdev_unmap_grant_ref()
707 		vm_object_deallocate(gmap->map->mem);  in gntdev_unmap_grant_ref()
710 	put_file_offset(priv_user, arg->count, arg->index);  in gntdev_unmap_grant_ref()
717  * Get file-offset and count for a given mapping, from the virtual address
739 	map = &td->td_proc->p_vmspace->vm_map;  in gntdev_get_offset_for_vaddr()
740 	error = vm_map_lookup(&map, arg->vaddr, VM_PROT_NONE, &entry,  in gntdev_get_offset_for_vaddr()
745 	if ((mem->type != OBJT_MGTDEVICE) ||  in gntdev_get_offset_for_vaddr()
746 	    (mem->un_pager.devp.ops != &gntdev_gmap_pg_ops)) {  in gntdev_get_offset_for_vaddr()
751 	gmap = mem->handle;  in gntdev_get_offset_for_vaddr()
753 	    (entry->end - entry->start) != (gmap->count * PAGE_SIZE)) {  in gntdev_get_offset_for_vaddr()
758 	arg->count = gmap->count;  in gntdev_get_offset_for_vaddr()
759 	arg->offset = gmap->file_index;  in gntdev_get_offset_for_vaddr()
767 /*-------------------- Grant Mapping Pager  ----------------------------------*/
785 gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset, int prot,  in gntdev_gmap_pg_fault()  argument
788 	struct gntdev_gmap *gmap = object->handle;  in gntdev_gmap_pg_fault()
790 	vm_page_t page;  in gntdev_gmap_pg_fault()  local
793 	if (gmap->map == NULL)  in gntdev_gmap_pg_fault()
796 	relative_offset = offset - gmap->file_index;  in gntdev_gmap_pg_fault()
798 	pidx = OFF_TO_IDX(offset);  in gntdev_gmap_pg_fault()
800 	if (ridx >= gmap->count ||  in gntdev_gmap_pg_fault()
801 	    gmap->grant_map_ops[ridx].status != GNTST_okay)  in gntdev_gmap_pg_fault()
804 	page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + relative_offset);  in gntdev_gmap_pg_fault()
805 	if (page == NULL)  in gntdev_gmap_pg_fault()
808 	KASSERT((page->flags & PG_FICTITIOUS) != 0,  in gntdev_gmap_pg_fault()
809 	    ("not fictitious %p", page));  in gntdev_gmap_pg_fault()
810 	KASSERT(vm_page_wired(page), ("page %p is not wired", page));  in gntdev_gmap_pg_fault()
811 	KASSERT(!vm_page_busied(page), ("page %p is busy", page));  in gntdev_gmap_pg_fault()
813 	vm_page_busy_acquire(page, 0);  in gntdev_gmap_pg_fault()
814 	vm_page_valid(page);  in gntdev_gmap_pg_fault()
816 		vm_page_replace(page, object, pidx, *mres);  in gntdev_gmap_pg_fault()
818 		vm_page_insert(page, object, pidx);  in gntdev_gmap_pg_fault()
819 	*mres = page;  in gntdev_gmap_pg_fault()
823 /*------------------ Grant Table Methods  ------------------------------------*/
826 notify(struct notify_data *notify, vm_page_t page)  in notify()  argument
828 	if (notify->action & UNMAP_NOTIFY_CLEAR_BYTE) {  in notify()
830 		uint64_t offset;  in notify()  local
832 		offset = notify->index & PAGE_MASK;  in notify()
833 		mem = (uint8_t *)pmap_quick_enter_page(page);  in notify()
834 		mem[offset] = 0;  in notify()
837 	if (notify->action & UNMAP_NOTIFY_SEND_EVENT) {  in notify()
838 		xen_intr_signal(notify->notify_evtchn_handle);  in notify()
839 		xen_intr_unbind(&notify->notify_evtchn_handle);  in notify()
841 	notify->action = 0;  in notify()
858 	if (source->action & UNMAP_NOTIFY_SEND_EVENT)  in copy_notify_helper()
859 		if (xen_intr_get_evtchn_from_port(source->event_channel_port,  in copy_notify_helper()
863 	if (destination->action & UNMAP_NOTIFY_SEND_EVENT)  in copy_notify_helper()
864 		xen_intr_unbind(&destination->notify_evtchn_handle);  in copy_notify_helper()
866 	destination->action = source->action;  in copy_notify_helper()
867 	destination->event_channel_port = source->event_channel_port;  in copy_notify_helper()
868 	destination->index = source->index;  in copy_notify_helper()
869 	destination->notify_evtchn_handle = handlep;  in copy_notify_helper()
893 	if (arg->action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT))  in gntdev_set_unmap_notify()
896 	index = arg->index & ~PAGE_MASK;  in gntdev_set_unmap_notify()
899 		if (gref->notify == NULL)  in gntdev_set_unmap_notify()
900 			gref->notify = malloc(sizeof(*arg), M_GNTDEV,  in gntdev_set_unmap_notify()
902 		return (copy_notify_helper(gref->notify, arg));  in gntdev_set_unmap_notify()
906 	mtx_lock(&priv_user->user_data_lock);  in gntdev_set_unmap_notify()
907 	RB_FOREACH(gmap, gmap_tree_head, &priv_user->gmap_tree) {  in gntdev_set_unmap_notify()
908 		if (arg->index >= gmap->file_index &&  in gntdev_set_unmap_notify()
909 		    arg->index < gmap->file_index + gmap->count * PAGE_SIZE) {  in gntdev_set_unmap_notify()
910 			if (gmap->notify == NULL)  in gntdev_set_unmap_notify()
911 				gmap->notify = malloc(sizeof(*arg), M_GNTDEV,  in gntdev_set_unmap_notify()
913 			error = copy_notify_helper(gmap->notify, arg);  in gntdev_set_unmap_notify()
917 	mtx_unlock(&priv_user->user_data_lock);  in gntdev_set_unmap_notify()
922 /*------------------ Gntdev Char Device Methods  -----------------------------*/
937 	struct file_offset_struct *offset, *offset_tmp;  in per_user_data_dtor()  local
942 	mtx_lock(&priv_user->user_data_lock);  in per_user_data_dtor()
945 	RB_FOREACH_SAFE(gref, gref_tree_head, &priv_user->gref_tree, gref_tmp) {  in per_user_data_dtor()
946 		RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref);  in per_user_data_dtor()
953 	RB_FOREACH_SAFE(gmap, gmap_tree_head, &priv_user->gmap_tree, gmap_tmp) {  in per_user_data_dtor()
954 		RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap);  in per_user_data_dtor()
957 		if (gmap->map)  in per_user_data_dtor()
958 			vm_object_deallocate(gmap->map->mem);  in per_user_data_dtor()
962 	RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset,  in per_user_data_dtor()
964 		RB_REMOVE(file_offset_head, &priv_user->file_offset, offset);  in per_user_data_dtor()
965 		free(offset, M_GNTDEV);  in per_user_data_dtor()
968 	mtx_unlock(&priv_user->user_data_lock);  in per_user_data_dtor()
972 	mtx_destroy(&priv_user->user_data_lock);  in per_user_data_dtor()
981 	struct file_offset_struct *offset;  in gntdev_open()  local
984 	RB_INIT(&priv_user->gref_tree);  in gntdev_open()
985 	RB_INIT(&priv_user->gmap_tree);  in gntdev_open()
986 	RB_INIT(&priv_user->file_offset);  in gntdev_open()
987 	offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO);  in gntdev_open()
988 	offset->file_offset = 0;  in gntdev_open()
989 	offset->count = MAX_OFFSET_COUNT;  in gntdev_open()
990 	RB_INSERT(file_offset_head, &priv_user->file_offset, offset);  in gntdev_open()
991 	mtx_init(&priv_user->user_data_lock,  in gntdev_open()
1053 	    curthread->td_ucred);  in mmap_gref()
1057 	mtx_lock(&priv_user->user_data_lock);  in mmap_gref()
1060 	    RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) {  in mmap_gref()
1061 		if (gref->page->object)  in mmap_gref()
1064 		vm_page_insert(gref->page, mem_obj,  in mmap_gref()
1065 		    OFF_TO_IDX(gref->file_index));  in mmap_gref()
1067 		count--;  in mmap_gref()
1070 	mtx_unlock(&priv_user->user_data_lock);  in mmap_gref()
1088     vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot)  in mmap_gmap()  argument
1098 	if (gmap_start->map) {  in mmap_gmap()
1099 		vm_object_reference(gmap_start->map->mem);  in mmap_gmap()
1100 		*object = gmap_start->map->mem;  in mmap_gmap()
1104 	gmap_start->map = malloc(sizeof(*(gmap_start->map)), M_GNTDEV,  in mmap_gmap()
1108 	gmap_start->map->pseudo_phys_res_id = 0;  in mmap_gmap()
1109 	gmap_start->map->pseudo_phys_res = xenmem_alloc(gntdev_dev,  in mmap_gmap()
1110 	    &gmap_start->map->pseudo_phys_res_id, size);  in mmap_gmap()
1111 	if (gmap_start->map->pseudo_phys_res == NULL) {  in mmap_gmap()
1112 		free(gmap_start->map, M_GNTDEV);  in mmap_gmap()
1113 		gmap_start->map = NULL;  in mmap_gmap()
1116 	gmap_start->map->phys_base_addr =  in mmap_gmap()
1117 	    rman_get_start(gmap_start->map->pseudo_phys_res);  in mmap_gmap()
1120 	gmap_start->map->mem = cdev_pager_allocate(gmap_start, OBJT_MGTDEVICE,  in mmap_gmap()
1121 	    &gntdev_gmap_pg_ops, size, nprot, *offset, NULL);  in mmap_gmap()
1122 	if (gmap_start->map->mem == NULL) {  in mmap_gmap()
1123 		xenmem_free(gntdev_dev, gmap_start->map->pseudo_phys_res_id,  in mmap_gmap()
1124 		    gmap_start->map->pseudo_phys_res);  in mmap_gmap()
1125 		free(gmap_start->map, M_GNTDEV);  in mmap_gmap()
1126 		gmap_start->map = NULL;  in mmap_gmap()
1130 	for (i = 0; i < gmap_start->count; i++) {  in mmap_gmap()
1131 		gmap_start->grant_map_ops[i].host_addr =  in mmap_gmap()
1132 		    gmap_start->map->phys_base_addr + i * PAGE_SIZE;  in mmap_gmap()
1135 			gmap_start->grant_map_ops[i].flags |= GNTMAP_readonly;  in mmap_gmap()
1139 	    gmap_start->grant_map_ops, gmap_start->count);  in mmap_gmap()
1146 		vm_object_deallocate(gmap_start->map->mem);  in mmap_gmap()
1152 	for (i = 0; i < gmap_start->count; i++) {  in mmap_gmap()
1155 		    gmap_start->grant_map_ops[i].status == GNTST_eagain) {  in mmap_gmap()
1157 			    &gmap_start->grant_map_ops[i], 1);  in mmap_gmap()
1161 		if (gmap_start->grant_map_ops[i].status == GNTST_eagain)  in mmap_gmap()
1162 			gmap_start->grant_map_ops[i].status = GNTST_bad_page;  in mmap_gmap()
1164 		if (gmap_start->grant_map_ops[i].status != GNTST_okay) {  in mmap_gmap()
1171 			vm_object_deallocate(gmap_start->map->mem);  in mmap_gmap()
1180 	 * unmapped, because it may be re-mmapped. Instead,  in mmap_gmap()
1182 	 * munmaps, we have also processed the unmap-ioctl.  in mmap_gmap()
1184 	vm_object_reference(gmap_start->map->mem);  in mmap_gmap()
1186 	*object = gmap_start->map->mem;  in mmap_gmap()
1192 gntdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size,  in gntdev_mmap_single()  argument
1207 	gref_start = gntdev_find_grefs(priv_user, *offset, count);  in gntdev_mmap_single()
1213 	gmap_start = gntdev_find_gmap(priv_user, *offset, count);  in gntdev_mmap_single()
1215 		error = mmap_gmap(priv_user, gmap_start, offset, size, object,  in gntdev_mmap_single()
1223 /*------------------ Private Device Attachment Functions  --------------------*/
1232 		panic("unable to attach gntdev user-space device");  in gntdev_identify()
1240 	device_set_desc(dev, "Xen grant-table user-space device");  in gntdev_probe()
1253 /*-------------------- Private Device Attachment Data  -----------------------*/