258 					  queue->kernel_if->u.g.vas[i],  in qp_free_queue()
259 					  queue->kernel_if->u.g.pas[i]);  in qp_free_queue()
277 	size_t queue_size = sizeof(*queue) + sizeof(*queue->kernel_if);  in qp_alloc_queue()
285 		 (sizeof(*queue->kernel_if->u.g.pas) +  in qp_alloc_queue()
286 		  sizeof(*queue->kernel_if->u.g.vas)))  in qp_alloc_queue()
289 	pas_size = num_pages * sizeof(*queue->kernel_if->u.g.pas);  in qp_alloc_queue()
290 	vas_size = num_pages * sizeof(*queue->kernel_if->u.g.vas);  in qp_alloc_queue()
299 	queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);  in qp_alloc_queue()
300 	queue->kernel_if->mutex = NULL;  in qp_alloc_queue()
301 	queue->kernel_if->num_pages = num_pages;  in qp_alloc_queue()
302 	queue->kernel_if->u.g.pas = (dma_addr_t *)(queue->kernel_if + 1);  in qp_alloc_queue()
303 	queue->kernel_if->u.g.vas =  in qp_alloc_queue()
304 		(void **)((u8 *)queue->kernel_if->u.g.pas + pas_size);  in qp_alloc_queue()
305 	queue->kernel_if->host = false;  in qp_alloc_queue()
308 		queue->kernel_if->u.g.vas[i] =  in qp_alloc_queue()
310 					   &queue->kernel_if->u.g.pas[i],  in qp_alloc_queue()
312 		if (!queue->kernel_if->u.g.vas[i]) {  in qp_alloc_queue()
320 	queue->q_header = queue->kernel_if->u.g.vas[0];  in qp_alloc_queue()
336 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;  in qp_memcpy_to_queue_iter()  local
347 		if (kernel_if->host)  in qp_memcpy_to_queue_iter()
348 			va = kmap_local_page(kernel_if->u.h.page[page_index]);  in qp_memcpy_to_queue_iter()
350 			va = kernel_if->u.g.vas[page_index + 1];  in qp_memcpy_to_queue_iter()
361 			if (kernel_if->host)  in qp_memcpy_to_queue_iter()
366 		if (kernel_if->host)  in qp_memcpy_to_queue_iter()
383 	struct vmci_queue_kern_if *kernel_if = queue->kernel_if;  in qp_memcpy_from_queue_iter()  local
395 		if (kernel_if->host)  in qp_memcpy_from_queue_iter()
396 			va = kmap_local_page(kernel_if->u.h.page[page_index]);  in qp_memcpy_from_queue_iter()
398 			va = kernel_if->u.g.vas[page_index + 1];  in qp_memcpy_from_queue_iter()
409 			if (kernel_if->host)  in qp_memcpy_from_queue_iter()
414 		if (kernel_if->host)  in qp_memcpy_from_queue_iter()
461 			produce_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;  in qp_alloc_ppn_set()
465 			consume_q->kernel_if->u.g.pas[i] >> PAGE_SHIFT;  in qp_alloc_ppn_set()
531 	const size_t queue_size = sizeof(*queue) + sizeof(*(queue->kernel_if));  in qp_host_alloc_queue()
537 		 sizeof(*queue->kernel_if->u.h.page))  in qp_host_alloc_queue()
540 	queue_page_size = num_pages * sizeof(*queue->kernel_if->u.h.page);  in qp_host_alloc_queue()
549 		queue->kernel_if = (struct vmci_queue_kern_if *)(queue + 1);  in qp_host_alloc_queue()
550 		queue->kernel_if->host = true;  in qp_host_alloc_queue()
551 		queue->kernel_if->mutex = NULL;  in qp_host_alloc_queue()
552 		queue->kernel_if->num_pages = num_pages;  in qp_host_alloc_queue()
553 		queue->kernel_if->u.h.header_page =  in qp_host_alloc_queue()
555 		queue->kernel_if->u.h.page =  in qp_host_alloc_queue()
556 			&queue->kernel_if->u.h.header_page[1];  in qp_host_alloc_queue()
586 	if (produce_q->kernel_if->host) {  in qp_init_queue_mutex()
587 		produce_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
588 		consume_q->kernel_if->mutex = &produce_q->kernel_if->__mutex;  in qp_init_queue_mutex()
589 		mutex_init(produce_q->kernel_if->mutex);  in qp_init_queue_mutex()
599 	if (produce_q->kernel_if->host) {  in qp_cleanup_queue_mutex()
600 		produce_q->kernel_if->mutex = NULL;  in qp_cleanup_queue_mutex()
601 		consume_q->kernel_if->mutex = NULL;  in qp_cleanup_queue_mutex()
612 	if (queue->kernel_if->host)  in qp_acquire_queue_mutex()
613 		mutex_lock(queue->kernel_if->mutex);  in qp_acquire_queue_mutex()
623 	if (queue->kernel_if->host)  in qp_release_queue_mutex()
624 		mutex_unlock(queue->kernel_if->mutex);  in qp_release_queue_mutex()
659 				     produce_q->kernel_if->num_pages,  in qp_host_get_user_memory()
661 				     produce_q->kernel_if->u.h.header_page);  in qp_host_get_user_memory()
662 	if (retval < (int)produce_q->kernel_if->num_pages) {  in qp_host_get_user_memory()
666 			qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
673 				     consume_q->kernel_if->num_pages,  in qp_host_get_user_memory()
675 				     consume_q->kernel_if->u.h.header_page);  in qp_host_get_user_memory()
676 	if (retval < (int)consume_q->kernel_if->num_pages) {  in qp_host_get_user_memory()
680 			qp_release_pages(consume_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
682 		qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_get_user_memory()
683 				 produce_q->kernel_if->num_pages, false);  in qp_host_get_user_memory()
710 	    produce_q->kernel_if->num_pages * PAGE_SIZE;  in qp_host_register_user_memory()
723 	qp_release_pages(produce_q->kernel_if->u.h.header_page,  in qp_host_unregister_user_memory()
724 			 produce_q->kernel_if->num_pages, true);  in qp_host_unregister_user_memory()
725 	memset(produce_q->kernel_if->u.h.header_page, 0,  in qp_host_unregister_user_memory()
726 	       sizeof(*produce_q->kernel_if->u.h.header_page) *  in qp_host_unregister_user_memory()
727 	       produce_q->kernel_if->num_pages);  in qp_host_unregister_user_memory()
728 	qp_release_pages(consume_q->kernel_if->u.h.header_page,  in qp_host_unregister_user_memory()
729 			 consume_q->kernel_if->num_pages, true);  in qp_host_unregister_user_memory()
730 	memset(consume_q->kernel_if->u.h.header_page, 0,  in qp_host_unregister_user_memory()
731 	       sizeof(*consume_q->kernel_if->u.h.header_page) *  in qp_host_unregister_user_memory()
732 	       consume_q->kernel_if->num_pages);  in qp_host_unregister_user_memory()
754 		if (produce_q->kernel_if->u.h.header_page == NULL ||  in qp_host_map_queues()
755 		    *produce_q->kernel_if->u.h.header_page == NULL)  in qp_host_map_queues()
758 		headers[0] = *produce_q->kernel_if->u.h.header_page;  in qp_host_map_queues()
759 		headers[1] = *consume_q->kernel_if->u.h.header_page;  in qp_host_map_queues()