106 	struct virtproc_info *vrp;  member
148 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,
203 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,  in __rpmsg_create_ept()  argument
210 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
233 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
236 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
243 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
248 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
257 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()  local
259 	return __rpmsg_create_channel(vrp, chinfo);  in virtio_rpmsg_create_channel()
266 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()  local
268 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
278 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
292 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)  in __rpmsg_destroy_ept()  argument
295 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
296 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
297 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
311 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
317 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()  local
323 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
341 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()  local
347 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
384 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,  in __rpmsg_create_channel()  argument
389 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
407 	vch->vrp = vrp;  in __rpmsg_create_channel()
414 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
424 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
434 static void *get_a_tx_buf(struct virtproc_info *vrp)  in get_a_tx_buf()  argument
440 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
446 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
447 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
450 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
452 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
473 static void rpmsg_upref_sleepers(struct virtproc_info *vrp)  in rpmsg_upref_sleepers()  argument
476 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
479 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
481 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
483 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
500 static void rpmsg_downref_sleepers(struct virtproc_info *vrp)  in rpmsg_downref_sleepers()  argument
503 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
506 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
508 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
510 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
552 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()  local
573 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
579 	msg = get_a_tx_buf(vrp);  in rpmsg_send_offchannel_raw()
586 		rpmsg_upref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
594 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
595 					(msg = get_a_tx_buf(vrp)),  in rpmsg_send_offchannel_raw()
599 		rpmsg_downref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
624 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
627 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
639 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
641 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
684 	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);  in virtio_rpmsg_get_mtu()
687 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,  in rpmsg_recv_single()  argument
692 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
710 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
717 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
719 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
725 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
743 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
746 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
758 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()  local
771 		err = rpmsg_recv_single(vrp, dev, msg, len);  in rpmsg_recv_done()
784 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
796 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()  local
801 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
811 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()  local
821 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
827 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
829 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
854 	struct virtproc_info *vrp;  in rpmsg_probe()  local
862 	vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);  in rpmsg_probe()
863 	if (!vrp)  in rpmsg_probe()
866 	vrp->vdev = vdev;  in rpmsg_probe()
868 	idr_init(&vrp->endpoints);  in rpmsg_probe()
869 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
870 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
871 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
878 	vrp->rvq = vqs[0];  in rpmsg_probe()
879 	vrp->svq = vqs[1];  in rpmsg_probe()
882 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
883 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
886 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
887 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
889 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
891 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
893 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
897 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
905 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
908 	vrp->rbufs = bufs_va;  in rpmsg_probe()
911 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
914 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
916 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
918 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
920 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
926 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
928 	vdev->priv = vrp;  in rpmsg_probe()
945 		vch->vrp = vrp;  in rpmsg_probe()
950 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
952 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
965 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
976 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
986 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
988 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
990 	kfree(vrp);  in rpmsg_probe()
1003 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()  local
1004 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1013 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1015 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1018 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1020 	kfree(vrp);  in rpmsg_remove()