1 // SPDX-License-Identifier: GPL-2.0
3  * Virtio-based remote processor messaging bus
8  * Ohad Ben-Cohen <ohad@wizery.com>
14 #include <linux/dma-mapping.h>
20 #include <linux/rpmsg.h>
21 #include <linux/rpmsg/byteorder.h>
22 #include <linux/rpmsg/ns.h>
34  * struct virtproc_info - virtual remote processor state
52  * This structure stores the rpmsg state of a given virtio remote processor
71 /* The feature bitmap for virtio rpmsg */
75  * struct rpmsg_hdr - common header for all rpmsg messages
83  * Every message sent(/received) on the rpmsg bus begins with this header.
96  * struct virtio_rpmsg_channel - rpmsg channel descriptor
97  * @rpdev: the rpmsg channel device
100  * This structure stores the channel that links the rpmsg device to the virtio
122  * We might also want to add support for user-provided buffers in time.
124  * to achieve zero-copy messaging.
126  * Note that these numbers are purely a decision of this driver - we
134  * Local addresses are dynamically allocated on-demand.
161  * rpmsg_sg_init - initialize scatterlist according to cpu address location
183  * __ept_release() - deallocate an rpmsg endpoint
210 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
216 	kref_init(&ept->refcount);  in __rpmsg_create_ept()
217 	mutex_init(&ept->cb_lock);  in __rpmsg_create_ept()
219 	ept->rpdev = rpdev;  in __rpmsg_create_ept()
220 	ept->cb = cb;  in __rpmsg_create_ept()
221 	ept->priv = priv;  in __rpmsg_create_ept()
222 	ept->ops = &virtio_endpoint_ops;  in __rpmsg_create_ept()
233 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
235 	/* bind the endpoint to an rpmsg address (and allocate one if needed) */  in __rpmsg_create_ept()
236 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
241 	ept->addr = id;  in __rpmsg_create_ept()
243 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
248 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
249 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_create_ept()
257 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()
266 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()
268 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
278 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
282  * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint
287  * bound to an rpmsg channel. This is needed for handling the internal
288  * name service endpoint, which isn't bound to an rpmsg channel.
295 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
296 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
297 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
299 	/* make sure in-flight inbound messages won't invoke cb anymore */  in __rpmsg_destroy_ept()
300 	mutex_lock(&ept->cb_lock);  in __rpmsg_destroy_ept()
301 	ept->cb = NULL;  in __rpmsg_destroy_ept()
302 	mutex_unlock(&ept->cb_lock);  in __rpmsg_destroy_ept()
304 	kref_put(&ept->refcount, __ept_release);  in __rpmsg_destroy_ept()
309 	struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev);  in virtio_rpmsg_destroy_ept()
311 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
317 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()
318 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_create()
322 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_create()
323 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
326 		strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));  in virtio_rpmsg_announce_create()
327 		nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);  in virtio_rpmsg_announce_create()
330 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_create()
341 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()
342 	struct device *dev = &rpdev->dev;  in virtio_rpmsg_announce_destroy()
346 	if (rpdev->announce && rpdev->ept &&  in virtio_rpmsg_announce_destroy()
347 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
350 		strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name));  in virtio_rpmsg_announce_destroy()
351 		nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr);  in virtio_rpmsg_announce_destroy()
354 		err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR);  in virtio_rpmsg_announce_destroy()
375 	kfree(rpdev->driver_override);  in virtio_rpmsg_release_device()
380  * create an rpmsg channel using its name and address info.
389 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
398 				chinfo->name, chinfo->src, chinfo->dst);  in __rpmsg_create_channel()
407 	vch->vrp = vrp;  in __rpmsg_create_channel()
410 	rpdev = &vch->rpdev;  in __rpmsg_create_channel()
411 	rpdev->src = chinfo->src;  in __rpmsg_create_channel()
412 	rpdev->dst = chinfo->dst;  in __rpmsg_create_channel()
413 	rpdev->ops = &virtio_rpmsg_ops;  in __rpmsg_create_channel()
414 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
417 	 * rpmsg server channels has predefined local address (for now),  in __rpmsg_create_channel()
420 	rpdev->announce = rpdev->src != RPMSG_ADDR_ANY;  in __rpmsg_create_channel()
422 	strscpy(rpdev->id.name, chinfo->name, sizeof(rpdev->id.name));  in __rpmsg_create_channel()
424 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
425 	rpdev->dev.release = virtio_rpmsg_release_device;  in __rpmsg_create_channel()
440 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
446 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
447 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
450 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
452 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
458  * rpmsg_upref_sleepers() - enable "tx-complete" interrupts, if needed
476 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
479 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
480 		/* enable "tx-complete" interrupts before dozing off */  in rpmsg_upref_sleepers()
481 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
483 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
487  * rpmsg_downref_sleepers() - disable "tx-complete" interrupts, if needed
503 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
506 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
507 		/* disable "tx-complete" interrupts */  in rpmsg_downref_sleepers()
508 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
510 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
514  * rpmsg_send_offchannel_raw() - send a message across to the remote processor
515  * @rpdev: the rpmsg channel
522  * This function is the base implementation for all of the rpmsg sending API.
534  * case -ERESTARTSYS is returned. The number '15' itself was picked
539  * the function will immediately fail, and -ENOMEM will be returned.
543  * (see include/linux/rpmsg.h).
552 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()
553 	struct device *dev = &rpdev->dev;  in rpmsg_send_offchannel_raw()
561 		return -EINVAL;  in rpmsg_send_offchannel_raw()
565 	 * We currently use fixed-sized buffers, and therefore the payload  in rpmsg_send_offchannel_raw()
569 	 * user-provided buffers (and then we can also support zero-copy  in rpmsg_send_offchannel_raw()
571 	 * variable-length buffer sizes.  in rpmsg_send_offchannel_raw()
573 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
575 		return -EMSGSIZE;  in rpmsg_send_offchannel_raw()
581 		return -ENOMEM;  in rpmsg_send_offchannel_raw()
585 		/* enable "tx-complete" interrupts, if not already enabled */  in rpmsg_send_offchannel_raw()
594 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
598 		/* disable "tx-complete" interrupts if we're the last sleeper */  in rpmsg_send_offchannel_raw()
604 			return -ERESTARTSYS;  in rpmsg_send_offchannel_raw()
608 	msg->len = cpu_to_rpmsg16(rpdev, len);  in rpmsg_send_offchannel_raw()
609 	msg->flags = 0;  in rpmsg_send_offchannel_raw()
610 	msg->src = cpu_to_rpmsg32(rpdev, src);  in rpmsg_send_offchannel_raw()
611 	msg->dst = cpu_to_rpmsg32(rpdev, dst);  in rpmsg_send_offchannel_raw()
612 	msg->reserved = 0;  in rpmsg_send_offchannel_raw()
613 	memcpy(msg->data, data, len);  in rpmsg_send_offchannel_raw()
616 		src, dst, len, msg->flags, msg->reserved);  in rpmsg_send_offchannel_raw()
624 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
627 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
631 		 * (memory won't leak, but rpmsg won't use it again for TX).  in rpmsg_send_offchannel_raw()
635 		goto out;  in rpmsg_send_offchannel_raw()
639 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
640 out:  in rpmsg_send_offchannel_raw()
641 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
647 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_send()
648 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_send()
656 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_sendto()
657 	u32 src = ept->addr;  in virtio_rpmsg_sendto()
664 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysend()
665 	u32 src = ept->addr, dst = rpdev->dst;  in virtio_rpmsg_trysend()
673 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_trysendto()
674 	u32 src = ept->addr;  in virtio_rpmsg_trysendto()
681 	struct rpmsg_device *rpdev = ept->rpdev;  in virtio_rpmsg_get_mtu()
684 	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);  in virtio_rpmsg_get_mtu()
692 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
693 	unsigned int msg_len = __rpmsg16_to_cpu(little_endian, msg->len);  in rpmsg_recv_single()
697 		__rpmsg32_to_cpu(little_endian, msg->src),  in rpmsg_recv_single()
698 		__rpmsg32_to_cpu(little_endian, msg->dst), msg_len,  in rpmsg_recv_single()
699 		__rpmsg16_to_cpu(little_endian, msg->flags),  in rpmsg_recv_single()
700 		__rpmsg32_to_cpu(little_endian, msg->reserved));  in rpmsg_recv_single()
707 	 * We currently use fixed-sized buffers, so trivially sanitize  in rpmsg_recv_single()
710 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
711 	    msg_len > (len - sizeof(struct rpmsg_hdr))) {  in rpmsg_recv_single()
713 		return -EINVAL;  in rpmsg_recv_single()
717 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
719 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
723 		kref_get(&ept->refcount);  in rpmsg_recv_single()
725 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
728 		/* make sure ept->cb doesn't go away while we use it */  in rpmsg_recv_single()
729 		mutex_lock(&ept->cb_lock);  in rpmsg_recv_single()
731 		if (ept->cb)  in rpmsg_recv_single()
732 			ept->cb(ept->rpdev, msg->data, msg_len, ept->priv,  in rpmsg_recv_single()
733 				__rpmsg32_to_cpu(little_endian, msg->src));  in rpmsg_recv_single()
735 		mutex_unlock(&ept->cb_lock);  in rpmsg_recv_single()
738 		kref_put(&ept->refcount, __ept_release);  in rpmsg_recv_single()
743 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
746 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
758 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()
759 	struct device *dev = &rvq->vdev->dev;  in rpmsg_recv_done()
784 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
796 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()
798 	dev_dbg(&svq->vdev->dev, "%s\n", __func__);  in rpmsg_xmit_done()
801 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
806  * create endpoint-to-endpoint communication without associated RPMsg channel.
807  * The endpoints are rattached to the ctrldev RPMsg device.
811 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()
818 		return ERR_PTR(-ENOMEM);  in rpmsg_virtio_add_ctrl_dev()
821 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
824 	rpdev_ctrl = &vch->rpdev;  in rpmsg_virtio_add_ctrl_dev()
825 	rpdev_ctrl->ops = &virtio_rpmsg_ops;  in rpmsg_virtio_add_ctrl_dev()
827 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
828 	rpdev_ctrl->dev.release = virtio_rpmsg_release_device;  in rpmsg_virtio_add_ctrl_dev()
829 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
844 	device_unregister(&rpdev_ctrl->dev);  in rpmsg_virtio_del_ctrl_dev()
864 		return -ENOMEM;  in rpmsg_probe()
866 	vrp->vdev = vdev;  in rpmsg_probe()
868 	idr_init(&vrp->endpoints);  in rpmsg_probe()
869 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
870 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
871 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
878 	vrp->rvq = vqs[0];  in rpmsg_probe()
879 	vrp->svq = vqs[1];  in rpmsg_probe()
882 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
883 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
886 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
887 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
889 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
891 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
893 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
896 	bufs_va = dma_alloc_coherent(vdev->dev.parent,  in rpmsg_probe()
897 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
900 		err = -ENOMEM;  in rpmsg_probe()
904 	dev_dbg(&vdev->dev, "buffers: va %p, dma %pad\n",  in rpmsg_probe()
905 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
908 	vrp->rbufs = bufs_va;  in rpmsg_probe()
911 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
914 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
916 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
918 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
920 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
925 	/* suppress "tx-complete" interrupts */  in rpmsg_probe()
926 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
928 	vdev->priv = vrp;  in rpmsg_probe()
940 			err = -ENOMEM;  in rpmsg_probe()
945 		vch->vrp = vrp;  in rpmsg_probe()
948 		rpdev_ns = &vch->rpdev;  in rpmsg_probe()
949 		rpdev_ns->ops = &virtio_rpmsg_ops;  in rpmsg_probe()
950 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
952 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
953 		rpdev_ns->dev.release = virtio_rpmsg_release_device;  in rpmsg_probe()
962 	 * Prepare to kick but don't notify yet - we can't do this before  in rpmsg_probe()
965 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
976 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
978 	dev_info(&vdev->dev, "rpmsg host is online\n");  in rpmsg_probe()
985 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_probe()
986 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
988 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1003 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()
1004 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1009 	ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device);  in rpmsg_remove()
1011 		dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret);  in rpmsg_remove()
1013 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1015 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1017 	dma_free_coherent(vdev->dev.parent, total_buf_space,  in rpmsg_remove()
1018 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1060 MODULE_DESCRIPTION("Virtio-based remote processor messaging bus");