1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Virtio-based remote processor messaging bus 4 * 5 * Copyright (C) 2011 Texas Instruments, Inc. 6 * Copyright (C) 2011 Google, Inc. 7 * 8 * Ohad Ben-Cohen <ohad@wizery.com> 9 * Brian Swetland <swetland@google.com> 10 */ 11 12 #define pr_fmt(fmt) "%s: " fmt, __func__ 13 14 #include <linux/dma-mapping.h> 15 #include <linux/idr.h> 16 #include <linux/jiffies.h> 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/mutex.h> 20 #include <linux/rpmsg.h> 21 #include <linux/rpmsg/byteorder.h> 22 #include <linux/rpmsg/ns.h> 23 #include <linux/scatterlist.h> 24 #include <linux/slab.h> 25 #include <linux/sched.h> 26 #include <linux/virtio.h> 27 #include <linux/virtio_ids.h> 28 #include <linux/virtio_config.h> 29 #include <linux/wait.h> 30 31 #include "rpmsg_internal.h" 32 33 /** 34 * struct virtproc_info - virtual remote processor state 35 * @vdev: the virtio device 36 * @rvq: rx virtqueue 37 * @svq: tx virtqueue 38 * @rbufs: kernel address of rx buffers 39 * @sbufs: kernel address of tx buffers 40 * @num_bufs: total number of buffers for rx and tx 41 * @buf_size: size of one rx or tx buffer 42 * @last_sbuf: index of last tx buffer used 43 * @bufs_dma: dma base addr of the buffers 44 * @tx_lock: protects svq and sbufs, to allow concurrent senders. 45 * sending a message might require waking up a dozing remote 46 * processor, which involves sleeping, hence the mutex. 47 * @endpoints: idr of local endpoints, allows fast retrieval 48 * @endpoints_lock: lock of the endpoints set 49 * @sendq: wait queue of sending contexts waiting for a tx buffers 50 * 51 * This structure stores the rpmsg state of a given virtio remote processor 52 * device (there might be several virtio proc devices for each physical 53 * remote processor). 54 */ 55 struct virtproc_info { 56 struct virtio_device *vdev; 57 struct virtqueue *rvq, *svq; 58 void *rbufs, *sbufs; 59 unsigned int num_bufs; 60 unsigned int buf_size; 61 int last_sbuf; 62 dma_addr_t bufs_dma; 63 struct mutex tx_lock; 64 struct idr endpoints; 65 struct mutex endpoints_lock; 66 wait_queue_head_t sendq; 67 }; 68 69 /* The feature bitmap for virtio rpmsg */ 70 #define VIRTIO_RPMSG_F_NS 0 /* RP supports name service notifications */ 71 72 /** 73 * struct rpmsg_hdr - common header for all rpmsg messages 74 * @src: source address 75 * @dst: destination address 76 * @reserved: reserved for future use 77 * @len: length of payload (in bytes) 78 * @flags: message flags 79 * @data: @len bytes of message payload data 80 * 81 * Every message sent(/received) on the rpmsg bus begins with this header. 82 */ 83 struct rpmsg_hdr { 84 __rpmsg32 src; 85 __rpmsg32 dst; 86 __rpmsg32 reserved; 87 __rpmsg16 len; 88 __rpmsg16 flags; 89 u8 data[]; 90 } __packed; 91 92 93 /** 94 * struct virtio_rpmsg_channel - rpmsg channel descriptor 95 * @rpdev: the rpmsg channel device 96 * @vrp: the virtio remote processor device this channel belongs to 97 * 98 * This structure stores the channel that links the rpmsg device to the virtio 99 * remote processor device. 100 */ 101 struct virtio_rpmsg_channel { 102 struct rpmsg_device rpdev; 103 104 struct virtproc_info *vrp; 105 }; 106 107 #define to_virtio_rpmsg_channel(_rpdev) \ 108 container_of(_rpdev, struct virtio_rpmsg_channel, rpdev) 109 110 /* 111 * We're allocating buffers of 512 bytes each for communications. The 112 * number of buffers will be computed from the number of buffers supported 113 * by the vring, upto a maximum of 512 buffers (256 in each direction). 114 * 115 * Each buffer will have 16 bytes for the msg header and 496 bytes for 116 * the payload. 117 * 118 * This will utilize a maximum total space of 256KB for the buffers. 119 * 120 * We might also want to add support for user-provided buffers in time. 121 * This will allow bigger buffer size flexibility, and can also be used 122 * to achieve zero-copy messaging. 123 * 124 * Note that these numbers are purely a decision of this driver - we 125 * can change this without changing anything in the firmware of the remote 126 * processor. 127 */ 128 #define MAX_RPMSG_NUM_BUFS (512) 129 #define MAX_RPMSG_BUF_SIZE (512) 130 131 /* 132 * Local addresses are dynamically allocated on-demand. 133 * We do not dynamically assign addresses from the low 1024 range, 134 * in order to reserve that address range for predefined services. 135 */ 136 #define RPMSG_RESERVED_ADDRESSES (1024) 137 138 static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept); 139 static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, const void *data, int len); 140 static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, const void *data, 141 int len, u32 dst); 142 static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, const void *data, 143 int len); 144 static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, const void *data, 145 int len, u32 dst); 146 static __poll_t virtio_rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp, 147 poll_table *wait); 148 static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept); 149 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp, 150 struct rpmsg_channel_info *chinfo); 151 152 static const struct rpmsg_endpoint_ops virtio_endpoint_ops = { 153 .destroy_ept = virtio_rpmsg_destroy_ept, 154 .send = virtio_rpmsg_send, 155 .sendto = virtio_rpmsg_sendto, 156 .trysend = virtio_rpmsg_trysend, 157 .trysendto = virtio_rpmsg_trysendto, 158 .poll = virtio_rpmsg_poll, 159 .get_mtu = virtio_rpmsg_get_mtu, 160 }; 161 162 /** 163 * rpmsg_sg_init - initialize scatterlist according to cpu address location 164 * @sg: scatterlist to fill 165 * @cpu_addr: virtual address of the buffer 166 * @len: buffer length 167 * 168 * An internal function filling scatterlist according to virtual address 169 * location (in vmalloc or in kernel). 170 */ 171 static void 172 rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len) 173 { 174 if (is_vmalloc_addr(cpu_addr)) { 175 sg_init_table(sg, 1); 176 sg_set_page(sg, vmalloc_to_page(cpu_addr), len, 177 offset_in_page(cpu_addr)); 178 } else { 179 WARN_ON(!virt_addr_valid(cpu_addr)); 180 sg_init_one(sg, cpu_addr, len); 181 } 182 } 183 184 /** 185 * __ept_release() - deallocate an rpmsg endpoint 186 * @kref: the ept's reference count 187 * 188 * This function deallocates an ept, and is invoked when its @kref refcount 189 * drops to zero. 190 * 191 * Never invoke this function directly! 192 */ 193 static void __ept_release(struct kref *kref) 194 { 195 struct rpmsg_endpoint *ept = container_of(kref, struct rpmsg_endpoint, 196 refcount); 197 /* 198 * At this point no one holds a reference to ept anymore, 199 * so we can directly free it 200 */ 201 kfree(ept); 202 } 203 204 /* for more info, see below documentation of rpmsg_create_ept() */ 205 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp, 206 struct rpmsg_device *rpdev, 207 rpmsg_rx_cb_t cb, 208 void *priv, u32 addr) 209 { 210 int id_min, id_max, id; 211 struct rpmsg_endpoint *ept; 212 struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev; 213 214 ept = kzalloc_obj(*ept); 215 if (!ept) 216 return NULL; 217 218 kref_init(&ept->refcount); 219 mutex_init(&ept->cb_lock); 220 221 ept->rpdev = rpdev; 222 ept->cb = cb; 223 ept->priv = priv; 224 ept->ops = &virtio_endpoint_ops; 225 226 /* do we need to allocate a local address ? */ 227 if (addr == RPMSG_ADDR_ANY) { 228 id_min = RPMSG_RESERVED_ADDRESSES; 229 id_max = 0; 230 } else { 231 id_min = addr; 232 id_max = addr + 1; 233 } 234 235 mutex_lock(&vrp->endpoints_lock); 236 237 /* bind the endpoint to an rpmsg address (and allocate one if needed) */ 238 id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL); 239 if (id < 0) { 240 dev_err(dev, "idr_alloc failed: %d\n", id); 241 goto free_ept; 242 } 243 ept->addr = id; 244 245 mutex_unlock(&vrp->endpoints_lock); 246 247 return ept; 248 249 free_ept: 250 mutex_unlock(&vrp->endpoints_lock); 251 kref_put(&ept->refcount, __ept_release); 252 return NULL; 253 } 254 255 static struct rpmsg_device *virtio_rpmsg_create_channel(struct rpmsg_device *rpdev, 256 struct rpmsg_channel_info *chinfo) 257 { 258 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 259 struct virtproc_info *vrp = vch->vrp; 260 261 return __rpmsg_create_channel(vrp, chinfo); 262 } 263 264 static int virtio_rpmsg_release_channel(struct rpmsg_device *rpdev, 265 struct rpmsg_channel_info *chinfo) 266 { 267 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 268 struct virtproc_info *vrp = vch->vrp; 269 270 return rpmsg_unregister_device(&vrp->vdev->dev, chinfo); 271 } 272 273 static struct rpmsg_endpoint *virtio_rpmsg_create_ept(struct rpmsg_device *rpdev, 274 rpmsg_rx_cb_t cb, 275 void *priv, 276 struct rpmsg_channel_info chinfo) 277 { 278 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 279 280 return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src); 281 } 282 283 /** 284 * __rpmsg_destroy_ept() - destroy an existing rpmsg endpoint 285 * @vrp: virtproc which owns this ept 286 * @ept: endpoing to destroy 287 * 288 * An internal function which destroy an ept without assuming it is 289 * bound to an rpmsg channel. This is needed for handling the internal 290 * name service endpoint, which isn't bound to an rpmsg channel. 291 * See also __rpmsg_create_ept(). 292 */ 293 static void 294 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept) 295 { 296 /* make sure new inbound messages can't find this ept anymore */ 297 mutex_lock(&vrp->endpoints_lock); 298 idr_remove(&vrp->endpoints, ept->addr); 299 mutex_unlock(&vrp->endpoints_lock); 300 301 /* make sure in-flight inbound messages won't invoke cb anymore */ 302 mutex_lock(&ept->cb_lock); 303 ept->cb = NULL; 304 mutex_unlock(&ept->cb_lock); 305 306 kref_put(&ept->refcount, __ept_release); 307 } 308 309 static void virtio_rpmsg_destroy_ept(struct rpmsg_endpoint *ept) 310 { 311 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(ept->rpdev); 312 313 __rpmsg_destroy_ept(vch->vrp, ept); 314 } 315 316 static int virtio_rpmsg_announce_create(struct rpmsg_device *rpdev) 317 { 318 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 319 struct virtproc_info *vrp = vch->vrp; 320 struct device *dev = &rpdev->dev; 321 int err = 0; 322 323 /* need to tell remote processor's name service about this channel ? */ 324 if (rpdev->announce && rpdev->ept && 325 virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { 326 struct rpmsg_ns_msg nsm; 327 328 strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name)); 329 nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr); 330 nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_CREATE); 331 332 err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); 333 if (err) 334 dev_err(dev, "failed to announce service %d\n", err); 335 } 336 337 return err; 338 } 339 340 static int virtio_rpmsg_announce_destroy(struct rpmsg_device *rpdev) 341 { 342 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 343 struct virtproc_info *vrp = vch->vrp; 344 struct device *dev = &rpdev->dev; 345 int err = 0; 346 347 /* tell remote processor's name service we're removing this channel */ 348 if (rpdev->announce && rpdev->ept && 349 virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) { 350 struct rpmsg_ns_msg nsm; 351 352 strscpy_pad(nsm.name, rpdev->id.name, sizeof(nsm.name)); 353 nsm.addr = cpu_to_rpmsg32(rpdev, rpdev->ept->addr); 354 nsm.flags = cpu_to_rpmsg32(rpdev, RPMSG_NS_DESTROY); 355 356 err = rpmsg_sendto(rpdev->ept, &nsm, sizeof(nsm), RPMSG_NS_ADDR); 357 if (err) 358 dev_err(dev, "failed to announce service %d\n", err); 359 } 360 361 return err; 362 } 363 364 static const struct rpmsg_device_ops virtio_rpmsg_ops = { 365 .create_channel = virtio_rpmsg_create_channel, 366 .release_channel = virtio_rpmsg_release_channel, 367 .create_ept = virtio_rpmsg_create_ept, 368 .announce_create = virtio_rpmsg_announce_create, 369 .announce_destroy = virtio_rpmsg_announce_destroy, 370 }; 371 372 static void virtio_rpmsg_release_device(struct device *dev) 373 { 374 struct rpmsg_device *rpdev = to_rpmsg_device(dev); 375 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 376 377 kfree(vch); 378 } 379 380 /* 381 * create an rpmsg channel using its name and address info. 382 * this function will be used to create both static and dynamic 383 * channels. 384 */ 385 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp, 386 struct rpmsg_channel_info *chinfo) 387 { 388 struct virtio_rpmsg_channel *vch; 389 struct rpmsg_device *rpdev; 390 struct device *tmp, *dev = &vrp->vdev->dev; 391 int ret; 392 393 /* make sure a similar channel doesn't already exist */ 394 tmp = rpmsg_find_device(dev, chinfo); 395 if (tmp) { 396 /* decrement the matched device's refcount back */ 397 put_device(tmp); 398 dev_err(dev, "channel %s:%x:%x already exist\n", 399 chinfo->name, chinfo->src, chinfo->dst); 400 return NULL; 401 } 402 403 vch = kzalloc_obj(*vch); 404 if (!vch) 405 return NULL; 406 407 /* Link the channel to our vrp */ 408 vch->vrp = vrp; 409 410 /* Assign public information to the rpmsg_device */ 411 rpdev = &vch->rpdev; 412 rpdev->src = chinfo->src; 413 rpdev->dst = chinfo->dst; 414 rpdev->ops = &virtio_rpmsg_ops; 415 rpdev->little_endian = virtio_is_little_endian(vrp->vdev); 416 417 /* 418 * rpmsg server channels has predefined local address (for now), 419 * and their existence needs to be announced remotely 420 */ 421 rpdev->announce = rpdev->src != RPMSG_ADDR_ANY; 422 423 strscpy(rpdev->id.name, chinfo->name, sizeof(rpdev->id.name)); 424 425 rpdev->dev.parent = &vrp->vdev->dev; 426 rpdev->dev.release = virtio_rpmsg_release_device; 427 ret = rpmsg_register_device(rpdev); 428 if (ret) 429 return NULL; 430 431 return rpdev; 432 } 433 434 /* super simple buffer "allocator" that is just enough for now */ 435 static void *get_a_tx_buf(struct virtproc_info *vrp) 436 { 437 unsigned int len; 438 void *ret; 439 440 mutex_lock(&vrp->tx_lock); 441 442 /* 443 * either pick the next unused tx buffer 444 * (half of our buffers are used for sending messages) 445 */ 446 if (vrp->last_sbuf < vrp->num_bufs / 2) 447 ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++; 448 /* or recycle a used one */ 449 else 450 ret = virtqueue_get_buf(vrp->svq, &len); 451 452 mutex_unlock(&vrp->tx_lock); 453 454 return ret; 455 } 456 457 /** 458 * rpmsg_send_offchannel_raw() - send a message across to the remote processor 459 * @rpdev: the rpmsg channel 460 * @src: source address 461 * @dst: destination address 462 * @data: payload of message 463 * @len: length of payload 464 * @wait: indicates whether caller should block in case no TX buffers available 465 * 466 * This function is the base implementation for all of the rpmsg sending API. 467 * 468 * It will send @data of length @len to @dst, and say it's from @src. The 469 * message will be sent to the remote processor which the @rpdev channel 470 * belongs to. 471 * 472 * The message is sent using one of the TX buffers that are available for 473 * communication with this remote processor. 474 * 475 * If @wait is true, the caller will be blocked until either a TX buffer is 476 * available, or 15 seconds elapses (we don't want callers to 477 * sleep indefinitely due to misbehaving remote processors), and in that 478 * case -ERESTARTSYS is returned. The number '15' itself was picked 479 * arbitrarily; there's little point in asking drivers to provide a timeout 480 * value themselves. 481 * 482 * Otherwise, if @wait is false, and there are no TX buffers available, 483 * the function will immediately fail, and -ENOMEM will be returned. 484 * 485 * Normally drivers shouldn't use this function directly; instead, drivers 486 * should use the appropriate rpmsg_{try}send{to} API 487 * (see include/linux/rpmsg.h). 488 * 489 * Return: 0 on success and an appropriate error value on failure. 490 */ 491 static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev, 492 u32 src, u32 dst, 493 const void *data, int len, bool wait) 494 { 495 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 496 struct virtproc_info *vrp = vch->vrp; 497 struct device *dev = &rpdev->dev; 498 struct scatterlist sg; 499 struct rpmsg_hdr *msg; 500 int err; 501 502 /* bcasting isn't allowed */ 503 if (src == RPMSG_ADDR_ANY || dst == RPMSG_ADDR_ANY) { 504 dev_err(dev, "invalid addr (src 0x%x, dst 0x%x)\n", src, dst); 505 return -EINVAL; 506 } 507 508 /* 509 * We currently use fixed-sized buffers, and therefore the payload 510 * length is limited. 511 * 512 * One of the possible improvements here is either to support 513 * user-provided buffers (and then we can also support zero-copy 514 * messaging), or to improve the buffer allocator, to support 515 * variable-length buffer sizes. 516 */ 517 if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) { 518 dev_err(dev, "message is too big (%d)\n", len); 519 return -EMSGSIZE; 520 } 521 522 /* grab a buffer */ 523 msg = get_a_tx_buf(vrp); 524 if (!msg && !wait) 525 return -ENOMEM; 526 527 /* no free buffer ? wait for one (but bail after 15 seconds) */ 528 while (!msg) { 529 /* 530 * sleep until a free buffer is available or 15 secs elapse. 531 * the timeout period is not configurable because there's 532 * little point in asking drivers to specify that. 533 * if later this happens to be required, it'd be easy to add. 534 */ 535 err = wait_event_interruptible_timeout(vrp->sendq, 536 (msg = get_a_tx_buf(vrp)), 537 msecs_to_jiffies(15000)); 538 539 /* timeout ? */ 540 if (!err) { 541 dev_err(dev, "timeout waiting for a tx buffer\n"); 542 return -ERESTARTSYS; 543 } 544 } 545 546 msg->len = cpu_to_rpmsg16(rpdev, len); 547 msg->flags = 0; 548 msg->src = cpu_to_rpmsg32(rpdev, src); 549 msg->dst = cpu_to_rpmsg32(rpdev, dst); 550 msg->reserved = 0; 551 memcpy(msg->data, data, len); 552 553 dev_dbg(dev, "TX From 0x%x, To 0x%x, Len %d, Flags %d, Reserved %d\n", 554 src, dst, len, msg->flags, msg->reserved); 555 #if defined(CONFIG_DYNAMIC_DEBUG) 556 dynamic_hex_dump("rpmsg_virtio TX: ", DUMP_PREFIX_NONE, 16, 1, 557 msg, sizeof(*msg) + len, true); 558 #endif 559 560 rpmsg_sg_init(&sg, msg, sizeof(*msg) + len); 561 562 mutex_lock(&vrp->tx_lock); 563 564 /* add message to the remote processor's virtqueue */ 565 err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL); 566 if (err) { 567 /* 568 * need to reclaim the buffer here, otherwise it's lost 569 * (memory won't leak, but rpmsg won't use it again for TX). 570 * this will wait for a buffer management overhaul. 571 */ 572 dev_err(dev, "virtqueue_add_outbuf failed: %d\n", err); 573 goto out; 574 } 575 576 /* tell the remote processor it has a pending message to read */ 577 virtqueue_kick(vrp->svq); 578 out: 579 mutex_unlock(&vrp->tx_lock); 580 return err; 581 } 582 583 static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, const void *data, int len) 584 { 585 struct rpmsg_device *rpdev = ept->rpdev; 586 u32 src = ept->addr, dst = rpdev->dst; 587 588 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); 589 } 590 591 static int virtio_rpmsg_sendto(struct rpmsg_endpoint *ept, const void *data, 592 int len, u32 dst) 593 { 594 struct rpmsg_device *rpdev = ept->rpdev; 595 u32 src = ept->addr; 596 597 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, true); 598 } 599 600 static int virtio_rpmsg_trysend(struct rpmsg_endpoint *ept, const void *data, 601 int len) 602 { 603 struct rpmsg_device *rpdev = ept->rpdev; 604 u32 src = ept->addr, dst = rpdev->dst; 605 606 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); 607 } 608 609 static int virtio_rpmsg_trysendto(struct rpmsg_endpoint *ept, const void *data, 610 int len, u32 dst) 611 { 612 struct rpmsg_device *rpdev = ept->rpdev; 613 u32 src = ept->addr; 614 615 return rpmsg_send_offchannel_raw(rpdev, src, dst, data, len, false); 616 } 617 618 static __poll_t virtio_rpmsg_poll(struct rpmsg_endpoint *ept, struct file *filp, 619 poll_table *wait) 620 { 621 struct rpmsg_device *rpdev = ept->rpdev; 622 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 623 struct virtproc_info *vrp = vch->vrp; 624 __poll_t mask = 0; 625 626 poll_wait(filp, &vrp->sendq, wait); 627 628 /* support multiple concurrent senders */ 629 mutex_lock(&vrp->tx_lock); 630 631 /* 632 * check for a free buffer, either: 633 * - we haven't used all of the available transmit buffers (half of the 634 * allocated buffers are used for transmit, hence num_bufs / 2), or, 635 * - we ask the virtqueue if there's a buffer available 636 */ 637 if (vrp->last_sbuf < vrp->num_bufs / 2 || 638 !virtqueue_enable_cb(vrp->svq)) 639 mask |= EPOLLOUT; 640 641 mutex_unlock(&vrp->tx_lock); 642 643 return mask; 644 } 645 646 static ssize_t virtio_rpmsg_get_mtu(struct rpmsg_endpoint *ept) 647 { 648 struct rpmsg_device *rpdev = ept->rpdev; 649 struct virtio_rpmsg_channel *vch = to_virtio_rpmsg_channel(rpdev); 650 651 return vch->vrp->buf_size - sizeof(struct rpmsg_hdr); 652 } 653 654 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, 655 struct rpmsg_hdr *msg, unsigned int len) 656 { 657 struct rpmsg_endpoint *ept; 658 struct scatterlist sg; 659 bool little_endian = virtio_is_little_endian(vrp->vdev); 660 unsigned int msg_len = __rpmsg16_to_cpu(little_endian, msg->len); 661 int err; 662 663 dev_dbg(dev, "From: 0x%x, To: 0x%x, Len: %d, Flags: %d, Reserved: %d\n", 664 __rpmsg32_to_cpu(little_endian, msg->src), 665 __rpmsg32_to_cpu(little_endian, msg->dst), msg_len, 666 __rpmsg16_to_cpu(little_endian, msg->flags), 667 __rpmsg32_to_cpu(little_endian, msg->reserved)); 668 #if defined(CONFIG_DYNAMIC_DEBUG) 669 dynamic_hex_dump("rpmsg_virtio RX: ", DUMP_PREFIX_NONE, 16, 1, 670 msg, sizeof(*msg) + msg_len, true); 671 #endif 672 673 /* 674 * We currently use fixed-sized buffers, so trivially sanitize 675 * the reported payload length. 676 */ 677 if (len > vrp->buf_size || 678 msg_len > (len - sizeof(struct rpmsg_hdr))) { 679 dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg_len); 680 return -EINVAL; 681 } 682 683 /* use the dst addr to fetch the callback of the appropriate user */ 684 mutex_lock(&vrp->endpoints_lock); 685 686 ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst)); 687 688 /* let's make sure no one deallocates ept while we use it */ 689 if (ept) 690 kref_get(&ept->refcount); 691 692 mutex_unlock(&vrp->endpoints_lock); 693 694 if (ept) { 695 /* make sure ept->cb doesn't go away while we use it */ 696 mutex_lock(&ept->cb_lock); 697 698 if (ept->cb) 699 ept->cb(ept->rpdev, msg->data, msg_len, ept->priv, 700 __rpmsg32_to_cpu(little_endian, msg->src)); 701 702 mutex_unlock(&ept->cb_lock); 703 704 /* farewell, ept, we don't need you anymore */ 705 kref_put(&ept->refcount, __ept_release); 706 } else 707 dev_warn_ratelimited(dev, "msg received with no recipient\n"); 708 709 /* publish the real size of the buffer */ 710 rpmsg_sg_init(&sg, msg, vrp->buf_size); 711 712 /* add the buffer back to the remote processor's virtqueue */ 713 err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL); 714 if (err < 0) { 715 dev_err(dev, "failed to add a virtqueue buffer: %d\n", err); 716 return err; 717 } 718 719 return 0; 720 } 721 722 /* called when an rx buffer is used, and it's time to digest a message */ 723 static void rpmsg_recv_done(struct virtqueue *rvq) 724 { 725 struct virtproc_info *vrp = rvq->vdev->priv; 726 struct device *dev = &rvq->vdev->dev; 727 struct rpmsg_hdr *msg; 728 unsigned int len, msgs_received = 0; 729 int err; 730 731 msg = virtqueue_get_buf(rvq, &len); 732 if (!msg) { 733 dev_err(dev, "uhm, incoming signal, but no used buffer ?\n"); 734 return; 735 } 736 737 while (msg) { 738 err = rpmsg_recv_single(vrp, dev, msg, len); 739 if (err) 740 break; 741 742 msgs_received++; 743 744 msg = virtqueue_get_buf(rvq, &len); 745 } 746 747 dev_dbg(dev, "Received %u messages\n", msgs_received); 748 749 /* tell the remote processor we added another available rx buffer */ 750 if (msgs_received) 751 virtqueue_kick(vrp->rvq); 752 } 753 754 /* 755 * This is invoked whenever the remote processor completed processing 756 * a TX msg we just sent it, and the buffer is put back to the used ring. 757 * 758 * Normally, though, we suppress this "tx complete" interrupt in order to 759 * avoid the incurred overhead. 760 */ 761 static void rpmsg_xmit_done(struct virtqueue *svq) 762 { 763 struct virtproc_info *vrp = svq->vdev->priv; 764 765 dev_dbg(&svq->vdev->dev, "%s\n", __func__); 766 767 /* wake up potential senders that are waiting for a tx buffer */ 768 wake_up_interruptible(&vrp->sendq); 769 } 770 771 /* 772 * Called to expose to user a /dev/rpmsg_ctrlX interface allowing to 773 * create endpoint-to-endpoint communication without associated RPMsg channel. 774 * The endpoints are rattached to the ctrldev RPMsg device. 775 */ 776 static struct rpmsg_device *rpmsg_virtio_add_ctrl_dev(struct virtio_device *vdev) 777 { 778 struct virtproc_info *vrp = vdev->priv; 779 struct virtio_rpmsg_channel *vch; 780 struct rpmsg_device *rpdev_ctrl; 781 int err = 0; 782 783 vch = kzalloc_obj(*vch); 784 if (!vch) 785 return ERR_PTR(-ENOMEM); 786 787 /* Link the channel to the vrp */ 788 vch->vrp = vrp; 789 790 /* Assign public information to the rpmsg_device */ 791 rpdev_ctrl = &vch->rpdev; 792 rpdev_ctrl->ops = &virtio_rpmsg_ops; 793 794 rpdev_ctrl->dev.parent = &vrp->vdev->dev; 795 rpdev_ctrl->dev.release = virtio_rpmsg_release_device; 796 rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev); 797 798 err = rpmsg_ctrldev_register_device(rpdev_ctrl); 799 if (err) { 800 /* vch will be free in virtio_rpmsg_release_device() */ 801 return ERR_PTR(err); 802 } 803 804 return rpdev_ctrl; 805 } 806 807 static void rpmsg_virtio_del_ctrl_dev(struct rpmsg_device *rpdev_ctrl) 808 { 809 if (!rpdev_ctrl) 810 return; 811 device_unregister(&rpdev_ctrl->dev); 812 } 813 814 static int rpmsg_probe(struct virtio_device *vdev) 815 { 816 struct virtqueue_info vqs_info[] = { 817 { "input", rpmsg_recv_done }, 818 { "output", rpmsg_xmit_done }, 819 }; 820 struct virtqueue *vqs[2]; 821 struct virtproc_info *vrp; 822 struct virtio_rpmsg_channel *vch = NULL; 823 struct rpmsg_device *rpdev_ns, *rpdev_ctrl; 824 void *bufs_va; 825 int err = 0, i; 826 size_t total_buf_space; 827 bool notify; 828 829 vrp = kzalloc_obj(*vrp); 830 if (!vrp) 831 return -ENOMEM; 832 833 vrp->vdev = vdev; 834 835 idr_init(&vrp->endpoints); 836 mutex_init(&vrp->endpoints_lock); 837 mutex_init(&vrp->tx_lock); 838 init_waitqueue_head(&vrp->sendq); 839 840 /* We expect two virtqueues, rx and tx (and in this order) */ 841 err = virtio_find_vqs(vdev, 2, vqs, vqs_info, NULL); 842 if (err) 843 goto free_vrp; 844 845 vrp->rvq = vqs[0]; 846 vrp->svq = vqs[1]; 847 848 /* we expect symmetric tx/rx vrings */ 849 WARN_ON(virtqueue_get_vring_size(vrp->rvq) != 850 virtqueue_get_vring_size(vrp->svq)); 851 852 /* we need less buffers if vrings are small */ 853 if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2) 854 vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2; 855 else 856 vrp->num_bufs = MAX_RPMSG_NUM_BUFS; 857 858 vrp->buf_size = MAX_RPMSG_BUF_SIZE; 859 860 total_buf_space = vrp->num_bufs * vrp->buf_size; 861 862 /* allocate coherent memory for the buffers */ 863 bufs_va = dma_alloc_coherent(vdev->dev.parent, 864 total_buf_space, &vrp->bufs_dma, 865 GFP_KERNEL); 866 if (!bufs_va) { 867 err = -ENOMEM; 868 goto vqs_del; 869 } 870 871 dev_dbg(&vdev->dev, "buffers: va %p, dma %pad\n", 872 bufs_va, &vrp->bufs_dma); 873 874 /* half of the buffers is dedicated for RX */ 875 vrp->rbufs = bufs_va; 876 877 /* and half is dedicated for TX */ 878 vrp->sbufs = bufs_va + total_buf_space / 2; 879 880 /* set up the receive buffers */ 881 for (i = 0; i < vrp->num_bufs / 2; i++) { 882 struct scatterlist sg; 883 void *cpu_addr = vrp->rbufs + i * vrp->buf_size; 884 885 rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size); 886 887 err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, 888 GFP_KERNEL); 889 WARN_ON(err); /* sanity check; this can't really happen */ 890 } 891 892 vdev->priv = vrp; 893 894 rpdev_ctrl = rpmsg_virtio_add_ctrl_dev(vdev); 895 if (IS_ERR(rpdev_ctrl)) { 896 err = PTR_ERR(rpdev_ctrl); 897 goto free_coherent; 898 } 899 900 /* if supported by the remote processor, enable the name service */ 901 if (virtio_has_feature(vdev, VIRTIO_RPMSG_F_NS)) { 902 vch = kzalloc_obj(*vch); 903 if (!vch) { 904 err = -ENOMEM; 905 goto free_ctrldev; 906 } 907 908 /* Link the channel to our vrp */ 909 vch->vrp = vrp; 910 911 /* Assign public information to the rpmsg_device */ 912 rpdev_ns = &vch->rpdev; 913 rpdev_ns->ops = &virtio_rpmsg_ops; 914 rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev); 915 916 rpdev_ns->dev.parent = &vrp->vdev->dev; 917 rpdev_ns->dev.release = virtio_rpmsg_release_device; 918 919 err = rpmsg_ns_register_device(rpdev_ns); 920 if (err) 921 /* vch will be free in virtio_rpmsg_release_device() */ 922 goto free_ctrldev; 923 } 924 925 /* 926 * Prepare to kick but don't notify yet - we can't do this before 927 * device is ready. 928 */ 929 notify = virtqueue_kick_prepare(vrp->rvq); 930 931 /* From this point on, we can notify and get callbacks. */ 932 virtio_device_ready(vdev); 933 934 /* tell the remote processor it can start sending messages */ 935 /* 936 * this might be concurrent with callbacks, but we are only 937 * doing notify, not a full kick here, so that's ok. 938 */ 939 if (notify) 940 virtqueue_notify(vrp->rvq); 941 942 dev_info(&vdev->dev, "rpmsg host is online\n"); 943 944 return 0; 945 946 free_ctrldev: 947 rpmsg_virtio_del_ctrl_dev(rpdev_ctrl); 948 free_coherent: 949 dma_free_coherent(vdev->dev.parent, total_buf_space, 950 bufs_va, vrp->bufs_dma); 951 vqs_del: 952 vdev->config->del_vqs(vrp->vdev); 953 free_vrp: 954 kfree(vrp); 955 return err; 956 } 957 958 static int rpmsg_remove_device(struct device *dev, void *data) 959 { 960 device_unregister(dev); 961 962 return 0; 963 } 964 965 static void rpmsg_remove(struct virtio_device *vdev) 966 { 967 struct virtproc_info *vrp = vdev->priv; 968 size_t total_buf_space = vrp->num_bufs * vrp->buf_size; 969 int ret; 970 971 virtio_reset_device(vdev); 972 973 ret = device_for_each_child(&vdev->dev, NULL, rpmsg_remove_device); 974 if (ret) 975 dev_warn(&vdev->dev, "can't remove rpmsg device: %d\n", ret); 976 977 idr_destroy(&vrp->endpoints); 978 979 vdev->config->del_vqs(vrp->vdev); 980 981 dma_free_coherent(vdev->dev.parent, total_buf_space, 982 vrp->rbufs, vrp->bufs_dma); 983 984 kfree(vrp); 985 } 986 987 static struct virtio_device_id id_table[] = { 988 { VIRTIO_ID_RPMSG, VIRTIO_DEV_ANY_ID }, 989 { 0 }, 990 }; 991 992 static unsigned int features[] = { 993 VIRTIO_RPMSG_F_NS, 994 }; 995 996 static struct virtio_driver virtio_ipc_driver = { 997 .feature_table = features, 998 .feature_table_size = ARRAY_SIZE(features), 999 .driver.name = KBUILD_MODNAME, 1000 .id_table = id_table, 1001 .probe = rpmsg_probe, 1002 .remove = rpmsg_remove, 1003 }; 1004 1005 static int __init rpmsg_init(void) 1006 { 1007 int ret; 1008 1009 ret = register_virtio_driver(&virtio_ipc_driver); 1010 if (ret) 1011 pr_err("failed to register virtio driver: %d\n", ret); 1012 1013 return ret; 1014 } 1015 subsys_initcall(rpmsg_init); 1016 1017 static void __exit rpmsg_fini(void) 1018 { 1019 unregister_virtio_driver(&virtio_ipc_driver); 1020 } 1021 module_exit(rpmsg_fini); 1022 1023 MODULE_DEVICE_TABLE(virtio, id_table); 1024 MODULE_DESCRIPTION("Virtio-based remote processor messaging bus"); 1025 MODULE_LICENSE("GPL v2"); 1026