1 /* 2 * Public API and common code for kernel->userspace relay file support. 3 * 4 * See Documentation/filesystems/relayfs.txt for an overview of relayfs. 5 * 6 * Copyright (C) 2002-2005 - Tom Zanussi (zanussi@us.ibm.com), IBM Corp 7 * Copyright (C) 1999-2005 - Karim Yaghmour (karim@opersys.com) 8 * 9 * Moved to kernel/relay.c by Paul Mundt, 2006. 10 * 11 * This file is released under the GPL. 12 */ 13 #include <linux/errno.h> 14 #include <linux/stddef.h> 15 #include <linux/slab.h> 16 #include <linux/module.h> 17 #include <linux/string.h> 18 #include <linux/relay.h> 19 #include <linux/vmalloc.h> 20 #include <linux/mm.h> 21 22 /* 23 * close() vm_op implementation for relay file mapping. 24 */ 25 static void relay_file_mmap_close(struct vm_area_struct *vma) 26 { 27 struct rchan_buf *buf = vma->vm_private_data; 28 buf->chan->cb->buf_unmapped(buf, vma->vm_file); 29 } 30 31 /* 32 * nopage() vm_op implementation for relay file mapping. 33 */ 34 static struct page *relay_buf_nopage(struct vm_area_struct *vma, 35 unsigned long address, 36 int *type) 37 { 38 struct page *page; 39 struct rchan_buf *buf = vma->vm_private_data; 40 unsigned long offset = address - vma->vm_start; 41 42 if (address > vma->vm_end) 43 return NOPAGE_SIGBUS; /* Disallow mremap */ 44 if (!buf) 45 return NOPAGE_OOM; 46 47 page = vmalloc_to_page(buf->start + offset); 48 if (!page) 49 return NOPAGE_OOM; 50 get_page(page); 51 52 if (type) 53 *type = VM_FAULT_MINOR; 54 55 return page; 56 } 57 58 /* 59 * vm_ops for relay file mappings. 60 */ 61 static struct vm_operations_struct relay_file_mmap_ops = { 62 .nopage = relay_buf_nopage, 63 .close = relay_file_mmap_close, 64 }; 65 66 /** 67 * relay_mmap_buf: - mmap channel buffer to process address space 68 * @buf: relay channel buffer 69 * @vma: vm_area_struct describing memory to be mapped 70 * 71 * Returns 0 if ok, negative on error 72 * 73 * Caller should already have grabbed mmap_sem. 74 */ 75 int relay_mmap_buf(struct rchan_buf *buf, struct vm_area_struct *vma) 76 { 77 unsigned long length = vma->vm_end - vma->vm_start; 78 struct file *filp = vma->vm_file; 79 80 if (!buf) 81 return -EBADF; 82 83 if (length != (unsigned long)buf->chan->alloc_size) 84 return -EINVAL; 85 86 vma->vm_ops = &relay_file_mmap_ops; 87 vma->vm_private_data = buf; 88 buf->chan->cb->buf_mapped(buf, filp); 89 90 return 0; 91 } 92 93 /** 94 * relay_alloc_buf - allocate a channel buffer 95 * @buf: the buffer struct 96 * @size: total size of the buffer 97 * 98 * Returns a pointer to the resulting buffer, NULL if unsuccessful. The 99 * passed in size will get page aligned, if it isn't already. 100 */ 101 static void *relay_alloc_buf(struct rchan_buf *buf, size_t *size) 102 { 103 void *mem; 104 unsigned int i, j, n_pages; 105 106 *size = PAGE_ALIGN(*size); 107 n_pages = *size >> PAGE_SHIFT; 108 109 buf->page_array = kcalloc(n_pages, sizeof(struct page *), GFP_KERNEL); 110 if (!buf->page_array) 111 return NULL; 112 113 for (i = 0; i < n_pages; i++) { 114 buf->page_array[i] = alloc_page(GFP_KERNEL); 115 if (unlikely(!buf->page_array[i])) 116 goto depopulate; 117 } 118 mem = vmap(buf->page_array, n_pages, VM_MAP, PAGE_KERNEL); 119 if (!mem) 120 goto depopulate; 121 122 memset(mem, 0, *size); 123 buf->page_count = n_pages; 124 return mem; 125 126 depopulate: 127 for (j = 0; j < i; j++) 128 __free_page(buf->page_array[j]); 129 kfree(buf->page_array); 130 return NULL; 131 } 132 133 /** 134 * relay_create_buf - allocate and initialize a channel buffer 135 * @alloc_size: size of the buffer to allocate 136 * @n_subbufs: number of sub-buffers in the channel 137 * 138 * Returns channel buffer if successful, NULL otherwise 139 */ 140 struct rchan_buf *relay_create_buf(struct rchan *chan) 141 { 142 struct rchan_buf *buf = kcalloc(1, sizeof(struct rchan_buf), GFP_KERNEL); 143 if (!buf) 144 return NULL; 145 146 buf->padding = kmalloc(chan->n_subbufs * sizeof(size_t *), GFP_KERNEL); 147 if (!buf->padding) 148 goto free_buf; 149 150 buf->start = relay_alloc_buf(buf, &chan->alloc_size); 151 if (!buf->start) 152 goto free_buf; 153 154 buf->chan = chan; 155 kref_get(&buf->chan->kref); 156 return buf; 157 158 free_buf: 159 kfree(buf->padding); 160 kfree(buf); 161 return NULL; 162 } 163 164 /** 165 * relay_destroy_channel - free the channel struct 166 * 167 * Should only be called from kref_put(). 168 */ 169 void relay_destroy_channel(struct kref *kref) 170 { 171 struct rchan *chan = container_of(kref, struct rchan, kref); 172 kfree(chan); 173 } 174 175 /** 176 * relay_destroy_buf - destroy an rchan_buf struct and associated buffer 177 * @buf: the buffer struct 178 */ 179 void relay_destroy_buf(struct rchan_buf *buf) 180 { 181 struct rchan *chan = buf->chan; 182 unsigned int i; 183 184 if (likely(buf->start)) { 185 vunmap(buf->start); 186 for (i = 0; i < buf->page_count; i++) 187 __free_page(buf->page_array[i]); 188 kfree(buf->page_array); 189 } 190 kfree(buf->padding); 191 kfree(buf); 192 kref_put(&chan->kref, relay_destroy_channel); 193 } 194 195 /** 196 * relay_remove_buf - remove a channel buffer 197 * 198 * Removes the file from the fileystem, which also frees the 199 * rchan_buf_struct and the channel buffer. Should only be called from 200 * kref_put(). 201 */ 202 void relay_remove_buf(struct kref *kref) 203 { 204 struct rchan_buf *buf = container_of(kref, struct rchan_buf, kref); 205 buf->chan->cb->remove_buf_file(buf->dentry); 206 relay_destroy_buf(buf); 207 } 208 209 /** 210 * relay_buf_empty - boolean, is the channel buffer empty? 211 * @buf: channel buffer 212 * 213 * Returns 1 if the buffer is empty, 0 otherwise. 214 */ 215 int relay_buf_empty(struct rchan_buf *buf) 216 { 217 return (buf->subbufs_produced - buf->subbufs_consumed) ? 0 : 1; 218 } 219 EXPORT_SYMBOL_GPL(relay_buf_empty); 220 221 /** 222 * relay_buf_full - boolean, is the channel buffer full? 223 * @buf: channel buffer 224 * 225 * Returns 1 if the buffer is full, 0 otherwise. 226 */ 227 int relay_buf_full(struct rchan_buf *buf) 228 { 229 size_t ready = buf->subbufs_produced - buf->subbufs_consumed; 230 return (ready >= buf->chan->n_subbufs) ? 1 : 0; 231 } 232 EXPORT_SYMBOL_GPL(relay_buf_full); 233 234 /* 235 * High-level relay kernel API and associated functions. 236 */ 237 238 /* 239 * rchan_callback implementations defining default channel behavior. Used 240 * in place of corresponding NULL values in client callback struct. 241 */ 242 243 /* 244 * subbuf_start() default callback. Does nothing. 245 */ 246 static int subbuf_start_default_callback (struct rchan_buf *buf, 247 void *subbuf, 248 void *prev_subbuf, 249 size_t prev_padding) 250 { 251 if (relay_buf_full(buf)) 252 return 0; 253 254 return 1; 255 } 256 257 /* 258 * buf_mapped() default callback. Does nothing. 259 */ 260 static void buf_mapped_default_callback(struct rchan_buf *buf, 261 struct file *filp) 262 { 263 } 264 265 /* 266 * buf_unmapped() default callback. Does nothing. 267 */ 268 static void buf_unmapped_default_callback(struct rchan_buf *buf, 269 struct file *filp) 270 { 271 } 272 273 /* 274 * create_buf_file_create() default callback. Does nothing. 275 */ 276 static struct dentry *create_buf_file_default_callback(const char *filename, 277 struct dentry *parent, 278 int mode, 279 struct rchan_buf *buf, 280 int *is_global) 281 { 282 return NULL; 283 } 284 285 /* 286 * remove_buf_file() default callback. Does nothing. 287 */ 288 static int remove_buf_file_default_callback(struct dentry *dentry) 289 { 290 return -EINVAL; 291 } 292 293 /* relay channel default callbacks */ 294 static struct rchan_callbacks default_channel_callbacks = { 295 .subbuf_start = subbuf_start_default_callback, 296 .buf_mapped = buf_mapped_default_callback, 297 .buf_unmapped = buf_unmapped_default_callback, 298 .create_buf_file = create_buf_file_default_callback, 299 .remove_buf_file = remove_buf_file_default_callback, 300 }; 301 302 /** 303 * wakeup_readers - wake up readers waiting on a channel 304 * @private: the channel buffer 305 * 306 * This is the work function used to defer reader waking. The 307 * reason waking is deferred is that calling directly from write 308 * causes problems if you're writing from say the scheduler. 309 */ 310 static void wakeup_readers(void *private) 311 { 312 struct rchan_buf *buf = private; 313 wake_up_interruptible(&buf->read_wait); 314 } 315 316 /** 317 * __relay_reset - reset a channel buffer 318 * @buf: the channel buffer 319 * @init: 1 if this is a first-time initialization 320 * 321 * See relay_reset for description of effect. 322 */ 323 static inline void __relay_reset(struct rchan_buf *buf, unsigned int init) 324 { 325 size_t i; 326 327 if (init) { 328 init_waitqueue_head(&buf->read_wait); 329 kref_init(&buf->kref); 330 INIT_WORK(&buf->wake_readers, NULL, NULL); 331 } else { 332 cancel_delayed_work(&buf->wake_readers); 333 flush_scheduled_work(); 334 } 335 336 buf->subbufs_produced = 0; 337 buf->subbufs_consumed = 0; 338 buf->bytes_consumed = 0; 339 buf->finalized = 0; 340 buf->data = buf->start; 341 buf->offset = 0; 342 343 for (i = 0; i < buf->chan->n_subbufs; i++) 344 buf->padding[i] = 0; 345 346 buf->chan->cb->subbuf_start(buf, buf->data, NULL, 0); 347 } 348 349 /** 350 * relay_reset - reset the channel 351 * @chan: the channel 352 * 353 * This has the effect of erasing all data from all channel buffers 354 * and restarting the channel in its initial state. The buffers 355 * are not freed, so any mappings are still in effect. 356 * 357 * NOTE: Care should be taken that the channel isn't actually 358 * being used by anything when this call is made. 359 */ 360 void relay_reset(struct rchan *chan) 361 { 362 unsigned int i; 363 struct rchan_buf *prev = NULL; 364 365 if (!chan) 366 return; 367 368 for (i = 0; i < NR_CPUS; i++) { 369 if (!chan->buf[i] || chan->buf[i] == prev) 370 break; 371 __relay_reset(chan->buf[i], 0); 372 prev = chan->buf[i]; 373 } 374 } 375 EXPORT_SYMBOL_GPL(relay_reset); 376 377 /** 378 * relay_open_buf - create a new relay channel buffer 379 * 380 * Internal - used by relay_open(). 381 */ 382 static struct rchan_buf *relay_open_buf(struct rchan *chan, 383 const char *filename, 384 struct dentry *parent, 385 int *is_global) 386 { 387 struct rchan_buf *buf; 388 struct dentry *dentry; 389 390 if (*is_global) 391 return chan->buf[0]; 392 393 buf = relay_create_buf(chan); 394 if (!buf) 395 return NULL; 396 397 /* Create file in fs */ 398 dentry = chan->cb->create_buf_file(filename, parent, S_IRUSR, 399 buf, is_global); 400 if (!dentry) { 401 relay_destroy_buf(buf); 402 return NULL; 403 } 404 405 buf->dentry = dentry; 406 __relay_reset(buf, 1); 407 408 return buf; 409 } 410 411 /** 412 * relay_close_buf - close a channel buffer 413 * @buf: channel buffer 414 * 415 * Marks the buffer finalized and restores the default callbacks. 416 * The channel buffer and channel buffer data structure are then freed 417 * automatically when the last reference is given up. 418 */ 419 static inline void relay_close_buf(struct rchan_buf *buf) 420 { 421 buf->finalized = 1; 422 cancel_delayed_work(&buf->wake_readers); 423 flush_scheduled_work(); 424 kref_put(&buf->kref, relay_remove_buf); 425 } 426 427 static inline void setup_callbacks(struct rchan *chan, 428 struct rchan_callbacks *cb) 429 { 430 if (!cb) { 431 chan->cb = &default_channel_callbacks; 432 return; 433 } 434 435 if (!cb->subbuf_start) 436 cb->subbuf_start = subbuf_start_default_callback; 437 if (!cb->buf_mapped) 438 cb->buf_mapped = buf_mapped_default_callback; 439 if (!cb->buf_unmapped) 440 cb->buf_unmapped = buf_unmapped_default_callback; 441 if (!cb->create_buf_file) 442 cb->create_buf_file = create_buf_file_default_callback; 443 if (!cb->remove_buf_file) 444 cb->remove_buf_file = remove_buf_file_default_callback; 445 chan->cb = cb; 446 } 447 448 /** 449 * relay_open - create a new relay channel 450 * @base_filename: base name of files to create 451 * @parent: dentry of parent directory, NULL for root directory 452 * @subbuf_size: size of sub-buffers 453 * @n_subbufs: number of sub-buffers 454 * @cb: client callback functions 455 * 456 * Returns channel pointer if successful, NULL otherwise. 457 * 458 * Creates a channel buffer for each cpu using the sizes and 459 * attributes specified. The created channel buffer files 460 * will be named base_filename0...base_filenameN-1. File 461 * permissions will be S_IRUSR. 462 */ 463 struct rchan *relay_open(const char *base_filename, 464 struct dentry *parent, 465 size_t subbuf_size, 466 size_t n_subbufs, 467 struct rchan_callbacks *cb) 468 { 469 unsigned int i; 470 struct rchan *chan; 471 char *tmpname; 472 int is_global = 0; 473 474 if (!base_filename) 475 return NULL; 476 477 if (!(subbuf_size && n_subbufs)) 478 return NULL; 479 480 chan = kcalloc(1, sizeof(struct rchan), GFP_KERNEL); 481 if (!chan) 482 return NULL; 483 484 chan->version = RELAYFS_CHANNEL_VERSION; 485 chan->n_subbufs = n_subbufs; 486 chan->subbuf_size = subbuf_size; 487 chan->alloc_size = FIX_SIZE(subbuf_size * n_subbufs); 488 setup_callbacks(chan, cb); 489 kref_init(&chan->kref); 490 491 tmpname = kmalloc(NAME_MAX + 1, GFP_KERNEL); 492 if (!tmpname) 493 goto free_chan; 494 495 for_each_online_cpu(i) { 496 sprintf(tmpname, "%s%d", base_filename, i); 497 chan->buf[i] = relay_open_buf(chan, tmpname, parent, 498 &is_global); 499 if (!chan->buf[i]) 500 goto free_bufs; 501 502 chan->buf[i]->cpu = i; 503 } 504 505 kfree(tmpname); 506 return chan; 507 508 free_bufs: 509 for (i = 0; i < NR_CPUS; i++) { 510 if (!chan->buf[i]) 511 break; 512 relay_close_buf(chan->buf[i]); 513 if (is_global) 514 break; 515 } 516 kfree(tmpname); 517 518 free_chan: 519 kref_put(&chan->kref, relay_destroy_channel); 520 return NULL; 521 } 522 EXPORT_SYMBOL_GPL(relay_open); 523 524 /** 525 * relay_switch_subbuf - switch to a new sub-buffer 526 * @buf: channel buffer 527 * @length: size of current event 528 * 529 * Returns either the length passed in or 0 if full. 530 * 531 * Performs sub-buffer-switch tasks such as invoking callbacks, 532 * updating padding counts, waking up readers, etc. 533 */ 534 size_t relay_switch_subbuf(struct rchan_buf *buf, size_t length) 535 { 536 void *old, *new; 537 size_t old_subbuf, new_subbuf; 538 539 if (unlikely(length > buf->chan->subbuf_size)) 540 goto toobig; 541 542 if (buf->offset != buf->chan->subbuf_size + 1) { 543 buf->prev_padding = buf->chan->subbuf_size - buf->offset; 544 old_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; 545 buf->padding[old_subbuf] = buf->prev_padding; 546 buf->subbufs_produced++; 547 buf->dentry->d_inode->i_size += buf->chan->subbuf_size - 548 buf->padding[old_subbuf]; 549 smp_mb(); 550 if (waitqueue_active(&buf->read_wait)) { 551 PREPARE_WORK(&buf->wake_readers, wakeup_readers, buf); 552 schedule_delayed_work(&buf->wake_readers, 1); 553 } 554 } 555 556 old = buf->data; 557 new_subbuf = buf->subbufs_produced % buf->chan->n_subbufs; 558 new = buf->start + new_subbuf * buf->chan->subbuf_size; 559 buf->offset = 0; 560 if (!buf->chan->cb->subbuf_start(buf, new, old, buf->prev_padding)) { 561 buf->offset = buf->chan->subbuf_size + 1; 562 return 0; 563 } 564 buf->data = new; 565 buf->padding[new_subbuf] = 0; 566 567 if (unlikely(length + buf->offset > buf->chan->subbuf_size)) 568 goto toobig; 569 570 return length; 571 572 toobig: 573 buf->chan->last_toobig = length; 574 return 0; 575 } 576 EXPORT_SYMBOL_GPL(relay_switch_subbuf); 577 578 /** 579 * relay_subbufs_consumed - update the buffer's sub-buffers-consumed count 580 * @chan: the channel 581 * @cpu: the cpu associated with the channel buffer to update 582 * @subbufs_consumed: number of sub-buffers to add to current buf's count 583 * 584 * Adds to the channel buffer's consumed sub-buffer count. 585 * subbufs_consumed should be the number of sub-buffers newly consumed, 586 * not the total consumed. 587 * 588 * NOTE: kernel clients don't need to call this function if the channel 589 * mode is 'overwrite'. 590 */ 591 void relay_subbufs_consumed(struct rchan *chan, 592 unsigned int cpu, 593 size_t subbufs_consumed) 594 { 595 struct rchan_buf *buf; 596 597 if (!chan) 598 return; 599 600 if (cpu >= NR_CPUS || !chan->buf[cpu]) 601 return; 602 603 buf = chan->buf[cpu]; 604 buf->subbufs_consumed += subbufs_consumed; 605 if (buf->subbufs_consumed > buf->subbufs_produced) 606 buf->subbufs_consumed = buf->subbufs_produced; 607 } 608 EXPORT_SYMBOL_GPL(relay_subbufs_consumed); 609 610 /** 611 * relay_close - close the channel 612 * @chan: the channel 613 * 614 * Closes all channel buffers and frees the channel. 615 */ 616 void relay_close(struct rchan *chan) 617 { 618 unsigned int i; 619 struct rchan_buf *prev = NULL; 620 621 if (!chan) 622 return; 623 624 for (i = 0; i < NR_CPUS; i++) { 625 if (!chan->buf[i] || chan->buf[i] == prev) 626 break; 627 relay_close_buf(chan->buf[i]); 628 prev = chan->buf[i]; 629 } 630 631 if (chan->last_toobig) 632 printk(KERN_WARNING "relay: one or more items not logged " 633 "[item size (%Zd) > sub-buffer size (%Zd)]\n", 634 chan->last_toobig, chan->subbuf_size); 635 636 kref_put(&chan->kref, relay_destroy_channel); 637 } 638 EXPORT_SYMBOL_GPL(relay_close); 639 640 /** 641 * relay_flush - close the channel 642 * @chan: the channel 643 * 644 * Flushes all channel buffers i.e. forces buffer switch. 645 */ 646 void relay_flush(struct rchan *chan) 647 { 648 unsigned int i; 649 struct rchan_buf *prev = NULL; 650 651 if (!chan) 652 return; 653 654 for (i = 0; i < NR_CPUS; i++) { 655 if (!chan->buf[i] || chan->buf[i] == prev) 656 break; 657 relay_switch_subbuf(chan->buf[i], 0); 658 prev = chan->buf[i]; 659 } 660 } 661 EXPORT_SYMBOL_GPL(relay_flush); 662 663 /** 664 * relay_file_open - open file op for relay files 665 * @inode: the inode 666 * @filp: the file 667 * 668 * Increments the channel buffer refcount. 669 */ 670 static int relay_file_open(struct inode *inode, struct file *filp) 671 { 672 struct rchan_buf *buf = inode->u.generic_ip; 673 kref_get(&buf->kref); 674 filp->private_data = buf; 675 676 return 0; 677 } 678 679 /** 680 * relay_file_mmap - mmap file op for relay files 681 * @filp: the file 682 * @vma: the vma describing what to map 683 * 684 * Calls upon relay_mmap_buf to map the file into user space. 685 */ 686 static int relay_file_mmap(struct file *filp, struct vm_area_struct *vma) 687 { 688 struct rchan_buf *buf = filp->private_data; 689 return relay_mmap_buf(buf, vma); 690 } 691 692 /** 693 * relay_file_poll - poll file op for relay files 694 * @filp: the file 695 * @wait: poll table 696 * 697 * Poll implemention. 698 */ 699 static unsigned int relay_file_poll(struct file *filp, poll_table *wait) 700 { 701 unsigned int mask = 0; 702 struct rchan_buf *buf = filp->private_data; 703 704 if (buf->finalized) 705 return POLLERR; 706 707 if (filp->f_mode & FMODE_READ) { 708 poll_wait(filp, &buf->read_wait, wait); 709 if (!relay_buf_empty(buf)) 710 mask |= POLLIN | POLLRDNORM; 711 } 712 713 return mask; 714 } 715 716 /** 717 * relay_file_release - release file op for relay files 718 * @inode: the inode 719 * @filp: the file 720 * 721 * Decrements the channel refcount, as the filesystem is 722 * no longer using it. 723 */ 724 static int relay_file_release(struct inode *inode, struct file *filp) 725 { 726 struct rchan_buf *buf = filp->private_data; 727 kref_put(&buf->kref, relay_remove_buf); 728 729 return 0; 730 } 731 732 /** 733 * relay_file_read_consume - update the consumed count for the buffer 734 */ 735 static void relay_file_read_consume(struct rchan_buf *buf, 736 size_t read_pos, 737 size_t bytes_consumed) 738 { 739 size_t subbuf_size = buf->chan->subbuf_size; 740 size_t n_subbufs = buf->chan->n_subbufs; 741 size_t read_subbuf; 742 743 if (buf->bytes_consumed + bytes_consumed > subbuf_size) { 744 relay_subbufs_consumed(buf->chan, buf->cpu, 1); 745 buf->bytes_consumed = 0; 746 } 747 748 buf->bytes_consumed += bytes_consumed; 749 read_subbuf = read_pos / buf->chan->subbuf_size; 750 if (buf->bytes_consumed + buf->padding[read_subbuf] == subbuf_size) { 751 if ((read_subbuf == buf->subbufs_produced % n_subbufs) && 752 (buf->offset == subbuf_size)) 753 return; 754 relay_subbufs_consumed(buf->chan, buf->cpu, 1); 755 buf->bytes_consumed = 0; 756 } 757 } 758 759 /** 760 * relay_file_read_avail - boolean, are there unconsumed bytes available? 761 */ 762 static int relay_file_read_avail(struct rchan_buf *buf, size_t read_pos) 763 { 764 size_t subbuf_size = buf->chan->subbuf_size; 765 size_t n_subbufs = buf->chan->n_subbufs; 766 size_t produced = buf->subbufs_produced; 767 size_t consumed = buf->subbufs_consumed; 768 769 relay_file_read_consume(buf, read_pos, 0); 770 771 if (unlikely(buf->offset > subbuf_size)) { 772 if (produced == consumed) 773 return 0; 774 return 1; 775 } 776 777 if (unlikely(produced - consumed >= n_subbufs)) { 778 consumed = (produced / n_subbufs) * n_subbufs; 779 buf->subbufs_consumed = consumed; 780 } 781 782 produced = (produced % n_subbufs) * subbuf_size + buf->offset; 783 consumed = (consumed % n_subbufs) * subbuf_size + buf->bytes_consumed; 784 785 if (consumed > produced) 786 produced += n_subbufs * subbuf_size; 787 788 if (consumed == produced) 789 return 0; 790 791 return 1; 792 } 793 794 /** 795 * relay_file_read_subbuf_avail - return bytes available in sub-buffer 796 */ 797 static size_t relay_file_read_subbuf_avail(size_t read_pos, 798 struct rchan_buf *buf) 799 { 800 size_t padding, avail = 0; 801 size_t read_subbuf, read_offset, write_subbuf, write_offset; 802 size_t subbuf_size = buf->chan->subbuf_size; 803 804 write_subbuf = (buf->data - buf->start) / subbuf_size; 805 write_offset = buf->offset > subbuf_size ? subbuf_size : buf->offset; 806 read_subbuf = read_pos / subbuf_size; 807 read_offset = read_pos % subbuf_size; 808 padding = buf->padding[read_subbuf]; 809 810 if (read_subbuf == write_subbuf) { 811 if (read_offset + padding < write_offset) 812 avail = write_offset - (read_offset + padding); 813 } else 814 avail = (subbuf_size - padding) - read_offset; 815 816 return avail; 817 } 818 819 /** 820 * relay_file_read_start_pos - find the first available byte to read 821 * 822 * If the read_pos is in the middle of padding, return the 823 * position of the first actually available byte, otherwise 824 * return the original value. 825 */ 826 static size_t relay_file_read_start_pos(size_t read_pos, 827 struct rchan_buf *buf) 828 { 829 size_t read_subbuf, padding, padding_start, padding_end; 830 size_t subbuf_size = buf->chan->subbuf_size; 831 size_t n_subbufs = buf->chan->n_subbufs; 832 833 read_subbuf = read_pos / subbuf_size; 834 padding = buf->padding[read_subbuf]; 835 padding_start = (read_subbuf + 1) * subbuf_size - padding; 836 padding_end = (read_subbuf + 1) * subbuf_size; 837 if (read_pos >= padding_start && read_pos < padding_end) { 838 read_subbuf = (read_subbuf + 1) % n_subbufs; 839 read_pos = read_subbuf * subbuf_size; 840 } 841 842 return read_pos; 843 } 844 845 /** 846 * relay_file_read_end_pos - return the new read position 847 */ 848 static size_t relay_file_read_end_pos(struct rchan_buf *buf, 849 size_t read_pos, 850 size_t count) 851 { 852 size_t read_subbuf, padding, end_pos; 853 size_t subbuf_size = buf->chan->subbuf_size; 854 size_t n_subbufs = buf->chan->n_subbufs; 855 856 read_subbuf = read_pos / subbuf_size; 857 padding = buf->padding[read_subbuf]; 858 if (read_pos % subbuf_size + count + padding == subbuf_size) 859 end_pos = (read_subbuf + 1) * subbuf_size; 860 else 861 end_pos = read_pos + count; 862 if (end_pos >= subbuf_size * n_subbufs) 863 end_pos = 0; 864 865 return end_pos; 866 } 867 868 /** 869 * subbuf_read_actor - read up to one subbuf's worth of data 870 */ 871 static int subbuf_read_actor(size_t read_start, 872 struct rchan_buf *buf, 873 size_t avail, 874 read_descriptor_t *desc, 875 read_actor_t actor) 876 { 877 void *from; 878 int ret = 0; 879 880 from = buf->start + read_start; 881 ret = avail; 882 if (copy_to_user(desc->arg.data, from, avail)) { 883 desc->error = -EFAULT; 884 ret = 0; 885 } 886 desc->arg.data += ret; 887 desc->written += ret; 888 desc->count -= ret; 889 890 return ret; 891 } 892 893 /** 894 * subbuf_send_actor - send up to one subbuf's worth of data 895 */ 896 static int subbuf_send_actor(size_t read_start, 897 struct rchan_buf *buf, 898 size_t avail, 899 read_descriptor_t *desc, 900 read_actor_t actor) 901 { 902 unsigned long pidx, poff; 903 unsigned int subbuf_pages; 904 int ret = 0; 905 906 subbuf_pages = buf->chan->alloc_size >> PAGE_SHIFT; 907 pidx = (read_start / PAGE_SIZE) % subbuf_pages; 908 poff = read_start & ~PAGE_MASK; 909 while (avail) { 910 struct page *p = buf->page_array[pidx]; 911 unsigned int len; 912 913 len = PAGE_SIZE - poff; 914 if (len > avail) 915 len = avail; 916 917 len = actor(desc, p, poff, len); 918 if (desc->error) 919 break; 920 921 avail -= len; 922 ret += len; 923 poff = 0; 924 pidx = (pidx + 1) % subbuf_pages; 925 } 926 927 return ret; 928 } 929 930 typedef int (*subbuf_actor_t) (size_t read_start, 931 struct rchan_buf *buf, 932 size_t avail, 933 read_descriptor_t *desc, 934 read_actor_t actor); 935 936 /** 937 * relay_file_read_subbufs - read count bytes, bridging subbuf boundaries 938 */ 939 static inline ssize_t relay_file_read_subbufs(struct file *filp, 940 loff_t *ppos, 941 size_t count, 942 subbuf_actor_t subbuf_actor, 943 read_actor_t actor, 944 void *target) 945 { 946 struct rchan_buf *buf = filp->private_data; 947 size_t read_start, avail; 948 read_descriptor_t desc; 949 int ret; 950 951 if (!count) 952 return 0; 953 954 desc.written = 0; 955 desc.count = count; 956 desc.arg.data = target; 957 desc.error = 0; 958 959 mutex_lock(&filp->f_dentry->d_inode->i_mutex); 960 do { 961 if (!relay_file_read_avail(buf, *ppos)) 962 break; 963 964 read_start = relay_file_read_start_pos(*ppos, buf); 965 avail = relay_file_read_subbuf_avail(read_start, buf); 966 if (!avail) 967 break; 968 969 avail = min(desc.count, avail); 970 ret = subbuf_actor(read_start, buf, avail, &desc, actor); 971 if (desc.error < 0) 972 break; 973 974 if (ret) { 975 relay_file_read_consume(buf, read_start, ret); 976 *ppos = relay_file_read_end_pos(buf, read_start, ret); 977 } 978 } while (desc.count && ret); 979 mutex_unlock(&filp->f_dentry->d_inode->i_mutex); 980 981 return desc.written; 982 } 983 984 static ssize_t relay_file_read(struct file *filp, 985 char __user *buffer, 986 size_t count, 987 loff_t *ppos) 988 { 989 return relay_file_read_subbufs(filp, ppos, count, subbuf_read_actor, 990 NULL, buffer); 991 } 992 993 static ssize_t relay_file_sendfile(struct file *filp, 994 loff_t *ppos, 995 size_t count, 996 read_actor_t actor, 997 void *target) 998 { 999 return relay_file_read_subbufs(filp, ppos, count, subbuf_send_actor, 1000 actor, target); 1001 } 1002 1003 struct file_operations relay_file_operations = { 1004 .open = relay_file_open, 1005 .poll = relay_file_poll, 1006 .mmap = relay_file_mmap, 1007 .read = relay_file_read, 1008 .llseek = no_llseek, 1009 .release = relay_file_release, 1010 .sendfile = relay_file_sendfile, 1011 }; 1012 EXPORT_SYMBOL_GPL(relay_file_operations); 1013