Lines Matching full:buffer
3 * Generic ring buffer
38 * The "absolute" timestamp in the buffer is only 59 bits.
63 * The ring buffer header is special. We must manually up keep it.
85 * The ring buffer is made up of a list of pages. A separate list of pages is
86 * allocated for each CPU. A writer may only write to a buffer that is
88 * from any per cpu buffer.
90 * The reader is special. For each per cpu buffer, the reader has its own
92 * page is swapped with another page in the ring buffer.
96 * again (as long as it is out of the ring buffer).
101 * |reader| RING BUFFER
112 * |reader| RING BUFFER
123 * |reader| RING BUFFER
134 * |buffer| RING BUFFER
146 * and swap that into the ring buffer.
284 /* inline for ring buffer fast paths */
308 #define for_each_buffer_cpu(buffer, cpu) \ argument
309 for_each_cpu(cpu, buffer->cpumask)
311 #define for_each_online_buffer_cpu(buffer, cpu) \ argument
312 for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
339 unsigned char data[] RB_ALIGN_DATA; /* data of buffer page */
348 * Note, the buffer_page list must be first. The buffer pages
349 * are allocated in cache lines, which means that each buffer
352 * add flags in the list struct pointers, to make the ring buffer
356 struct list_head list; /* list of buffer pages */
368 * The buffer page counters, write and entries, must be reset
435 * ABSOLUTE - the buffer requests all events to have absolute time stamps
471 * head_page == tail_page && head == tail then buffer is empty.
477 struct trace_buffer *buffer; member
522 /* ring buffer pages to update, > 0 to add, < 0 to remove */
576 int ring_buffer_print_page_header(struct trace_buffer *buffer, struct trace_seq *s) in ring_buffer_print_page_header() argument
600 (unsigned int)buffer->subbuf_size, in ring_buffer_print_page_header()
618 * is on the buffer that it passed in.
675 static inline u64 rb_time_stamp(struct trace_buffer *buffer);
679 * @buffer: The buffer that the event is on
683 * committed to the ring buffer. And must be called from the same
690 * the max nesting, then the write_stamp of the buffer is returned,
694 u64 ring_buffer_event_time_stamp(struct trace_buffer *buffer, in ring_buffer_event_time_stamp() argument
697 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()]; in ring_buffer_event_time_stamp()
726 * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
727 * @buffer: The ring_buffer to get the number of pages from
730 * Returns the number of pages that have content in the ring buffer.
732 size_t ring_buffer_nr_dirty_pages(struct trace_buffer *buffer, int cpu) in ring_buffer_nr_dirty_pages() argument
738 read = local_read(&buffer->buffers[cpu]->pages_read); in ring_buffer_nr_dirty_pages()
739 lost = local_read(&buffer->buffers[cpu]->pages_lost); in ring_buffer_nr_dirty_pages()
740 cnt = local_read(&buffer->buffers[cpu]->pages_touched); in ring_buffer_nr_dirty_pages()
756 static __always_inline bool full_hit(struct trace_buffer *buffer, int cpu, int full) in full_hit() argument
758 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in full_hit()
767 * Add one as dirty will never equal nr_pages, as the sub-buffer in full_hit()
771 dirty = ring_buffer_nr_dirty_pages(buffer, cpu) + 1; in full_hit()
777 * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
780 * ring buffer waiters queue.
809 * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
810 * @buffer: The ring buffer to wake waiters on
811 * @cpu: The CPU buffer to wake waiters on
813 * In the case of a file that represents a ring buffer is closing,
816 void ring_buffer_wake_waiters(struct trace_buffer *buffer, int cpu) in ring_buffer_wake_waiters() argument
821 if (!buffer) in ring_buffer_wake_waiters()
827 for_each_buffer_cpu(buffer, cpu) in ring_buffer_wake_waiters()
828 ring_buffer_wake_waiters(buffer, cpu); in ring_buffer_wake_waiters()
830 rbwork = &buffer->irq_work; in ring_buffer_wake_waiters()
832 if (WARN_ON_ONCE(!buffer->buffers)) in ring_buffer_wake_waiters()
837 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wake_waiters()
838 /* The CPU buffer may not have been initialized yet */ in ring_buffer_wake_waiters()
848 static bool rb_watermark_hit(struct trace_buffer *buffer, int cpu, int full) in rb_watermark_hit() argument
855 return !ring_buffer_empty(buffer); in rb_watermark_hit()
857 cpu_buffer = buffer->buffers[cpu]; in rb_watermark_hit()
859 if (!ring_buffer_empty_cpu(buffer, cpu)) { in rb_watermark_hit()
868 ret = !pagebusy && full_hit(buffer, cpu, full); in rb_watermark_hit()
880 rb_wait_cond(struct rb_irq_work *rbwork, struct trace_buffer *buffer, in rb_wait_cond() argument
883 if (rb_watermark_hit(buffer, cpu, full)) in rb_wait_cond()
896 * We don't clear it even if the buffer is no longer in rb_wait_cond()
935 * ring_buffer_wait - wait for input to the ring buffer
936 * @buffer: buffer to wait on
937 * @cpu: the cpu buffer to wait on
943 * as data is added to any of the @buffer's cpu buffers. Otherwise
944 * it will wait for data to be added to a specific cpu buffer.
946 int ring_buffer_wait(struct trace_buffer *buffer, int cpu, int full, in ring_buffer_wait() argument
957 * data in any cpu buffer, or a specific buffer, put the in ring_buffer_wait()
961 rbwork = &buffer->irq_work; in ring_buffer_wait()
965 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_wait()
967 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_wait()
985 rb_wait_cond(rbwork, buffer, cpu, full, cond, data)); in ring_buffer_wait()
991 * ring_buffer_poll_wait - poll on buffer input
992 * @buffer: buffer to wait on
993 * @cpu: the cpu buffer to wait on
999 * as data is added to any of the @buffer's cpu buffers. Otherwise
1000 * it will wait for data to be added to a specific cpu buffer.
1005 __poll_t ring_buffer_poll_wait(struct trace_buffer *buffer, int cpu, in ring_buffer_poll_wait() argument
1012 rbwork = &buffer->irq_work; in ring_buffer_poll_wait()
1015 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_poll_wait()
1018 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_poll_wait()
1025 if (rb_watermark_hit(buffer, cpu, full)) in ring_buffer_poll_wait()
1031 * compare the amount in the ring buffer to shortest_full. in ring_buffer_poll_wait()
1032 * If the amount in the ring buffer is greater than the in ring_buffer_poll_wait()
1049 * checking if the ring buffer is empty. Once the waiters_pending bit in ring_buffer_poll_wait()
1056 * the buffer goes from empty to having content. But as this race is in ring_buffer_poll_wait()
1062 if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) || in ring_buffer_poll_wait()
1063 (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu))) in ring_buffer_poll_wait()
1068 /* buffer may be either ring_buffer or ring_buffer_per_cpu */
1076 atomic_inc(&__b->buffer->record_disabled); \
1087 static inline u64 rb_time_stamp(struct trace_buffer *buffer) in rb_time_stamp() argument
1092 if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && likely(buffer->clock == trace_clock_local)) in rb_time_stamp()
1095 ts = buffer->clock(); in rb_time_stamp()
1101 u64 ring_buffer_time_stamp(struct trace_buffer *buffer) in ring_buffer_time_stamp() argument
1106 time = rb_time_stamp(buffer); in ring_buffer_time_stamp()
1113 void ring_buffer_normalize_time_stamp(struct trace_buffer *buffer, in ring_buffer_normalize_time_stamp() argument
1122 * Making the ring buffer lockless makes things tricky.
1127 * The reader page is always off the ring buffer, but when the
1129 * a new one from the buffer. The reader needs to take from
1179 * the reader page with a page in the buffer, but before it
1495 * rb_check_pages - integrity check of buffer pages
1496 * @cpu_buffer: CPU buffer with pages to test
1509 * Walk the linked list underpinning the ring buffer and validate all in rb_check_pages()
1516 * ring buffer size. Therefore, the code releases and re-acquires the in rb_check_pages()
1580 static void *rb_range_meta(struct trace_buffer *buffer, int nr_pages, int cpu) in rb_range_meta() argument
1582 int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in rb_range_meta()
1583 unsigned long ptr = buffer->range_addr_start; in rb_range_meta()
1639 * Return a specific sub-buffer for a given @cpu defined by @idx.
1647 meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu); in rb_range_buffer()
1656 /* Map this buffer to the order that's in meta->buffers[] */ in rb_range_buffer()
1662 if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end) in rb_range_buffer()
1669 * See if the existing memory contains valid ring buffer data.
1675 struct trace_buffer *buffer, int nr_pages, in rb_meta_valid() argument
1690 pr_info("Ring buffer boot meta[%d] mismatch of magic or struct size\n", cpu); in rb_meta_valid()
1697 pr_info("Ring buffer boot meta [%d] mismatch of subbuf_size/nr_pages\n", cpu); in rb_meta_valid()
1707 pr_info("Ring buffer boot meta [%d] head buffer out of range\n", cpu); in rb_meta_valid()
1713 pr_info("Ring buffer boot meta [%d] commit buffer out of range\n", cpu); in rb_meta_valid()
1725 pr_info("Ring buffer boot meta [%d] array out of range\n", cpu); in rb_meta_valid()
1730 pr_info("Ring buffer boot meta [%d] buffer invalid commit\n", cpu); in rb_meta_valid()
1735 pr_info("Ring buffer boot meta [%d] array has duplicates\n", cpu); in rb_meta_valid()
1826 pr_info("Ring buffer reader page is invalid\n"); in rb_meta_validate_events()
1849 pr_info("Ring buffer meta [%d] invalid buffer page\n", in rb_meta_validate_events()
1854 /* If the buffer has content, update pages_touched */ in rb_meta_validate_events()
1867 pr_info("Ring buffer meta [%d] commit page not found\n", in rb_meta_validate_events()
1875 pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu); in rb_meta_validate_events()
1906 static void rb_range_meta_init(struct trace_buffer *buffer, int nr_pages) in rb_range_meta_init() argument
1922 meta = rb_range_meta(buffer, nr_pages, cpu); in rb_range_meta_init()
1924 if (rb_meta_valid(meta, cpu, buffer, nr_pages, subbuf_mask)) { in rb_range_meta_init()
1931 buffer->last_text_delta = THIS_TEXT_PTR - meta->text_addr; in rb_range_meta_init()
1932 buffer->last_data_delta = THIS_DATA_PTR - meta->data_addr; in rb_range_meta_init()
1937 next_meta = rb_range_meta(buffer, nr_pages, cpu + 1); in rb_range_meta_init()
1939 next_meta = (void *)buffer->range_addr_end; in rb_range_meta_init()
1958 * location of the ring buffer. Although their addresses in rb_range_meta_init()
2013 seq_printf(m, "buffer[%ld]: %d\n", val, meta->buffers[val]); in rbm_show()
2029 int ring_buffer_meta_seq_init(struct file *file, struct trace_buffer *buffer, int cpu) in ring_buffer_meta_seq_init() argument
2039 m->private = buffer->buffers[cpu]; in ring_buffer_meta_seq_init()
2062 struct trace_buffer *buffer = cpu_buffer->buffer; in __rb_allocate_pages() local
2099 if (buffer->range_addr_start) in __rb_allocate_pages()
2100 meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu); in __rb_allocate_pages()
2119 /* A range was given. Use that for the buffer page */ in __rb_allocate_pages()
2131 cpu_buffer->buffer->subbuf_order); in __rb_allocate_pages()
2137 bpage->order = cpu_buffer->buffer->subbuf_order; in __rb_allocate_pages()
2169 * The ring buffer page list is a circular list that does not in rb_allocate_pages()
2184 rb_allocate_cpu_buffer(struct trace_buffer *buffer, long nr_pages, int cpu) in rb_allocate_cpu_buffer() argument
2198 cpu_buffer->buffer = buffer; in rb_allocate_cpu_buffer()
2200 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key); in rb_allocate_cpu_buffer()
2218 if (buffer->range_addr_start) { in rb_allocate_cpu_buffer()
2224 cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu); in rb_allocate_cpu_buffer()
2234 cpu_buffer->buffer->subbuf_order); in rb_allocate_cpu_buffer()
2256 pr_warn("Ring buffer meta buffers not all mapped\n"); in rb_allocate_cpu_buffer()
2274 /* The valid meta buffer still needs to activate the head page */ in rb_allocate_cpu_buffer()
2318 struct trace_buffer *buffer; in alloc_buffer() local
2326 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()), in alloc_buffer()
2328 if (!buffer) in alloc_buffer()
2331 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL)) in alloc_buffer()
2334 buffer->subbuf_order = order; in alloc_buffer()
2336 buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE; in alloc_buffer()
2338 /* Max payload is buffer page size - header (8bytes) */ in alloc_buffer()
2339 buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2); in alloc_buffer()
2341 buffer->flags = flags; in alloc_buffer()
2342 buffer->clock = trace_clock_local; in alloc_buffer()
2343 buffer->reader_lock_key = key; in alloc_buffer()
2345 init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters); in alloc_buffer()
2346 init_waitqueue_head(&buffer->irq_work.waiters); in alloc_buffer()
2348 buffer->cpus = nr_cpu_ids; in alloc_buffer()
2351 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()), in alloc_buffer()
2353 if (!buffer->buffers) in alloc_buffer()
2394 buffer->range_addr_start = start; in alloc_buffer()
2395 buffer->range_addr_end = end; in alloc_buffer()
2397 rb_range_meta_init(buffer, nr_pages); in alloc_buffer()
2401 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); in alloc_buffer()
2407 cpumask_set_cpu(cpu, buffer->cpumask); in alloc_buffer()
2408 buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in alloc_buffer()
2409 if (!buffer->buffers[cpu]) in alloc_buffer()
2412 ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in alloc_buffer()
2416 mutex_init(&buffer->mutex); in alloc_buffer()
2418 return buffer; in alloc_buffer()
2421 for_each_buffer_cpu(buffer, cpu) { in alloc_buffer()
2422 if (buffer->buffers[cpu]) in alloc_buffer()
2423 rb_free_cpu_buffer(buffer->buffers[cpu]); in alloc_buffer()
2425 kfree(buffer->buffers); in alloc_buffer()
2428 free_cpumask_var(buffer->cpumask); in alloc_buffer()
2431 kfree(buffer); in alloc_buffer()
2438 * @flags: attributes to set for the ring buffer.
2439 * @key: ring buffer reader_lock_key.
2442 * flag. This flag means that the buffer will overwrite old data
2443 * when the buffer wraps. If this flag is not set, the buffer will
2449 /* Default buffer page size - one system page */ in __ring_buffer_alloc()
2458 * @flags: attributes to set for the ring buffer.
2459 * @order: sub-buffer order
2462 * @key: ring buffer reader_lock_key.
2465 * flag. This flag means that the buffer will overwrite old data
2466 * when the buffer wraps. If this flag is not set, the buffer will
2479 * @buffer: The buffer to return the delta from
2485 bool ring_buffer_last_boot_delta(struct trace_buffer *buffer, long *text, in ring_buffer_last_boot_delta() argument
2488 if (!buffer) in ring_buffer_last_boot_delta()
2491 if (!buffer->last_text_delta) in ring_buffer_last_boot_delta()
2494 *text = buffer->last_text_delta; in ring_buffer_last_boot_delta()
2495 *data = buffer->last_data_delta; in ring_buffer_last_boot_delta()
2501 * ring_buffer_free - free a ring buffer.
2502 * @buffer: the buffer to free.
2505 ring_buffer_free(struct trace_buffer *buffer) in ring_buffer_free() argument
2509 cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node); in ring_buffer_free()
2511 irq_work_sync(&buffer->irq_work.work); in ring_buffer_free()
2513 for_each_buffer_cpu(buffer, cpu) in ring_buffer_free()
2514 rb_free_cpu_buffer(buffer->buffers[cpu]); in ring_buffer_free()
2516 kfree(buffer->buffers); in ring_buffer_free()
2517 free_cpumask_var(buffer->cpumask); in ring_buffer_free()
2519 kfree(buffer); in ring_buffer_free()
2523 void ring_buffer_set_clock(struct trace_buffer *buffer, in ring_buffer_set_clock() argument
2526 buffer->clock = clock; in ring_buffer_set_clock()
2529 void ring_buffer_set_time_stamp_abs(struct trace_buffer *buffer, bool abs) in ring_buffer_set_time_stamp_abs() argument
2531 buffer->time_stamp_abs = abs; in ring_buffer_set_time_stamp_abs()
2534 bool ring_buffer_time_stamp_abs(struct trace_buffer *buffer) in ring_buffer_time_stamp_abs() argument
2536 return buffer->time_stamp_abs; in ring_buffer_time_stamp_abs()
2576 * from the ring buffer in rb_remove_pages()
2605 /* make sure pages points to a valid page in the ring buffer */ in rb_remove_pages()
2620 /* last buffer page to remove */ in rb_remove_pages()
2637 * bytes consumed in ring buffer from here. in rb_remove_pages()
2671 * in the ring buffer. Now we are racing with the writer trying to in rb_insert_pages()
2764 * ring_buffer_resize - resize the ring buffer
2765 * @buffer: the buffer to resize.
2767 * @cpu_id: the cpu buffer to resize
2769 * Minimum size is 2 * buffer->subbuf_size.
2773 int ring_buffer_resize(struct trace_buffer *buffer, unsigned long size, in ring_buffer_resize() argument
2781 * Always succeed at resizing a non-existent buffer: in ring_buffer_resize()
2783 if (!buffer) in ring_buffer_resize()
2786 /* Make sure the requested buffer exists */ in ring_buffer_resize()
2788 !cpumask_test_cpu(cpu_id, buffer->cpumask)) in ring_buffer_resize()
2791 nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size); in ring_buffer_resize()
2797 /* prevent another thread from changing buffer sizes */ in ring_buffer_resize()
2798 mutex_lock(&buffer->mutex); in ring_buffer_resize()
2799 atomic_inc(&buffer->resizing); in ring_buffer_resize()
2804 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
2807 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2808 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2816 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2817 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2845 * since we can change their buffer sizes without any race. in ring_buffer_resize()
2847 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2848 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2871 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2872 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2883 cpu_buffer = buffer->buffers[cpu_id]; in ring_buffer_resize()
2890 * manipulating the ring buffer and is expecting a sane state while in ring_buffer_resize()
2934 * The ring buffer resize can happen with the ring buffer in ring_buffer_resize()
2936 * as possible. But if the buffer is disabled, we do not need in ring_buffer_resize()
2938 * that the buffer is not corrupt. in ring_buffer_resize()
2940 if (atomic_read(&buffer->record_disabled)) { in ring_buffer_resize()
2941 atomic_inc(&buffer->record_disabled); in ring_buffer_resize()
2943 * Even though the buffer was disabled, we must make sure in ring_buffer_resize()
2949 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2950 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2953 atomic_dec(&buffer->record_disabled); in ring_buffer_resize()
2956 atomic_dec(&buffer->resizing); in ring_buffer_resize()
2957 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2961 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_resize()
2964 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_resize()
2977 atomic_dec(&buffer->resizing); in ring_buffer_resize()
2978 mutex_unlock(&buffer->mutex); in ring_buffer_resize()
2983 void ring_buffer_change_overwrite(struct trace_buffer *buffer, int val) in ring_buffer_change_overwrite() argument
2985 mutex_lock(&buffer->mutex); in ring_buffer_change_overwrite()
2987 buffer->flags |= RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2989 buffer->flags &= ~RB_FL_OVERWRITE; in ring_buffer_change_overwrite()
2990 mutex_unlock(&buffer->mutex); in ring_buffer_change_overwrite()
3082 addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1; in rb_event_index()
3107 /* Return the index into the sub-buffers for a given sub-buffer */
3321 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); in rb_reset_tail()
3390 /* Set write to end of buffer */ in rb_reset_tail()
3406 struct trace_buffer *buffer = cpu_buffer->buffer; in rb_move_tail() local
3416 * it all the way around the buffer, bail, and warn in rb_move_tail()
3429 * page with the buffer head. in rb_move_tail()
3435 * the buffer, unless the commit page is still on the in rb_move_tail()
3449 if (!(buffer->flags & RB_FL_OVERWRITE)) { in rb_move_tail()
3465 * page. We could have a small buffer, and in rb_move_tail()
3466 * have filled up the buffer with events in rb_move_tail()
3564 * is added to the buffer, it will lose those bits. in rb_add_timestamp()
3580 pr_warn("Ring buffer clock went backwards: %llu -> %llu\n", in rb_add_timestamp()
3595 * @cpu_buffer: The per cpu buffer of the @event
3600 * is the actual size that is written to the ring buffer,
3675 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); in rb_try_to_discard()
3849 rb_wakeups(struct trace_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer) in rb_wakeups() argument
3851 if (buffer->irq_work.waiters_pending) { in rb_wakeups()
3852 buffer->irq_work.waiters_pending = false; in rb_wakeups()
3854 irq_work_queue(&buffer->irq_work.work); in rb_wakeups()
3874 if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full)) in rb_wakeups()
3939 * if an interrupt comes in while NORMAL bit is set and the ring buffer
3991 * @buffer: The ring buffer to modify
3993 * The ring buffer has a safety mechanism to prevent recursion.
4002 void ring_buffer_nest_start(struct trace_buffer *buffer) in ring_buffer_nest_start() argument
4010 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_start()
4017 * @buffer: The ring buffer to modify
4022 void ring_buffer_nest_end(struct trace_buffer *buffer) in ring_buffer_nest_end() argument
4029 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_nest_end()
4037 * @buffer: The buffer to commit to
4039 * This commits the data to the ring buffer, and releases any locks held.
4043 int ring_buffer_unlock_commit(struct trace_buffer *buffer) in ring_buffer_unlock_commit() argument
4048 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unlock_commit()
4052 rb_wakeups(buffer, cpu_buffer); in ring_buffer_unlock_commit()
4209 * the buffer page.
4287 info->ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
4298 /* Use the sub-buffer timestamp */ in __rb_reserve_next()
4321 /* See if we shot pass the end of this buffer page */ in __rb_reserve_next()
4322 if (unlikely(write > cpu_buffer->buffer->subbuf_size)) { in __rb_reserve_next()
4357 ts = rb_time_stamp(cpu_buffer->buffer); in __rb_reserve_next()
4394 /* We reserved something on the buffer */ in __rb_reserve_next()
4415 rb_reserve_next_event(struct trace_buffer *buffer, in rb_reserve_next_event() argument
4425 * ring buffer does cmpxchg as well as atomic64 operations in rb_reserve_next_event()
4440 * Due to the ability to swap a cpu buffer from a buffer in rb_reserve_next_event()
4446 if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) { in rb_reserve_next_event()
4455 if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) { in rb_reserve_next_event()
4458 if (info.length > cpu_buffer->buffer->max_data_size) in rb_reserve_next_event()
4496 * ring_buffer_lock_reserve - reserve a part of the buffer
4497 * @buffer: the ring buffer to reserve from
4500 * Returns a reserved event on the ring buffer to copy directly to.
4511 ring_buffer_lock_reserve(struct trace_buffer *buffer, unsigned long length) in ring_buffer_lock_reserve() argument
4520 if (unlikely(atomic_read(&buffer->record_disabled))) in ring_buffer_lock_reserve()
4525 if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask))) in ring_buffer_lock_reserve()
4528 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_lock_reserve()
4533 if (unlikely(length > buffer->max_data_size)) in ring_buffer_lock_reserve()
4539 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_lock_reserve()
4567 addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1); in rb_decrement_entry()
4589 /* commit not part of this buffer?? */ in rb_decrement_entry()
4595 * @buffer: the ring buffer
4598 * Sometimes an event that is in the ring buffer needs to be ignored.
4599 * This function lets the user discard an event in the ring buffer
4603 * committed. It will try to free the event from the ring buffer
4612 void ring_buffer_discard_commit(struct trace_buffer *buffer, in ring_buffer_discard_commit() argument
4622 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_discard_commit()
4629 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing)); in ring_buffer_discard_commit()
4646 * ring_buffer_write - write data to the buffer without reserving
4647 * @buffer: The ring buffer to write to.
4649 * @data: The data to write to the buffer.
4652 * one function. If you already have the data to write to the buffer, it
4658 int ring_buffer_write(struct trace_buffer *buffer, in ring_buffer_write() argument
4670 if (atomic_read(&buffer->record_disabled)) in ring_buffer_write()
4675 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_write()
4678 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_write()
4683 if (length > buffer->max_data_size) in ring_buffer_write()
4689 event = rb_reserve_next_event(buffer, cpu_buffer, length); in ring_buffer_write()
4699 rb_wakeups(buffer, cpu_buffer); in ring_buffer_write()
4714 * The total entries in the ring buffer is the running counter
4715 * of entries entered into the ring buffer, minus the sum of
4716 * the entries read from the ring buffer and the number of
4732 * ring_buffer_record_disable - stop all writes into the buffer
4733 * @buffer: The ring buffer to stop writes to.
4735 * This prevents all writes to the buffer. Any attempt to write
4736 * to the buffer after this will fail and return NULL.
4740 void ring_buffer_record_disable(struct trace_buffer *buffer) in ring_buffer_record_disable() argument
4742 atomic_inc(&buffer->record_disabled); in ring_buffer_record_disable()
4747 * ring_buffer_record_enable - enable writes to the buffer
4748 * @buffer: The ring buffer to enable writes
4753 void ring_buffer_record_enable(struct trace_buffer *buffer) in ring_buffer_record_enable() argument
4755 atomic_dec(&buffer->record_disabled); in ring_buffer_record_enable()
4760 * ring_buffer_record_off - stop all writes into the buffer
4761 * @buffer: The ring buffer to stop writes to.
4763 * This prevents all writes to the buffer. Any attempt to write
4764 * to the buffer after this will fail and return NULL.
4770 void ring_buffer_record_off(struct trace_buffer *buffer) in ring_buffer_record_off() argument
4775 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_off()
4778 } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd)); in ring_buffer_record_off()
4783 * ring_buffer_record_on - restart writes into the buffer
4784 * @buffer: The ring buffer to start writes to.
4786 * This enables all writes to the buffer that was disabled by
4793 void ring_buffer_record_on(struct trace_buffer *buffer) in ring_buffer_record_on() argument
4798 rd = atomic_read(&buffer->record_disabled); in ring_buffer_record_on()
4801 } while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd)); in ring_buffer_record_on()
4806 * ring_buffer_record_is_on - return true if the ring buffer can write
4807 * @buffer: The ring buffer to see if write is enabled
4809 * Returns true if the ring buffer is in a state that it accepts writes.
4811 bool ring_buffer_record_is_on(struct trace_buffer *buffer) in ring_buffer_record_is_on() argument
4813 return !atomic_read(&buffer->record_disabled); in ring_buffer_record_is_on()
4817 * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
4818 * @buffer: The ring buffer to see if write is set enabled
4820 * Returns true if the ring buffer is set writable by ring_buffer_record_on().
4823 * It may return true when the ring buffer has been disabled by
4825 * the ring buffer.
4827 bool ring_buffer_record_is_set_on(struct trace_buffer *buffer) in ring_buffer_record_is_set_on() argument
4829 return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF); in ring_buffer_record_is_set_on()
4834 * @buffer: The ring buffer to stop writes to.
4835 * @cpu: The CPU buffer to stop
4837 * This prevents all writes to the buffer. Any attempt to write
4838 * to the buffer after this will fail and return NULL.
4842 void ring_buffer_record_disable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_disable_cpu() argument
4846 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_disable_cpu()
4849 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_disable_cpu()
4855 * ring_buffer_record_enable_cpu - enable writes to the buffer
4856 * @buffer: The ring buffer to enable writes
4862 void ring_buffer_record_enable_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_record_enable_cpu() argument
4866 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_record_enable_cpu()
4869 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_record_enable_cpu()
4875 * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
4876 * @buffer: The ring buffer
4877 * @cpu: The per CPU buffer to read from.
4879 u64 ring_buffer_oldest_event_ts(struct trace_buffer *buffer, int cpu) in ring_buffer_oldest_event_ts() argument
4886 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_oldest_event_ts()
4889 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_oldest_event_ts()
4908 * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
4909 * @buffer: The ring buffer
4910 * @cpu: The per CPU buffer to read from.
4912 unsigned long ring_buffer_bytes_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_bytes_cpu() argument
4917 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_bytes_cpu()
4920 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_bytes_cpu()
4928 * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
4929 * @buffer: The ring buffer
4930 * @cpu: The per CPU buffer to get the entries from.
4932 unsigned long ring_buffer_entries_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_entries_cpu() argument
4936 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_entries_cpu()
4939 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries_cpu()
4947 * buffer wrapping around (only if RB_FL_OVERWRITE is on).
4948 * @buffer: The ring buffer
4949 * @cpu: The per CPU buffer to get the number of overruns from
4951 unsigned long ring_buffer_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_overrun_cpu() argument
4956 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_overrun_cpu()
4959 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overrun_cpu()
4968 * commits failing due to the buffer wrapping around while there are uncommitted
4970 * @buffer: The ring buffer
4971 * @cpu: The per CPU buffer to get the number of overruns from
4974 ring_buffer_commit_overrun_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_commit_overrun_cpu() argument
4979 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_commit_overrun_cpu()
4982 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_commit_overrun_cpu()
4991 * the ring buffer filling up (only if RB_FL_OVERWRITE is off).
4992 * @buffer: The ring buffer
4993 * @cpu: The per CPU buffer to get the number of overruns from
4996 ring_buffer_dropped_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_dropped_events_cpu() argument
5001 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_dropped_events_cpu()
5004 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_dropped_events_cpu()
5013 * @buffer: The ring buffer
5014 * @cpu: The per CPU buffer to get the number of events read
5017 ring_buffer_read_events_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_read_events_cpu() argument
5021 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_events_cpu()
5024 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_events_cpu()
5030 * ring_buffer_entries - get the number of entries in a buffer
5031 * @buffer: The ring buffer
5033 * Returns the total number of entries in the ring buffer
5036 unsigned long ring_buffer_entries(struct trace_buffer *buffer) in ring_buffer_entries() argument
5042 /* if you care about this being correct, lock the buffer */ in ring_buffer_entries()
5043 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_entries()
5044 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_entries()
5053 * ring_buffer_overruns - get the number of overruns in buffer
5054 * @buffer: The ring buffer
5056 * Returns the total number of overruns in the ring buffer
5059 unsigned long ring_buffer_overruns(struct trace_buffer *buffer) in ring_buffer_overruns() argument
5065 /* if you care about this being correct, lock the buffer */ in ring_buffer_overruns()
5066 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_overruns()
5067 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_overruns()
5232 unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size); in rb_get_reader_page()
5269 /* Don't bother swapping if the ring buffer is empty */ in rb_get_reader_page()
5292 * cpu_buffer->pages just needs to point to the buffer, it in rb_get_reader_page()
5293 * has no specific buffer page to point to. Lets move it out in rb_get_reader_page()
5408 /* This function should not be called when buffer is empty */ in rb_advance_reader()
5440 * Check if we are at the end of the buffer. in rb_advance_iter()
5507 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
5517 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_buffer_peek()
5535 struct trace_buffer *buffer; in rb_iter_peek() local
5544 buffer = cpu_buffer->buffer; in rb_iter_peek()
5547 * Check if someone performed a consuming read to the buffer in rb_iter_peek()
5548 * or removed some pages from the buffer. In these cases, in rb_iter_peek()
5564 * the ring buffer with an active write as the consumer is. in rb_iter_peek()
5600 ring_buffer_normalize_time_stamp(cpu_buffer->buffer, in rb_iter_peek()
5610 ring_buffer_normalize_time_stamp(buffer, in rb_iter_peek()
5631 * If an NMI die dumps out the content of the ring buffer in rb_reader_lock()
5633 * preempted a task that holds the ring buffer locks. If in rb_reader_lock()
5635 * to do the read, but this can corrupt the ring buffer, in rb_reader_lock()
5642 /* Continue without locking, but disable the ring buffer */ in rb_reader_lock()
5656 * @buffer: The ring buffer to read
5665 ring_buffer_peek(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_peek() argument
5668 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_peek()
5673 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_peek()
5692 * @iter: The ring buffer iterator
5707 * @iter: The ring buffer iterator
5733 * @buffer: The ring buffer to get the next event from
5734 * @cpu: the cpu to read the buffer from
5738 * Returns the next event in the ring buffer, and that event is consumed.
5740 * and eventually empty the ring buffer if the producer is slower.
5743 ring_buffer_consume(struct trace_buffer *buffer, int cpu, u64 *ts, in ring_buffer_consume() argument
5755 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_consume()
5758 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_consume()
5782 * ring_buffer_read_prepare - Prepare for a non consuming read of the buffer
5783 * @buffer: The ring buffer to read from
5784 * @cpu: The cpu buffer to iterate over
5788 * through the buffer. Memory is allocated, buffer resizing
5799 ring_buffer_read_prepare(struct trace_buffer *buffer, int cpu, gfp_t flags) in ring_buffer_read_prepare() argument
5804 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_prepare()
5812 iter->event_size = buffer->subbuf_size; in ring_buffer_read_prepare()
5819 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_prepare()
5844 * ring_buffer_read_start - start a non consuming read of the buffer
5847 * This finalizes the startup of an iteration through the buffer.
5874 * ring_buffer_read_finish - finish reading the iterator of the buffer
5877 * This re-enables resizing of the buffer, and frees the iterator.
5884 /* Use this opportunity to check the integrity of the ring buffer. */ in ring_buffer_read_finish()
5895 * @iter: The ring buffer iterator
5914 * ring_buffer_size - return the size of the ring buffer (in bytes)
5915 * @buffer: The ring buffer.
5916 * @cpu: The CPU to get ring buffer size from.
5918 unsigned long ring_buffer_size(struct trace_buffer *buffer, int cpu) in ring_buffer_size() argument
5920 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_size()
5923 return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages; in ring_buffer_size()
5929 * @buffer: The ring buffer.
5933 unsigned long ring_buffer_max_event_size(struct trace_buffer *buffer) in ring_buffer_max_event_size() argument
5936 if (ring_buffer_time_stamp_abs(buffer)) in ring_buffer_max_event_size()
5937 return buffer->max_data_size - RB_LEN_TIME_EXTEND; in ring_buffer_max_event_size()
5938 return buffer->max_data_size; in ring_buffer_max_event_size()
6025 /* Must have disabled the cpu buffer then done a synchronize_rcu */
6046 * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
6047 * @buffer: The ring buffer to reset a per cpu buffer of
6048 * @cpu: The CPU buffer to be reset
6050 void ring_buffer_reset_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_reset_cpu() argument
6052 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_cpu()
6055 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_reset_cpu()
6058 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_cpu()
6059 mutex_lock(&buffer->mutex); in ring_buffer_reset_cpu()
6072 /* Make sure persistent meta now uses this buffer's addresses */ in ring_buffer_reset_cpu()
6073 meta = rb_range_meta(buffer, 0, cpu_buffer->cpu); in ring_buffer_reset_cpu()
6077 mutex_unlock(&buffer->mutex); in ring_buffer_reset_cpu()
6085 * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
6086 * @buffer: The ring buffer to reset a per cpu buffer of
6088 void ring_buffer_reset_online_cpus(struct trace_buffer *buffer) in ring_buffer_reset_online_cpus() argument
6094 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset_online_cpus()
6095 mutex_lock(&buffer->mutex); in ring_buffer_reset_online_cpus()
6097 for_each_online_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
6098 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
6107 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset_online_cpus()
6108 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset_online_cpus()
6119 /* Make sure persistent meta now uses this buffer's addresses */ in ring_buffer_reset_online_cpus()
6120 meta = rb_range_meta(buffer, 0, cpu_buffer->cpu); in ring_buffer_reset_online_cpus()
6128 mutex_unlock(&buffer->mutex); in ring_buffer_reset_online_cpus()
6132 * ring_buffer_reset - reset a ring buffer
6133 * @buffer: The ring buffer to reset all cpu buffers
6135 void ring_buffer_reset(struct trace_buffer *buffer) in ring_buffer_reset() argument
6140 /* prevent another thread from changing buffer sizes */ in ring_buffer_reset()
6141 mutex_lock(&buffer->mutex); in ring_buffer_reset()
6143 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
6144 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
6153 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_reset()
6154 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_reset()
6162 mutex_unlock(&buffer->mutex); in ring_buffer_reset()
6167 * ring_buffer_empty - is the ring buffer empty?
6168 * @buffer: The ring buffer to test
6170 bool ring_buffer_empty(struct trace_buffer *buffer) in ring_buffer_empty() argument
6178 /* yes this is racy, but if you don't like the race, lock the buffer */ in ring_buffer_empty()
6179 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_empty()
6180 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty()
6196 * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
6197 * @buffer: The ring buffer
6198 * @cpu: The CPU buffer to test
6200 bool ring_buffer_empty_cpu(struct trace_buffer *buffer, int cpu) in ring_buffer_empty_cpu() argument
6207 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_empty_cpu()
6210 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_empty_cpu()
6223 * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
6224 * @buffer_a: One buffer to swap with
6225 * @buffer_b: The other buffer to swap with
6229 * of a CPU buffer and has another back up buffer lying around.
6230 * it is expected that the tracer handles the cpu buffer not being
6291 * it will mess the state of the cpu buffer. in ring_buffer_swap_cpu()
6301 cpu_buffer_b->buffer = buffer_a; in ring_buffer_swap_cpu()
6302 cpu_buffer_a->buffer = buffer_b; in ring_buffer_swap_cpu()
6316 * ring_buffer_alloc_read_page - allocate a page to read from buffer
6317 * @buffer: the buffer to allocate for.
6318 * @cpu: the cpu buffer to allocate.
6321 * When reading a full page from the ring buffer, these functions
6324 * needs to get pages from the ring buffer, it passes the result
6326 * the page that was allocated, with the read page of the buffer.
6332 ring_buffer_alloc_read_page(struct trace_buffer *buffer, int cpu) in ring_buffer_alloc_read_page() argument
6339 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_alloc_read_page()
6346 bpage->order = buffer->subbuf_order; in ring_buffer_alloc_read_page()
6347 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_alloc_read_page()
6364 cpu_buffer->buffer->subbuf_order); in ring_buffer_alloc_read_page()
6381 * @buffer: the buffer the page was allocate for
6382 * @cpu: the cpu buffer the page came from
6387 void ring_buffer_free_read_page(struct trace_buffer *buffer, int cpu, in ring_buffer_free_read_page() argument
6395 if (!buffer || !buffer->buffers || !buffer->buffers[cpu]) in ring_buffer_free_read_page()
6398 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_free_read_page()
6402 * is different from the subbuffer order of the buffer - in ring_buffer_free_read_page()
6405 if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order) in ring_buffer_free_read_page()
6426 * ring_buffer_read_page - extract a page from the ring buffer
6427 * @buffer: buffer to extract from
6430 * @cpu: the cpu of the buffer to extract
6433 * This function will pull out a page from the ring buffer and consume it.
6436 * to swap with a page in the ring buffer.
6439 * rpage = ring_buffer_alloc_read_page(buffer, cpu);
6442 * ret = ring_buffer_read_page(buffer, rpage, len, cpu, 0);
6445 * ring_buffer_free_read_page(buffer, cpu, rpage);
6451 * The ring buffer can be used anywhere in the kernel and can not
6452 * blindly call wake_up. The layer that uses the ring buffer must be
6459 int ring_buffer_read_page(struct trace_buffer *buffer, in ring_buffer_read_page() argument
6463 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu]; in ring_buffer_read_page()
6474 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_read_page()
6488 if (data_page->order != buffer->subbuf_order) in ring_buffer_read_page()
6513 * we must copy the data from the page to the buffer. in ring_buffer_read_page()
6612 if (buffer->subbuf_size - commit >= sizeof(missed_events)) { in ring_buffer_read_page()
6624 if (commit < buffer->subbuf_size) in ring_buffer_read_page()
6625 memset(&bpage->data[commit], 0, buffer->subbuf_size - commit); in ring_buffer_read_page()
6648 * ring_buffer_subbuf_size_get - get size of the sub buffer.
6649 * @buffer: the buffer to get the sub buffer size from
6651 * Returns size of the sub buffer, in bytes.
6653 int ring_buffer_subbuf_size_get(struct trace_buffer *buffer) in ring_buffer_subbuf_size_get() argument
6655 return buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in ring_buffer_subbuf_size_get()
6660 * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
6661 * @buffer: The ring_buffer to get the system sub page order from
6663 * By default, one ring buffer sub page equals to one system page. This parameter
6664 * is configurable, per ring buffer. The size of the ring buffer sub page can be
6667 * Returns the order of buffer sub page size, in system pages:
6668 * 0 means the sub buffer size is 1 system page and so forth.
6671 int ring_buffer_subbuf_order_get(struct trace_buffer *buffer) in ring_buffer_subbuf_order_get() argument
6673 if (!buffer) in ring_buffer_subbuf_order_get()
6676 return buffer->subbuf_order; in ring_buffer_subbuf_order_get()
6681 * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
6682 * @buffer: The ring_buffer to set the new page size.
6683 * @order: Order of the system pages in one sub buffer page
6685 * By default, one ring buffer pages equals to one system page. This API can be
6686 * used to set new size of the ring buffer page. The size must be order of
6688 * system pages that are allocated for one ring buffer page:
6696 int ring_buffer_subbuf_order_set(struct trace_buffer *buffer, int order) in ring_buffer_subbuf_order_set() argument
6706 if (!buffer || order < 0) in ring_buffer_subbuf_order_set()
6709 if (buffer->subbuf_order == order) in ring_buffer_subbuf_order_set()
6720 old_order = buffer->subbuf_order; in ring_buffer_subbuf_order_set()
6721 old_size = buffer->subbuf_size; in ring_buffer_subbuf_order_set()
6723 /* prevent another thread from changing buffer sizes */ in ring_buffer_subbuf_order_set()
6724 mutex_lock(&buffer->mutex); in ring_buffer_subbuf_order_set()
6725 atomic_inc(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6730 buffer->subbuf_order = order; in ring_buffer_subbuf_order_set()
6731 buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE; in ring_buffer_subbuf_order_set()
6734 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6736 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_subbuf_order_set()
6739 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
6747 nr_pages = old_size * buffer->buffers[cpu]->nr_pages; in ring_buffer_subbuf_order_set()
6748 nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size); in ring_buffer_subbuf_order_set()
6759 /* Allocate the new size buffer */ in ring_buffer_subbuf_order_set()
6769 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6774 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_subbuf_order_set()
6777 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
6828 atomic_dec(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6829 mutex_unlock(&buffer->mutex); in ring_buffer_subbuf_order_set()
6834 buffer->subbuf_order = old_order; in ring_buffer_subbuf_order_set()
6835 buffer->subbuf_size = old_size; in ring_buffer_subbuf_order_set()
6837 atomic_dec(&buffer->record_disabled); in ring_buffer_subbuf_order_set()
6838 mutex_unlock(&buffer->mutex); in ring_buffer_subbuf_order_set()
6840 for_each_buffer_cpu(buffer, cpu) { in ring_buffer_subbuf_order_set()
6841 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_subbuf_order_set()
6908 meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE; in rb_setup_ids_meta_page()
6915 rb_get_mapped_buffer(struct trace_buffer *buffer, int cpu) in rb_get_mapped_buffer() argument
6919 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in rb_get_mapped_buffer()
6922 cpu_buffer = buffer->buffers[cpu]; in rb_get_mapped_buffer()
6960 mutex_lock(&cpu_buffer->buffer->mutex); in __rb_inc_dec_mapped()
6972 mutex_unlock(&cpu_buffer->buffer->mutex); in __rb_inc_dec_mapped()
7003 subbuf_order = cpu_buffer->buffer->subbuf_order; in __rb_map_vma()
7093 int ring_buffer_map(struct trace_buffer *buffer, int cpu, in ring_buffer_map() argument
7100 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_map()
7103 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_map()
7115 /* prevent another thread from changing buffer/sub-buffer sizes */ in ring_buffer_map()
7116 mutex_lock(&buffer->mutex); in ring_buffer_map()
7156 mutex_unlock(&buffer->mutex); in ring_buffer_map()
7162 int ring_buffer_unmap(struct trace_buffer *buffer, int cpu) in ring_buffer_unmap() argument
7168 if (!cpumask_test_cpu(cpu, buffer->cpumask)) in ring_buffer_unmap()
7171 cpu_buffer = buffer->buffers[cpu]; in ring_buffer_unmap()
7183 mutex_lock(&buffer->mutex); in ring_buffer_unmap()
7198 mutex_unlock(&buffer->mutex); in ring_buffer_unmap()
7206 int ring_buffer_map_get_reader(struct trace_buffer *buffer, int cpu) in ring_buffer_map_get_reader() argument
7214 cpu_buffer = rb_get_mapped_buffer(buffer, cpu); in ring_buffer_map_get_reader()
7260 if (buffer->subbuf_size - commit >= sizeof(missed_events)) { in ring_buffer_map_get_reader()
7293 * If we were to free the buffer, then the user would lose any trace that was in
7294 * the buffer.
7298 struct trace_buffer *buffer; in trace_rb_cpu_prepare() local
7303 buffer = container_of(node, struct trace_buffer, node); in trace_rb_cpu_prepare()
7304 if (cpumask_test_cpu(cpu, buffer->cpumask)) in trace_rb_cpu_prepare()
7310 for_each_buffer_cpu(buffer, cpu_i) { in trace_rb_cpu_prepare()
7313 nr_pages = buffer->buffers[cpu_i]->nr_pages; in trace_rb_cpu_prepare()
7314 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) { in trace_rb_cpu_prepare()
7322 buffer->buffers[cpu] = in trace_rb_cpu_prepare()
7323 rb_allocate_cpu_buffer(buffer, nr_pages, cpu); in trace_rb_cpu_prepare()
7324 if (!buffer->buffers[cpu]) { in trace_rb_cpu_prepare()
7325 WARN(1, "failed to allocate ring buffer on CPU %u\n", in trace_rb_cpu_prepare()
7330 cpumask_set_cpu(cpu, buffer->cpumask); in trace_rb_cpu_prepare()
7336 * This is a basic integrity check of the ring buffer.
7339 * writing to the per cpu ring buffer various sizes of data.
7343 * IPIs to the other CPUs to also write into the ring buffer.
7344 * this is to test the nesting ability of the buffer.
7347 * ring buffer should happen that's not expected, a big warning
7353 struct trace_buffer *buffer; member
7406 /* read rb_test_started before checking buffer enabled */ in rb_write_something()
7409 event = ring_buffer_lock_reserve(data->buffer, len); in rb_write_something()
7423 if (RB_WARN_ON(data->buffer, event_len < len)) in rb_write_something()
7449 ring_buffer_unlock_commit(data->buffer); in rb_write_something()
7495 struct trace_buffer *buffer; in test_ringbuffer() local
7500 pr_warn("Lockdown is enabled, skipping ring buffer tests\n"); in test_ringbuffer()
7504 pr_info("Running ring buffer tests...\n"); in test_ringbuffer()
7506 buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE); in test_ringbuffer()
7507 if (WARN_ON(!buffer)) in test_ringbuffer()
7510 /* Disable buffer so that threads can't write to it yet */ in test_ringbuffer()
7511 ring_buffer_record_off(buffer); in test_ringbuffer()
7514 rb_data[cpu].buffer = buffer; in test_ringbuffer()
7534 ring_buffer_record_on(buffer); in test_ringbuffer()
7536 * Show buffer is enabled before setting rb_test_started. in test_ringbuffer()
7539 * buffer gets enabled, there will always be some kind of in test_ringbuffer()
7542 * the threads see that the buffer is active. in test_ringbuffer()
7560 ring_buffer_free(buffer); in test_ringbuffer()
7600 if (RB_WARN_ON(buffer, total_dropped)) in test_ringbuffer()
7605 while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) { in test_ringbuffer()
7612 pr_info("buffer had: %.*s\n", item->size, item->str); in test_ringbuffer()
7614 RB_WARN_ON(buffer, 1); in test_ringbuffer()
7634 if (RB_WARN_ON(buffer, total_len != total_alloc || in test_ringbuffer()
7638 if (RB_WARN_ON(buffer, total_lost + total_read != total_events)) in test_ringbuffer()
7644 pr_info("Ring buffer PASSED!\n"); in test_ringbuffer()
7646 ring_buffer_free(buffer); in test_ringbuffer()