1 // SPDX-License-Identifier: GPL-2.0
108  *   +------+
111  *   +------+        +---+   +---+   +---+
112  *                   |   |-->|   |-->|   |
113  *                   +---+   +---+   +---+
116  *                     +---------------+
119  *   +------+
121  *   |page  |------------------v
122  *   +------+        +---+   +---+   +---+
123  *                   |   |-->|   |-->|   |
124  *                   +---+   +---+   +---+
127  *                     +---------------+
130  *   +------+
132  *   |page  |------------------v
133  *   +------+        +---+   +---+   +---+
134  *      ^            |   |-->|   |-->|   |
135  *      |            +---+   +---+   +---+
138  *      +------------------------------+
141  *   +------+
143  *   |page  |------------------v
144  *   +------+        +---+   +---+   +---+
145  *      ^            |   |   |   |-->|   |
146  *      |   New      +---+   +---+   +---+
147  *      |  Reader------^               |
149  *      +------------------------------+
175 	(event->type_len >= RINGBUF_TYPE_TIME_EXTEND)
179 	return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;  in rb_null_event()
185 	event->type_len = RINGBUF_TYPE_PADDING;  in rb_event_set_padding()
186 	event->time_delta = 0;  in rb_event_set_padding()
194 	if (event->type_len)  in rb_event_data_length()
195 		length = event->type_len * RB_ALIGNMENT;  in rb_event_data_length()
197 		length = event->array[0];  in rb_event_data_length()
209 	switch (event->type_len) {  in rb_event_length()
213 			return -1;  in rb_event_length()
214 		return  event->array[0] + RB_EVNT_HDR_SIZE;  in rb_event_length()
249  * ring_buffer_event_length - return the length of the event
266 	if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)  in ring_buffer_event_length()
268 	length -= RB_EVNT_HDR_SIZE;  in ring_buffer_event_length()
269 	if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))  in ring_buffer_event_length()
270                 length -= sizeof(event->array[0]);  in ring_buffer_event_length()
281 	WARN_ON_ONCE(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);  in rb_event_data()
283 	if (event->type_len)  in rb_event_data()
284 		return (void *)&event->array[0];  in rb_event_data()
286 	return (void *)&event->array[1];  in rb_event_data()
290  * ring_buffer_event_data - return the data of the event
300 	for_each_cpu(cpu, buffer->cpumask)
303 	for_each_cpu_and(cpu, buffer->cpumask, cpu_online_mask)
309 	ts = event->array[0];  in rb_event_time_stamp()
311 	ts += event->time_delta;  in rb_event_time_stamp()
365 	local_set(&bpage->commit, 0);  in rb_init_page()
370 	return local_read(&bpage->page->commit);  in rb_page_commit()
376 	if (!bpage->range)  in free_buffer_page()
377 		free_pages((unsigned long)bpage->page, bpage->order);  in free_buffer_page()
401 	 * gracefully without invoking oom-killer and the system is not  in alloc_cpu_data()
420 	atomic_t			seq;  member
442  *  EXTEND - wants a time extend
443  *  ABSOLUTE - the buffer requests all events to have absolute time stamps
444  *  FORCE - force a full time stamp.
612 			 (unsigned int)(buffer ? buffer->subbuf_size :  in ring_buffer_print_page_header()
613 						 PAGE_SIZE - BUF_PAGE_HDR_SIZE),  in ring_buffer_print_page_header()
621 	*ret = local64_read(&t->time);  in rb_time_read()
625 	local64_set(&t->time, val);  in rb_time_set()
639 	struct buffer_page *page = cpu_buffer->commit_page;  in verify_event()
640 	struct buffer_page *tail_page = READ_ONCE(cpu_buffer->tail_page);  in verify_event()
651 		commit = local_read(&page->page->commit);  in verify_event()
652 		write = local_read(&page->write);  in verify_event()
653 		if (addr >= (unsigned long)&page->page->data[commit] &&  in verify_event()
654 		    addr < (unsigned long)&page->page->data[write])  in verify_event()
657 		next = rb_list_head(page->list.next);  in verify_event()
691  * ring_buffer_event_time_stamp - return the event's current time stamp
710 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[smp_processor_id()];  in ring_buffer_event_time_stamp()
715 	if (event->type_len == RINGBUF_TYPE_TIME_STAMP) {  in ring_buffer_event_time_stamp()
717 		return rb_fix_abs_ts(ts, cpu_buffer->tail_page->page->time_stamp);  in ring_buffer_event_time_stamp()
720 	nest = local_read(&cpu_buffer->committing);  in ring_buffer_event_time_stamp()
726 	if (likely(--nest < MAX_NEST))  in ring_buffer_event_time_stamp()
727 		return cpu_buffer->event_stamp[nest];  in ring_buffer_event_time_stamp()
733 	rb_time_read(&cpu_buffer->write_stamp, &ts);  in ring_buffer_event_time_stamp()
739  * ring_buffer_nr_dirty_pages - get the number of used pages in the ring buffer
751 	read = local_read(&buffer->buffers[cpu]->pages_read);  in ring_buffer_nr_dirty_pages()
752 	lost = local_read(&buffer->buffers[cpu]->pages_lost);  in ring_buffer_nr_dirty_pages()
753 	cnt = local_read(&buffer->buffers[cpu]->pages_touched);  in ring_buffer_nr_dirty_pages()
758 	cnt -= lost;  in ring_buffer_nr_dirty_pages()
766 	return cnt - read;  in ring_buffer_nr_dirty_pages()
771 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in full_hit()
775 	nr_pages = cpu_buffer->nr_pages;  in full_hit()
780 	 * Add one as dirty will never equal nr_pages, as the sub-buffer  in full_hit()
790  * rb_wake_up_waiters - wake up tasks waiting for ring buffer input
800 	(void)atomic_fetch_inc_release(&rbwork->seq);  in rb_wake_up_waiters()
802 	wake_up_all(&rbwork->waiters);  in rb_wake_up_waiters()
803 	if (rbwork->full_waiters_pending || rbwork->wakeup_full) {  in rb_wake_up_waiters()
809 		raw_spin_lock(&cpu_buffer->reader_lock);  in rb_wake_up_waiters()
810 		rbwork->wakeup_full = false;  in rb_wake_up_waiters()
811 		rbwork->full_waiters_pending = false;  in rb_wake_up_waiters()
814 		cpu_buffer->shortest_full = 0;  in rb_wake_up_waiters()
815 		raw_spin_unlock(&cpu_buffer->reader_lock);  in rb_wake_up_waiters()
817 		wake_up_all(&rbwork->full_waiters);  in rb_wake_up_waiters()
822  * ring_buffer_wake_waiters - wake up any waiters on this ring buffer
843 		rbwork = &buffer->irq_work;  in ring_buffer_wake_waiters()
845 		if (WARN_ON_ONCE(!buffer->buffers))  in ring_buffer_wake_waiters()
850 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wake_waiters()
854 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_wake_waiters()
858 	irq_work_queue(&rbwork->work);  in ring_buffer_wake_waiters()
870 	cpu_buffer = buffer->buffers[cpu];  in rb_watermark_hit()
879 		raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in rb_watermark_hit()
880 		pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;  in rb_watermark_hit()
883 		if (!ret && (!cpu_buffer->shortest_full ||  in rb_watermark_hit()
884 			     cpu_buffer->shortest_full > full)) {  in rb_watermark_hit()
885 		    cpu_buffer->shortest_full = full;  in rb_watermark_hit()
887 		raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_watermark_hit()
923 		rbwork->full_waiters_pending = true;  in rb_wait_cond()
925 		rbwork->waiters_pending = true;  in rb_wait_cond()
932 	int				seq;  member
937  * wait loop the first time it is woken up.
942 	struct rb_irq_work *rbwork = rdata->irq_work;  in rb_wait_once()
944 	return atomic_read_acquire(&rbwork->seq) != rdata->seq;  in rb_wait_once()
948  * ring_buffer_wait - wait for input to the ring buffer
974 		rbwork = &buffer->irq_work;  in ring_buffer_wait()
978 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_wait()
979 			return -ENODEV;  in ring_buffer_wait()
980 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_wait()
981 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_wait()
985 		waitq = &rbwork->full_waiters;  in ring_buffer_wait()
987 		waitq = &rbwork->waiters;  in ring_buffer_wait()
989 	/* Set up to exit loop as soon as it is woken */  in ring_buffer_wait()
993 		rdata.seq = atomic_read_acquire(&rbwork->seq);  in ring_buffer_wait()
1004  * ring_buffer_poll_wait - poll on buffer input
1025 		rbwork = &buffer->irq_work;  in ring_buffer_poll_wait()
1028 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_poll_wait()
1031 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_wait()
1032 		rbwork = &cpu_buffer->irq_work;  in ring_buffer_poll_wait()
1036 		poll_wait(filp, &rbwork->full_waiters, poll_table);  in ring_buffer_poll_wait()
1053 		rbwork->full_waiters_pending = true;  in ring_buffer_poll_wait()
1057 	poll_wait(filp, &rbwork->waiters, poll_table);  in ring_buffer_poll_wait()
1058 	rbwork->waiters_pending = true;  in ring_buffer_poll_wait()
1089 				atomic_inc(&__b->buffer->record_disabled); \
1091 				atomic_inc(&b->record_disabled);	\
1104 	/* Skip retpolines :-( */  in rb_time_stamp()
1105 	if (IS_ENABLED(CONFIG_MITIGATION_RETPOLINE) && likely(buffer->clock == trace_clock_local))  in rb_time_stamp()
1108 		ts = buffer->clock();  in rb_time_stamp()
1166  * head->list->prev->next        bit 1          bit 0
1167  *                              -------        -------
1174  * +----+       +-----+        +-----+
1175  * |    |------>|  T  |---X--->|  N  |
1176  * |    |<------|     |        |     |
1177  * +----+       +-----+        +-----+
1179  *   |          +-----+          | |
1180  *   +----------|  R  |----------+ |
1181  *              |     |<-----------+
1182  *              +-----+
1184  * Key:  ---X-->  HEAD flag set in pointer
1214  * rb_list_head - remove any bit
1224  * rb_is_head_page - test if the given page is the head page
1236 	val = (unsigned long)list->next;  in rb_is_head_page()
1238 	if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)  in rb_is_head_page()
1253 	struct list_head *list = page->list.prev;  in rb_is_reader_page()
1255 	return rb_list_head(list->next) != &page->list;  in rb_is_reader_page()
1259  * rb_set_list_to_head - set a list_head to be pointing to head.
1265 	ptr = (unsigned long *)&list->next;  in rb_set_list_to_head()
1271  * rb_head_page_activate - sets up head page
1277 	head = cpu_buffer->head_page;  in rb_head_page_activate()
1284 	rb_set_list_to_head(head->list.prev);  in rb_head_page_activate()
1286 	if (cpu_buffer->ring_meta) {  in rb_head_page_activate()
1287 		struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_head_page_activate()
1288 		meta->head_buffer = (unsigned long)head->page;  in rb_head_page_activate()
1294 	unsigned long *ptr = (unsigned long *)&list->next;  in rb_list_head_clear()
1300  * rb_head_page_deactivate - clears head page ptr (for free list)
1308 	rb_list_head_clear(cpu_buffer->pages);  in rb_head_page_deactivate()
1310 	list_for_each(hd, cpu_buffer->pages)  in rb_head_page_deactivate()
1320 	unsigned long val = (unsigned long)&head->list;  in rb_head_page_set()
1323 	list = &prev->list;  in rb_head_page_set()
1327 	ret = cmpxchg((unsigned long *)&list->next,  in rb_head_page_set()
1366 	struct list_head *p = rb_list_head((*bpage)->list.next);  in rb_inc_page()
1373 	struct list_head *p = rb_list_head((*bpage)->list.prev);  in rb_dec_page()
1386 	if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page))  in rb_set_head_page()
1390 	list = cpu_buffer->pages;  in rb_set_head_page()
1391 	if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))  in rb_set_head_page()
1394 	page = head = cpu_buffer->head_page;  in rb_set_head_page()
1397 	 * where we started, and we miss in one loop.  in rb_set_head_page()
1398 	 * A second loop should grab the header, but we'll do  in rb_set_head_page()
1403 			if (rb_is_head_page(page, page->list.prev)) {  in rb_set_head_page()
1404 				cpu_buffer->head_page = page;  in rb_set_head_page()
1419 	unsigned long *ptr = (unsigned long *)&old->list.prev->next;  in rb_head_page_replace()
1425 	return try_cmpxchg(ptr, &val, (unsigned long)&new->list);  in rb_head_page_replace()
1429  * rb_tail_page_update - move the tail page forward
1447 	old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);  in rb_tail_page_update()
1448 	old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);  in rb_tail_page_update()
1461 	if (tail_page == READ_ONCE(cpu_buffer->tail_page)) {  in rb_tail_page_update()
1476 		(void)local_cmpxchg(&next_page->write, old_write, val);  in rb_tail_page_update()
1477 		(void)local_cmpxchg(&next_page->entries, old_entries, eval);  in rb_tail_page_update()
1484 		local_set(&next_page->page->commit, 0);  in rb_tail_page_update()
1487 		if (try_cmpxchg(&cpu_buffer->tail_page, &tail_page, next_page))  in rb_tail_page_update()
1488 			local_inc(&cpu_buffer->pages_touched);  in rb_tail_page_update()
1504 		       rb_list_head(rb_list_head(list->next)->prev) != list))  in rb_check_links()
1508 		       rb_list_head(rb_list_head(list->prev)->next) != list))  in rb_check_links()
1515  * rb_check_pages - integrity check of buffer pages
1535 	 * time when interrupts are disabled non-deterministic, dependent on the  in rb_check_pages()
1536 	 * ring buffer size. Therefore, the code releases and re-acquires the  in rb_check_pages()
1542 	 * giving up. This is acceptable because this is only a self-validation  in rb_check_pages()
1551 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1552 	head = rb_list_head(cpu_buffer->pages);  in rb_check_pages()
1555 	buffer_cnt = cpu_buffer->cnt;  in rb_check_pages()
1557 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1560 		raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1562 		if (buffer_cnt != cpu_buffer->cnt) {  in rb_check_pages()
1564 			raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1568 		tmp = rb_list_head(tmp->next);  in rb_check_pages()
1576 		raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1580 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_check_pages()
1602 	int subbuf_size = buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in rb_range_meta()
1608 	bmeta = buffer->meta;  in rb_range_meta()
1612 	ptr = (unsigned long)bmeta + bmeta->buffers_offset;  in rb_range_meta()
1617 		nr_subbufs = meta->nr_subbufs;  in rb_range_meta()
1642 			size = ptr - p;  in rb_range_meta()
1643 			ptr += size * (cpu - 2);  in rb_range_meta()
1652 	int subbuf_size = meta->subbuf_size;  in rb_subbufs_from_meta()
1656 	ptr = rb_range_align_subbuf(ptr, subbuf_size, meta->nr_subbufs);  in rb_subbufs_from_meta()
1662  * Return a specific sub-buffer for a given @cpu defined by @idx.
1670 	meta = rb_range_meta(cpu_buffer->buffer, 0, cpu_buffer->cpu);  in rb_range_buffer()
1674 	if (WARN_ON_ONCE(idx >= meta->nr_subbufs))  in rb_range_buffer()
1677 	subbuf_size = meta->subbuf_size;  in rb_range_buffer()
1679 	/* Map this buffer to the order that's in meta->buffers[] */  in rb_range_buffer()
1680 	idx = meta->buffers[idx];  in rb_range_buffer()
1685 	if (ptr + subbuf_size > cpu_buffer->buffer->range_addr_end)  in rb_range_buffer()
1697 	unsigned long ptr = buffer->range_addr_start;  in rb_meta_init()
1703 	buffer->meta = bmeta;  in rb_meta_init()
1705 	total_size = buffer->range_addr_end - buffer->range_addr_start;  in rb_meta_init()
1715 	if (bmeta->magic != RING_BUFFER_META_MAGIC) {  in rb_meta_init()
1720 	if (bmeta->struct_sizes != struct_sizes) {  in rb_meta_init()
1725 	if (bmeta->total_size != total_size) {  in rb_meta_init()
1730 	if (bmeta->buffers_offset > bmeta->total_size) {  in rb_meta_init()
1735 	if (bmeta->buffers_offset != (void *)ptr - (void *)bmeta) {  in rb_meta_init()
1743 	bmeta->magic = RING_BUFFER_META_MAGIC;  in rb_meta_init()
1744 	bmeta->struct_sizes = struct_sizes;  in rb_meta_init()
1745 	bmeta->total_size = total_size;  in rb_meta_init()
1746 	bmeta->buffers_offset = (void *)ptr - (void *)bmeta;  in rb_meta_init()
1749 	memset((void *)bmeta + sizeof(*bmeta), 0, bmeta->buffers_offset - sizeof(*bmeta));  in rb_meta_init()
1773 	buffers_start = meta->first_buffer;  in rb_cpu_meta_valid()
1774 	buffers_end = meta->first_buffer + (subbuf_size * meta->nr_subbufs);  in rb_cpu_meta_valid()
1777 	if (meta->head_buffer < buffers_start ||  in rb_cpu_meta_valid()
1778 	    meta->head_buffer >= buffers_end) {  in rb_cpu_meta_valid()
1783 	if (meta->commit_buffer < buffers_start ||  in rb_cpu_meta_valid()
1784 	    meta->commit_buffer >= buffers_end) {  in rb_cpu_meta_valid()
1791 	bitmap_clear(subbuf_mask, 0, meta->nr_subbufs);  in rb_cpu_meta_valid()
1794 	for (i = 0; i < meta->nr_subbufs; i++) {  in rb_cpu_meta_valid()
1795 		if (meta->buffers[i] < 0 ||  in rb_cpu_meta_valid()
1796 		    meta->buffers[i] >= meta->nr_subbufs) {  in rb_cpu_meta_valid()
1801 		if ((unsigned)local_read(&subbuf->commit) > subbuf_size) {  in rb_cpu_meta_valid()
1806 		if (test_bit(meta->buffers[i], subbuf_mask)) {  in rb_cpu_meta_valid()
1811 		set_bit(meta->buffers[i], subbuf_mask);  in rb_cpu_meta_valid()
1832 	ts = dpage->time_stamp;  in rb_read_data_buffer()
1836 		event = (struct ring_buffer_event *)(dpage->data + e);  in rb_read_data_buffer()
1838 		if (len <= 0 || len > tail - e)  in rb_read_data_buffer()
1839 			return -1;  in rb_read_data_buffer()
1841 		switch (event->type_len) {  in rb_read_data_buffer()
1854 				return -1;  in rb_read_data_buffer()
1860 			if (event->time_delta == 1)  in rb_read_data_buffer()
1865 			ts += event->time_delta;  in rb_read_data_buffer()
1869 			return -1;  in rb_read_data_buffer()
1882 	tail = local_read(&dpage->commit);  in rb_validate_buffer()
1889 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_meta_validate_events()
1897 	if (!meta || !meta->head_buffer)  in rb_meta_validate_events()
1900 	orig_head = head_page = cpu_buffer->head_page;  in rb_meta_validate_events()
1901 	orig_reader = cpu_buffer->reader_page;  in rb_meta_validate_events()
1904 	ret = rb_validate_buffer(orig_reader->page, cpu_buffer->cpu);  in rb_meta_validate_events()
1910 	entry_bytes += local_read(&orig_reader->page->commit);  in rb_meta_validate_events()
1911 	local_set(&orig_reader->entries, ret);  in rb_meta_validate_events()
1913 	ts = head_page->page->time_stamp;  in rb_meta_validate_events()
1919 	if (head_page == cpu_buffer->tail_page)  in rb_meta_validate_events()
1923 	for (i = 0; i < meta->nr_subbufs + 1; i++, rb_dec_page(&head_page)) {  in rb_meta_validate_events()
1926 		if (head_page == cpu_buffer->tail_page)  in rb_meta_validate_events()
1930 		if (ts < head_page->page->time_stamp)  in rb_meta_validate_events()
1933 		ts = head_page->page->time_stamp;  in rb_meta_validate_events()
1939 		ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu);  in rb_meta_validate_events()
1944 		local_set(&head_page->entries, ret);  in rb_meta_validate_events()
1946 			local_inc(&cpu_buffer->pages_touched);  in rb_meta_validate_events()
1951 		pr_info("Ring buffer [%d] rewound %d pages\n", cpu_buffer->cpu, i);  in rb_meta_validate_events()
1970 		cpu_buffer->reader_page->list.next = &orig_head->list;  in rb_meta_validate_events()
1971 		cpu_buffer->reader_page->list.prev = orig_head->list.prev;  in rb_meta_validate_events()
1972 		orig_head->list.prev = &cpu_buffer->reader_page->list;  in rb_meta_validate_events()
1973 		bpage->list.next = &cpu_buffer->reader_page->list;  in rb_meta_validate_events()
1976 		cpu_buffer->reader_page = head_page;  in rb_meta_validate_events()
1979 		head_page->list.prev = bpage->list.prev;  in rb_meta_validate_events()
1981 		bpage->list.next = &head_page->list;  in rb_meta_validate_events()
1982 		rb_set_list_to_head(&bpage->list);  in rb_meta_validate_events()
1983 		cpu_buffer->pages = &head_page->list;  in rb_meta_validate_events()
1985 		cpu_buffer->head_page = head_page;  in rb_meta_validate_events()
1986 		meta->head_buffer = (unsigned long)head_page->page;  in rb_meta_validate_events()
1989 		bpage = cpu_buffer->reader_page;  in rb_meta_validate_events()
1990 		meta->buffers[0] = rb_meta_subbuf_idx(meta, bpage->page);  in rb_meta_validate_events()
1991 		bpage->id = 0;  in rb_meta_validate_events()
1993 		for (i = 1, bpage = head_page; i < meta->nr_subbufs;  in rb_meta_validate_events()
1995 			meta->buffers[i] = rb_meta_subbuf_idx(meta, bpage->page);  in rb_meta_validate_events()
1996 			bpage->id = i;  in rb_meta_validate_events()
2005 	if (meta->commit_buffer == (unsigned long)cpu_buffer->reader_page->page) {  in rb_meta_validate_events()
2006 		cpu_buffer->commit_page = cpu_buffer->reader_page;  in rb_meta_validate_events()
2012 	for (i = 0; i < meta->nr_subbufs + 1; i++, rb_inc_page(&head_page)) {  in rb_meta_validate_events()
2018 		ret = rb_validate_buffer(head_page->page, cpu_buffer->cpu);  in rb_meta_validate_events()
2021 				cpu_buffer->cpu);  in rb_meta_validate_events()
2027 			local_inc(&cpu_buffer->pages_touched);  in rb_meta_validate_events()
2030 		entry_bytes += local_read(&head_page->page->commit);  in rb_meta_validate_events()
2031 		local_set(&head_page->entries, ret);  in rb_meta_validate_events()
2033 		if (head_page == cpu_buffer->commit_page)  in rb_meta_validate_events()
2037 	if (head_page != cpu_buffer->commit_page) {  in rb_meta_validate_events()
2039 			cpu_buffer->cpu);  in rb_meta_validate_events()
2043 	local_set(&cpu_buffer->entries, entries);  in rb_meta_validate_events()
2044 	local_set(&cpu_buffer->entries_bytes, entry_bytes);  in rb_meta_validate_events()
2046 	pr_info("Ring buffer meta [%d] is from previous boot!\n", cpu_buffer->cpu);  in rb_meta_validate_events()
2051 	meta->head_buffer = 0;  in rb_meta_validate_events()
2052 	meta->commit_buffer = 0;  in rb_meta_validate_events()
2055 	local_set(&cpu_buffer->reader_page->entries, 0);  in rb_meta_validate_events()
2056 	local_set(&cpu_buffer->reader_page->page->commit, 0);  in rb_meta_validate_events()
2059 	for (i = 0; i < meta->nr_subbufs - 1; i++, rb_inc_page(&head_page)) {  in rb_meta_validate_events()
2060 		local_set(&head_page->entries, 0);  in rb_meta_validate_events()
2061 		local_set(&head_page->page->commit, 0);  in rb_meta_validate_events()
2090 			delta = (unsigned long)subbuf - meta->first_buffer;  in rb_range_meta_init()
2091 			meta->first_buffer += delta;  in rb_range_meta_init()
2092 			meta->head_buffer += delta;  in rb_range_meta_init()
2093 			meta->commit_buffer += delta;  in rb_range_meta_init()
2097 		if (cpu < nr_cpu_ids - 1)  in rb_range_meta_init()
2100 			next_meta = (void *)buffer->range_addr_end;  in rb_range_meta_init()
2102 		memset(meta, 0, next_meta - (void *)meta);  in rb_range_meta_init()
2104 		meta->nr_subbufs = nr_pages + 1;  in rb_range_meta_init()
2105 		meta->subbuf_size = PAGE_SIZE;  in rb_range_meta_init()
2109 		meta->first_buffer = (unsigned long)subbuf;  in rb_range_meta_init()
2112 		 * The buffers[] array holds the order of the sub-buffers  in rb_range_meta_init()
2113 		 * that are after the meta data. The sub-buffers may  in rb_range_meta_init()
2117 		 * index into the sub-buffers holding their actual order.  in rb_range_meta_init()
2119 		for (i = 0; i < meta->nr_subbufs; i++) {  in rb_range_meta_init()
2120 			meta->buffers[i] = i;  in rb_range_meta_init()
2122 			subbuf += meta->subbuf_size;  in rb_range_meta_init()
2130 	struct ring_buffer_per_cpu *cpu_buffer = m->private;  in rbm_start()
2131 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rbm_start()
2137 	if (*pos > meta->nr_subbufs)  in rbm_start()
2155 	struct ring_buffer_per_cpu *cpu_buffer = m->private;  in rbm_show()
2156 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rbm_show()
2161 			   rb_meta_subbuf_idx(meta, (void *)meta->head_buffer));  in rbm_show()
2163 			   rb_meta_subbuf_idx(meta, (void *)meta->commit_buffer));  in rbm_show()
2164 		seq_printf(m, "subbuf_size:   %d\n", meta->subbuf_size);  in rbm_show()
2165 		seq_printf(m, "nr_subbufs:    %d\n", meta->nr_subbufs);  in rbm_show()
2169 	val -= 2;  in rbm_show()
2170 	seq_printf(m, "buffer[%ld]:    %d\n", val, meta->buffers[val]);  in rbm_show()
2195 	m = file->private_data;  in ring_buffer_meta_seq_init()
2196 	m->private = buffer->buffers[cpu];  in ring_buffer_meta_seq_init()
2205 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_meta_buffer_update()
2207 	if (meta->head_buffer == (unsigned long)bpage->page)  in rb_meta_buffer_update()
2208 		cpu_buffer->head_page = bpage;  in rb_meta_buffer_update()
2210 	if (meta->commit_buffer == (unsigned long)bpage->page) {  in rb_meta_buffer_update()
2211 		cpu_buffer->commit_page = bpage;  in rb_meta_buffer_update()
2212 		cpu_buffer->tail_page = bpage;  in rb_meta_buffer_update()
2225 	if (cpu >= trace_desc->nr_cpus)  in ring_buffer_desc()
2228 	end = (struct ring_buffer_desc *)((void *)trace_desc + trace_desc->struct_len);  in ring_buffer_desc()
2230 	len = struct_size(desc, page_va, desc->nr_page_va);  in ring_buffer_desc()
2233 	if (desc < end && desc->cpu == cpu)  in ring_buffer_desc()
2238 		if (desc->cpu == cpu)  in ring_buffer_desc()
2247 	return page_id >= desc->nr_page_va ? NULL : (void *)desc->page_va[page_id];  in ring_buffer_desc_page()
2253 	struct trace_buffer *buffer = cpu_buffer->buffer;  in __rb_allocate_pages()
2256 	bool user_thread = current->mm != NULL;  in __rb_allocate_pages()
2269 		return -ENOMEM;  in __rb_allocate_pages()
2278 	 * loop, even if it was triggered by an allocation somewhere else.  in __rb_allocate_pages()
2283 	if (buffer->range_addr_start)  in __rb_allocate_pages()
2284 		meta = rb_range_meta(buffer, nr_pages, cpu_buffer->cpu);  in __rb_allocate_pages()
2286 	if (buffer->remote) {  in __rb_allocate_pages()
2287 		desc = ring_buffer_desc(buffer->remote->desc, cpu_buffer->cpu);  in __rb_allocate_pages()
2288 		if (!desc || WARN_ON(desc->nr_page_va != (nr_pages + 1)))  in __rb_allocate_pages()
2289 			return -EINVAL;  in __rb_allocate_pages()
2294 		bpage = alloc_cpu_page(cpu_buffer->cpu);  in __rb_allocate_pages()
2304 		list_add_tail(&bpage->list, pages);  in __rb_allocate_pages()
2308 			bpage->page = rb_range_buffer(cpu_buffer, i + 1);  in __rb_allocate_pages()
2309 			if (!bpage->page)  in __rb_allocate_pages()
2312 			if (meta->head_buffer)  in __rb_allocate_pages()
2314 			bpage->range = 1;  in __rb_allocate_pages()
2315 			bpage->id = i + 1;  in __rb_allocate_pages()
2322 			bpage->page = p;  in __rb_allocate_pages()
2323 			bpage->range = 1; /* bpage->page can't be freed */  in __rb_allocate_pages()
2324 			bpage->id = i + 1;  in __rb_allocate_pages()
2325 			cpu_buffer->subbuf_ids[i + 1] = bpage;  in __rb_allocate_pages()
2327 			int order = cpu_buffer->buffer->subbuf_order;  in __rb_allocate_pages()
2328 			bpage->page = alloc_cpu_data(cpu_buffer->cpu, order);  in __rb_allocate_pages()
2329 			if (!bpage->page)  in __rb_allocate_pages()
2332 		bpage->order = cpu_buffer->buffer->subbuf_order;  in __rb_allocate_pages()
2344 		list_del_init(&bpage->list);  in __rb_allocate_pages()
2350 	return -ENOMEM;  in __rb_allocate_pages()
2361 		return -ENOMEM;  in rb_allocate_pages()
2368 	cpu_buffer->pages = pages.next;  in rb_allocate_pages()
2371 	cpu_buffer->nr_pages = nr_pages;  in rb_allocate_pages()
2390 	cpu_buffer->cpu = cpu;  in rb_allocate_cpu_buffer()
2391 	cpu_buffer->buffer = buffer;  in rb_allocate_cpu_buffer()
2392 	raw_spin_lock_init(&cpu_buffer->reader_lock);  in rb_allocate_cpu_buffer()
2393 	lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);  in rb_allocate_cpu_buffer()
2394 	cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;  in rb_allocate_cpu_buffer()
2395 	INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);  in rb_allocate_cpu_buffer()
2396 	init_completion(&cpu_buffer->update_done);  in rb_allocate_cpu_buffer()
2397 	init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);  in rb_allocate_cpu_buffer()
2398 	init_waitqueue_head(&cpu_buffer->irq_work.waiters);  in rb_allocate_cpu_buffer()
2399 	init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);  in rb_allocate_cpu_buffer()
2400 	mutex_init(&cpu_buffer->mapping_lock);  in rb_allocate_cpu_buffer()
2408 	cpu_buffer->reader_page = bpage;  in rb_allocate_cpu_buffer()
2410 	if (buffer->range_addr_start) {  in rb_allocate_cpu_buffer()
2415 		cpu_buffer->mapped = 1;  in rb_allocate_cpu_buffer()
2416 		cpu_buffer->ring_meta = rb_range_meta(buffer, nr_pages, cpu);  in rb_allocate_cpu_buffer()
2417 		bpage->page = rb_range_buffer(cpu_buffer, 0);  in rb_allocate_cpu_buffer()
2418 		if (!bpage->page)  in rb_allocate_cpu_buffer()
2420 		if (cpu_buffer->ring_meta->head_buffer)  in rb_allocate_cpu_buffer()
2422 		bpage->range = 1;  in rb_allocate_cpu_buffer()
2423 	} else if (buffer->remote) {  in rb_allocate_cpu_buffer()
2424 		struct ring_buffer_desc *desc = ring_buffer_desc(buffer->remote->desc, cpu);  in rb_allocate_cpu_buffer()
2429 		cpu_buffer->remote = buffer->remote;  in rb_allocate_cpu_buffer()
2430 		cpu_buffer->meta_page = (struct trace_buffer_meta *)(void *)desc->meta_va;  in rb_allocate_cpu_buffer()
2431 		cpu_buffer->nr_pages = nr_pages;  in rb_allocate_cpu_buffer()
2432 		cpu_buffer->subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1,  in rb_allocate_cpu_buffer()
2433 						 sizeof(*cpu_buffer->subbuf_ids), GFP_KERNEL);  in rb_allocate_cpu_buffer()
2434 		if (!cpu_buffer->subbuf_ids)  in rb_allocate_cpu_buffer()
2437 		/* Remote buffers are read-only and immutable */  in rb_allocate_cpu_buffer()
2438 		atomic_inc(&cpu_buffer->record_disabled);  in rb_allocate_cpu_buffer()
2439 		atomic_inc(&cpu_buffer->resize_disabled);  in rb_allocate_cpu_buffer()
2441 		bpage->page = ring_buffer_desc_page(desc, cpu_buffer->meta_page->reader.id);  in rb_allocate_cpu_buffer()
2442 		if (!bpage->page)  in rb_allocate_cpu_buffer()
2445 		bpage->range = 1;  in rb_allocate_cpu_buffer()
2446 		cpu_buffer->subbuf_ids[0] = bpage;  in rb_allocate_cpu_buffer()
2448 		int order = cpu_buffer->buffer->subbuf_order;  in rb_allocate_cpu_buffer()
2449 		bpage->page = alloc_cpu_data(cpu, order);  in rb_allocate_cpu_buffer()
2450 		if (!bpage->page)  in rb_allocate_cpu_buffer()
2454 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);  in rb_allocate_cpu_buffer()
2455 	INIT_LIST_HEAD(&cpu_buffer->new_pages);  in rb_allocate_cpu_buffer()
2464 	meta = cpu_buffer->ring_meta;  in rb_allocate_cpu_buffer()
2465 	if (!meta || !meta->head_buffer ||  in rb_allocate_cpu_buffer()
2466 	    !cpu_buffer->head_page || !cpu_buffer->commit_page || !cpu_buffer->tail_page) {  in rb_allocate_cpu_buffer()
2467 		if (meta && meta->head_buffer &&  in rb_allocate_cpu_buffer()
2468 		    (cpu_buffer->head_page || cpu_buffer->commit_page || cpu_buffer->tail_page)) {  in rb_allocate_cpu_buffer()
2470 			if (!cpu_buffer->head_page)  in rb_allocate_cpu_buffer()
2472 			if (!cpu_buffer->commit_page)  in rb_allocate_cpu_buffer()
2474 			if (!cpu_buffer->tail_page)  in rb_allocate_cpu_buffer()
2478 		cpu_buffer->head_page  in rb_allocate_cpu_buffer()
2479 			= list_entry(cpu_buffer->pages, struct buffer_page, list);  in rb_allocate_cpu_buffer()
2480 		cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;  in rb_allocate_cpu_buffer()
2484 		if (cpu_buffer->ring_meta)  in rb_allocate_cpu_buffer()
2485 			meta->commit_buffer = meta->head_buffer;  in rb_allocate_cpu_buffer()
2494 	free_buffer_page(cpu_buffer->reader_page);  in rb_allocate_cpu_buffer()
2501 	struct list_head *head = cpu_buffer->pages;  in rb_free_cpu_buffer()
2504 	irq_work_sync(&cpu_buffer->irq_work.work);  in rb_free_cpu_buffer()
2506 	if (cpu_buffer->remote)  in rb_free_cpu_buffer()
2507 		kfree(cpu_buffer->subbuf_ids);  in rb_free_cpu_buffer()
2509 	free_buffer_page(cpu_buffer->reader_page);  in rb_free_cpu_buffer()
2515 			list_del_init(&bpage->list);  in rb_free_cpu_buffer()
2522 	free_page((unsigned long)cpu_buffer->free_page);  in rb_free_cpu_buffer()
2533 	arch_ring_buffer_flush_range(buffer->range_addr_start, buffer->range_addr_end);  in rb_flush_buffer_cb()
2557 	if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))  in alloc_buffer()
2560 	buffer->subbuf_order = order;  in alloc_buffer()
2562 	buffer->subbuf_size = subbuf_size - BUF_PAGE_HDR_SIZE;  in alloc_buffer()
2564 	/* Max payload is buffer page size - header (8bytes) */  in alloc_buffer()
2565 	buffer->max_data_size = buffer->subbuf_size - (sizeof(u32) * 2);  in alloc_buffer()
2567 	buffer->flags = flags;  in alloc_buffer()
2568 	buffer->clock = trace_clock_local;  in alloc_buffer()
2569 	buffer->reader_lock_key = key;  in alloc_buffer()
2571 	init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);  in alloc_buffer()
2572 	init_waitqueue_head(&buffer->irq_work.waiters);  in alloc_buffer()
2574 	buffer->cpus = nr_cpu_ids;  in alloc_buffer()
2577 	buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),  in alloc_buffer()
2579 	if (!buffer->buffers)  in alloc_buffer()
2602 		size = end - buffers_start;  in alloc_buffer()
2606 		 * The number of sub-buffers (nr_pages) is determined by the  in alloc_buffer()
2612 		nr_pages = (size - sizeof(struct ring_buffer_cpu_meta)) /  in alloc_buffer()
2629 			nr_pages--;  in alloc_buffer()
2634 		nr_pages--;  in alloc_buffer()
2635 		buffer->range_addr_start = start;  in alloc_buffer()
2636 		buffer->range_addr_end = end;  in alloc_buffer()
2640 		struct ring_buffer_desc *desc = ring_buffer_desc(remote->desc, cpu);  in alloc_buffer()
2642 		buffer->remote = remote;  in alloc_buffer()
2643 		/* The writer is remote. This ring-buffer is read-only */  in alloc_buffer()
2644 		atomic_inc(&buffer->record_disabled);  in alloc_buffer()
2645 		nr_pages = desc->nr_page_va - 1;  in alloc_buffer()
2651 		nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);  in alloc_buffer()
2656 	cpumask_set_cpu(cpu, buffer->cpumask);  in alloc_buffer()
2657 	buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);  in alloc_buffer()
2658 	if (!buffer->buffers[cpu])  in alloc_buffer()
2661 	ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in alloc_buffer()
2665 	mutex_init(&buffer->mutex);  in alloc_buffer()
2669 		buffer->flush_nb.notifier_call = rb_flush_buffer_cb;  in alloc_buffer()
2670 		atomic_notifier_chain_register(&panic_notifier_list, &buffer->flush_nb);  in alloc_buffer()
2677 		if (buffer->buffers[cpu])  in alloc_buffer()
2678 			rb_free_cpu_buffer(buffer->buffers[cpu]);  in alloc_buffer()
2680 	kfree(buffer->buffers);  in alloc_buffer()
2683 	free_cpumask_var(buffer->cpumask);  in alloc_buffer()
2689  * __ring_buffer_alloc - allocate a new ring_buffer
2702 	/* Default buffer page size - one system page */  in __ring_buffer_alloc()
2709  * __ring_buffer_alloc_range - allocate a new ring_buffer from existing memory
2712  * @order: sub-buffer order
2734  * __ring_buffer_alloc_remote - allocate a new ring_buffer from a remote
2735  * @remote: Contains a description of the ring-buffer pages and remote callbacks.
2749 	if (!buffer || !buffer->meta)  in ring_buffer_meta_scratch()
2752 	meta = buffer->meta;  in ring_buffer_meta_scratch()
2757 		*size = (void *)meta + meta->buffers_offset - ptr;  in ring_buffer_meta_scratch()
2763  * ring_buffer_free - free a ring buffer.
2771 	if (buffer->range_addr_start && buffer->range_addr_end)  in ring_buffer_free()
2772 		atomic_notifier_chain_unregister(&panic_notifier_list, &buffer->flush_nb);  in ring_buffer_free()
2774 	cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);  in ring_buffer_free()
2776 	irq_work_sync(&buffer->irq_work.work);  in ring_buffer_free()
2779 		rb_free_cpu_buffer(buffer->buffers[cpu]);  in ring_buffer_free()
2781 	kfree(buffer->buffers);  in ring_buffer_free()
2782 	free_cpumask_var(buffer->cpumask);  in ring_buffer_free()
2791 	buffer->clock = clock;  in ring_buffer_set_clock()
2796 	buffer->time_stamp_abs = abs;  in ring_buffer_set_time_stamp_abs()
2801 	return buffer->time_stamp_abs;  in ring_buffer_time_stamp_abs()
2806 	return local_read(&bpage->entries) & RB_WRITE_MASK;  in rb_page_entries()
2811 	return local_read(&bpage->write) & RB_WRITE_MASK;  in rb_page_write()
2826 	raw_spin_lock_irq(&cpu_buffer->reader_lock);  in rb_remove_pages()
2827 	atomic_inc(&cpu_buffer->record_disabled);  in rb_remove_pages()
2833 	 * the first and last pages. This is done in a busy loop so that we  in rb_remove_pages()
2837 	tail_page = &cpu_buffer->tail_page->list;  in rb_remove_pages()
2843 	if (cpu_buffer->tail_page == cpu_buffer->reader_page)  in rb_remove_pages()
2844 		tail_page = rb_list_head(tail_page->next);  in rb_remove_pages()
2848 	first_page = list_entry(rb_list_head(to_remove->next),  in rb_remove_pages()
2852 		to_remove = rb_list_head(to_remove)->next;  in rb_remove_pages()
2856 	cpu_buffer->pages_removed += nr_removed;  in rb_remove_pages()
2858 	next_page = rb_list_head(to_remove)->next;  in rb_remove_pages()
2865 	tail_page->next = (struct list_head *)((unsigned long)next_page |  in rb_remove_pages()
2868 	next_page->prev = tail_page;  in rb_remove_pages()
2871 	cpu_buffer->pages = next_page;  in rb_remove_pages()
2872 	cpu_buffer->cnt++;  in rb_remove_pages()
2876 		cpu_buffer->head_page = list_entry(next_page,  in rb_remove_pages()
2880 	atomic_dec(&cpu_buffer->record_disabled);  in rb_remove_pages()
2881 	raw_spin_unlock_irq(&cpu_buffer->reader_lock);  in rb_remove_pages()
2883 	RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));  in rb_remove_pages()
2905 			local_add(page_entries, &cpu_buffer->overrun);  in rb_remove_pages()
2906 			local_sub(rb_page_commit(to_remove_page), &cpu_buffer->entries_bytes);  in rb_remove_pages()
2907 			local_inc(&cpu_buffer->pages_lost);  in rb_remove_pages()
2915 		nr_removed--;  in rb_remove_pages()
2927 	struct list_head *pages = &cpu_buffer->new_pages;  in rb_insert_pages()
2933 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in rb_insert_pages()
2941 	 * 2. We cmpxchg the prev_page->next to point from head page to the  in rb_insert_pages()
2943 	 * 3. Finally, we update the head->prev to the end of new list.  in rb_insert_pages()
2950 	while (retries--) {  in rb_insert_pages()
2958 		head_page = &hpage->list;  in rb_insert_pages()
2959 		prev_page = head_page->prev;  in rb_insert_pages()
2961 		first_page = pages->next;  in rb_insert_pages()
2962 		last_page  = pages->prev;  in rb_insert_pages()
2967 		last_page->next = head_page_with_bit;  in rb_insert_pages()
2968 		first_page->prev = prev_page;  in rb_insert_pages()
2971 		if (try_cmpxchg(&prev_page->next,  in rb_insert_pages()
2978 			head_page->prev = last_page;  in rb_insert_pages()
2979 			cpu_buffer->cnt++;  in rb_insert_pages()
2992 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in rb_insert_pages()
2997 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in rb_insert_pages()
2999 			list_del_init(&bpage->list);  in rb_insert_pages()
3010 	if (cpu_buffer->nr_pages_to_update > 0)  in rb_update_pages()
3014 					-cpu_buffer->nr_pages_to_update);  in rb_update_pages()
3017 		cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;  in rb_update_pages()
3025 	complete(&cpu_buffer->update_done);  in update_pages_handler()
3029  * ring_buffer_resize - resize the ring buffer
3034  * Minimum size is 2 * buffer->subbuf_size.
3046 	 * Always succeed at resizing a non-existent buffer:  in ring_buffer_resize()
3053 	    !cpumask_test_cpu(cpu_id, buffer->cpumask))  in ring_buffer_resize()
3056 	nr_pages = DIV_ROUND_UP(size, buffer->subbuf_size);  in ring_buffer_resize()
3069 	mutex_lock(&buffer->mutex);  in ring_buffer_resize()
3070 	atomic_inc(&buffer->resizing);  in ring_buffer_resize()
3079 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3080 			if (atomic_read(&cpu_buffer->resize_disabled)) {  in ring_buffer_resize()
3081 				err = -EBUSY;  in ring_buffer_resize()
3088 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3090 			cpu_buffer->nr_pages_to_update = nr_pages -  in ring_buffer_resize()
3091 							cpu_buffer->nr_pages;  in ring_buffer_resize()
3095 			if (cpu_buffer->nr_pages_to_update <= 0)  in ring_buffer_resize()
3101 			INIT_LIST_HEAD(&cpu_buffer->new_pages);  in ring_buffer_resize()
3102 			if (__rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update,  in ring_buffer_resize()
3103 						&cpu_buffer->new_pages)) {  in ring_buffer_resize()
3105 				err = -ENOMEM;  in ring_buffer_resize()
3118 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3119 			if (!cpu_buffer->nr_pages_to_update)  in ring_buffer_resize()
3125 				cpu_buffer->nr_pages_to_update = 0;  in ring_buffer_resize()
3132 							 &cpu_buffer->update_pages_work);  in ring_buffer_resize()
3134 					update_pages_handler(&cpu_buffer->update_pages_work);  in ring_buffer_resize()
3142 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3143 			if (!cpu_buffer->nr_pages_to_update)  in ring_buffer_resize()
3147 				wait_for_completion(&cpu_buffer->update_done);  in ring_buffer_resize()
3148 			cpu_buffer->nr_pages_to_update = 0;  in ring_buffer_resize()
3152 		cpu_buffer = buffer->buffers[cpu_id];  in ring_buffer_resize()
3154 		if (nr_pages == cpu_buffer->nr_pages)  in ring_buffer_resize()
3162 		if (atomic_read(&cpu_buffer->resize_disabled)) {  in ring_buffer_resize()
3163 			err = -EBUSY;  in ring_buffer_resize()
3167 		cpu_buffer->nr_pages_to_update = nr_pages -  in ring_buffer_resize()
3168 						cpu_buffer->nr_pages;  in ring_buffer_resize()
3170 		INIT_LIST_HEAD(&cpu_buffer->new_pages);  in ring_buffer_resize()
3171 		if (cpu_buffer->nr_pages_to_update > 0 &&  in ring_buffer_resize()
3172 			__rb_allocate_pages(cpu_buffer, cpu_buffer->nr_pages_to_update,  in ring_buffer_resize()
3173 					    &cpu_buffer->new_pages)) {  in ring_buffer_resize()
3174 			err = -ENOMEM;  in ring_buffer_resize()
3190 						 &cpu_buffer->update_pages_work);  in ring_buffer_resize()
3191 				wait_for_completion(&cpu_buffer->update_done);  in ring_buffer_resize()
3195 		cpu_buffer->nr_pages_to_update = 0;  in ring_buffer_resize()
3206 	if (atomic_read(&buffer->record_disabled)) {  in ring_buffer_resize()
3207 		atomic_inc(&buffer->record_disabled);  in ring_buffer_resize()
3216 			cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3219 		atomic_dec(&buffer->record_disabled);  in ring_buffer_resize()
3222 	atomic_dec(&buffer->resizing);  in ring_buffer_resize()
3223 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
3230 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_resize()
3231 		cpu_buffer->nr_pages_to_update = 0;  in ring_buffer_resize()
3233 		if (list_empty(&cpu_buffer->new_pages))  in ring_buffer_resize()
3236 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,  in ring_buffer_resize()
3238 			list_del_init(&bpage->list);  in ring_buffer_resize()
3245 	atomic_dec(&buffer->resizing);  in ring_buffer_resize()
3246 	mutex_unlock(&buffer->mutex);  in ring_buffer_resize()
3253 	mutex_lock(&buffer->mutex);  in ring_buffer_change_overwrite()
3255 		buffer->flags |= RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
3257 		buffer->flags &= ~RB_FL_OVERWRITE;  in ring_buffer_change_overwrite()
3258 	mutex_unlock(&buffer->mutex);  in ring_buffer_change_overwrite()
3264 	return bpage->page->data + index;  in __rb_page_index()
3270 	return __rb_page_index(cpu_buffer->reader_page,  in rb_reader_event()
3271 			       cpu_buffer->reader_page->read);  in rb_reader_event()
3278 	struct buffer_page *iter_head_page = iter->head_page;  in rb_iter_head_event()
3282 	if (iter->head != iter->next_event)  in rb_iter_head_event()
3283 		return iter->event;  in rb_iter_head_event()
3294 	if (iter->head > commit - 8)  in rb_iter_head_event()
3297 	event = __rb_page_index(iter_head_page, iter->head);  in rb_iter_head_event()
3306 	if ((iter->head + length) > commit || length > iter->event_size)  in rb_iter_head_event()
3310 	memcpy(iter->event, event, length);  in rb_iter_head_event()
3318 	if (iter->page_stamp != iter_head_page->page->time_stamp ||  in rb_iter_head_event()
3322 	iter->next_event = iter->head + length;  in rb_iter_head_event()
3323 	return iter->event;  in rb_iter_head_event()
3326 	iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;  in rb_iter_head_event()
3327 	iter->head = 0;  in rb_iter_head_event()
3328 	iter->next_event = 0;  in rb_iter_head_event()
3329 	iter->missed_events = 1;  in rb_iter_head_event()
3342 	return rb_page_commit(cpu_buffer->commit_page);  in rb_commit_index()
3350 	addr &= (PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1;  in rb_event_index()
3352 	return addr - BUF_PAGE_HDR_SIZE;  in rb_event_index()
3357 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_inc_iter()
3365 	if (iter->head_page == cpu_buffer->reader_page)  in rb_inc_iter()
3366 		iter->head_page = rb_set_head_page(cpu_buffer);  in rb_inc_iter()
3368 		rb_inc_page(&iter->head_page);  in rb_inc_iter()
3370 	iter->page_stamp = iter->read_stamp = iter->head_page->page->time_stamp;  in rb_inc_iter()
3371 	iter->head = 0;  in rb_inc_iter()
3372 	iter->next_event = 0;  in rb_inc_iter()
3375 /* Return the index into the sub-buffers for a given sub-buffer */
3380 	subbuf_array = (void *)meta + sizeof(int) * meta->nr_subbufs;  in rb_meta_subbuf_idx()
3381 	subbuf_array = (void *)ALIGN((unsigned long)subbuf_array, meta->subbuf_size);  in rb_meta_subbuf_idx()
3382 	return (subbuf - subbuf_array) / meta->subbuf_size;  in rb_meta_subbuf_idx()
3388 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_update_meta_head()
3389 	unsigned long old_head = (unsigned long)next_page->page;  in rb_update_meta_head()
3393 	new_head = (unsigned long)next_page->page;  in rb_update_meta_head()
3399 	(void)cmpxchg(&meta->head_buffer, old_head, new_head);  in rb_update_meta_head()
3405 	struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_update_meta_reader()
3406 	void *old_reader = cpu_buffer->reader_page->page;  in rb_update_meta_reader()
3407 	void *new_reader = reader->page;  in rb_update_meta_reader()
3410 	id = reader->id;  in rb_update_meta_reader()
3411 	cpu_buffer->reader_page->id = id;  in rb_update_meta_reader()
3412 	reader->id = 0;  in rb_update_meta_reader()
3414 	meta->buffers[0] = rb_meta_subbuf_idx(meta, new_reader);  in rb_update_meta_reader()
3415 	meta->buffers[id] = rb_meta_subbuf_idx(meta, old_reader);  in rb_update_meta_reader()
3422  * rb_handle_head_page - writer hit the head page
3426  *          -1 on error
3450 	 *  NORMAL - an interrupt already moved it for us  in rb_handle_head_page()
3451 	 *  HEAD   - we are the first to get here.  in rb_handle_head_page()
3452 	 *  UPDATE - we are the interrupt interrupting  in rb_handle_head_page()
3454 	 *  MOVED  - a reader on another CPU moved the next  in rb_handle_head_page()
3466 		local_add(entries, &cpu_buffer->overrun);  in rb_handle_head_page()
3467 		local_sub(rb_page_commit(next_page), &cpu_buffer->entries_bytes);  in rb_handle_head_page()
3468 		local_inc(&cpu_buffer->pages_lost);  in rb_handle_head_page()
3470 		if (cpu_buffer->ring_meta)  in rb_handle_head_page()
3502 		return -1;  in rb_handle_head_page()
3527 	 *  HEAD   - an interrupt came in and already set it.  in rb_handle_head_page()
3528 	 *  NORMAL - One of two things:  in rb_handle_head_page()
3540 		return -1;  in rb_handle_head_page()
3556 		buffer_tail_page = READ_ONCE(cpu_buffer->tail_page);  in rb_handle_head_page()
3579 			return -1;  in rb_handle_head_page()
3589 	unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);  in rb_reset_tail()
3590 	struct buffer_page *tail_page = info->tail_page;  in rb_reset_tail()
3592 	unsigned long length = info->length;  in rb_reset_tail()
3605 			tail_page->real_end = 0;  in rb_reset_tail()
3607 		local_sub(length, &tail_page->write);  in rb_reset_tail()
3618 	tail_page->real_end = tail;  in rb_reset_tail()
3632 	if (tail > (bsize - RB_EVNT_MIN_SIZE)) {  in rb_reset_tail()
3642 		local_sub(length, &tail_page->write);  in rb_reset_tail()
3647 	event->array[0] = (bsize - tail) - RB_EVNT_HDR_SIZE;  in rb_reset_tail()
3648 	event->type_len = RINGBUF_TYPE_PADDING;  in rb_reset_tail()
3650 	event->time_delta = 1;  in rb_reset_tail()
3653 	local_add(bsize - tail, &cpu_buffer->entries_bytes);  in rb_reset_tail()
3655 	/* Make sure the padding is visible before the tail_page->write update */  in rb_reset_tail()
3659 	length = (tail + length) - bsize;  in rb_reset_tail()
3660 	local_sub(length, &tail_page->write);  in rb_reset_tail()
3672 	struct buffer_page *tail_page = info->tail_page;  in rb_move_tail()
3673 	struct buffer_page *commit_page = cpu_buffer->commit_page;  in rb_move_tail()
3674 	struct trace_buffer *buffer = cpu_buffer->buffer;  in rb_move_tail()
3688 		local_inc(&cpu_buffer->commit_overrun);  in rb_move_tail()
3706 	if (rb_is_head_page(next_page, &tail_page->list)) {  in rb_move_tail()
3712 		if (!rb_is_reader_page(cpu_buffer->commit_page)) {  in rb_move_tail()
3717 			if (!(buffer->flags & RB_FL_OVERWRITE)) {  in rb_move_tail()
3718 				local_inc(&cpu_buffer->dropped_events);  in rb_move_tail()
3740 			if (unlikely((cpu_buffer->commit_page !=  in rb_move_tail()
3741 				      cpu_buffer->tail_page) &&  in rb_move_tail()
3742 				     (cpu_buffer->commit_page ==  in rb_move_tail()
3743 				      cpu_buffer->reader_page))) {  in rb_move_tail()
3744 				local_inc(&cpu_buffer->commit_overrun);  in rb_move_tail()
3759 	local_inc(&cpu_buffer->committing);  in rb_move_tail()
3762 	return ERR_PTR(-EAGAIN);  in rb_move_tail()
3777 		event->type_len = RINGBUF_TYPE_TIME_STAMP;  in rb_add_time_stamp()
3779 		event->type_len = RINGBUF_TYPE_TIME_EXTEND;  in rb_add_time_stamp()
3783 		event->time_delta = delta & TS_MASK;  in rb_add_time_stamp()
3784 		event->array[0] = delta >> TS_SHIFT;  in rb_add_time_stamp()
3787 		event->time_delta = 0;  in rb_add_time_stamp()
3788 		event->array[0] = 0;  in rb_add_time_stamp()
3808 		  (unsigned long long)info->delta,  in rb_check_timestamp()
3809 		  (unsigned long long)info->ts,  in rb_check_timestamp()
3810 		  (unsigned long long)info->before,  in rb_check_timestamp()
3811 		  (unsigned long long)info->after,  in rb_check_timestamp()
3812 		  (unsigned long long)({rb_time_read(&cpu_buffer->write_stamp, &write_stamp); write_stamp;}),  in rb_check_timestamp()
3826 	bool abs = info->add_timestamp &  in rb_add_timestamp()
3829 	if (unlikely(info->delta > (1ULL << 59))) {  in rb_add_timestamp()
3834 		if (abs && (info->ts & TS_MSB)) {  in rb_add_timestamp()
3835 			info->delta &= ABS_TS_MASK;  in rb_add_timestamp()
3838 		} else if (info->before == info->after && info->before > info->ts) {  in rb_add_timestamp()
3848 				pr_warn("Ring buffer clock went backwards: %llu -> %llu\n",  in rb_add_timestamp()
3849 					info->before, info->ts);  in rb_add_timestamp()
3854 			info->delta = 0;  in rb_add_timestamp()
3856 	*event = rb_add_time_stamp(cpu_buffer, *event, info->delta, abs);  in rb_add_timestamp()
3857 	*length -= RB_LEN_TIME_EXTEND;  in rb_add_timestamp()
3862  * rb_update_event - update event type and data
3877 	unsigned length = info->length;  in rb_update_event()
3878 	u64 delta = info->delta;  in rb_update_event()
3879 	unsigned int nest = local_read(&cpu_buffer->committing) - 1;  in rb_update_event()
3882 		cpu_buffer->event_stamp[nest] = info->ts;  in rb_update_event()
3888 	if (unlikely(info->add_timestamp))  in rb_update_event()
3891 	event->time_delta = delta;  in rb_update_event()
3892 	length -= RB_EVNT_HDR_SIZE;  in rb_update_event()
3894 		event->type_len = 0;  in rb_update_event()
3895 		event->array[0] = length;  in rb_update_event()
3897 		event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);  in rb_update_event()
3943 	addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);  in rb_try_to_discard()
3945 	bpage = READ_ONCE(cpu_buffer->tail_page);  in rb_try_to_discard()
3951 	if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {  in rb_try_to_discard()
3953 			local_read(&bpage->write) & ~RB_WRITE_MASK;  in rb_try_to_discard()
3967 		rb_time_set(&cpu_buffer->before_stamp, 0);  in rb_try_to_discard()
3987 		if (local_try_cmpxchg(&bpage->write, &old_index, new_index)) {  in rb_try_to_discard()
3989 			local_sub(event_length, &cpu_buffer->entries_bytes);  in rb_try_to_discard()
4000 	local_inc(&cpu_buffer->committing);  in rb_start_commit()
4001 	local_inc(&cpu_buffer->commits);  in rb_start_commit()
4014 	 * back to us). This allows us to do a simple loop to  in rb_set_commit_to_write()
4018 	max_count = cpu_buffer->nr_pages * 100;  in rb_set_commit_to_write()
4020 	while (cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)) {  in rb_set_commit_to_write()
4021 		if (RB_WARN_ON(cpu_buffer, !(--max_count)))  in rb_set_commit_to_write()
4024 			       rb_is_reader_page(cpu_buffer->tail_page)))  in rb_set_commit_to_write()
4030 		local_set(&cpu_buffer->commit_page->page->commit,  in rb_set_commit_to_write()
4031 			  rb_page_write(cpu_buffer->commit_page));  in rb_set_commit_to_write()
4032 		rb_inc_page(&cpu_buffer->commit_page);  in rb_set_commit_to_write()
4033 		if (cpu_buffer->ring_meta) {  in rb_set_commit_to_write()
4034 			struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_set_commit_to_write()
4035 			meta->commit_buffer = (unsigned long)cpu_buffer->commit_page->page;  in rb_set_commit_to_write()
4041 	       rb_page_write(cpu_buffer->commit_page)) {  in rb_set_commit_to_write()
4045 		local_set(&cpu_buffer->commit_page->page->commit,  in rb_set_commit_to_write()
4046 			  rb_page_write(cpu_buffer->commit_page));  in rb_set_commit_to_write()
4048 			   local_read(&cpu_buffer->commit_page->page->commit) &  in rb_set_commit_to_write()
4057 	 * If an interrupt came in just after the first while loop  in rb_set_commit_to_write()
4061 	if (unlikely(cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)))  in rb_set_commit_to_write()
4070 		       !local_read(&cpu_buffer->committing)))  in rb_end_commit()
4074 	commits = local_read(&cpu_buffer->commits);  in rb_end_commit()
4077 	if (local_read(&cpu_buffer->committing) == 1)  in rb_end_commit()
4080 	local_dec(&cpu_buffer->committing);  in rb_end_commit()
4090 	if (unlikely(local_read(&cpu_buffer->commits) != commits) &&  in rb_end_commit()
4091 	    !local_read(&cpu_buffer->committing)) {  in rb_end_commit()
4092 		local_inc(&cpu_buffer->committing);  in rb_end_commit()
4103 	event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;  in rb_event_discard()
4104 	event->type_len = RINGBUF_TYPE_PADDING;  in rb_event_discard()
4106 	if (!event->time_delta)  in rb_event_discard()
4107 		event->time_delta = 1;  in rb_event_discard()
4112 	local_inc(&cpu_buffer->entries);  in rb_commit()
4121 	/* irq_work_queue_on() is not NMI-safe */  in rb_irq_work_queue()
4123 		return irq_work_queue(&irq_work->work);  in rb_irq_work_queue()
4130 	return irq_work_queue_on(&irq_work->work, cpu);  in rb_irq_work_queue()
4136 	if (buffer->irq_work.waiters_pending) {  in rb_wakeups()
4137 		buffer->irq_work.waiters_pending = false;  in rb_wakeups()
4139 		rb_irq_work_queue(&buffer->irq_work);  in rb_wakeups()
4142 	if (cpu_buffer->irq_work.waiters_pending) {  in rb_wakeups()
4143 		cpu_buffer->irq_work.waiters_pending = false;  in rb_wakeups()
4145 		rb_irq_work_queue(&cpu_buffer->irq_work);  in rb_wakeups()
4148 	if (cpu_buffer->last_pages_touch == local_read(&cpu_buffer->pages_touched))  in rb_wakeups()
4151 	if (cpu_buffer->reader_page == cpu_buffer->commit_page)  in rb_wakeups()
4154 	if (!cpu_buffer->irq_work.full_waiters_pending)  in rb_wakeups()
4157 	cpu_buffer->last_pages_touch = local_read(&cpu_buffer->pages_touched);  in rb_wakeups()
4159 	if (!full_hit(buffer, cpu_buffer->cpu, cpu_buffer->shortest_full))  in rb_wakeups()
4162 	cpu_buffer->irq_work.wakeup_full = true;  in rb_wakeups()
4163 	cpu_buffer->irq_work.full_waiters_pending = false;  in rb_wakeups()
4165 	rb_irq_work_queue(&cpu_buffer->irq_work);  in rb_wakeups()
4202  *  101 - 1 = 100
4205  *  1010 - 1 = 1001
4240 	unsigned int val = cpu_buffer->current_context;  in trace_recursive_lock()
4243 	bit = RB_CTX_NORMAL - bit;  in trace_recursive_lock()
4245 	if (unlikely(val & (1 << (bit + cpu_buffer->nest)))) {  in trace_recursive_lock()
4252 		if (val & (1 << (bit + cpu_buffer->nest))) {  in trace_recursive_lock()
4258 	val |= (1 << (bit + cpu_buffer->nest));  in trace_recursive_lock()
4259 	cpu_buffer->current_context = val;  in trace_recursive_lock()
4267 	cpu_buffer->current_context &=  in trace_recursive_unlock()
4268 		cpu_buffer->current_context - (1 << cpu_buffer->nest);  in trace_recursive_unlock()
4275  * ring_buffer_nest_start - Allow to trace while nested
4295 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_start()
4297 	cpu_buffer->nest += NESTED_BITS;  in ring_buffer_nest_start()
4301  * ring_buffer_nest_end - Allow to trace while nested
4314 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_nest_end()
4316 	cpu_buffer->nest -= NESTED_BITS;  in ring_buffer_nest_end()
4321  * ring_buffer_unlock_commit - commit a reserved
4333 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unlock_commit()
4374 	if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))  in show_flags()
4379 	if (entry->flags & TRACE_FLAG_SOFTIRQ)  in show_flags()
4382 	if (entry->flags & TRACE_FLAG_HARDIRQ)  in show_flags()
4385 	if (entry->flags & TRACE_FLAG_NMI)  in show_flags()
4395 	if (rb_event_data_length(event) - RB_EVNT_HDR_SIZE < sizeof(*entry))  in show_irq()
4399 	if (entry->flags & TRACE_FLAG_IRQS_OFF)  in show_irq()
4429 	ts = bpage->time_stamp;  in dump_buffer_page()
4434 		event = (struct ring_buffer_event *)(bpage->data + e);  in dump_buffer_page()
4436 		switch (event->type_len) {  in dump_buffer_page()
4453 			ts += event->time_delta;  in dump_buffer_page()
4455 				e, ts, event->time_delta);  in dump_buffer_page()
4459 			ts += event->time_delta;  in dump_buffer_page()
4461 				e, ts, event->time_delta,  in dump_buffer_page()
4482 		atomic_inc(&cpu_buffer->record_disabled);		\
4488 			/* Do not re-enable checking */			\
4505 	bpage = info->tail_page->page;  in check_buffer()
4509 		tail = local_read(&bpage->commit);  in check_buffer()
4510 	} else if (info->add_timestamp &  in check_buffer()
4520 	if (tail <= 8 || tail > local_read(&bpage->commit))  in check_buffer()
4529 	ret = rb_read_data_buffer(bpage, tail, cpu_buffer->cpu, &ts, &delta);  in check_buffer()
4532 …r_warn_return("[CPU: %d]ABSOLUTE TIME WENT BACKWARDS: last ts: %lld absolute ts: %lld clock:%pS\n",  in check_buffer()
4533 					   cpu_buffer->cpu, ts, delta,  in check_buffer()
4534 					   cpu_buffer->buffer->clock);  in check_buffer()
4538 	if ((full && ts > info->ts) ||  in check_buffer()
4539 	    (!full && ts + info->delta != info->ts)) {  in check_buffer()
4540 …T MATCH expected:%lld actual:%lld delta:%lld before:%lld after:%lld%s context:%s\ntrace clock:%pS",  in check_buffer()
4541 				   cpu_buffer->cpu,  in check_buffer()
4542 				   ts + info->delta, info->ts, info->delta,  in check_buffer()
4543 				   info->before, info->after,  in check_buffer()
4545 				   cpu_buffer->buffer->clock);  in check_buffer()
4566 	/* Don't let the compiler play games with cpu_buffer->tail_page */  in __rb_reserve_next()
4567 	tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);  in __rb_reserve_next()
4569  /*A*/	w = local_read(&tail_page->write) & RB_WRITE_MASK;  in __rb_reserve_next()
4571 	rb_time_read(&cpu_buffer->before_stamp, &info->before);  in __rb_reserve_next()
4572 	rb_time_read(&cpu_buffer->write_stamp, &info->after);  in __rb_reserve_next()
4574 	info->ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
4576 	if ((info->add_timestamp & RB_ADD_STAMP_ABSOLUTE)) {  in __rb_reserve_next()
4577 		info->delta = info->ts;  in __rb_reserve_next()
4585 			/* Use the sub-buffer timestamp */  in __rb_reserve_next()
4586 			info->delta = 0;  in __rb_reserve_next()
4587 		} else if (unlikely(info->before != info->after)) {  in __rb_reserve_next()
4588 			info->add_timestamp |= RB_ADD_STAMP_FORCE | RB_ADD_STAMP_EXTEND;  in __rb_reserve_next()
4589 			info->length += RB_LEN_TIME_EXTEND;  in __rb_reserve_next()
4591 			info->delta = info->ts - info->after;  in __rb_reserve_next()
4592 			if (unlikely(test_time_stamp(info->delta))) {  in __rb_reserve_next()
4593 				info->add_timestamp |= RB_ADD_STAMP_EXTEND;  in __rb_reserve_next()
4594 				info->length += RB_LEN_TIME_EXTEND;  in __rb_reserve_next()
4599  /*B*/	rb_time_set(&cpu_buffer->before_stamp, info->ts);  in __rb_reserve_next()
4601  /*C*/	write = local_add_return(info->length, &tail_page->write);  in __rb_reserve_next()
4606 	tail = write - info->length;  in __rb_reserve_next()
4609 	if (unlikely(write > cpu_buffer->buffer->subbuf_size)) {  in __rb_reserve_next()
4616  /*D*/		rb_time_set(&cpu_buffer->write_stamp, info->ts);  in __rb_reserve_next()
4623 		if (likely(!(info->add_timestamp &  in __rb_reserve_next()
4626 			info->delta = info->ts - info->after;  in __rb_reserve_next()
4629 			info->delta = info->ts;  in __rb_reserve_next()
4633 		/* SLOW PATH - Interrupted between A and C */  in __rb_reserve_next()
4636 		rb_time_read(&cpu_buffer->before_stamp, &info->before);  in __rb_reserve_next()
4644 		ts = rb_time_stamp(cpu_buffer->buffer);  in __rb_reserve_next()
4645 		rb_time_set(&cpu_buffer->before_stamp, ts);  in __rb_reserve_next()
4648  /*E*/		rb_time_read(&cpu_buffer->write_stamp, &info->after);  in __rb_reserve_next()
4650  /*F*/		if (write == (local_read(&tail_page->write) & RB_WRITE_MASK) &&  in __rb_reserve_next()
4651 		    info->after == info->before && info->after < ts) {  in __rb_reserve_next()
4654 			 * safe to use info->after for the delta as it  in __rb_reserve_next()
4655 			 * matched info->before and is still valid.  in __rb_reserve_next()
4657 			info->delta = ts - info->after;  in __rb_reserve_next()
4667 			info->delta = 0;  in __rb_reserve_next()
4669 		info->ts = ts;  in __rb_reserve_next()
4670 		info->add_timestamp &= ~RB_ADD_STAMP_FORCE;  in __rb_reserve_next()
4677 	if (unlikely(!tail && !(info->add_timestamp &  in __rb_reserve_next()
4679 		info->delta = 0;  in __rb_reserve_next()
4686 	local_inc(&tail_page->entries);  in __rb_reserve_next()
4693 		tail_page->page->time_stamp = info->ts;  in __rb_reserve_next()
4696 	local_add(info->length, &cpu_buffer->entries_bytes);  in __rb_reserve_next()
4733 	if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) {  in rb_reserve_next_event()
4734 		local_dec(&cpu_buffer->committing);  in rb_reserve_next_event()
4735 		local_dec(&cpu_buffer->commits);  in rb_reserve_next_event()
4742 	if (ring_buffer_time_stamp_abs(cpu_buffer->buffer)) {  in rb_reserve_next_event()
4745 		if (info.length > cpu_buffer->buffer->max_data_size)  in rb_reserve_next_event()
4757 	 * If one does, it will cause this original code to loop  in rb_reserve_next_event()
4769 	if (unlikely(PTR_ERR(event) == -EAGAIN)) {  in rb_reserve_next_event()
4771 			info.length -= RB_LEN_TIME_EXTEND;  in rb_reserve_next_event()
4783  * ring_buffer_lock_reserve - reserve a part of the buffer
4807 	if (unlikely(atomic_read(&buffer->record_disabled)))  in ring_buffer_lock_reserve()
4812 	if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))  in ring_buffer_lock_reserve()
4815 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_lock_reserve()
4817 	if (unlikely(atomic_read(&cpu_buffer->record_disabled)))  in ring_buffer_lock_reserve()
4820 	if (unlikely(length > buffer->max_data_size))  in ring_buffer_lock_reserve()
4851 	struct buffer_page *bpage = cpu_buffer->commit_page;  in rb_decrement_entry()
4854 	addr &= ~((PAGE_SIZE << cpu_buffer->buffer->subbuf_order) - 1);  in rb_decrement_entry()
4857 	if (likely(bpage->page == (void *)addr)) {  in rb_decrement_entry()
4858 		local_dec(&bpage->entries);  in rb_decrement_entry()
4864 	 * start with the next page and check the end loop there.  in rb_decrement_entry()
4869 		if (bpage->page == (void *)addr) {  in rb_decrement_entry()
4870 			local_dec(&bpage->entries);  in rb_decrement_entry()
4881  * ring_buffer_discard_commit - discard an event that has not been committed
4909 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_discard_commit()
4916 	RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));  in ring_buffer_discard_commit()
4930  * ring_buffer_write - write data to the buffer without reserving
4949 	int ret = -EBUSY;  in ring_buffer_write()
4954 	if (atomic_read(&buffer->record_disabled))  in ring_buffer_write()
4955 		return -EBUSY;  in ring_buffer_write()
4959 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_write()
4960 		return -EBUSY;  in ring_buffer_write()
4962 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_write()
4964 	if (atomic_read(&cpu_buffer->record_disabled))  in ring_buffer_write()
4965 		return -EBUSY;  in ring_buffer_write()
4967 	if (length > buffer->max_data_size)  in ring_buffer_write()
4968 		return -EBUSY;  in ring_buffer_write()
4971 		return -EBUSY;  in ring_buffer_write()
5002 	return local_read(&cpu_buffer->entries) -  in rb_num_of_entries()
5003 		(local_read(&cpu_buffer->overrun) + cpu_buffer->read);  in rb_num_of_entries()
5012  * ring_buffer_record_disable - stop all writes into the buffer
5022 	atomic_inc(&buffer->record_disabled);  in ring_buffer_record_disable()
5027  * ring_buffer_record_enable - enable writes to the buffer
5035 	atomic_dec(&buffer->record_disabled);  in ring_buffer_record_enable()
5040  * ring_buffer_record_off - stop all writes into the buffer
5055 	rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_off()
5058 	} while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));  in ring_buffer_record_off()
5063  * ring_buffer_record_on - restart writes into the buffer
5078 	rd = atomic_read(&buffer->record_disabled);  in ring_buffer_record_on()
5081 	} while (!atomic_try_cmpxchg(&buffer->record_disabled, &rd, new_rd));  in ring_buffer_record_on()
5086  * ring_buffer_record_is_on - return true if the ring buffer can write
5093 	return !atomic_read(&buffer->record_disabled);  in ring_buffer_record_is_on()
5097  * ring_buffer_record_is_set_on - return true if the ring buffer is set writable
5109 	return !(atomic_read(&buffer->record_disabled) & RB_BUFFER_OFF);  in ring_buffer_record_is_set_on()
5113  * ring_buffer_record_is_on_cpu - return true if the ring buffer can write
5124 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_is_on_cpu()
5127 		!atomic_read(&cpu_buffer->record_disabled);  in ring_buffer_record_is_on_cpu()
5131  * ring_buffer_record_disable_cpu - stop all writes into the cpu_buffer
5144 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_disable_cpu()
5147 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_disable_cpu()
5148 	atomic_inc(&cpu_buffer->record_disabled);  in ring_buffer_record_disable_cpu()
5153  * ring_buffer_record_enable_cpu - enable writes to the buffer
5164 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_record_enable_cpu()
5167 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_record_enable_cpu()
5168 	atomic_dec(&cpu_buffer->record_disabled);  in ring_buffer_record_enable_cpu()
5173  * ring_buffer_oldest_event_ts - get the oldest event timestamp from the buffer
5184 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_oldest_event_ts()
5187 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_oldest_event_ts()
5188 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_oldest_event_ts()
5193 	if (cpu_buffer->tail_page == cpu_buffer->reader_page)  in ring_buffer_oldest_event_ts()
5194 		bpage = cpu_buffer->reader_page;  in ring_buffer_oldest_event_ts()
5198 		ret = bpage->page->time_stamp;  in ring_buffer_oldest_event_ts()
5199 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_oldest_event_ts()
5206  * ring_buffer_bytes_cpu - get the number of bytes unconsumed in a cpu buffer
5215 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_bytes_cpu()
5218 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_bytes_cpu()
5219 	ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;  in ring_buffer_bytes_cpu()
5226  * ring_buffer_entries_cpu - get the number of entries in a cpu buffer
5234 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_entries_cpu()
5237 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries_cpu()
5244  * ring_buffer_overrun_cpu - get the number of overruns caused by the ring
5254 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_overrun_cpu()
5257 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overrun_cpu()
5258 	ret = local_read(&cpu_buffer->overrun);  in ring_buffer_overrun_cpu()
5265  * ring_buffer_commit_overrun_cpu - get the number of overruns caused by
5277 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_commit_overrun_cpu()
5280 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_commit_overrun_cpu()
5281 	ret = local_read(&cpu_buffer->commit_overrun);  in ring_buffer_commit_overrun_cpu()
5288  * ring_buffer_dropped_events_cpu - get the number of dropped events caused by
5299 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_dropped_events_cpu()
5302 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_dropped_events_cpu()
5303 	ret = local_read(&cpu_buffer->dropped_events);  in ring_buffer_dropped_events_cpu()
5310  * ring_buffer_read_events_cpu - get the number of events successfully read
5319 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_events_cpu()
5322 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_events_cpu()
5323 	return cpu_buffer->read;  in ring_buffer_read_events_cpu()
5328  * ring_buffer_entries - get the number of entries in a buffer
5342 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_entries()
5351  * ring_buffer_overruns - get the number of overruns in buffer
5365 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_overruns()
5366 		overruns += local_read(&cpu_buffer->overrun);  in ring_buffer_overruns()
5375 	local_set(&cpu_buffer->entries, READ_ONCE(cpu_buffer->meta_page->entries));  in rb_read_remote_meta_page()
5376 	local_set(&cpu_buffer->overrun, READ_ONCE(cpu_buffer->meta_page->overrun));  in rb_read_remote_meta_page()
5377 	local_set(&cpu_buffer->pages_touched, READ_ONCE(cpu_buffer->meta_page->pages_touched));  in rb_read_remote_meta_page()
5378 	local_set(&cpu_buffer->pages_lost, READ_ONCE(cpu_buffer->meta_page->pages_lost));  in rb_read_remote_meta_page()
5388 	orig = next = cpu_buffer->head_page;  in rb_update_remote_head()
5392 	while (cpu_buffer->head_page->page->time_stamp > next->page->time_stamp) {  in rb_update_remote_head()
5395 		rb_list_head_clear(cpu_buffer->head_page->list.prev);  in rb_update_remote_head()
5396 		rb_inc_page(&cpu_buffer->head_page);  in rb_update_remote_head()
5397 		rb_set_list_to_head(cpu_buffer->head_page->list.prev);  in rb_update_remote_head()
5399 		if (cpu_buffer->head_page == orig) {  in rb_update_remote_head()
5400 			if (WARN_ON_ONCE(!(--retry)))  in rb_update_remote_head()
5405 	orig = cpu_buffer->commit_page = cpu_buffer->head_page;  in rb_update_remote_head()
5408 	while (cpu_buffer->commit_page->page->time_stamp < next->page->time_stamp) {  in rb_update_remote_head()
5410 		rb_inc_page(&cpu_buffer->commit_page);  in rb_update_remote_head()
5412 		if (cpu_buffer->commit_page == orig) {  in rb_update_remote_head()
5413 			if (WARN_ON_ONCE(!(--retry)))  in rb_update_remote_head()
5421 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in rb_iter_reset()
5423 	if (cpu_buffer->remote) {  in rb_iter_reset()
5429 	iter->head_page = cpu_buffer->reader_page;  in rb_iter_reset()
5430 	iter->head = cpu_buffer->reader_page->read;  in rb_iter_reset()
5431 	iter->next_event = iter->head;  in rb_iter_reset()
5432 	iter->missed_events = 0;  in rb_iter_reset()
5434 	iter->cache_reader_page = iter->head_page;  in rb_iter_reset()
5435 	iter->cache_read = cpu_buffer->read;  in rb_iter_reset()
5436 	iter->cache_pages_removed = cpu_buffer->pages_removed;  in rb_iter_reset()
5438 	if (iter->head) {  in rb_iter_reset()
5439 		iter->read_stamp = cpu_buffer->read_stamp;  in rb_iter_reset()
5440 		iter->page_stamp = cpu_buffer->reader_page->page->time_stamp;  in rb_iter_reset()
5442 		iter->read_stamp = iter->head_page->page->time_stamp;  in rb_iter_reset()
5443 		iter->page_stamp = iter->read_stamp;  in rb_iter_reset()
5448  * ring_buffer_iter_reset - reset an iterator
5462 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_reset()
5464 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_reset()
5466 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_reset()
5471  * ring_buffer_iter_empty - check if an iterator has no more to read
5485 	cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_empty()
5486 	reader = cpu_buffer->reader_page;  in ring_buffer_iter_empty()
5487 	head_page = cpu_buffer->head_page;  in ring_buffer_iter_empty()
5488 	commit_page = READ_ONCE(cpu_buffer->commit_page);  in ring_buffer_iter_empty()
5489 	commit_ts = commit_page->page->time_stamp;  in ring_buffer_iter_empty()
5502 	curr_commit_page = READ_ONCE(cpu_buffer->commit_page);  in ring_buffer_iter_empty()
5503 	curr_commit_ts = READ_ONCE(curr_commit_page->page->time_stamp);  in ring_buffer_iter_empty()
5511 	return ((iter->head_page == commit_page && iter->head >= commit) ||  in ring_buffer_iter_empty()
5512 		(iter->head_page == reader && commit_page == head_page &&  in ring_buffer_iter_empty()
5513 		 head_page->read == commit &&  in ring_buffer_iter_empty()
5514 		 iter->head == rb_page_size(cpu_buffer->reader_page)));  in ring_buffer_iter_empty()
5524 	switch (event->type_len) {  in rb_update_read_stamp()
5530 		cpu_buffer->read_stamp += delta;  in rb_update_read_stamp()
5535 		delta = rb_fix_abs_ts(delta, cpu_buffer->read_stamp);  in rb_update_read_stamp()
5536 		cpu_buffer->read_stamp = delta;  in rb_update_read_stamp()
5540 		cpu_buffer->read_stamp += event->time_delta;  in rb_update_read_stamp()
5554 	switch (event->type_len) {  in rb_update_iter_read_stamp()
5560 		iter->read_stamp += delta;  in rb_update_iter_read_stamp()
5565 		delta = rb_fix_abs_ts(delta, iter->read_stamp);  in rb_update_iter_read_stamp()
5566 		iter->read_stamp = delta;  in rb_update_iter_read_stamp()
5570 		iter->read_stamp += event->time_delta;  in rb_update_iter_read_stamp()
5574 		RB_WARN_ON(iter->cpu_buffer, 1);  in rb_update_iter_read_stamp()
5587 	if (cpu_buffer->reader_page->read < rb_page_size(cpu_buffer->reader_page)) {  in __rb_get_reader_page_from_remote()
5588 		if (!cpu_buffer->reader_page->read)  in __rb_get_reader_page_from_remote()
5589 			cpu_buffer->read_stamp = cpu_buffer->reader_page->page->time_stamp;  in __rb_get_reader_page_from_remote()
5590 		return cpu_buffer->reader_page;  in __rb_get_reader_page_from_remote()
5593 	prev_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];  in __rb_get_reader_page_from_remote()
5595 	WARN_ON_ONCE(cpu_buffer->remote->swap_reader_page(cpu_buffer->cpu,  in __rb_get_reader_page_from_remote()
5596 							  cpu_buffer->remote->priv));  in __rb_get_reader_page_from_remote()
5598 	if (WARN_ON_ONCE(cpu_buffer->meta_page->reader.id > cpu_buffer->nr_pages))  in __rb_get_reader_page_from_remote()
5601 	new_reader = cpu_buffer->subbuf_ids[cpu_buffer->meta_page->reader.id];  in __rb_get_reader_page_from_remote()
5612 	rb_list_head_clear(cpu_buffer->head_page->list.prev);  in __rb_get_reader_page_from_remote()
5614 	prev_reader->list.next = prev_head->list.next;  in __rb_get_reader_page_from_remote()
5615 	prev_reader->list.prev = prev_head->list.prev;  in __rb_get_reader_page_from_remote()
5618 	last->list.next = &prev_reader->list;  in __rb_get_reader_page_from_remote()
5619 	new_head->list.prev = &prev_reader->list;  in __rb_get_reader_page_from_remote()
5621 	new_reader->list.prev = &new_reader->list;  in __rb_get_reader_page_from_remote()
5622 	new_reader->list.next = &new_head->list;  in __rb_get_reader_page_from_remote()
5625 	rb_set_list_to_head(&last->list);  in __rb_get_reader_page_from_remote()
5627 	cpu_buffer->head_page = new_head;  in __rb_get_reader_page_from_remote()
5628 	cpu_buffer->reader_page = new_reader;  in __rb_get_reader_page_from_remote()
5629 	cpu_buffer->pages = &new_head->list;  in __rb_get_reader_page_from_remote()
5630 	cpu_buffer->read_stamp = new_reader->page->time_stamp;  in __rb_get_reader_page_from_remote()
5631 	cpu_buffer->lost_events = cpu_buffer->meta_page->reader.lost_events;  in __rb_get_reader_page_from_remote()
5633 	return rb_page_size(cpu_buffer->reader_page) ? cpu_buffer->reader_page : NULL;  in __rb_get_reader_page_from_remote()
5640 	unsigned long bsize = READ_ONCE(cpu_buffer->buffer->subbuf_size);  in __rb_get_reader_page()
5647 	arch_spin_lock(&cpu_buffer->lock);  in __rb_get_reader_page()
5651 	 * This should normally only loop twice. But because the  in __rb_get_reader_page()
5653 	 * a case where we will loop three times. There should be no  in __rb_get_reader_page()
5654 	 * reason to loop four times (that I know of).  in __rb_get_reader_page()
5661 	reader = cpu_buffer->reader_page;  in __rb_get_reader_page()
5664 	if (cpu_buffer->reader_page->read < rb_page_size(reader))  in __rb_get_reader_page()
5669 		       cpu_buffer->reader_page->read > rb_page_size(reader)))  in __rb_get_reader_page()
5674 	if (cpu_buffer->commit_page == cpu_buffer->reader_page)  in __rb_get_reader_page()
5684 	local_set(&cpu_buffer->reader_page->write, 0);  in __rb_get_reader_page()
5685 	local_set(&cpu_buffer->reader_page->entries, 0);  in __rb_get_reader_page()
5686 	cpu_buffer->reader_page->real_end = 0;  in __rb_get_reader_page()
5695 	cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);  in __rb_get_reader_page()
5696 	cpu_buffer->reader_page->list.prev = reader->list.prev;  in __rb_get_reader_page()
5699 	 * cpu_buffer->pages just needs to point to the buffer, it  in __rb_get_reader_page()
5703 	cpu_buffer->pages = reader->list.prev;  in __rb_get_reader_page()
5706 	rb_set_list_to_head(&cpu_buffer->reader_page->list);  in __rb_get_reader_page()
5718 	overwrite = local_read(&(cpu_buffer->overrun));  in __rb_get_reader_page()
5731 	ret = rb_head_page_replace(reader, cpu_buffer->reader_page);  in __rb_get_reader_page()
5739 	if (cpu_buffer->ring_meta)  in __rb_get_reader_page()
5747 	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;  in __rb_get_reader_page()
5748 	rb_inc_page(&cpu_buffer->head_page);  in __rb_get_reader_page()
5750 	cpu_buffer->cnt++;  in __rb_get_reader_page()
5751 	local_inc(&cpu_buffer->pages_read);  in __rb_get_reader_page()
5754 	cpu_buffer->reader_page = reader;  in __rb_get_reader_page()
5755 	cpu_buffer->reader_page->read = 0;  in __rb_get_reader_page()
5757 	if (overwrite != cpu_buffer->last_overrun) {  in __rb_get_reader_page()
5758 		cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;  in __rb_get_reader_page()
5759 		cpu_buffer->last_overrun = overwrite;  in __rb_get_reader_page()
5766 	if (reader && reader->read == 0)  in __rb_get_reader_page()
5767 		cpu_buffer->read_stamp = reader->page->time_stamp;  in __rb_get_reader_page()
5769 	arch_spin_unlock(&cpu_buffer->lock);  in __rb_get_reader_page()
5810 	return cpu_buffer->remote ? __rb_get_reader_page_from_remote(cpu_buffer) :  in rb_get_reader_page()
5828 	if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)  in rb_advance_reader()
5829 		cpu_buffer->read++;  in rb_advance_reader()
5834 	cpu_buffer->reader_page->read += length;  in rb_advance_reader()
5835 	cpu_buffer->read_bytes += length;  in rb_advance_reader()
5842 	cpu_buffer = iter->cpu_buffer;  in rb_advance_iter()
5845 	if (iter->head == iter->next_event) {  in rb_advance_iter()
5851 	iter->head = iter->next_event;  in rb_advance_iter()
5856 	if (iter->next_event >= rb_page_size(iter->head_page)) {  in rb_advance_iter()
5858 		if (iter->head_page == cpu_buffer->commit_page)  in rb_advance_iter()
5864 	rb_update_iter_read_stamp(iter, iter->event);  in rb_advance_iter()
5869 	return cpu_buffer->lost_events;  in rb_lost_events()
5886 	 * we should never loop more than once.  in rb_buffer_peek()
5898 	switch (event->type_len) {  in rb_buffer_peek()
5920 			*ts = rb_fix_abs_ts(*ts, reader->page->time_stamp);  in rb_buffer_peek()
5921 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
5922 							 cpu_buffer->cpu, ts);  in rb_buffer_peek()
5930 			*ts = cpu_buffer->read_stamp + event->time_delta;  in rb_buffer_peek()
5931 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_buffer_peek()
5932 							 cpu_buffer->cpu, ts);  in rb_buffer_peek()
5957 	cpu_buffer = iter->cpu_buffer;  in rb_iter_peek()
5958 	buffer = cpu_buffer->buffer;  in rb_iter_peek()
5965 	if (unlikely(iter->cache_read != cpu_buffer->read ||  in rb_iter_peek()
5966 		     iter->cache_reader_page != cpu_buffer->reader_page ||  in rb_iter_peek()
5967 		     iter->cache_pages_removed != cpu_buffer->pages_removed))  in rb_iter_peek()
5987 	if (iter->head >= rb_page_size(iter->head_page)) {  in rb_iter_peek()
5996 	switch (event->type_len) {  in rb_iter_peek()
6013 			*ts = rb_fix_abs_ts(*ts, iter->head_page->page->time_stamp);  in rb_iter_peek()
6014 			ring_buffer_normalize_time_stamp(cpu_buffer->buffer,  in rb_iter_peek()
6015 							 cpu_buffer->cpu, ts);  in rb_iter_peek()
6023 			*ts = iter->read_stamp + event->time_delta;  in rb_iter_peek()
6025 							 cpu_buffer->cpu, ts);  in rb_iter_peek()
6040 		raw_spin_lock(&cpu_buffer->reader_lock);  in rb_reader_lock()
6053 	if (raw_spin_trylock(&cpu_buffer->reader_lock))  in rb_reader_lock()
6057 	atomic_inc(&cpu_buffer->record_disabled);  in rb_reader_lock()
6065 		raw_spin_unlock(&cpu_buffer->reader_lock);  in rb_reader_unlock()
6069  * ring_buffer_peek - peek at the next event to be read
6082 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_peek()
6087 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_peek()
6094 	if (event && event->type_len == RINGBUF_TYPE_PADDING)  in ring_buffer_peek()
6099 	if (event && event->type_len == RINGBUF_TYPE_PADDING)  in ring_buffer_peek()
6105 /** ring_buffer_iter_dropped - report if there are dropped events
6112 	return iter->missed_events != 0;  in ring_buffer_iter_dropped()
6117  * ring_buffer_iter_peek - peek at the next event to be read
6127 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_peek()
6132 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_peek()
6134 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_peek()
6136 	if (event && event->type_len == RINGBUF_TYPE_PADDING)  in ring_buffer_iter_peek()
6143  * ring_buffer_consume - return an event and consume it
6166 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_consume()
6169 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_consume()
6175 		cpu_buffer->lost_events = 0;  in ring_buffer_consume()
6185 	if (event && event->type_len == RINGBUF_TYPE_PADDING)  in ring_buffer_consume()
6193  * ring_buffer_read_start - start a non consuming read of the buffer
6198  * This creates an iterator to allow non-consuming iteration through
6211 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_start()
6219 	iter->event_size = buffer->subbuf_size;  in ring_buffer_read_start()
6220 	iter->event = kmalloc(iter->event_size, flags);  in ring_buffer_read_start()
6221 	if (!iter->event) {  in ring_buffer_read_start()
6226 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_start()
6228 	iter->cpu_buffer = cpu_buffer;  in ring_buffer_read_start()
6230 	atomic_inc(&cpu_buffer->resize_disabled);  in ring_buffer_read_start()
6232 	guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock);  in ring_buffer_read_start()
6233 	arch_spin_lock(&cpu_buffer->lock);  in ring_buffer_read_start()
6235 	arch_spin_unlock(&cpu_buffer->lock);  in ring_buffer_read_start()
6242  * ring_buffer_read_finish - finish reading the iterator of the buffer
6245  * This re-enables resizing of the buffer, and frees the iterator.
6250 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_read_finish()
6255 	atomic_dec(&cpu_buffer->resize_disabled);  in ring_buffer_read_finish()
6256 	kfree(iter->event);  in ring_buffer_read_finish()
6262  * ring_buffer_iter_advance - advance the iterator to the next location
6270 	struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;  in ring_buffer_iter_advance()
6273 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_advance()
6274 	iter->missed_events = 0;  in ring_buffer_iter_advance()
6277 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_iter_advance()
6282  * ring_buffer_size - return the size of the ring buffer (in bytes)
6288 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_size()
6291 	return buffer->subbuf_size * buffer->buffers[cpu]->nr_pages;  in ring_buffer_size()
6296  * ring_buffer_max_event_size - return the max data size of an event
6305 		return buffer->max_data_size - RB_LEN_TIME_EXTEND;  in ring_buffer_max_event_size()
6306 	return buffer->max_data_size;  in ring_buffer_max_event_size()
6312 	local_set(&page->write, 0);  in rb_clear_buffer_page()
6313 	local_set(&page->entries, 0);  in rb_clear_buffer_page()
6314 	rb_init_page(page->page);  in rb_clear_buffer_page()
6315 	page->read = 0;  in rb_clear_buffer_page()
6343 	if (cpu_buffer->ring_meta)  in rb_page_id()
6344 		id = rb_meta_subbuf_idx(cpu_buffer->ring_meta, bpage->page);  in rb_page_id()
6346 		bpage->id = id;  in rb_page_id()
6353 	struct trace_buffer_meta *meta = cpu_buffer->meta_page;  in rb_update_meta_page()
6358 	meta->reader.read = cpu_buffer->reader_page->read;  in rb_update_meta_page()
6359 	meta->reader.id = rb_page_id(cpu_buffer, cpu_buffer->reader_page,  in rb_update_meta_page()
6360 				     cpu_buffer->reader_page->id);  in rb_update_meta_page()
6362 	meta->reader.lost_events = cpu_buffer->lost_events;  in rb_update_meta_page()
6364 	meta->entries = local_read(&cpu_buffer->entries);  in rb_update_meta_page()
6365 	meta->overrun = local_read(&cpu_buffer->overrun);  in rb_update_meta_page()
6366 	meta->read = cpu_buffer->read;  in rb_update_meta_page()
6367 	meta->pages_lost = local_read(&cpu_buffer->pages_lost);  in rb_update_meta_page()
6368 	meta->pages_touched = local_read(&cpu_buffer->pages_touched);  in rb_update_meta_page()
6370 	/* Some archs do not have data cache coherency between kernel and user-space */  in rb_update_meta_page()
6371 	flush_kernel_vmap_range(cpu_buffer->meta_page, PAGE_SIZE);  in rb_update_meta_page()
6379 	if (cpu_buffer->remote) {  in rb_reset_cpu()
6380 		if (!cpu_buffer->remote->reset)  in rb_reset_cpu()
6383 		cpu_buffer->remote->reset(cpu_buffer->cpu, cpu_buffer->remote->priv);  in rb_reset_cpu()
6386 		/* Read related values, not covered by the meta-page */  in rb_reset_cpu()
6387 		local_set(&cpu_buffer->pages_read, 0);  in rb_reset_cpu()
6388 		cpu_buffer->read = 0;  in rb_reset_cpu()
6389 		cpu_buffer->read_bytes = 0;  in rb_reset_cpu()
6390 		cpu_buffer->last_overrun = 0;  in rb_reset_cpu()
6391 		cpu_buffer->reader_page->read = 0;  in rb_reset_cpu()
6398 	cpu_buffer->head_page  in rb_reset_cpu()
6399 		= list_entry(cpu_buffer->pages, struct buffer_page, list);  in rb_reset_cpu()
6400 	rb_clear_buffer_page(cpu_buffer->head_page);  in rb_reset_cpu()
6401 	list_for_each_entry(page, cpu_buffer->pages, list) {  in rb_reset_cpu()
6405 	cpu_buffer->tail_page = cpu_buffer->head_page;  in rb_reset_cpu()
6406 	cpu_buffer->commit_page = cpu_buffer->head_page;  in rb_reset_cpu()
6408 	INIT_LIST_HEAD(&cpu_buffer->reader_page->list);  in rb_reset_cpu()
6409 	INIT_LIST_HEAD(&cpu_buffer->new_pages);  in rb_reset_cpu()
6410 	rb_clear_buffer_page(cpu_buffer->reader_page);  in rb_reset_cpu()
6412 	local_set(&cpu_buffer->entries_bytes, 0);  in rb_reset_cpu()
6413 	local_set(&cpu_buffer->overrun, 0);  in rb_reset_cpu()
6414 	local_set(&cpu_buffer->commit_overrun, 0);  in rb_reset_cpu()
6415 	local_set(&cpu_buffer->dropped_events, 0);  in rb_reset_cpu()
6416 	local_set(&cpu_buffer->entries, 0);  in rb_reset_cpu()
6417 	local_set(&cpu_buffer->committing, 0);  in rb_reset_cpu()
6418 	local_set(&cpu_buffer->commits, 0);  in rb_reset_cpu()
6419 	local_set(&cpu_buffer->pages_touched, 0);  in rb_reset_cpu()
6420 	local_set(&cpu_buffer->pages_lost, 0);  in rb_reset_cpu()
6421 	local_set(&cpu_buffer->pages_read, 0);  in rb_reset_cpu()
6422 	cpu_buffer->last_pages_touch = 0;  in rb_reset_cpu()
6423 	cpu_buffer->shortest_full = 0;  in rb_reset_cpu()
6424 	cpu_buffer->read = 0;  in rb_reset_cpu()
6425 	cpu_buffer->read_bytes = 0;  in rb_reset_cpu()
6427 	rb_time_set(&cpu_buffer->write_stamp, 0);  in rb_reset_cpu()
6428 	rb_time_set(&cpu_buffer->before_stamp, 0);  in rb_reset_cpu()
6430 	memset(cpu_buffer->event_stamp, 0, sizeof(cpu_buffer->event_stamp));  in rb_reset_cpu()
6432 	cpu_buffer->lost_events = 0;  in rb_reset_cpu()
6433 	cpu_buffer->last_overrun = 0;  in rb_reset_cpu()
6436 	cpu_buffer->pages_removed = 0;  in rb_reset_cpu()
6438 	if (cpu_buffer->mapped) {  in rb_reset_cpu()
6440 		if (cpu_buffer->ring_meta) {  in rb_reset_cpu()
6441 			struct ring_buffer_cpu_meta *meta = cpu_buffer->ring_meta;  in rb_reset_cpu()
6442 			meta->commit_buffer = meta->head_buffer;  in rb_reset_cpu()
6450 	guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock);  in reset_disabled_cpu_buffer()
6452 	if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))  in reset_disabled_cpu_buffer()
6455 	arch_spin_lock(&cpu_buffer->lock);  in reset_disabled_cpu_buffer()
6459 	arch_spin_unlock(&cpu_buffer->lock);  in reset_disabled_cpu_buffer()
6463  * ring_buffer_reset_cpu - reset a ring buffer per CPU buffer
6469 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_cpu()
6471 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_reset_cpu()
6475 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_cpu()
6477 	atomic_inc(&cpu_buffer->resize_disabled);  in ring_buffer_reset_cpu()
6478 	atomic_inc(&cpu_buffer->record_disabled);  in ring_buffer_reset_cpu()
6485 	atomic_dec(&cpu_buffer->record_disabled);  in ring_buffer_reset_cpu()
6486 	atomic_dec(&cpu_buffer->resize_disabled);  in ring_buffer_reset_cpu()
6488 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_cpu()
6496  * ring_buffer_reset_online_cpus - reset a ring buffer per CPU buffer
6505 	mutex_lock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
6508 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
6510 		atomic_add(RESET_BIT, &cpu_buffer->resize_disabled);  in ring_buffer_reset_online_cpus()
6511 		atomic_inc(&cpu_buffer->record_disabled);  in ring_buffer_reset_online_cpus()
6518 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset_online_cpus()
6524 		if (!(atomic_read(&cpu_buffer->resize_disabled) & RESET_BIT))  in ring_buffer_reset_online_cpus()
6529 		atomic_dec(&cpu_buffer->record_disabled);  in ring_buffer_reset_online_cpus()
6530 		atomic_sub(RESET_BIT, &cpu_buffer->resize_disabled);  in ring_buffer_reset_online_cpus()
6533 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset_online_cpus()
6537  * ring_buffer_reset - reset a ring buffer
6546 	mutex_lock(&buffer->mutex);  in ring_buffer_reset()
6549 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
6551 		atomic_inc(&cpu_buffer->resize_disabled);  in ring_buffer_reset()
6552 		atomic_inc(&cpu_buffer->record_disabled);  in ring_buffer_reset()
6559 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_reset()
6563 		atomic_dec(&cpu_buffer->record_disabled);  in ring_buffer_reset()
6564 		atomic_dec(&cpu_buffer->resize_disabled);  in ring_buffer_reset()
6567 	mutex_unlock(&buffer->mutex);  in ring_buffer_reset()
6572  * ring_buffer_empty - is the ring buffer empty?
6585 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty()
6601  * ring_buffer_empty_cpu - is a cpu buffer of a ring buffer empty?
6612 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_empty_cpu()
6615 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_empty_cpu()
6631 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_poll_remote()
6632 			return -EINVAL;  in ring_buffer_poll_remote()
6634 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_remote()
6636 		guard(raw_spinlock)(&cpu_buffer->reader_lock);  in ring_buffer_poll_remote()
6650 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_remote()
6652 		guard(raw_spinlock)(&cpu_buffer->reader_lock);  in ring_buffer_poll_remote()
6657 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_poll_remote()
6668  * ring_buffer_swap_cpu - swap a CPU buffer between two ring buffers
6683 	int ret = -EINVAL;  in ring_buffer_swap_cpu()
6685 	if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||  in ring_buffer_swap_cpu()
6686 	    !cpumask_test_cpu(cpu, buffer_b->cpumask))  in ring_buffer_swap_cpu()
6687 		return -EINVAL;  in ring_buffer_swap_cpu()
6689 	cpu_buffer_a = buffer_a->buffers[cpu];  in ring_buffer_swap_cpu()
6690 	cpu_buffer_b = buffer_b->buffers[cpu];  in ring_buffer_swap_cpu()
6693 	if (WARN_ON_ONCE(cpu_buffer_a->mapped || cpu_buffer_b->mapped))  in ring_buffer_swap_cpu()
6694 		return -EBUSY;  in ring_buffer_swap_cpu()
6697 	if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)  in ring_buffer_swap_cpu()
6698 		return -EINVAL;  in ring_buffer_swap_cpu()
6700 	if (buffer_a->subbuf_order != buffer_b->subbuf_order)  in ring_buffer_swap_cpu()
6701 		return -EINVAL;  in ring_buffer_swap_cpu()
6703 	if (atomic_read(&buffer_a->record_disabled))  in ring_buffer_swap_cpu()
6704 		return -EAGAIN;  in ring_buffer_swap_cpu()
6706 	if (atomic_read(&buffer_b->record_disabled))  in ring_buffer_swap_cpu()
6707 		return -EAGAIN;  in ring_buffer_swap_cpu()
6709 	if (atomic_read(&cpu_buffer_a->record_disabled))  in ring_buffer_swap_cpu()
6710 		return -EAGAIN;  in ring_buffer_swap_cpu()
6712 	if (atomic_read(&cpu_buffer_b->record_disabled))  in ring_buffer_swap_cpu()
6713 		return -EAGAIN;  in ring_buffer_swap_cpu()
6721 	atomic_inc(&cpu_buffer_a->record_disabled);  in ring_buffer_swap_cpu()
6722 	atomic_inc(&cpu_buffer_b->record_disabled);  in ring_buffer_swap_cpu()
6724 	ret = -EBUSY;  in ring_buffer_swap_cpu()
6725 	if (local_read(&cpu_buffer_a->committing))  in ring_buffer_swap_cpu()
6727 	if (local_read(&cpu_buffer_b->committing))  in ring_buffer_swap_cpu()
6734 	if (atomic_read(&buffer_a->resizing))  in ring_buffer_swap_cpu()
6736 	if (atomic_read(&buffer_b->resizing))  in ring_buffer_swap_cpu()
6739 	buffer_a->buffers[cpu] = cpu_buffer_b;  in ring_buffer_swap_cpu()
6740 	buffer_b->buffers[cpu] = cpu_buffer_a;  in ring_buffer_swap_cpu()
6742 	cpu_buffer_b->buffer = buffer_a;  in ring_buffer_swap_cpu()
6743 	cpu_buffer_a->buffer = buffer_b;  in ring_buffer_swap_cpu()
6748 	atomic_dec(&cpu_buffer_a->record_disabled);  in ring_buffer_swap_cpu()
6749 	atomic_dec(&cpu_buffer_b->record_disabled);  in ring_buffer_swap_cpu()
6756  * ring_buffer_alloc_read_page - allocate a page to read from buffer
6778 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_alloc_read_page()
6779 		return ERR_PTR(-ENODEV);  in ring_buffer_alloc_read_page()
6783 		return ERR_PTR(-ENOMEM);  in ring_buffer_alloc_read_page()
6785 	bpage->order = buffer->subbuf_order;  in ring_buffer_alloc_read_page()
6786 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_alloc_read_page()
6788 	arch_spin_lock(&cpu_buffer->lock);  in ring_buffer_alloc_read_page()
6790 	if (cpu_buffer->free_page) {  in ring_buffer_alloc_read_page()
6791 		bpage->data = cpu_buffer->free_page;  in ring_buffer_alloc_read_page()
6792 		cpu_buffer->free_page = NULL;  in ring_buffer_alloc_read_page()
6795 	arch_spin_unlock(&cpu_buffer->lock);  in ring_buffer_alloc_read_page()
6798 	if (bpage->data) {  in ring_buffer_alloc_read_page()
6799 		rb_init_page(bpage->data);  in ring_buffer_alloc_read_page()
6801 		bpage->data = alloc_cpu_data(cpu, cpu_buffer->buffer->subbuf_order);  in ring_buffer_alloc_read_page()
6802 		if (!bpage->data) {  in ring_buffer_alloc_read_page()
6804 			return ERR_PTR(-ENOMEM);  in ring_buffer_alloc_read_page()
6813  * ring_buffer_free_read_page - free an allocated read page
6824 	struct buffer_data_page *bpage = data_page->data;  in ring_buffer_free_read_page()
6828 	if (!buffer || !buffer->buffers || !buffer->buffers[cpu])  in ring_buffer_free_read_page()
6831 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_free_read_page()
6835 	 * is different from the subbuffer order of the buffer -  in ring_buffer_free_read_page()
6838 	if (page_ref_count(page) > 1 || data_page->order != buffer->subbuf_order)  in ring_buffer_free_read_page()
6842 	arch_spin_lock(&cpu_buffer->lock);  in ring_buffer_free_read_page()
6844 	if (!cpu_buffer->free_page) {  in ring_buffer_free_read_page()
6845 		cpu_buffer->free_page = bpage;  in ring_buffer_free_read_page()
6849 	arch_spin_unlock(&cpu_buffer->lock);  in ring_buffer_free_read_page()
6853 	free_pages((unsigned long)bpage, data_page->order);  in ring_buffer_free_read_page()
6859  * ring_buffer_read_page - extract a page from the ring buffer
6896 	struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];  in ring_buffer_read_page()
6906 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_read_page()
6907 		return -1;  in ring_buffer_read_page()
6914 		return -1;  in ring_buffer_read_page()
6916 	len -= BUF_PAGE_HDR_SIZE;  in ring_buffer_read_page()
6918 	if (!data_page || !data_page->data)  in ring_buffer_read_page()
6919 		return -1;  in ring_buffer_read_page()
6921 	if (data_page->order != buffer->subbuf_order)  in ring_buffer_read_page()
6922 		return -1;  in ring_buffer_read_page()
6924 	bpage = data_page->data;  in ring_buffer_read_page()
6926 		return -1;  in ring_buffer_read_page()
6928 	guard(raw_spinlock_irqsave)(&cpu_buffer->reader_lock);  in ring_buffer_read_page()
6932 		return -1;  in ring_buffer_read_page()
6936 	read = reader->read;  in ring_buffer_read_page()
6940 	missed_events = cpu_buffer->lost_events;  in ring_buffer_read_page()
6942 	force_memcpy = cpu_buffer->mapped || cpu_buffer->remote;  in ring_buffer_read_page()
6951 	if (read || (len < (commit - read)) ||  in ring_buffer_read_page()
6952 	    cpu_buffer->reader_page == cpu_buffer->commit_page ||  in ring_buffer_read_page()
6954 		struct buffer_data_page *rpage = cpu_buffer->reader_page->page;  in ring_buffer_read_page()
6966 		    (!read || (len < (commit - read)) ||  in ring_buffer_read_page()
6967 		     cpu_buffer->reader_page == cpu_buffer->commit_page))  in ring_buffer_read_page()
6968 			return -1;  in ring_buffer_read_page()
6970 		if (len > (commit - read))  in ring_buffer_read_page()
6971 			len = (commit - read);  in ring_buffer_read_page()
6977 			return -1;  in ring_buffer_read_page()
6980 		save_timestamp = cpu_buffer->read_stamp;  in ring_buffer_read_page()
6991 			memcpy(bpage->data + pos, rpage->data + rpos, size);  in ring_buffer_read_page()
6993 			len -= size;  in ring_buffer_read_page()
6996 			rpos = reader->read;  in ring_buffer_read_page()
7008 		local_set(&bpage->commit, pos);  in ring_buffer_read_page()
7009 		bpage->time_stamp = save_timestamp;  in ring_buffer_read_page()
7015 		cpu_buffer->read += rb_page_entries(reader);  in ring_buffer_read_page()
7016 		cpu_buffer->read_bytes += rb_page_size(reader);  in ring_buffer_read_page()
7020 		bpage = reader->page;  in ring_buffer_read_page()
7021 		reader->page = data_page->data;  in ring_buffer_read_page()
7022 		local_set(&reader->write, 0);  in ring_buffer_read_page()
7023 		local_set(&reader->entries, 0);  in ring_buffer_read_page()
7024 		reader->read = 0;  in ring_buffer_read_page()
7025 		data_page->data = bpage;  in ring_buffer_read_page()
7032 		if (reader->real_end)  in ring_buffer_read_page()
7033 			local_set(&bpage->commit, reader->real_end);  in ring_buffer_read_page()
7036 	cpu_buffer->lost_events = 0;  in ring_buffer_read_page()
7038 	commit = local_read(&bpage->commit);  in ring_buffer_read_page()
7046 		if (buffer->subbuf_size - commit >= sizeof(missed_events)) {  in ring_buffer_read_page()
7047 			memcpy(&bpage->data[commit], &missed_events,  in ring_buffer_read_page()
7049 			local_add(RB_MISSED_STORED, &bpage->commit);  in ring_buffer_read_page()
7052 		local_add(RB_MISSED_EVENTS, &bpage->commit);  in ring_buffer_read_page()
7058 	if (commit < buffer->subbuf_size)  in ring_buffer_read_page()
7059 		memset(&bpage->data[commit], 0, buffer->subbuf_size - commit);  in ring_buffer_read_page()
7066  * ring_buffer_read_page_data - get pointer to the data in the page.
7073 	return page->data;  in ring_buffer_read_page_data()
7078  * ring_buffer_subbuf_size_get - get size of the sub buffer.
7085 	return buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in ring_buffer_subbuf_size_get()
7090  * ring_buffer_subbuf_order_get - get order of system sub pages in one buffer page.
7104 		return -EINVAL;  in ring_buffer_subbuf_order_get()
7106 	return buffer->subbuf_order;  in ring_buffer_subbuf_order_get()
7111  * ring_buffer_subbuf_order_set - set the size of ring buffer sub page.
7119  *  0 - 1 system page
7120  *  1 - 2 system pages
7121  *  3 - 4 system pages
7137 		return -EINVAL;  in ring_buffer_subbuf_order_set()
7139 	if (buffer->subbuf_order == order)  in ring_buffer_subbuf_order_set()
7144 		return -EINVAL;  in ring_buffer_subbuf_order_set()
7148 		return -EINVAL;  in ring_buffer_subbuf_order_set()
7150 	old_order = buffer->subbuf_order;  in ring_buffer_subbuf_order_set()
7151 	old_size = buffer->subbuf_size;  in ring_buffer_subbuf_order_set()
7154 	guard(mutex)(&buffer->mutex);  in ring_buffer_subbuf_order_set()
7155 	atomic_inc(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
7160 	buffer->subbuf_order = order;  in ring_buffer_subbuf_order_set()
7161 	buffer->subbuf_size = psize - BUF_PAGE_HDR_SIZE;  in ring_buffer_subbuf_order_set()
7166 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_subbuf_order_set()
7169 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
7171 		if (cpu_buffer->mapped) {  in ring_buffer_subbuf_order_set()
7172 			err = -EBUSY;  in ring_buffer_subbuf_order_set()
7177 		nr_pages = old_size * buffer->buffers[cpu]->nr_pages;  in ring_buffer_subbuf_order_set()
7178 		nr_pages = DIV_ROUND_UP(nr_pages, buffer->subbuf_size);  in ring_buffer_subbuf_order_set()
7184 		cpu_buffer->nr_pages_to_update = nr_pages;  in ring_buffer_subbuf_order_set()
7190 		INIT_LIST_HEAD(&cpu_buffer->new_pages);  in ring_buffer_subbuf_order_set()
7192 					&cpu_buffer->new_pages)) {  in ring_buffer_subbuf_order_set()
7194 			err = -ENOMEM;  in ring_buffer_subbuf_order_set()
7204 		if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_subbuf_order_set()
7207 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
7209 		raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_subbuf_order_set()
7221 		list_add(&old_pages, cpu_buffer->pages);  in ring_buffer_subbuf_order_set()
7222 		list_add(&cpu_buffer->reader_page->list, &old_pages);  in ring_buffer_subbuf_order_set()
7225 		cpu_buffer->reader_page = list_entry(cpu_buffer->new_pages.next,  in ring_buffer_subbuf_order_set()
7227 		list_del_init(&cpu_buffer->reader_page->list);  in ring_buffer_subbuf_order_set()
7230 		cpu_buffer->pages = cpu_buffer->new_pages.next;  in ring_buffer_subbuf_order_set()
7231 		list_del_init(&cpu_buffer->new_pages);  in ring_buffer_subbuf_order_set()
7232 		cpu_buffer->cnt++;  in ring_buffer_subbuf_order_set()
7234 		cpu_buffer->head_page  in ring_buffer_subbuf_order_set()
7235 			= list_entry(cpu_buffer->pages, struct buffer_page, list);  in ring_buffer_subbuf_order_set()
7236 		cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;  in ring_buffer_subbuf_order_set()
7238 		cpu_buffer->nr_pages = cpu_buffer->nr_pages_to_update;  in ring_buffer_subbuf_order_set()
7239 		cpu_buffer->nr_pages_to_update = 0;  in ring_buffer_subbuf_order_set()
7241 		old_free_data_page = cpu_buffer->free_page;  in ring_buffer_subbuf_order_set()
7242 		cpu_buffer->free_page = NULL;  in ring_buffer_subbuf_order_set()
7246 		raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_subbuf_order_set()
7250 			list_del_init(&bpage->list);  in ring_buffer_subbuf_order_set()
7258 	atomic_dec(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
7263 	buffer->subbuf_order = old_order;  in ring_buffer_subbuf_order_set()
7264 	buffer->subbuf_size = old_size;  in ring_buffer_subbuf_order_set()
7266 	atomic_dec(&buffer->record_disabled);  in ring_buffer_subbuf_order_set()
7269 		cpu_buffer = buffer->buffers[cpu];  in ring_buffer_subbuf_order_set()
7271 		if (!cpu_buffer->nr_pages_to_update)  in ring_buffer_subbuf_order_set()
7274 		list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages, list) {  in ring_buffer_subbuf_order_set()
7275 			list_del_init(&bpage->list);  in ring_buffer_subbuf_order_set()
7288 	if (cpu_buffer->meta_page)  in rb_alloc_meta_page()
7293 		return -ENOMEM;  in rb_alloc_meta_page()
7295 	cpu_buffer->meta_page = page_to_virt(page);  in rb_alloc_meta_page()
7302 	unsigned long addr = (unsigned long)cpu_buffer->meta_page;  in rb_free_meta_page()
7305 	cpu_buffer->meta_page = NULL;  in rb_free_meta_page()
7311 	struct trace_buffer_meta *meta = cpu_buffer->meta_page;  in rb_setup_ids_meta_page()
7312 	unsigned int nr_subbufs = cpu_buffer->nr_pages + 1;  in rb_setup_ids_meta_page()
7317 	id = rb_page_id(cpu_buffer, cpu_buffer->reader_page, id);  in rb_setup_ids_meta_page()
7318 	subbuf_ids[id++] = cpu_buffer->reader_page;  in rb_setup_ids_meta_page()
7338 	cpu_buffer->subbuf_ids = subbuf_ids;  in rb_setup_ids_meta_page()
7340 	meta->meta_struct_len = sizeof(*meta);  in rb_setup_ids_meta_page()
7341 	meta->nr_subbufs = nr_subbufs;  in rb_setup_ids_meta_page()
7342 	meta->subbuf_size = cpu_buffer->buffer->subbuf_size + BUF_PAGE_HDR_SIZE;  in rb_setup_ids_meta_page()
7343 	meta->meta_page_size = meta->subbuf_size;  in rb_setup_ids_meta_page()
7353 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in rb_get_mapped_buffer()
7354 		return ERR_PTR(-EINVAL);  in rb_get_mapped_buffer()
7356 	cpu_buffer = buffer->buffers[cpu];  in rb_get_mapped_buffer()
7358 	mutex_lock(&cpu_buffer->mapping_lock);  in rb_get_mapped_buffer()
7360 	if (!cpu_buffer->user_mapped) {  in rb_get_mapped_buffer()
7361 		mutex_unlock(&cpu_buffer->mapping_lock);  in rb_get_mapped_buffer()
7362 		return ERR_PTR(-ENODEV);  in rb_get_mapped_buffer()
7370 	mutex_unlock(&cpu_buffer->mapping_lock);  in rb_put_mapped_buffer()
7374  * Fast-path for rb_buffer_(un)map(). Called whenever the meta-page doesn't need
7375  * to be set-up or torn-down.
7382 	lockdep_assert_held(&cpu_buffer->mapping_lock);  in __rb_inc_dec_mapped()
7385 	if (WARN_ON(cpu_buffer->mapped < cpu_buffer->user_mapped))  in __rb_inc_dec_mapped()
7386 		return -EINVAL;  in __rb_inc_dec_mapped()
7388 	if (inc && cpu_buffer->mapped == UINT_MAX)  in __rb_inc_dec_mapped()
7389 		return -EBUSY;  in __rb_inc_dec_mapped()
7391 	if (WARN_ON(!inc && cpu_buffer->user_mapped == 0))  in __rb_inc_dec_mapped()
7392 		return -EINVAL;  in __rb_inc_dec_mapped()
7394 	mutex_lock(&cpu_buffer->buffer->mutex);  in __rb_inc_dec_mapped()
7395 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in __rb_inc_dec_mapped()
7398 		cpu_buffer->user_mapped++;  in __rb_inc_dec_mapped()
7399 		cpu_buffer->mapped++;  in __rb_inc_dec_mapped()
7401 		cpu_buffer->user_mapped--;  in __rb_inc_dec_mapped()
7402 		cpu_buffer->mapped--;  in __rb_inc_dec_mapped()
7405 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in __rb_inc_dec_mapped()
7406 	mutex_unlock(&cpu_buffer->buffer->mutex);  in __rb_inc_dec_mapped()
7412  *   +--------------+  pgoff == 0
7414  *   +--------------+  pgoff == 1
7417  *   +--------------+  pgoff == (1 + (1 << subbuf_order))
7426 	unsigned long nr_subbufs, nr_pages, nr_vma_pages, pgoff = vma->vm_pgoff;  in __rb_map_vma()
7433 	if (vma->vm_flags & VM_WRITE || vma->vm_flags & VM_EXEC ||  in __rb_map_vma()
7434 	    !(vma->vm_flags & VM_MAYSHARE))  in __rb_map_vma()
7435 		return -EPERM;  in __rb_map_vma()
7437 	subbuf_order = cpu_buffer->buffer->subbuf_order;  in __rb_map_vma()
7441 		return -EINVAL;  in __rb_map_vma()
7450 	lockdep_assert_held(&cpu_buffer->mapping_lock);  in __rb_map_vma()
7452 	nr_subbufs = cpu_buffer->nr_pages + 1; /* + reader-subbuf */  in __rb_map_vma()
7453 	nr_pages = ((nr_subbufs + 1) << subbuf_order); /* + meta-page */  in __rb_map_vma()
7455 		return -EINVAL;  in __rb_map_vma()
7457 	nr_pages -= pgoff;  in __rb_map_vma()
7461 		return -EINVAL;  in __rb_map_vma()
7467 		return -ENOMEM;  in __rb_map_vma()
7472 		pages[p++] = virt_to_page(cpu_buffer->meta_page);  in __rb_map_vma()
7475 		 * Pad with the zero-page to align the meta-page with the  in __rb_map_vma()
7476 		 * sub-buffers.  in __rb_map_vma()
7478 		meta_page_padding = subbuf_pages - 1;  in __rb_map_vma()
7479 		while (meta_page_padding-- && p < nr_pages) {  in __rb_map_vma()
7481 				vma->vm_start + (PAGE_SIZE * p);  in __rb_map_vma()
7486 		/* Skip the meta-page */  in __rb_map_vma()
7487 		pgoff -= subbuf_pages;  in __rb_map_vma()
7498 			return -EINVAL;  in __rb_map_vma()
7500 		subbuf = cpu_buffer->subbuf_ids[s];  in __rb_map_vma()
7501 		page = virt_to_page((void *)subbuf->page);  in __rb_map_vma()
7512 	err = vm_insert_pages(vma, vma->vm_start, pages, &nr_pages);  in __rb_map_vma()
7520 	return -EOPNOTSUPP;  in __rb_map_vma()
7532 	if (!cpumask_test_cpu(cpu, buffer->cpumask) || buffer->remote)  in ring_buffer_map()
7533 		return -EINVAL;  in ring_buffer_map()
7535 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_map()
7537 	guard(mutex)(&cpu_buffer->mapping_lock);  in ring_buffer_map()
7539 	if (cpu_buffer->user_mapped) {  in ring_buffer_map()
7546 	/* prevent another thread from changing buffer/sub-buffer sizes */  in ring_buffer_map()
7547 	guard(mutex)(&buffer->mutex);  in ring_buffer_map()
7554 	subbuf_ids = kcalloc(cpu_buffer->nr_pages + 1, sizeof(*subbuf_ids), GFP_KERNEL);  in ring_buffer_map()
7557 		return -ENOMEM;  in ring_buffer_map()
7560 	atomic_inc(&cpu_buffer->resize_disabled);  in ring_buffer_map()
7566 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_map()
7569 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_map()
7573 		raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_map()
7575 		cpu_buffer->mapped++;  in ring_buffer_map()
7576 		cpu_buffer->user_mapped = 1;  in ring_buffer_map()
7577 		raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_map()
7579 		kfree(cpu_buffer->subbuf_ids);  in ring_buffer_map()
7580 		cpu_buffer->subbuf_ids = NULL;  in ring_buffer_map()
7582 		atomic_dec(&cpu_buffer->resize_disabled);  in ring_buffer_map()
7596 	if (WARN_ON(!cpumask_test_cpu(cpu, buffer->cpumask)))  in ring_buffer_map_dup()
7599 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_map_dup()
7601 	guard(mutex)(&cpu_buffer->mapping_lock);  in ring_buffer_map_dup()
7603 	if (cpu_buffer->user_mapped)  in ring_buffer_map_dup()
7614 	if (!cpumask_test_cpu(cpu, buffer->cpumask))  in ring_buffer_unmap()
7615 		return -EINVAL;  in ring_buffer_unmap()
7617 	cpu_buffer = buffer->buffers[cpu];  in ring_buffer_unmap()
7619 	guard(mutex)(&cpu_buffer->mapping_lock);  in ring_buffer_unmap()
7621 	if (!cpu_buffer->user_mapped) {  in ring_buffer_unmap()
7622 		return -ENODEV;  in ring_buffer_unmap()
7623 	} else if (cpu_buffer->user_mapped > 1) {  in ring_buffer_unmap()
7628 	guard(mutex)(&buffer->mutex);  in ring_buffer_unmap()
7629 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_unmap()
7632 	if (!WARN_ON_ONCE(cpu_buffer->mapped < cpu_buffer->user_mapped))  in ring_buffer_unmap()
7633 		cpu_buffer->mapped--;  in ring_buffer_unmap()
7634 	cpu_buffer->user_mapped = 0;  in ring_buffer_unmap()
7636 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_unmap()
7638 	kfree(cpu_buffer->subbuf_ids);  in ring_buffer_unmap()
7639 	cpu_buffer->subbuf_ids = NULL;  in ring_buffer_unmap()
7641 	atomic_dec(&cpu_buffer->resize_disabled);  in ring_buffer_unmap()
7658 	raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);  in ring_buffer_map_get_reader()
7664 	reader_size = rb_page_size(cpu_buffer->reader_page);  in ring_buffer_map_get_reader()
7671 	if (cpu_buffer->reader_page->read < reader_size) {  in ring_buffer_map_get_reader()
7672 		while (cpu_buffer->reader_page->read < reader_size)  in ring_buffer_map_get_reader()
7678 	if (cpu_buffer->reader_page == cpu_buffer->commit_page)  in ring_buffer_map_get_reader()
7686 	missed_events = cpu_buffer->lost_events;  in ring_buffer_map_get_reader()
7689 		if (cpu_buffer->reader_page != cpu_buffer->commit_page) {  in ring_buffer_map_get_reader()
7690 			struct buffer_data_page *bpage = reader->page;  in ring_buffer_map_get_reader()
7697 			if (reader->real_end)  in ring_buffer_map_get_reader()
7698 				local_set(&bpage->commit, reader->real_end);  in ring_buffer_map_get_reader()
7704 			if (buffer->subbuf_size - commit >= sizeof(missed_events)) {  in ring_buffer_map_get_reader()
7705 				memcpy(&bpage->data[commit], &missed_events,  in ring_buffer_map_get_reader()
7707 				local_add(RB_MISSED_STORED, &bpage->commit);  in ring_buffer_map_get_reader()
7709 			local_add(RB_MISSED_EVENTS, &bpage->commit);  in ring_buffer_map_get_reader()
7710 		} else if (!WARN_ONCE(cpu_buffer->reader_page == cpu_buffer->tail_page,  in ring_buffer_map_get_reader()
7725 				cpu, missed_events, cpu_buffer->reader_page->page->time_stamp);  in ring_buffer_map_get_reader()
7729 	cpu_buffer->lost_events = 0;  in ring_buffer_map_get_reader()
7734 	/* Some archs do not have data cache coherency between kernel and user-space */  in ring_buffer_map_get_reader()
7735 	flush_kernel_vmap_range(cpu_buffer->reader_page->page,  in ring_buffer_map_get_reader()
7736 				buffer->subbuf_size + BUF_PAGE_HDR_SIZE);  in ring_buffer_map_get_reader()
7740 	raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);  in ring_buffer_map_get_reader()
7765 	if (cpumask_test_cpu(cpu, buffer->cpumask))  in trace_rb_cpu_prepare()
7774 			nr_pages = buffer->buffers[cpu_i]->nr_pages;  in trace_rb_cpu_prepare()
7775 		if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {  in trace_rb_cpu_prepare()
7783 	buffer->buffers[cpu] =  in trace_rb_cpu_prepare()
7785 	if (!buffer->buffers[cpu]) {  in trace_rb_cpu_prepare()
7788 		return -ENOMEM;  in trace_rb_cpu_prepare()
7802 	cpumask_set_cpu(cpu, buffer->cpumask);  in trace_rb_cpu_prepare()
7810  * It will kick off a thread per CPU that will go into a loop
7870 	cnt = data->cnt + (nested ? 27 : 0);  in rb_write_something()
7873 	size = (cnt * 68 / 25) % (sizeof(rb_string) - 1);  in rb_write_something()
7881 	event = ring_buffer_lock_reserve(data->buffer, len);  in rb_write_something()
7886 				data->bytes_dropped_nested += len;  in rb_write_something()
7888 				data->bytes_dropped += len;  in rb_write_something()
7895 	if (RB_WARN_ON(data->buffer, event_len < len))  in rb_write_something()
7899 	item->size = size;  in rb_write_something()
7900 	memcpy(item->str, rb_string, size);  in rb_write_something()
7903 		data->bytes_alloc_nested += event_len;  in rb_write_something()
7904 		data->bytes_written_nested += len;  in rb_write_something()
7905 		data->events_nested++;  in rb_write_something()
7906 		if (!data->min_size_nested || len < data->min_size_nested)  in rb_write_something()
7907 			data->min_size_nested = len;  in rb_write_something()
7908 		if (len > data->max_size_nested)  in rb_write_something()
7909 			data->max_size_nested = len;  in rb_write_something()
7911 		data->bytes_alloc += event_len;  in rb_write_something()
7912 		data->bytes_written += len;  in rb_write_something()
7913 		data->events++;  in rb_write_something()
7914 		if (!data->min_size || len < data->min_size)  in rb_write_something()
7915 			data->max_size = len;  in rb_write_something()
7916 		if (len > data->max_size)  in rb_write_something()
7917 			data->max_size = len;  in rb_write_something()
7921 	ring_buffer_unlock_commit(data->buffer);  in rb_write_something()
7932 		data->cnt++;  in rb_test()
7935 		/* Now sleep between a min of 100-300us and a max of 1ms */  in rb_test()
7936 		usleep_range(((data->cnt % 3) + 1) * 100, 1000);  in rb_test()
8054 		ret = -1;  in test_ringbuffer()
8056 		total_events = data->events + data->events_nested;  in test_ringbuffer()
8057 		total_written = data->bytes_written + data->bytes_written_nested;  in test_ringbuffer()
8058 		total_alloc = data->bytes_alloc + data->bytes_alloc_nested;  in test_ringbuffer()
8059 		total_dropped = data->bytes_dropped + data->bytes_dropped_nested;  in test_ringbuffer()
8061 		big_event_size = data->max_size + data->max_size_nested;  in test_ringbuffer()
8062 		small_event_size = data->min_size + data->min_size_nested;  in test_ringbuffer()
8081 			total_size += item->size + sizeof(struct rb_item);  in test_ringbuffer()
8082 			if (memcmp(&item->str[0], rb_string, item->size) != 0) {  in test_ringbuffer()
8084 				pr_info("buffer had: %.*s\n", item->size, item->str);  in test_ringbuffer()
8085 				pr_info("expected:   %.*s\n", item->size, rb_string);  in test_ringbuffer()
8087 				ret = -1;  in test_ringbuffer()
8095 		ret = -1;  in test_ringbuffer()