87  * associated with the CPU it is currently executing on.  A reader may read
90  * The reader is special. For each per cpu buffer, the reader has its own
91  * reader page. When a reader has read the entire reader page, this reader
94  * Now, as long as the writer is off the reader page, the reader can do what
101  *   |reader|          RING BUFFER
112  *   |reader|          RING BUFFER
123  *   |reader|          RING BUFFER
139  *      |  Reader------^               |
144  * After we make this swap, the reader can hand this page off to the splice
747 	/* The reader can read an empty page, but not more than that */  in ring_buffer_nr_dirty_pages()
1127  * The reader page is always off the ring buffer, but when the
1128  * reader finishes with a page, it needs to swap its page with
1129  * a new one from the buffer. The reader needs to take from
1135  * The reader must be careful to replace only the head page, and
1137  * ASCII art, the reader sets its old page to point to the next
1139  * the old reader page. But if the writer moves the head page
1140  * during this operation, the reader could end up with the tail.
1173  *         R      Reader page
1178  *  What the above shows is that the reader just swapped out
1179  *  the reader page with a page in the buffer, but before it
1182  *  the new page added by the reader and is about to move forward
1213  * Because the reader may move the head_page pointer, we can
1215  * the reader page). But if the next page is a header page,
1234  * The unique thing about the reader page, is that, if the
1236  * back to the reader page.
1317 	/* check if the reader took the page */  in rb_head_page_set()
1596 		/* Include the reader page */  in rb_range_meta()
1823 	/* Do the reader page first */  in rb_meta_validate_events()
1826 		pr_info("Ring buffer reader page is invalid\n");  in rb_meta_validate_events()
1843 		/* Reader page has already been done */  in rb_meta_validate_events()
1883 	/* Reset the reader page */  in rb_meta_validate_events()
2373 		/* Need at least two pages plus the reader page */  in alloc_buffer()
2392 		/* nr_pages should not count the reader page */  in alloc_buffer()
2564 	 * We don't race with the readers since we have acquired the reader  in rb_remove_pages()
2575 	 * tail page might be on reader page, we remove the next page  in rb_remove_pages()
2670 	 * We are holding the reader lock, so the reader page won't be swapped  in rb_insert_pages()
2673 	 * We are going to adapt the reader page update process where:  in rb_insert_pages()
2803 		 * Don't succeed if resizing is disabled, as a reader might be  in ring_buffer_resize()
2889 		 * Don't succeed if resizing is disabled, as a reader might be  in ring_buffer_resize()
3092 	 * The iterator could be on the reader page (it starts there).  in rb_inc_iter()
3093 	 * But the head could have moved, since the reader was  in rb_inc_iter()
3135 				  struct buffer_page *reader)  in rb_update_meta_reader()  argument
3139 	void *new_reader = reader->page;  in rb_update_meta_reader()
3142 	id = reader->id;  in rb_update_meta_reader()
3144 	reader->id = 0;  in rb_update_meta_reader()
3149 	/* The head pointer is the one after the reader */  in rb_update_meta_reader()
3150 	rb_update_meta_head(cpu_buffer, reader);  in rb_update_meta_reader()
3186 	 *  MOVED  - a reader on another CPU moved the next  in rb_handle_head_page()
3187 	 *           pointer to its reader page. Give up  in rb_handle_head_page()
3227 		 * The reader is on another CPU and just did  in rb_handle_head_page()
3239 	 * set to UPDATE. This will keep the reader from  in rb_handle_head_page()
3240 	 * swapping the head page with the reader page.  in rb_handle_head_page()
3241 	 * The reader (on another CPU) will spin till  in rb_handle_head_page()
3334 		 * and the reader will ignore it.  in rb_reset_tail()
3347 	 * This will be used by the reader to add lost event  in rb_reset_tail()
3427 	 * We are fighting against races between a reader that  in rb_move_tail()
3428 	 * could be on another CPU trying to swap its reader  in rb_move_tail()
3436 	 * reader page.  in rb_move_tail()
3441 		 * If the commit is not on the reader page, then  in rb_move_tail()
3464 			 * commit page could still be on the reader  in rb_move_tail()
4576 	 * Because the commit page may be on the reader page we  in rb_decrement_entry()
4892 	 * if the tail is on reader_page, oldest time stamp is on the reader  in ring_buffer_oldest_event_ts()
5127 	struct buffer_page *reader;  in ring_buffer_iter_empty()  local
5136 	reader = cpu_buffer->reader_page;  in ring_buffer_iter_empty()
5162 		(iter->head_page == reader && commit_page == head_page &&  in ring_buffer_iter_empty()
5231 	struct buffer_page *reader = NULL;  in rb_get_reader_page()  local
5244 	 * start of the reader inserts an empty page, it causes  in rb_get_reader_page()
5249 		reader = NULL;  in rb_get_reader_page()
5253 	reader = cpu_buffer->reader_page;  in rb_get_reader_page()
5256 	if (cpu_buffer->reader_page->read < rb_page_size(reader))  in rb_get_reader_page()
5261 		       cpu_buffer->reader_page->read > rb_page_size(reader)))  in rb_get_reader_page()
5265 	reader = NULL;  in rb_get_reader_page()
5274 	 * Reset the reader page to size zero.  in rb_get_reader_page()
5283 	 * Splice the empty reader page into the list around the head.  in rb_get_reader_page()
5285 	reader = rb_set_head_page(cpu_buffer);  in rb_get_reader_page()
5286 	if (!reader)  in rb_get_reader_page()
5288 	cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);  in rb_get_reader_page()
5289 	cpu_buffer->reader_page->list.prev = reader->list.prev;  in rb_get_reader_page()
5296 	cpu_buffer->pages = reader->list.prev;  in rb_get_reader_page()
5298 	/* The reader page will be pointing to the new head */  in rb_get_reader_page()
5305 	 * side. Note, the reader will constantly fail the swap  in rb_get_reader_page()
5324 	ret = rb_head_page_replace(reader, cpu_buffer->reader_page);  in rb_get_reader_page()
5333 		rb_update_meta_reader(cpu_buffer, reader);  in rb_get_reader_page()
5338 	 * Now make the new head point back to the reader page.  in rb_get_reader_page()
5340 	rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;  in rb_get_reader_page()
5346 	/* Finally update the reader page to the new head */  in rb_get_reader_page()
5347 	cpu_buffer->reader_page = reader;  in rb_get_reader_page()
5359 	if (reader && reader->read == 0)  in rb_get_reader_page()
5360 		cpu_buffer->read_stamp = reader->page->time_stamp;  in rb_get_reader_page()
5372 		if (likely(!reader || rb_page_write(reader) <= bsize))  in rb_get_reader_page()
5377 		/* Get the latest version of the reader write value */  in rb_get_reader_page()
5383 		reader = NULL;  in rb_get_reader_page()
5389 	 * In addition, a writer may be writing on the reader page  in rb_get_reader_page()
5397 	return reader;  in rb_get_reader_page()
5403 	struct buffer_page *reader;  in rb_advance_reader()  local
5406 	reader = rb_get_reader_page(cpu_buffer);  in rb_advance_reader()
5409 	if (RB_WARN_ON(cpu_buffer, !reader))  in rb_advance_reader()
5463 	struct buffer_page *reader;  in rb_buffer_peek()  local
5478 	reader = rb_get_reader_page(cpu_buffer);  in rb_buffer_peek()
5479 	if (!reader)  in rb_buffer_peek()
5506 			*ts = rb_fix_abs_ts(*ts, reader->page->time_stamp);  in rb_buffer_peek()
5957 	meta->reader.read = cpu_buffer->reader_page->read;  in rb_update_meta_page()
5958 	meta->reader.id = cpu_buffer->reader_page->id;  in rb_update_meta_page()
5959 	meta->reader.lost_events = cpu_buffer->lost_events;  in rb_update_meta_page()
6448  * the writer is off the reader page.
6466 	struct buffer_page *reader;  in ring_buffer_read_page()  local
6497 	reader = rb_get_reader_page(cpu_buffer);  in ring_buffer_read_page()
6498 	if (!reader)  in ring_buffer_read_page()
6503 	read = reader->read;  in ring_buffer_read_page()
6504 	commit = rb_page_size(reader);  in ring_buffer_read_page()
6528 		 * the reader page.  in ring_buffer_read_page()
6561 			rpos = reader->read;  in ring_buffer_read_page()
6580 		cpu_buffer->read += rb_page_entries(reader);  in ring_buffer_read_page()
6581 		cpu_buffer->read_bytes += rb_page_size(reader);  in ring_buffer_read_page()
6585 		bpage = reader->page;  in ring_buffer_read_page()
6586 		reader->page = data_page->data;  in ring_buffer_read_page()
6587 		local_set(&reader->write, 0);  in ring_buffer_read_page()
6588 		local_set(&reader->entries, 0);  in ring_buffer_read_page()
6589 		reader->read = 0;  in ring_buffer_read_page()
6597 		if (reader->real_end)  in ring_buffer_read_page()
6598 			local_set(&bpage->commit, reader->real_end);  in ring_buffer_read_page()
6756 		/* Include the reader page */  in ring_buffer_subbuf_order_set()
6794 		/* One page was allocated for the reader page */  in ring_buffer_subbuf_order_set()
7018 	nr_subbufs = cpu_buffer->nr_pages + 1; /* + reader-subbuf */  in __rb_map_vma()
7122 	/* subbuf_ids include the reader while nr_pages does not */  in ring_buffer_map()
7209 	struct buffer_page *reader;  in ring_buffer_map_get_reader()  local
7227 	 * There are data to be read on the current reader page, we can  in ring_buffer_map_get_reader()
7229 	 * everything. Let's update the kernel reader accordingly.  in ring_buffer_map_get_reader()
7237 	reader = rb_get_reader_page(cpu_buffer);  in ring_buffer_map_get_reader()
7238 	if (WARN_ON(!reader))  in ring_buffer_map_get_reader()
7246 			struct buffer_data_page *bpage = reader->page;  in ring_buffer_map_get_reader()
7253 			if (reader->real_end)  in ring_buffer_map_get_reader()
7254 				local_set(&bpage->commit, reader->real_end);  in ring_buffer_map_get_reader()
7259 			commit = rb_page_size(reader);  in ring_buffer_map_get_reader()
7270 		 * is on the reader page.  in ring_buffer_map_get_reader()