85 static bool simple_rb_loaded(struct simple_rb_per_cpu *cpu_buffer)  in simple_rb_loaded()  argument
87 	return !!cpu_buffer->bpages;  in simple_rb_loaded()
90 static int simple_rb_find_head(struct simple_rb_per_cpu *cpu_buffer)  in simple_rb_find_head()  argument
92 	int retry = cpu_buffer->nr_pages * 2;  in simple_rb_find_head()
95 	head = cpu_buffer->head_page;  in simple_rb_find_head()
107 			cpu_buffer->head_page = head;  in simple_rb_find_head()
130 int simple_ring_buffer_swap_reader_page(struct simple_rb_per_cpu *cpu_buffer)  in simple_ring_buffer_swap_reader_page()  argument
137 	if (!simple_rb_loaded(cpu_buffer))  in simple_ring_buffer_swap_reader_page()
140 	reader = cpu_buffer->reader_page;  in simple_ring_buffer_swap_reader_page()
144 		ret = simple_rb_find_head(cpu_buffer);  in simple_ring_buffer_swap_reader_page()
148 		head = cpu_buffer->head_page;  in simple_ring_buffer_swap_reader_page()
160 		overrun = cpu_buffer->meta->overrun;  in simple_ring_buffer_swap_reader_page()
166 	cpu_buffer->head_page = simple_bpage_from_link(reader->link.next);  in simple_ring_buffer_swap_reader_page()
167 	cpu_buffer->head_page->link.prev = &reader->link;  in simple_ring_buffer_swap_reader_page()
168 	cpu_buffer->reader_page = head;  in simple_ring_buffer_swap_reader_page()
169 	cpu_buffer->meta->reader.lost_events = overrun - cpu_buffer->last_overrun;  in simple_ring_buffer_swap_reader_page()
170 	cpu_buffer->meta->reader.id = cpu_buffer->reader_page->id;  in simple_ring_buffer_swap_reader_page()
171 	cpu_buffer->last_overrun = overrun;  in simple_ring_buffer_swap_reader_page()
177 static struct simple_buffer_page *simple_rb_move_tail(struct simple_rb_per_cpu *cpu_buffer)  in simple_rb_move_tail()  argument
181 	tail = cpu_buffer->tail_page;  in simple_rb_move_tail()
191 		simple_rb_meta_inc(cpu_buffer->meta->overrun, new_tail->entries);  in simple_rb_move_tail()
192 		simple_rb_meta_inc(cpu_buffer->meta->pages_lost, 1);  in simple_rb_move_tail()
199 	cpu_buffer->tail_page = new_tail;  in simple_rb_move_tail()
201 	simple_rb_meta_inc(cpu_buffer->meta->pages_touched, 1);  in simple_rb_move_tail()
224 simple_rb_reserve_next(struct simple_rb_per_cpu *cpu_buffer, unsigned long length, u64 timestamp)  in simple_rb_reserve_next()  argument
227 	struct simple_buffer_page *tail = cpu_buffer->tail_page;  in simple_rb_reserve_next()
232 	time_delta = timestamp - cpu_buffer->write_stamp;  in simple_rb_reserve_next()
241 		tail = simple_rb_move_tail(cpu_buffer);  in simple_rb_reserve_next()
254 	cpu_buffer->write_stamp = timestamp;  in simple_rb_reserve_next()
277 void *simple_ring_buffer_reserve(struct simple_rb_per_cpu *cpu_buffer, unsigned long length,  in simple_ring_buffer_reserve()  argument
282 	if (cmpxchg(&cpu_buffer->status, SIMPLE_RB_READY, SIMPLE_RB_WRITING) != SIMPLE_RB_READY)  in simple_ring_buffer_reserve()
285 	rb_event = simple_rb_reserve_next(cpu_buffer, length, timestamp);  in simple_ring_buffer_reserve()
295 void simple_ring_buffer_commit(struct simple_rb_per_cpu *cpu_buffer)  in simple_ring_buffer_commit()  argument
297 	local_set(&cpu_buffer->tail_page->page->commit,  in simple_ring_buffer_commit()
298 		  cpu_buffer->tail_page->write);  in simple_ring_buffer_commit()
299 	simple_rb_meta_inc(cpu_buffer->meta->entries, 1);  in simple_ring_buffer_commit()
305 	smp_store_release(&cpu_buffer->status, SIMPLE_RB_READY);  in simple_ring_buffer_commit()
309 static u32 simple_rb_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable)  in simple_rb_enable_tracing()  argument
314 		return cmpxchg(&cpu_buffer->status, SIMPLE_RB_UNAVAILABLE, SIMPLE_RB_READY);  in simple_rb_enable_tracing()
318 		prev_status = cmpxchg_acquire(&cpu_buffer->status,  in simple_rb_enable_tracing()
334 int simple_ring_buffer_reset(struct simple_rb_per_cpu *cpu_buffer)  in simple_ring_buffer_reset()  argument
340 	if (!simple_rb_loaded(cpu_buffer))  in simple_ring_buffer_reset()
343 	prev_status = simple_rb_enable_tracing(cpu_buffer, false);  in simple_ring_buffer_reset()
345 	ret = simple_rb_find_head(cpu_buffer);  in simple_ring_buffer_reset()
349 	bpage = cpu_buffer->tail_page = cpu_buffer->head_page;  in simple_ring_buffer_reset()
353 	} while (bpage != cpu_buffer->head_page);  in simple_ring_buffer_reset()
355 	simple_bpage_reset(cpu_buffer->reader_page);  in simple_ring_buffer_reset()
357 	cpu_buffer->last_overrun = 0;  in simple_ring_buffer_reset()
358 	cpu_buffer->write_stamp = 0;  in simple_ring_buffer_reset()
360 	cpu_buffer->meta->reader.read = 0;  in simple_ring_buffer_reset()
361 	cpu_buffer->meta->reader.lost_events = 0;  in simple_ring_buffer_reset()
362 	cpu_buffer->meta->entries = 0;  in simple_ring_buffer_reset()
363 	cpu_buffer->meta->overrun = 0;  in simple_ring_buffer_reset()
364 	cpu_buffer->meta->read = 0;  in simple_ring_buffer_reset()
365 	cpu_buffer->meta->pages_lost = 0;  in simple_ring_buffer_reset()
366 	cpu_buffer->meta->pages_touched = 0;  in simple_ring_buffer_reset()
369 		simple_rb_enable_tracing(cpu_buffer, true);  in simple_ring_buffer_reset()
375 int simple_ring_buffer_init_mm(struct simple_rb_per_cpu *cpu_buffer,  in simple_ring_buffer_init_mm()  argument
390 	memset(cpu_buffer, 0, sizeof(*cpu_buffer));  in simple_ring_buffer_init_mm()
392 	cpu_buffer->meta = load_page(desc->meta_va);  in simple_ring_buffer_init_mm()
393 	if (!cpu_buffer->meta)  in simple_ring_buffer_init_mm()
396 	memset(cpu_buffer->meta, 0, sizeof(*cpu_buffer->meta));  in simple_ring_buffer_init_mm()
397 	cpu_buffer->meta->meta_page_size = PAGE_SIZE;  in simple_ring_buffer_init_mm()
402 		unload_page(cpu_buffer->meta);  in simple_ring_buffer_init_mm()
409 	cpu_buffer->nr_pages = 1;  in simple_ring_buffer_init_mm()
411 	cpu_buffer->reader_page = bpage;  in simple_ring_buffer_init_mm()
412 	cpu_buffer->tail_page = bpage + 1;  in simple_ring_buffer_init_mm()
413 	cpu_buffer->head_page = bpage + 1;  in simple_ring_buffer_init_mm()
428 		cpu_buffer->nr_pages = i + 1;  in simple_ring_buffer_init_mm()
434 		unload_page(cpu_buffer->meta);  in simple_ring_buffer_init_mm()
439 	cpu_buffer->meta->nr_subbufs = cpu_buffer->nr_pages;  in simple_ring_buffer_init_mm()
441 	bpage->link.next = &cpu_buffer->tail_page->link;  in simple_ring_buffer_init_mm()
442 	cpu_buffer->tail_page->link.prev = &bpage->link;  in simple_ring_buffer_init_mm()
447 	cpu_buffer->bpages = bpages;  in simple_ring_buffer_init_mm()
467 int simple_ring_buffer_init(struct simple_rb_per_cpu *cpu_buffer, struct simple_buffer_page *bpages,  in simple_ring_buffer_init()  argument
470 	return simple_ring_buffer_init_mm(cpu_buffer, bpages, desc, __load_page, __unload_page);  in simple_ring_buffer_init()
474 void simple_ring_buffer_unload_mm(struct simple_rb_per_cpu *cpu_buffer,  in simple_ring_buffer_unload_mm()  argument
479 	if (!simple_rb_loaded(cpu_buffer))  in simple_ring_buffer_unload_mm()
482 	simple_rb_enable_tracing(cpu_buffer, false);  in simple_ring_buffer_unload_mm()
484 	unload_page(cpu_buffer->meta);  in simple_ring_buffer_unload_mm()
485 	for (p = 0; p < cpu_buffer->nr_pages; p++)  in simple_ring_buffer_unload_mm()
486 		unload_page(cpu_buffer->bpages[p].page);  in simple_ring_buffer_unload_mm()
488 	cpu_buffer->bpages = NULL;  in simple_ring_buffer_unload_mm()
495 void simple_ring_buffer_unload(struct simple_rb_per_cpu *cpu_buffer)  in simple_ring_buffer_unload()  argument
497 	return simple_ring_buffer_unload_mm(cpu_buffer, __unload_page);  in simple_ring_buffer_unload()
508 int simple_ring_buffer_enable_tracing(struct simple_rb_per_cpu *cpu_buffer, bool enable)  in simple_ring_buffer_enable_tracing()  argument
510 	if (!simple_rb_loaded(cpu_buffer))  in simple_ring_buffer_enable_tracing()
513 	simple_rb_enable_tracing(cpu_buffer, enable);  in simple_ring_buffer_enable_tracing()