20 static void perf_output_wakeup(struct perf_output_handle *handle)  in perf_output_wakeup()  argument
22 	atomic_set(&handle->rb->poll, EPOLLIN);  in perf_output_wakeup()
24 	handle->event->pending_wakeup = 1;  in perf_output_wakeup()
26 	if (*perf_event_fasync(handle->event) && !handle->event->pending_kill)  in perf_output_wakeup()
27 		handle->event->pending_kill = POLL_IN;  in perf_output_wakeup()
29 	irq_work_queue(&handle->event->pending_irq);  in perf_output_wakeup()
40 static void perf_output_get_handle(struct perf_output_handle *handle)  in perf_output_get_handle()  argument
42 	struct perf_buffer *rb = handle->rb;  in perf_output_get_handle()
51 	handle->wakeup = local_read(&rb->wakeup);  in perf_output_get_handle()
54 static void perf_output_put_handle(struct perf_output_handle *handle)  in perf_output_put_handle()  argument
56 	struct perf_buffer *rb = handle->rb;  in perf_output_put_handle()
134 	if (handle->wakeup != local_read(&rb->wakeup))  in perf_output_put_handle()
135 		perf_output_wakeup(handle);  in perf_output_put_handle()
153 __perf_output_begin(struct perf_output_handle *handle,  in __perf_output_begin()  argument
186 	handle->rb    = rb;  in __perf_output_begin()
187 	handle->event = event;  in __perf_output_begin()
196 	perf_output_get_handle(handle);  in __perf_output_begin()
242 	handle->page = (offset >> page_shift) & (rb->nr_pages - 1);  in __perf_output_begin()
244 	handle->addr = rb->data_pages[handle->page] + offset;  in __perf_output_begin()
245 	handle->size = (1UL << page_shift) - offset;  in __perf_output_begin()
256 		perf_output_put(handle, lost_event);  in __perf_output_begin()
257 		perf_event__output_id_sample(event, handle, data);  in __perf_output_begin()
265 	perf_output_put_handle(handle);  in __perf_output_begin()
272 int perf_output_begin_forward(struct perf_output_handle *handle,  in perf_output_begin_forward()  argument
276 	return __perf_output_begin(handle, data, event, size, false);  in perf_output_begin_forward()
279 int perf_output_begin_backward(struct perf_output_handle *handle,  in perf_output_begin_backward()  argument
283 	return __perf_output_begin(handle, data, event, size, true);  in perf_output_begin_backward()
286 int perf_output_begin(struct perf_output_handle *handle,  in perf_output_begin()  argument
291 	return __perf_output_begin(handle, data, event, size,  in perf_output_begin()
295 unsigned int perf_output_copy(struct perf_output_handle *handle,  in perf_output_copy()  argument
298 	return __output_copy(handle, buf, len);  in perf_output_copy()
301 unsigned int perf_output_skip(struct perf_output_handle *handle,  in perf_output_skip()  argument
304 	return __output_skip(handle, NULL, len);  in perf_output_skip()
307 void perf_output_end(struct perf_output_handle *handle)  in perf_output_end()  argument
309 	perf_output_put_handle(handle);  in perf_output_end()
344 void perf_aux_output_flag(struct perf_output_handle *handle, u64 flags)  in perf_aux_output_flag()  argument
353 	handle->aux_flags |= flags;  in perf_aux_output_flag()
371 void *perf_aux_output_begin(struct perf_output_handle *handle,  in perf_aux_output_begin()  argument
420 	handle->rb = rb;  in perf_aux_output_begin()
421 	handle->event = event;  in perf_aux_output_begin()
422 	handle->head = aux_head;  in perf_aux_output_begin()
423 	handle->size = 0;  in perf_aux_output_begin()
424 	handle->aux_flags = 0;  in perf_aux_output_begin()
433 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_begin()
435 			handle->size = CIRC_SPACE(aux_head, aux_tail, perf_aux_size(rb));  in perf_aux_output_begin()
442 		if (!handle->size) { /* A, matches D */  in perf_aux_output_begin()
444 			perf_output_wakeup(handle);  in perf_aux_output_begin()
450 	return handle->rb->aux_priv;  in perf_aux_output_begin()
458 	handle->event = NULL;  in perf_aux_output_begin()
487 void perf_aux_output_end(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_end()  argument
489 	bool wakeup = !!(handle->aux_flags & PERF_AUX_FLAG_TRUNCATED);  in perf_aux_output_end()
490 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_end()
495 		handle->aux_flags |= PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
497 		aux_head = handle->head;  in perf_aux_output_end()
500 		handle->aux_flags &= ~PERF_AUX_FLAG_OVERWRITE;  in perf_aux_output_end()
518 	if (size || (handle->aux_flags & ~(u64)PERF_AUX_FLAG_OVERWRITE))  in perf_aux_output_end()
519 		perf_event_aux_event(handle->event, aux_head, size,  in perf_aux_output_end()
520 				     handle->aux_flags);  in perf_aux_output_end()
527 		if (handle->aux_flags & PERF_AUX_FLAG_TRUNCATED)  in perf_aux_output_end()
528 			handle->event->pending_disable = smp_processor_id();  in perf_aux_output_end()
529 		perf_output_wakeup(handle);  in perf_aux_output_end()
532 	handle->event = NULL;  in perf_aux_output_end()
545 int perf_aux_output_skip(struct perf_output_handle *handle, unsigned long size)  in perf_aux_output_skip()  argument
547 	struct perf_buffer *rb = handle->rb;  in perf_aux_output_skip()
549 	if (size > handle->size)  in perf_aux_output_skip()
556 		perf_output_wakeup(handle);  in perf_aux_output_skip()
557 		handle->wakeup = rb->aux_wakeup + rb->aux_watermark;  in perf_aux_output_skip()
560 	handle->head = rb->aux_head;  in perf_aux_output_skip()
561 	handle->size -= size;  in perf_aux_output_skip()
567 void *perf_get_aux(struct perf_output_handle *handle)  in perf_get_aux()  argument
570 	if (!handle->event)  in perf_get_aux()
573 	return handle->rb->aux_priv;  in perf_get_aux()
581 			  struct perf_output_handle *handle,  in perf_output_copy_aux()  argument
601 		remainder = perf_output_copy(handle, addr, tocopy);  in perf_output_copy_aux()