84 	struct bts_buffer *bb;  in bts_buffer_setup_aux()  local
105 	bb = kzalloc_node(struct_size(bb, buf, nr_buf), GFP_KERNEL, node);  in bts_buffer_setup_aux()
106 	if (!bb)  in bts_buffer_setup_aux()
109 	bb->nr_pages = nr_pages;  in bts_buffer_setup_aux()
110 	bb->nr_bufs = nr_buf;  in bts_buffer_setup_aux()
111 	bb->snapshot = overwrite;  in bts_buffer_setup_aux()
112 	bb->data_pages = pages;  in bts_buffer_setup_aux()
113 	bb->real_size = size - size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
115 	for (pg = 0, nr_buf = 0, offset = 0, pad = 0; nr_buf < bb->nr_bufs; nr_buf++) {  in bts_buffer_setup_aux()
120 		bb->buf[nr_buf].page = page;  in bts_buffer_setup_aux()
121 		bb->buf[nr_buf].offset = offset;  in bts_buffer_setup_aux()
122 		bb->buf[nr_buf].displacement = (pad ? BTS_RECORD_SIZE - pad : 0);  in bts_buffer_setup_aux()
123 		bb->buf[nr_buf].size = buf_size(page) - bb->buf[nr_buf].displacement;  in bts_buffer_setup_aux()
124 		pad = bb->buf[nr_buf].size % BTS_RECORD_SIZE;  in bts_buffer_setup_aux()
125 		bb->buf[nr_buf].size -= pad;  in bts_buffer_setup_aux()
131 	return bb;  in bts_buffer_setup_aux()
139 static unsigned long bts_buffer_offset(struct bts_buffer *bb, unsigned int idx)  in bts_buffer_offset()  argument
141 	return bb->buf[idx].offset + bb->buf[idx].displacement;  in bts_buffer_offset()
145 bts_config_buffer(struct bts_buffer *bb)  in bts_config_buffer()  argument
149 	struct bts_phys *phys = &bb->buf[bb->cur_buf];  in bts_config_buffer()
153 	index = local_read(&bb->head);  in bts_config_buffer()
155 	if (!bb->snapshot) {  in bts_config_buffer()
156 		if (bb->end < phys->offset + buf_size(page))  in bts_config_buffer()
157 			end = bb->end - phys->offset - phys->displacement;  in bts_config_buffer()
172 	ds->bts_interrupt_threshold = !bb->snapshot  in bts_config_buffer()
188 	struct bts_buffer *bb = perf_get_aux(&bts->handle);  in bts_update()  local
191 	if (!bb)  in bts_update()
194 	head = index + bts_buffer_offset(bb, bb->cur_buf);  in bts_update()
195 	old = local_xchg(&bb->head, head);  in bts_update()
197 	if (!bb->snapshot) {  in bts_update()
209 		local_add(head - old, &bb->data_size);  in bts_update()
211 		local_set(&bb->data_size, head);  in bts_update()
222 bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle);
236 	struct bts_buffer *bb = perf_get_aux(&bts->handle);  in __bts_event_start()  local
239 	if (!bb->snapshot)  in __bts_event_start()
246 	bts_config_buffer(bb);  in __bts_event_start()
265 	struct bts_buffer *bb;  in bts_event_start()  local
267 	bb = perf_aux_output_begin(&bts->handle, event);  in bts_event_start()
268 	if (!bb)  in bts_event_start()
271 	if (bts_buffer_reset(bb, &bts->handle))  in bts_event_start()
310 	struct bts_buffer *bb = NULL;  in bts_event_stop()  local
317 		bb = perf_get_aux(&bts->handle);  in bts_event_stop()
324 		if (bb) {  in bts_event_stop()
325 			if (bb->snapshot)  in bts_event_stop()
327 					local_xchg(&bb->data_size,  in bts_event_stop()
328 						   bb->nr_pages << PAGE_SHIFT);  in bts_event_stop()
330 					    local_xchg(&bb->data_size, 0));  in bts_event_stop()
386 bts_buffer_reset(struct bts_buffer *bb, struct perf_output_handle *handle)  in bts_buffer_reset()  argument
393 	if (bb->snapshot)  in bts_buffer_reset()
396 	head = handle->head & ((bb->nr_pages << PAGE_SHIFT) - 1);  in bts_buffer_reset()
398 	phys = &bb->buf[bb->cur_buf];  in bts_buffer_reset()
407 		next_buf = bb->cur_buf + 1;  in bts_buffer_reset()
408 		if (next_buf >= bb->nr_bufs)  in bts_buffer_reset()
410 		next_phys = &bb->buf[next_buf];  in bts_buffer_reset()
435 				bb->cur_buf = next_buf;  in bts_buffer_reset()
436 				local_set(&bb->head, head);  in bts_buffer_reset()
449 	bb->end = head + space;  in bts_buffer_reset()
466 	struct bts_buffer *bb;  in intel_bts_interrupt()  local
489 	bb = perf_get_aux(&bts->handle);  in intel_bts_interrupt()
490 	if (!bb)  in intel_bts_interrupt()
498 	if (bb->snapshot)  in intel_bts_interrupt()
501 	old_head = local_read(&bb->head);  in intel_bts_interrupt()
505 	if (old_head == local_read(&bb->head))  in intel_bts_interrupt()
508 	perf_aux_output_end(&bts->handle, local_xchg(&bb->data_size, 0));  in intel_bts_interrupt()
510 	bb = perf_aux_output_begin(&bts->handle, event);  in intel_bts_interrupt()
511 	if (bb)  in intel_bts_interrupt()
512 		err = bts_buffer_reset(bb, &bts->handle);  in intel_bts_interrupt()
517 		if (bb) {  in intel_bts_interrupt()