30 static DEFINE_PER_CPU(struct pt, pt_ctx);
106 PMU_FORMAT_ATTR(pt,		"config:0"	);
418 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_start()  local
425 	if (READ_ONCE(pt->vmx_on))  in pt_config_start()
426 		perf_aux_output_flag(&pt->handle, PERF_AUX_FLAG_PARTIAL);  in pt_config_start()
464 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_filters()  local
486 		if (pt->filters.filter[range].msr_a != filter->msr_a) {  in pt_config_filters()
488 			pt->filters.filter[range].msr_a = filter->msr_a;  in pt_config_filters()
491 		if (pt->filters.filter[range].msr_b != filter->msr_b) {  in pt_config_filters()
493 			pt->filters.filter[range].msr_b = filter->msr_b;  in pt_config_filters()
504 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config()  local
505 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_config()
546 	WRITE_ONCE(pt->resume_allowed, 1);  in pt_config()
549 	WRITE_ONCE(pt->handle_nmi, 1);  in pt_config()
557 	WRITE_ONCE(pt->pause_allowed, 1);  in pt_config()
562 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_stop()  local
570 	if (!READ_ONCE(pt->vmx_on))  in pt_config_stop()
645 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_config_buffer()  local
658 	if (pt->output_base != reg) {  in pt_config_buffer()
659 		pt->output_base = reg;  in pt_config_buffer()
664 	if (pt->output_mask != reg) {  in pt_config_buffer()
665 		pt->output_mask = reg;  in pt_config_buffer()
868 static void pt_update_head(struct pt *pt)  in pt_update_head()  argument
870 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_update_head()
922 static void pt_handle_status(struct pt *pt)  in pt_handle_status()  argument
924 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_handle_status()
947 			perf_aux_output_flag(&pt->handle,  in pt_handle_status()
983 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_read_offset()  local
987 		rdmsrl(MSR_IA32_RTIT_OUTPUT_BASE, pt->output_base);  in pt_read_offset()
988 		tp = phys_to_virt(pt->output_base);  in pt_read_offset()
992 	rdmsrl(MSR_IA32_RTIT_OUTPUT_MASK, pt->output_mask);  in pt_read_offset()
994 	buf->output_off = pt->output_mask >> 32;  in pt_read_offset()
997 		buf->cur_idx = (pt->output_mask & 0xffffff80) >> 7;  in pt_read_offset()
1504 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_interrupt()  local
1506 	struct perf_event *event = pt->handle.event;  in intel_pt_interrupt()
1513 	if (!READ_ONCE(pt->handle_nmi))  in intel_pt_interrupt()
1521 	buf = perf_get_aux(&pt->handle);  in intel_pt_interrupt()
1527 	pt_handle_status(pt);  in intel_pt_interrupt()
1529 	pt_update_head(pt);  in intel_pt_interrupt()
1531 	perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in intel_pt_interrupt()
1536 		buf = perf_aux_output_begin(&pt->handle, event);  in intel_pt_interrupt()
1539 			WRITE_ONCE(pt->resume_allowed, 0);  in intel_pt_interrupt()
1543 		pt_buffer_reset_offsets(buf, pt->handle.head);  in intel_pt_interrupt()
1545 		ret = pt_buffer_reset_markers(buf, &pt->handle);  in intel_pt_interrupt()
1547 			perf_aux_output_end(&pt->handle, 0);  in intel_pt_interrupt()
1548 			WRITE_ONCE(pt->resume_allowed, 0);  in intel_pt_interrupt()
1559 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in intel_pt_handle_vmx()  local
1574 	WRITE_ONCE(pt->vmx_on, on);  in intel_pt_handle_vmx()
1580 	event = pt->handle.event;  in intel_pt_handle_vmx()
1582 		perf_aux_output_flag(&pt->handle,  in intel_pt_handle_vmx()
1600 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_start()  local
1604 		if (READ_ONCE(pt->resume_allowed)) {  in pt_event_start()
1617 			   READ_ONCE(pt->resume_allowed))  in pt_event_start()
1623 	buf = perf_aux_output_begin(&pt->handle, event);  in pt_event_start()
1627 	pt_buffer_reset_offsets(buf, pt->handle.head);  in pt_event_start()
1629 		if (pt_buffer_reset_markers(buf, &pt->handle))  in pt_event_start()
1641 	perf_aux_output_end(&pt->handle, 0);  in pt_event_start()
1648 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_stop()  local
1651 		if (READ_ONCE(pt->pause_allowed))  in pt_event_stop()
1660 	WRITE_ONCE(pt->handle_nmi, 0);  in pt_event_stop()
1668 	WRITE_ONCE(pt->pause_allowed, 0);  in pt_event_stop()
1669 	WRITE_ONCE(pt->resume_allowed, 0);  in pt_event_stop()
1680 		struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_stop()
1685 		if (WARN_ON_ONCE(pt->handle.event != event))  in pt_event_stop()
1690 		pt_handle_status(pt);  in pt_event_stop()
1692 		pt_update_head(pt);  in pt_event_stop()
1695 			pt->handle.head =  in pt_event_stop()
1698 		perf_aux_output_end(&pt->handle, local_xchg(&buf->data_size, 0));  in pt_event_stop()
1706 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_snapshot_aux()  local
1707 	struct pt_buffer *buf = perf_get_aux(&pt->handle);  in pt_event_snapshot_aux()
1722 	WRITE_ONCE(pt->pause_allowed, 0);  in pt_event_snapshot_aux()
1723 	WRITE_ONCE(pt->resume_allowed, 0);  in pt_event_snapshot_aux()
1731 	pt_update_head(pt);  in pt_event_snapshot_aux()
1738 	ret = perf_output_copy_aux(&pt->handle, handle, from, to);  in pt_event_snapshot_aux()
1744 	if (READ_ONCE(pt->handle_nmi)) {  in pt_event_snapshot_aux()
1745 		WRITE_ONCE(pt->resume_allowed, 1);  in pt_event_snapshot_aux()
1749 		WRITE_ONCE(pt->pause_allowed, 1);  in pt_event_snapshot_aux()
1762 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in pt_event_add()  local
1766 	if (pt->handle.event)  in pt_event_add()
1817 	struct pt *pt = this_cpu_ptr(&pt_ctx);  in cpu_emergency_stop_pt()  local
1819 	if (pt->handle.event)  in cpu_emergency_stop_pt()
1820 		pt_event_stop(pt->handle.event, PERF_EF_UPDATE);  in cpu_emergency_stop_pt()