107 	struct cs_etm_auxtrace *etm;  member
129 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm);
130 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,
135 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu);
231 	return etmq->etm->pid_fmt;  in cs_etm__get_pid_fmt()
256 			if (etmq->etm->per_thread_decoding)  in cs_etm__insert_trace_id_node()
284 static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int cpu)  in cs_etm__get_queue()  argument
286 	if (etm->per_thread_decoding)  in cs_etm__get_queue()
287 		return etm->queues.queue_array[0].priv;  in cs_etm__get_queue()
289 		return etm->queues.queue_array[cpu].priv;  in cs_etm__get_queue()
292 static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id,  in cs_etm__map_trace_id_v0()  argument
301 	etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]);  in cs_etm__map_trace_id_v0()
310 	for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) {  in cs_etm__map_trace_id_v0()
313 		etmq = etm->queues.queue_array[i].priv;  in cs_etm__map_trace_id_v0()
323 static int cs_etm__process_trace_id_v0(struct cs_etm_auxtrace *etm, int cpu,  in cs_etm__process_trace_id_v0()  argument
330 	cpu_data = get_cpu_data(etm, cpu);  in cs_etm__process_trace_id_v0()
334 	err = cs_etm__map_trace_id_v0(etm, trace_chan_id, cpu_data);  in cs_etm__process_trace_id_v0()
345 static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu,  in cs_etm__process_trace_id_v0_1()  argument
348 	struct cs_etm_queue *etmq = cs_etm__get_queue(etm, cpu);  in cs_etm__process_trace_id_v0_1()
359 	if (!etmq->etm->per_thread_decoding && etmq->sink_id != SINK_UNSET &&  in cs_etm__process_trace_id_v0_1()
368 	for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) {  in cs_etm__process_trace_id_v0_1()
369 		struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv;  in cs_etm__process_trace_id_v0_1()
391 	cpu_data = get_cpu_data(etm, cpu);  in cs_etm__process_trace_id_v0_1()
449 static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu)  in get_cpu_data_idx()  argument
453 	for (i = 0; i < etm->num_cpu; i++) {  in get_cpu_data_idx()
454 		if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) {  in get_cpu_data_idx()
466 static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu)  in get_cpu_data()  argument
468 	int idx = get_cpu_data_idx(etm, cpu);  in get_cpu_data()
470 	return (idx != -1) ? etm->metadata[idx] : NULL;  in get_cpu_data()
483 	struct cs_etm_auxtrace *etm;  in cs_etm__process_aux_output_hw_id()  local
501 	etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace);  in cs_etm__process_aux_output_hw_id()
502 	if (!etm || !etm->metadata)  in cs_etm__process_aux_output_hw_id()
520 		return cs_etm__process_trace_id_v0(etm, cpu, hw_id);  in cs_etm__process_aux_output_hw_id()
522 	return cs_etm__process_trace_id_v0_1(etm, cpu, hw_id);  in cs_etm__process_aux_output_hw_id()
605 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__init_traceid_queue()  local
609 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];  in cs_etm__init_traceid_queue()
612 	tidq->thread = machine__findnew_thread(&etm->session->machines.host, -1,  in cs_etm__init_traceid_queue()
614 	tidq->prev_packet_thread = machine__idle_thread(&etm->session->machines.host);  in cs_etm__init_traceid_queue()
624 	if (etm->synth_opts.last_branch) {  in cs_etm__init_traceid_queue()
627 		sz += etm->synth_opts.last_branch_sz *  in cs_etm__init_traceid_queue()
659 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__etmq_get_traceid_queue()  local
661 	if (etm->per_thread_decoding)  in cs_etm__etmq_get_traceid_queue()
737 static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm,  in cs_etm__packet_swap()  argument
742 	if (etm->synth_opts.branches || etm->synth_opts.last_branch ||  in cs_etm__packet_swap()
743 	    etm->synth_opts.instructions) {  in cs_etm__packet_swap()
900 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__flush_events()  local
909 	if (etm->timeless_decoding) {  in cs_etm__flush_events()
914 		return cs_etm__process_timeless_queues(etm, -1);  in cs_etm__flush_events()
917 	return cs_etm__process_timestamped_queues(etm);  in cs_etm__flush_events()
1033 		return &etmq->etm->session->machines.host;  in cs_etm__get_machine()
1045 		return machines__find_guest(&etmq->etm->session->machines,  in cs_etm__get_machine()
1052 		return &etmq->etm->session->machines.host;  in cs_etm__get_machine()
1183 static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm,  in cs_etm__setup_queue()  argument
1198 	etmq->etm = etm;  in cs_etm__setup_queue()
1207 static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm,  in cs_etm__queue_first_cs_timestamp()  argument
1273 	ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);  in cs_etm__queue_first_cs_timestamp()
1303 	nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos;  in cs_etm__copy_last_branch_rb()
1315 	if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) {  in cs_etm__copy_last_branch_rb()
1400 		tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz;  in cs_etm__update_last_branch_rb()
1415 	if (bs->nr < etmq->etm->synth_opts.last_branch_sz)  in cs_etm__update_last_branch_rb()
1434 	queue = &etmq->etm->queues.queue_array[etmq->queue_nr];  in cs_etm__get_trace()
1451 		int fd = perf_data__fd(etmq->etm->session->data);  in cs_etm__get_trace()
1502 	return !!etmq->etm->timeless_decoding;  in cs_etm__etmq_is_timeless()
1536 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__convert_sample_time()  local
1538 	if (etm->has_virtual_ts)  in cs_etm__convert_sample_time()
1539 		return tsc_to_perf_time(cs_timestamp, &etm->tc);  in cs_etm__convert_sample_time()
1547 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__resolve_sample_time()  local
1550 	if (!etm->timeless_decoding && etm->has_virtual_ts)  in cs_etm__resolve_sample_time()
1553 		return etm->latest_kernel_timestamp;  in cs_etm__resolve_sample_time()
1561 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__synth_instruction_sample()  local
1575 	sample.id = etmq->etm->instructions_id;  in cs_etm__synth_instruction_sample()
1576 	sample.stream_id = etmq->etm->instructions_id;  in cs_etm__synth_instruction_sample()
1584 	if (etm->synth_opts.last_branch)  in cs_etm__synth_instruction_sample()
1587 	if (etm->synth_opts.inject) {  in cs_etm__synth_instruction_sample()
1589 					   etm->instructions_sample_type);  in cs_etm__synth_instruction_sample()
1594 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);  in cs_etm__synth_instruction_sample()
1612 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__synth_branch_sample()  local
1636 	sample.id = etmq->etm->branches_id;  in cs_etm__synth_branch_sample()
1637 	sample.stream_id = etmq->etm->branches_id;  in cs_etm__synth_branch_sample()
1649 	if (etm->synth_opts.last_branch) {  in cs_etm__synth_branch_sample()
1661 	if (etm->synth_opts.inject) {  in cs_etm__synth_branch_sample()
1663 					   etm->branches_sample_type);  in cs_etm__synth_branch_sample()
1668 	ret = perf_session__deliver_synth_event(etm->session, event, &sample);  in cs_etm__synth_branch_sample()
1678 static int cs_etm__synth_events(struct cs_etm_auxtrace *etm,  in cs_etm__synth_events()  argument
1689 		if (evsel->core.attr.type == etm->pmu_type) {  in cs_etm__synth_events()
1706 	if (etm->timeless_decoding)  in cs_etm__synth_events()
1725 	if (etm->synth_opts.branches) {  in cs_etm__synth_events()
1732 		etm->branches_sample_type = attr.sample_type;  in cs_etm__synth_events()
1733 		etm->branches_id = id;  in cs_etm__synth_events()
1738 	if (etm->synth_opts.last_branch) {  in cs_etm__synth_events()
1748 	if (etm->synth_opts.instructions) {  in cs_etm__synth_events()
1750 		attr.sample_period = etm->synth_opts.period;  in cs_etm__synth_events()
1751 		etm->instructions_sample_period = attr.sample_period;  in cs_etm__synth_events()
1755 		etm->instructions_sample_type = attr.sample_type;  in cs_etm__synth_events()
1756 		etm->instructions_id = id;  in cs_etm__synth_events()
1766 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__sample()  local
1780 	if (etm->synth_opts.last_branch &&  in cs_etm__sample()
1785 	if (etm->synth_opts.instructions &&  in cs_etm__sample()
1786 	    tidq->period_instructions >= etm->instructions_sample_period) {  in cs_etm__sample()
1839 		u64 offset = etm->instructions_sample_period - instrs_prev;  in cs_etm__sample()
1843 		if (etm->synth_opts.last_branch)  in cs_etm__sample()
1847 				etm->instructions_sample_period) {  in cs_etm__sample()
1858 				etm->instructions_sample_period);  in cs_etm__sample()
1862 			offset += etm->instructions_sample_period;  in cs_etm__sample()
1864 				etm->instructions_sample_period;  in cs_etm__sample()
1868 	if (etm->synth_opts.branches) {  in cs_etm__sample()
1887 	cs_etm__packet_swap(etm, tidq);  in cs_etm__sample()
1915 	struct cs_etm_auxtrace *etm = etmq->etm;  in cs_etm__flush()  local
1921 	if (etmq->etm->synth_opts.last_branch &&  in cs_etm__flush()
1922 	    etmq->etm->synth_opts.instructions &&  in cs_etm__flush()
1948 	if (etm->synth_opts.branches &&  in cs_etm__flush()
1956 	cs_etm__packet_swap(etm, tidq);  in cs_etm__flush()
1959 	if (etm->synth_opts.last_branch)  in cs_etm__flush()
1979 	if (etmq->etm->synth_opts.last_branch &&  in cs_etm__end_block()
1980 	    etmq->etm->synth_opts.instructions &&  in cs_etm__end_block()
2579 static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm,  in cs_etm__process_timeless_queues()  argument
2583 	struct auxtrace_queues *queues = &etm->queues;  in cs_etm__process_timeless_queues()
2586 		struct auxtrace_queue *queue = &etm->queues.queue_array[i];  in cs_etm__process_timeless_queues()
2593 		if (etm->per_thread_decoding) {  in cs_etm__process_timeless_queues()
2609 static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm)  in cs_etm__process_timestamped_queues()  argument
2623 	for (i = 0; i < etm->queues.nr_queues; i++) {  in cs_etm__process_timestamped_queues()
2624 		etmq = etm->queues.queue_array[i].priv;  in cs_etm__process_timestamped_queues()
2628 		ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i);  in cs_etm__process_timestamped_queues()
2634 		if (!etm->heap.heap_cnt)  in cs_etm__process_timestamped_queues()
2638 		cs_queue_nr = etm->heap.heap_array[0].queue_nr;  in cs_etm__process_timestamped_queues()
2641 		queue = &etm->queues.queue_array[queue_nr];  in cs_etm__process_timestamped_queues()
2648 		auxtrace_heap__pop(&etm->heap);  in cs_etm__process_timestamped_queues()
2715 		ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp);  in cs_etm__process_timestamped_queues()
2718 	for (i = 0; i < etm->queues.nr_queues; i++) {  in cs_etm__process_timestamped_queues()
2721 		etmq = etm->queues.queue_array[i].priv;  in cs_etm__process_timestamped_queues()
2739 static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm,  in cs_etm__process_itrace_start()  argument
2744 	if (etm->timeless_decoding)  in cs_etm__process_itrace_start()
2753 	th = machine__findnew_thread(&etm->session->machines.host,  in cs_etm__process_itrace_start()
2764 static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm,  in cs_etm__process_switch_cpu_wide()  argument
2774 	if (etm->timeless_decoding)  in cs_etm__process_switch_cpu_wide()
2791 	th = machine__findnew_thread(&etm->session->machines.host,  in cs_etm__process_switch_cpu_wide()
2807 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__process_event()  local
2829 		if (etm->per_thread_decoding && etm->timeless_decoding)  in cs_etm__process_event()
2830 			return cs_etm__process_timeless_queues(etm,  in cs_etm__process_event()
2835 		return cs_etm__process_itrace_start(etm, event);  in cs_etm__process_event()
2838 		return cs_etm__process_switch_cpu_wide(etm, event);  in cs_etm__process_event()
2847 			etm->latest_kernel_timestamp = sample->time;  in cs_etm__process_event()
2857 static void dump_queued_data(struct cs_etm_auxtrace *etm,  in dump_queued_data()  argument
2867 	for (i = 0; i < etm->queues.nr_queues; ++i)  in dump_queued_data()
2868 		list_for_each_entry(buf, &etm->queues.queue_array[i].head, list)  in dump_queued_data()
2870 				cs_etm__dump_event(etm->queues.queue_array[i].priv, buf);  in dump_queued_data()
2877 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__process_auxtrace_event()  local
2880 	if (!etm->data_queued) {  in cs_etm__process_auxtrace_event()
2896 		err = auxtrace_queues__add_event(&etm->queues, session,  in cs_etm__process_auxtrace_event()
2903 				cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer);  in cs_etm__process_auxtrace_event()
2907 		dump_queued_data(etm, &event->auxtrace);  in cs_etm__process_auxtrace_event()
2912 static int cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm)  in cs_etm__setup_timeless_decoding()  argument
2915 	struct evlist *evlist = etm->session->evlist;  in cs_etm__setup_timeless_decoding()
2918 	if (etm->synth_opts.timeless_decoding) {  in cs_etm__setup_timeless_decoding()
2919 		etm->timeless_decoding = true;  in cs_etm__setup_timeless_decoding()
2927 		if (cs_etm__evsel_is_auxtrace(etm->session, evsel)) {  in cs_etm__setup_timeless_decoding()
2928 			etm->timeless_decoding =  in cs_etm__setup_timeless_decoding()
3019 	struct cs_etm_auxtrace *etm = container_of(session->auxtrace,  in cs_etm__queue_aux_fragment()  local
3048 		etm->per_thread_decoding = true;  in cs_etm__queue_aux_fragment()
3052 		if (etm->per_thread_decoding) {  in cs_etm__queue_aux_fragment()
3083 		struct cs_etm_queue *etmq = etm->queues.queue_array[auxtrace_event->idx].priv;  in cs_etm__queue_aux_fragment()
3096 		err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment,  in cs_etm__queue_aux_fragment()
3234 static int cs_etm__map_trace_ids_metadata(struct cs_etm_auxtrace *etm, int num_cpu,  in cs_etm__map_trace_ids_metadata()  argument
3257 		err = cs_etm__map_trace_id_v0(etm, trace_chan_id, metadata[i]);  in cs_etm__map_trace_ids_metadata()
3324 static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm)  in cs_etm__create_decoders()  argument
3326 	struct auxtrace_queues *queues = &etm->queues;  in cs_etm__create_decoders()
3352 	struct cs_etm_auxtrace *etm = NULL;  in cs_etm__process_auxtrace_info_full()  local
3422 	etm = zalloc(sizeof(*etm));  in cs_etm__process_auxtrace_info_full()
3424 	if (!etm) {  in cs_etm__process_auxtrace_info_full()
3434 	etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]);  in cs_etm__process_auxtrace_info_full()
3436 	err = auxtrace_queues__init_nr(&etm->queues, max_cpu + 1);  in cs_etm__process_auxtrace_info_full()
3440 	for (unsigned int j = 0; j < etm->queues.nr_queues; ++j) {  in cs_etm__process_auxtrace_info_full()
3441 		err = cs_etm__setup_queue(etm, &etm->queues.queue_array[j], j);  in cs_etm__process_auxtrace_info_full()
3447 		etm->synth_opts = *session->itrace_synth_opts;  in cs_etm__process_auxtrace_info_full()
3449 		itrace_synth_opts__set_default(&etm->synth_opts,  in cs_etm__process_auxtrace_info_full()
3451 		etm->synth_opts.callchain = false;  in cs_etm__process_auxtrace_info_full()
3454 	etm->session = session;  in cs_etm__process_auxtrace_info_full()
3456 	etm->num_cpu = num_cpu;  in cs_etm__process_auxtrace_info_full()
3457 	etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff);  in cs_etm__process_auxtrace_info_full()
3458 	etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0);  in cs_etm__process_auxtrace_info_full()
3459 	etm->metadata = metadata;  in cs_etm__process_auxtrace_info_full()
3460 	etm->auxtrace_type = auxtrace_info->type;  in cs_etm__process_auxtrace_info_full()
3462 	if (etm->synth_opts.use_timestamp)  in cs_etm__process_auxtrace_info_full()
3472 		etm->has_virtual_ts = true;  in cs_etm__process_auxtrace_info_full()
3475 		etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu);  in cs_etm__process_auxtrace_info_full()
3477 	if (!etm->has_virtual_ts)  in cs_etm__process_auxtrace_info_full()
3484 	etm->auxtrace.process_event = cs_etm__process_event;  in cs_etm__process_auxtrace_info_full()
3485 	etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event;  in cs_etm__process_auxtrace_info_full()
3486 	etm->auxtrace.flush_events = cs_etm__flush_events;  in cs_etm__process_auxtrace_info_full()
3487 	etm->auxtrace.free_events = cs_etm__free_events;  in cs_etm__process_auxtrace_info_full()
3488 	etm->auxtrace.free = cs_etm__free;  in cs_etm__process_auxtrace_info_full()
3489 	etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace;  in cs_etm__process_auxtrace_info_full()
3490 	session->auxtrace = &etm->auxtrace;  in cs_etm__process_auxtrace_info_full()
3492 	err = cs_etm__setup_timeless_decoding(etm);  in cs_etm__process_auxtrace_info_full()
3496 	etm->tc.time_shift = tc->time_shift;  in cs_etm__process_auxtrace_info_full()
3497 	etm->tc.time_mult = tc->time_mult;  in cs_etm__process_auxtrace_info_full()
3498 	etm->tc.time_zero = tc->time_zero;  in cs_etm__process_auxtrace_info_full()
3500 		etm->tc.time_cycles = tc->time_cycles;  in cs_etm__process_auxtrace_info_full()
3501 		etm->tc.time_mask = tc->time_mask;  in cs_etm__process_auxtrace_info_full()
3502 		etm->tc.cap_user_time_zero = tc->cap_user_time_zero;  in cs_etm__process_auxtrace_info_full()
3503 		etm->tc.cap_user_time_short = tc->cap_user_time_short;  in cs_etm__process_auxtrace_info_full()
3505 	err = cs_etm__synth_events(etm, session);  in cs_etm__process_auxtrace_info_full()
3544 		err = cs_etm__map_trace_ids_metadata(etm, num_cpu, metadata);  in cs_etm__process_auxtrace_info_full()
3549 	err = cs_etm__create_decoders(etm);  in cs_etm__process_auxtrace_info_full()
3553 	etm->data_queued = etm->queues.populated;  in cs_etm__process_auxtrace_info_full()
3557 	auxtrace_queues__free(&etm->queues);  in cs_etm__process_auxtrace_info_full()
3560 	zfree(&etm);  in cs_etm__process_auxtrace_info_full()