500 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc);
1179 	struct perf_cpu_pmu_context *cpc;  in perf_mux_hrtimer_handler()  local
1184 	cpc = container_of(hr, struct perf_cpu_pmu_context, hrtimer);  in perf_mux_hrtimer_handler()
1185 	rotations = perf_rotate_context(cpc);  in perf_mux_hrtimer_handler()
1187 	raw_spin_lock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1189 		hrtimer_forward_now(hr, cpc->hrtimer_interval);  in perf_mux_hrtimer_handler()
1191 		cpc->hrtimer_active = 0;  in perf_mux_hrtimer_handler()
1192 	raw_spin_unlock(&cpc->hrtimer_lock);  in perf_mux_hrtimer_handler()
1197 static void __perf_mux_hrtimer_init(struct perf_cpu_pmu_context *cpc, int cpu)  in __perf_mux_hrtimer_init()  argument
1199 	struct hrtimer *timer = &cpc->hrtimer;  in __perf_mux_hrtimer_init()
1200 	struct pmu *pmu = cpc->epc.pmu;  in __perf_mux_hrtimer_init()
1211 	cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * interval);  in __perf_mux_hrtimer_init()
1213 	raw_spin_lock_init(&cpc->hrtimer_lock);  in __perf_mux_hrtimer_init()
1218 static int perf_mux_hrtimer_restart(struct perf_cpu_pmu_context *cpc)  in perf_mux_hrtimer_restart()  argument
1220 	struct hrtimer *timer = &cpc->hrtimer;  in perf_mux_hrtimer_restart()
1223 	raw_spin_lock_irqsave(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
1224 	if (!cpc->hrtimer_active) {  in perf_mux_hrtimer_restart()
1225 		cpc->hrtimer_active = 1;  in perf_mux_hrtimer_restart()
1226 		hrtimer_forward_now(timer, cpc->hrtimer_interval);  in perf_mux_hrtimer_restart()
1229 	raw_spin_unlock_irqrestore(&cpc->hrtimer_lock, flags);  in perf_mux_hrtimer_restart()
2364 	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);  in event_sched_out()  local
2367 	// XXX cpc serialization, probably per-cpu IRQ disabled  in event_sched_out()
2396 		cpc->active_oncpu--;  in event_sched_out()
2401 	if (event->attr.exclusive || !cpc->active_oncpu)  in event_sched_out()
2402 		cpc->exclusive = 0;  in event_sched_out()
2517 			struct perf_cpu_pmu_context *cpc = this_cpc(pmu_ctx->pmu);  in __perf_remove_from_context()  local
2519 			WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __perf_remove_from_context()
2520 			cpc->task_epc = NULL;  in __perf_remove_from_context()
2711 	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);  in event_sched_in()  local
2750 		cpc->active_oncpu++;  in event_sched_in()
2756 		cpc->exclusive = 1;  in event_sched_in()
2816 	struct perf_cpu_pmu_context *cpc = this_cpc(epc->pmu);  in group_can_go_on()  local
2827 	if (cpc->exclusive)  in group_can_go_on()
3439 		struct perf_cpu_pmu_context *cpc = this_cpc(pmu);  in __pmu_ctx_sched_out()  local
3441 		WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __pmu_ctx_sched_out()
3442 		cpc->task_epc = NULL;  in __pmu_ctx_sched_out()
3649 	struct perf_cpu_pmu_context *cpc;  in perf_ctx_sched_task_cb()  local
3652 		cpc = this_cpc(pmu_ctx->pmu);  in perf_ctx_sched_task_cb()
3654 		if (cpc->sched_cb_usage && pmu_ctx->pmu->sched_task)  in perf_ctx_sched_task_cb()
3760 	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);  in perf_sched_cb_dec()  local
3765 	if (!--cpc->sched_cb_usage)  in perf_sched_cb_dec()
3766 		list_del(&cpc->sched_cb_entry);  in perf_sched_cb_dec()
3772 	struct perf_cpu_pmu_context *cpc = this_cpc(pmu);  in perf_sched_cb_inc()  local
3774 	if (!cpc->sched_cb_usage++)  in perf_sched_cb_inc()
3775 		list_add(&cpc->sched_cb_entry, this_cpu_ptr(&sched_cb_list));  in perf_sched_cb_inc()
3789 static void __perf_pmu_sched_task(struct perf_cpu_pmu_context *cpc,  in __perf_pmu_sched_task()  argument
3795 	pmu = cpc->epc.pmu;  in __perf_pmu_sched_task()
3804 	pmu->sched_task(cpc->task_epc, task, sched_in);  in __perf_pmu_sched_task()
3815 	struct perf_cpu_pmu_context *cpc;  in perf_pmu_sched_task()  local
3821 	list_for_each_entry(cpc, this_cpu_ptr(&sched_cb_list), sched_cb_entry)  in perf_pmu_sched_task()
3822 		__perf_pmu_sched_task(cpc, sched_in ? next : prev, sched_in);  in perf_pmu_sched_task()
3885 	struct perf_cpu_pmu_context *cpc;  in __link_epc()  local
3890 	cpc = this_cpc(pmu_ctx->pmu);  in __link_epc()
3891 	WARN_ON_ONCE(cpc->task_epc && cpc->task_epc != pmu_ctx);  in __link_epc()
3892 	cpc->task_epc = pmu_ctx;  in __link_epc()
4024 			struct perf_cpu_pmu_context *cpc = this_cpc(event->pmu_ctx->pmu);  in merge_sched_in()  local
4027 			perf_mux_hrtimer_restart(cpc);  in merge_sched_in()
4447 static bool perf_rotate_context(struct perf_cpu_pmu_context *cpc)  in perf_rotate_context()  argument
4460 	cpu_epc = &cpc->epc;  in perf_rotate_context()
4462 	task_epc = cpc->task_epc;  in perf_rotate_context()
5040 		struct perf_cpu_pmu_context *cpc;  in find_get_pmu_context()  local
5042 		cpc = *per_cpu_ptr(pmu->cpu_pmu_context, event->cpu);  in find_get_pmu_context()
5043 		epc = &cpc->epc;  in find_get_pmu_context()
5112 	struct perf_cpu_pmu_context *cpc =  in free_cpc_rcu()  local
5113 		container_of(head, typeof(*cpc), epc.rcu_head);  in free_cpc_rcu()
5115 	kfree(cpc);  in free_cpc_rcu()
12151 		struct perf_cpu_pmu_context *cpc;  in perf_event_mux_interval_ms_store()  local
12152 		cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_event_mux_interval_ms_store()
12153 		cpc->hrtimer_interval = ns_to_ktime(NSEC_PER_MSEC * timer);  in perf_event_mux_interval_ms_store()
12155 		cpu_function_call(cpu, perf_mux_hrtimer_restart_ipi, cpc);  in perf_event_mux_interval_ms_store()
12330 			struct perf_cpu_pmu_context *cpc;  in perf_pmu_free()  local
12332 			cpc = *per_cpu_ptr(pmu->cpu_pmu_context, cpu);  in perf_pmu_free()
12333 			if (!cpc)  in perf_pmu_free()
12335 			if (cpc->epc.embedded) {  in perf_pmu_free()
12337 				put_pmu_ctx(&cpc->epc);  in perf_pmu_free()
12340 			kfree(cpc);  in perf_pmu_free()
12387 		struct perf_cpu_pmu_context *cpc =  in DEFINE_FREE()  local
12392 		if (!cpc)  in DEFINE_FREE()
12395 		*per_cpu_ptr(pmu->cpu_pmu_context, cpu) = cpc;  in DEFINE_FREE()
12396 		__perf_init_event_pmu_context(&cpc->epc, pmu);  in DEFINE_FREE()
12397 		__perf_mux_hrtimer_init(cpc, cpu);  in DEFINE_FREE()