1 // SPDX-License-Identifier: GPL-2.0-only
30  * sampling_rate_store - update sampling rate effective immediately if needed.
34  * original sampling_rate was 1 second and the requested new sampling rate is 10
37  * change the sampling rate too late; up to 1 second later. Thus, if we are
38  * reducing the sampling rate, we need to make the new value effective
41  * This must be called with dbs_data->mutex held, otherwise traversing
54 		return -EINVAL;  in sampling_rate_store()
56 	dbs_data->sampling_rate = sampling_interval;  in sampling_rate_store()
59 	 * We are operating under dbs_data->mutex and so the list and its  in sampling_rate_store()
62 	list_for_each_entry(policy_dbs, &attr_set->policy_list, list) {  in sampling_rate_store()
63 		mutex_lock(&policy_dbs->update_mutex);  in sampling_rate_store()
65 		 * On 32-bit architectures this may race with the  in sampling_rate_store()
78 		mutex_unlock(&policy_dbs->update_mutex);  in sampling_rate_store()
86  * gov_update_cpu_data - Update CPU load data.
87  * @dbs_data: Top-level governor data pointer.
91  * system-wide).
99 	list_for_each_entry(policy_dbs, &dbs_data->attr_set.policy_list, list) {  in gov_update_cpu_data()
102 		for_each_cpu(j, policy_dbs->policy->cpus) {  in gov_update_cpu_data()
105 			j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time,  in gov_update_cpu_data()
106 								  dbs_data->io_is_busy);  in gov_update_cpu_data()
107 			if (dbs_data->ignore_nice_load)  in gov_update_cpu_data()
108 				j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);  in gov_update_cpu_data()
116 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in dbs_update()
117 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in dbs_update()
118 	unsigned int ignore_nice = dbs_data->ignore_nice_load;  in dbs_update()
125 	 * so as to keep the wake-up-from-idle detection logic a bit  in dbs_update()
128 	sampling_rate = dbs_data->sampling_rate * policy_dbs->rate_mult;  in dbs_update()
134 	io_busy = dbs_data->io_is_busy;  in dbs_update()
137 	for_each_cpu(j, policy->cpus) {  in dbs_update()
145 		time_elapsed = update_time - j_cdbs->prev_update_time;  in dbs_update()
146 		j_cdbs->prev_update_time = update_time;  in dbs_update()
148 		idle_time = cur_idle_time - j_cdbs->prev_cpu_idle;  in dbs_update()
149 		j_cdbs->prev_cpu_idle = cur_idle_time;  in dbs_update()
154 			idle_time += div_u64(cur_nice - j_cdbs->prev_cpu_nice, NSEC_PER_USEC);  in dbs_update()
155 			j_cdbs->prev_cpu_nice = cur_nice;  in dbs_update()
164 			load = j_cdbs->prev_load;  in dbs_update()
166 				    j_cdbs->prev_load)) {  in dbs_update()
171 			 * time-window, because it would show near-zero load,  in dbs_update()
173 			 * was. This is undesirable for latency-sensitive bursty  in dbs_update()
177 			 * time-window and give this task a chance to start with  in dbs_update()
179 			 * shouldn't be over-done, lest we get stuck at a high  in dbs_update()
186 			 * 'idle_time' (as compared to the sampling rate)  in dbs_update()
189 			load = j_cdbs->prev_load;  in dbs_update()
190 			j_cdbs->prev_load = 0;  in dbs_update()
193 				load = 100 * (time_elapsed - idle_time) / time_elapsed;  in dbs_update()
207 				 * last sampling interval and 100 can be  in dbs_update()
212 			j_cdbs->prev_load = load;  in dbs_update()
226 	policy_dbs->idle_periods = idle_periods;  in dbs_update()
239 	policy = policy_dbs->policy;  in dbs_work_handler()
244 	 * ondemand governor isn't updating the sampling rate in parallel.  in dbs_work_handler()
246 	mutex_lock(&policy_dbs->update_mutex);  in dbs_work_handler()
247 	gov_update_sample_delay(policy_dbs, gov->gov_dbs_update(policy));  in dbs_work_handler()
248 	mutex_unlock(&policy_dbs->update_mutex);  in dbs_work_handler()
251 	atomic_set(&policy_dbs->work_count, 0);  in dbs_work_handler()
258 	policy_dbs->work_in_progress = false;  in dbs_work_handler()
266 	schedule_work_on(smp_processor_id(), &policy_dbs->work);  in dbs_irq_work()
273 	struct policy_dbs_info *policy_dbs = cdbs->policy_dbs;  in dbs_update_util_handler()
276 	if (!cpufreq_this_cpu_can_update(policy_dbs->policy))  in dbs_update_util_handler()
282 	 * - Work has already been queued up or is in progress.  in dbs_update_util_handler()
283 	 * - It is too early (too little time from the previous sample).  in dbs_update_util_handler()
285 	if (policy_dbs->work_in_progress)  in dbs_update_util_handler()
293 	lst = READ_ONCE(policy_dbs->last_sample_time);  in dbs_update_util_handler()
294 	delta_ns = time - lst;  in dbs_update_util_handler()
295 	if ((s64)delta_ns < policy_dbs->sample_delay_ns)  in dbs_update_util_handler()
303 	if (policy_dbs->is_shared) {  in dbs_update_util_handler()
304 		if (!atomic_add_unless(&policy_dbs->work_count, 1, 1))  in dbs_update_util_handler()
311 		if (unlikely(lst != READ_ONCE(policy_dbs->last_sample_time))) {  in dbs_update_util_handler()
312 			atomic_set(&policy_dbs->work_count, 0);  in dbs_update_util_handler()
317 	policy_dbs->last_sample_time = time;  in dbs_update_util_handler()
318 	policy_dbs->work_in_progress = true;  in dbs_update_util_handler()
319 	irq_work_queue(&policy_dbs->irq_work);  in dbs_update_util_handler()
325 	struct cpufreq_policy *policy = policy_dbs->policy;  in gov_set_update_util()
329 	policy_dbs->last_sample_time = 0;  in gov_set_update_util()
331 	for_each_cpu(cpu, policy->cpus) {  in gov_set_update_util()
334 		cpufreq_add_update_util_hook(cpu, &cdbs->update_util,  in gov_set_update_util()
343 	for_each_cpu(i, policy->cpus)  in gov_clear_update_util()
355 	/* Allocate memory for per-policy governor data. */  in alloc_policy_dbs_info()
356 	policy_dbs = gov->alloc();  in alloc_policy_dbs_info()
360 	policy_dbs->policy = policy;  in alloc_policy_dbs_info()
361 	mutex_init(&policy_dbs->update_mutex);  in alloc_policy_dbs_info()
362 	atomic_set(&policy_dbs->work_count, 0);  in alloc_policy_dbs_info()
363 	init_irq_work(&policy_dbs->irq_work, dbs_irq_work);  in alloc_policy_dbs_info()
364 	INIT_WORK(&policy_dbs->work, dbs_work_handler);  in alloc_policy_dbs_info()
367 	for_each_cpu(j, policy->related_cpus) {  in alloc_policy_dbs_info()
370 		j_cdbs->policy_dbs = policy_dbs;  in alloc_policy_dbs_info()
380 	mutex_destroy(&policy_dbs->update_mutex);  in free_policy_dbs_info()
382 	for_each_cpu(j, policy_dbs->policy->related_cpus) {  in free_policy_dbs_info()
385 		j_cdbs->policy_dbs = NULL;  in free_policy_dbs_info()
386 		j_cdbs->update_util.func = NULL;  in free_policy_dbs_info()
388 	gov->free(policy_dbs);  in free_policy_dbs_info()
394 	struct dbs_governor *gov = dbs_data->gov;  in cpufreq_dbs_data_release()
396 	gov->exit(dbs_data);  in cpufreq_dbs_data_release()
408 	if (policy->governor_data)  in cpufreq_dbs_governor_init()
409 		return -EBUSY;  in cpufreq_dbs_governor_init()
413 		return -ENOMEM;  in cpufreq_dbs_governor_init()
415 	/* Protect gov->gdbs_data against concurrent updates. */  in cpufreq_dbs_governor_init()
418 	dbs_data = gov->gdbs_data;  in cpufreq_dbs_governor_init()
421 			ret = -EINVAL;  in cpufreq_dbs_governor_init()
424 		policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
425 		policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
427 		gov_attr_set_get(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
433 		ret = -ENOMEM;  in cpufreq_dbs_governor_init()
437 	dbs_data->gov = gov;  in cpufreq_dbs_governor_init()
438 	gov_attr_set_init(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_init()
440 	ret = gov->init(dbs_data);  in cpufreq_dbs_governor_init()
445 	 * The sampling interval should not be less than the transition latency  in cpufreq_dbs_governor_init()
449 	dbs_data->sampling_rate = max_t(unsigned int,  in cpufreq_dbs_governor_init()
454 		gov->gdbs_data = dbs_data;  in cpufreq_dbs_governor_init()
456 	policy_dbs->dbs_data = dbs_data;  in cpufreq_dbs_governor_init()
457 	policy->governor_data = policy_dbs;  in cpufreq_dbs_governor_init()
459 	gov->kobj_type.sysfs_ops = &governor_sysfs_ops;  in cpufreq_dbs_governor_init()
460 	gov->kobj_type.release = cpufreq_dbs_data_release;  in cpufreq_dbs_governor_init()
461 	ret = kobject_init_and_add(&dbs_data->attr_set.kobj, &gov->kobj_type,  in cpufreq_dbs_governor_init()
463 				   "%s", gov->gov.name);  in cpufreq_dbs_governor_init()
470 	kobject_put(&dbs_data->attr_set.kobj);  in cpufreq_dbs_governor_init()
472 	policy->governor_data = NULL;  in cpufreq_dbs_governor_init()
475 		gov->gdbs_data = NULL;  in cpufreq_dbs_governor_init()
476 	gov->exit(dbs_data);  in cpufreq_dbs_governor_init()
493 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_exit()
494 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_exit()
497 	/* Protect gov->gdbs_data against concurrent updates. */  in cpufreq_dbs_governor_exit()
500 	count = gov_attr_set_put(&dbs_data->attr_set, &policy_dbs->list);  in cpufreq_dbs_governor_exit()
502 	policy->governor_data = NULL;  in cpufreq_dbs_governor_exit()
505 		gov->gdbs_data = NULL;  in cpufreq_dbs_governor_exit()
516 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_start()
517 	struct dbs_data *dbs_data = policy_dbs->dbs_data;  in cpufreq_dbs_governor_start()
521 	if (!policy->cur)  in cpufreq_dbs_governor_start()
522 		return -EINVAL;  in cpufreq_dbs_governor_start()
524 	policy_dbs->is_shared = policy_is_shared(policy);  in cpufreq_dbs_governor_start()
525 	policy_dbs->rate_mult = 1;  in cpufreq_dbs_governor_start()
527 	sampling_rate = dbs_data->sampling_rate;  in cpufreq_dbs_governor_start()
528 	ignore_nice = dbs_data->ignore_nice_load;  in cpufreq_dbs_governor_start()
529 	io_busy = dbs_data->io_is_busy;  in cpufreq_dbs_governor_start()
531 	for_each_cpu(j, policy->cpus) {  in cpufreq_dbs_governor_start()
534 		j_cdbs->prev_cpu_idle = get_cpu_idle_time(j, &j_cdbs->prev_update_time, io_busy);  in cpufreq_dbs_governor_start()
538 		j_cdbs->prev_load = 0;  in cpufreq_dbs_governor_start()
541 			j_cdbs->prev_cpu_nice = kcpustat_field(&kcpustat_cpu(j), CPUTIME_NICE, j);  in cpufreq_dbs_governor_start()
544 	gov->start(policy);  in cpufreq_dbs_governor_start()
553 	struct policy_dbs_info *policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_stop()
555 	gov_clear_update_util(policy_dbs->policy);  in cpufreq_dbs_governor_stop()
556 	irq_work_sync(&policy_dbs->irq_work);  in cpufreq_dbs_governor_stop()
557 	cancel_work_sync(&policy_dbs->work);  in cpufreq_dbs_governor_stop()
558 	atomic_set(&policy_dbs->work_count, 0);  in cpufreq_dbs_governor_stop()
559 	policy_dbs->work_in_progress = false;  in cpufreq_dbs_governor_stop()
567 	/* Protect gov->gdbs_data against cpufreq_dbs_governor_exit() */  in cpufreq_dbs_governor_limits()
569 	policy_dbs = policy->governor_data;  in cpufreq_dbs_governor_limits()
573 	mutex_lock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()
576 	mutex_unlock(&policy_dbs->update_mutex);  in cpufreq_dbs_governor_limits()