1 // SPDX-License-Identifier: GPL-2.0
7 #include <linux/clk-provider.h>
47 	void __iomem *base;  member
50 	 * Mutex to synchronize between de-init sequence and re-starting LMh
76 	struct dev_pm_opp *opp;  in qcom_cpufreq_set_bw()  local
80 	dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_set_bw()
82 		return -ENODEV;  in qcom_cpufreq_set_bw()
84 	opp = dev_pm_opp_find_freq_exact(dev, freq_hz, true);  in qcom_cpufreq_set_bw()
85 	if (IS_ERR(opp))  in qcom_cpufreq_set_bw()
86 		return PTR_ERR(opp);  in qcom_cpufreq_set_bw()
88 	ret = dev_pm_opp_set_opp(dev, opp);  in qcom_cpufreq_set_bw()
89 	dev_pm_opp_put(opp);  in qcom_cpufreq_set_bw()
100 	/* Skip voltage update if the opp table is not available */  in qcom_cpufreq_update_opp()
116 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_target_index()
118 	unsigned long freq = policy->freq_table[index].frequency;  in qcom_cpufreq_hw_target_index()
121 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_target_index()
123 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_target_index()
124 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_target_index()
125 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_target_index()
137 	if (qcom_cpufreq.soc_data->reg_current_vote)  in qcom_lmh_get_throttle_freq()
138 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_current_vote) & 0x3ff;  in qcom_lmh_get_throttle_freq()
140 		lval = readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_domain_state) & 0xff;  in qcom_lmh_get_throttle_freq()
157 	data = policy->driver_data;  in qcom_cpufreq_get_freq()
160 	index = readl_relaxed(data->base + soc_data->reg_perf_state);  in qcom_cpufreq_get_freq()
161 	index = min(index, LUT_MAX_ENTRIES - 1);  in qcom_cpufreq_get_freq()
163 	return policy->freq_table[index].frequency;  in qcom_cpufreq_get_freq()
175 	data = policy->driver_data;  in qcom_cpufreq_hw_get()
177 	if (data->throttle_irq >= 0)  in qcom_cpufreq_hw_get()
186 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_fast_switch()
191 	index = policy->cached_resolved_idx;  in qcom_cpufreq_hw_fast_switch()
192 	writel_relaxed(index, data->base + soc_data->reg_perf_state);  in qcom_cpufreq_hw_fast_switch()
194 	if (data->per_core_dcvs)  in qcom_cpufreq_hw_fast_switch()
195 		for (i = 1; i < cpumask_weight(policy->related_cpus); i++)  in qcom_cpufreq_hw_fast_switch()
196 			writel_relaxed(index, data->base + soc_data->reg_perf_state + i * 4);  in qcom_cpufreq_hw_fast_switch()
198 	return policy->freq_table[index].frequency;  in qcom_cpufreq_hw_fast_switch()
207 	struct dev_pm_opp *opp;  in qcom_cpufreq_hw_read_lut()  local
210 	struct qcom_cpufreq_data *drv_data = policy->driver_data;  in qcom_cpufreq_hw_read_lut()
215 		return -ENOMEM;  in qcom_cpufreq_hw_read_lut()
219 		/* Disable all opps and cross-validate against LUT later */  in qcom_cpufreq_hw_read_lut()
222 			opp = dev_pm_opp_find_freq_ceil(cpu_dev, &rate);  in qcom_cpufreq_hw_read_lut()
223 			if (IS_ERR(opp))  in qcom_cpufreq_hw_read_lut()
226 			dev_pm_opp_put(opp);  in qcom_cpufreq_hw_read_lut()
229 	} else if (ret != -ENODEV) {  in qcom_cpufreq_hw_read_lut()
230 		dev_err(cpu_dev, "Invalid opp table in device tree\n");  in qcom_cpufreq_hw_read_lut()
234 		policy->fast_switch_possible = true;  in qcom_cpufreq_hw_read_lut()
239 		data = readl_relaxed(drv_data->base + soc_data->reg_freq_lut +  in qcom_cpufreq_hw_read_lut()
240 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
245 		data = readl_relaxed(drv_data->base + soc_data->reg_volt_lut +  in qcom_cpufreq_hw_read_lut()
246 				      i * soc_data->lut_row_size);  in qcom_cpufreq_hw_read_lut()
260 				dev_warn(cpu_dev, "failed to update OPP for freq=%d\n", freq);  in qcom_cpufreq_hw_read_lut()
273 			struct cpufreq_frequency_table *prev = &table[i - 1];  in qcom_cpufreq_hw_read_lut()
279 			if (prev->frequency == CPUFREQ_ENTRY_INVALID) {  in qcom_cpufreq_hw_read_lut()
281 					prev->frequency = prev_freq;  in qcom_cpufreq_hw_read_lut()
282 					prev->flags = CPUFREQ_BOOST_FREQ;  in qcom_cpufreq_hw_read_lut()
284 					dev_warn(cpu_dev, "failed to update OPP for freq=%d\n",  in qcom_cpufreq_hw_read_lut()
296 	policy->freq_table = table;  in qcom_cpufreq_hw_read_lut()
297 	dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus);  in qcom_cpufreq_hw_read_lut()
313 		ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_get_related_cpus()
314 						 "#freq-domain-cells", 0,  in qcom_get_related_cpus()
327 	struct cpufreq_policy *policy = data->policy;  in qcom_lmh_dcvs_notify()
328 	int cpu = cpumask_first(policy->related_cpus);  in qcom_lmh_dcvs_notify()
331 	struct dev_pm_opp *opp;  in qcom_lmh_dcvs_notify()  local
335 	 * registered opp table and use it to calculate thermal pressure.  in qcom_lmh_dcvs_notify()
339 	opp = dev_pm_opp_find_freq_floor(dev, &freq_hz);  in qcom_lmh_dcvs_notify()
340 	if (IS_ERR(opp) && PTR_ERR(opp) == -ERANGE)  in qcom_lmh_dcvs_notify()
341 		opp = dev_pm_opp_find_freq_ceil(dev, &freq_hz);  in qcom_lmh_dcvs_notify()
343 	if (IS_ERR(opp)) {  in qcom_lmh_dcvs_notify()
344 		dev_warn(dev, "Can't find the OPP for throttling: %pe!\n", opp);  in qcom_lmh_dcvs_notify()
346 		dev_pm_opp_put(opp);  in qcom_lmh_dcvs_notify()
352 	arch_update_hw_pressure(policy->related_cpus, throttled_freq);  in qcom_lmh_dcvs_notify()
358 	mutex_lock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
359 	if (data->cancel_throttle)  in qcom_lmh_dcvs_notify()
367 		enable_irq(data->throttle_irq);  in qcom_lmh_dcvs_notify()
369 		mod_delayed_work(system_highpri_wq, &data->throttle_work,  in qcom_lmh_dcvs_notify()
373 	mutex_unlock(&data->throttle_lock);  in qcom_lmh_dcvs_notify()
389 	disable_irq_nosync(c_data->throttle_irq);  in qcom_lmh_dcvs_handle_irq()
390 	schedule_delayed_work(&c_data->throttle_work, 0);  in qcom_lmh_dcvs_handle_irq()
392 	if (qcom_cpufreq.soc_data->reg_intr_clr)  in qcom_lmh_dcvs_handle_irq()
394 			       c_data->base + qcom_cpufreq.soc_data->reg_intr_clr);  in qcom_lmh_dcvs_handle_irq()
421 	{ .compatible = "qcom,cpufreq-hw", .data = &qcom_soc_data },
422 	{ .compatible = "qcom,cpufreq-epss", .data = &epss_soc_data },
429 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_lmh_init()
437 	data->throttle_irq = platform_get_irq_optional(pdev, index);  in qcom_cpufreq_hw_lmh_init()
438 	if (data->throttle_irq == -ENXIO)  in qcom_cpufreq_hw_lmh_init()
440 	if (data->throttle_irq < 0)  in qcom_cpufreq_hw_lmh_init()
441 		return data->throttle_irq;  in qcom_cpufreq_hw_lmh_init()
443 	data->cancel_throttle = false;  in qcom_cpufreq_hw_lmh_init()
444 	data->policy = policy;  in qcom_cpufreq_hw_lmh_init()
446 	mutex_init(&data->throttle_lock);  in qcom_cpufreq_hw_lmh_init()
447 	INIT_DEFERRABLE_WORK(&data->throttle_work, qcom_lmh_dcvs_poll);  in qcom_cpufreq_hw_lmh_init()
449 	snprintf(data->irq_name, sizeof(data->irq_name), "dcvsh-irq-%u", policy->cpu);  in qcom_cpufreq_hw_lmh_init()
450 	ret = request_threaded_irq(data->throttle_irq, NULL, qcom_lmh_dcvs_handle_irq,  in qcom_cpufreq_hw_lmh_init()
451 				   IRQF_ONESHOT | IRQF_NO_AUTOEN, data->irq_name, data);  in qcom_cpufreq_hw_lmh_init()
453 		dev_err(&pdev->dev, "Error registering %s: %d\n", data->irq_name, ret);  in qcom_cpufreq_hw_lmh_init()
457 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_lmh_init()
459 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_lmh_init()
460 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_lmh_init()
467 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_online()
471 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_online()
474 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
475 	data->cancel_throttle = false;  in qcom_cpufreq_hw_cpu_online()
476 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_online()
478 	ret = irq_set_affinity_and_hint(data->throttle_irq, policy->cpus);  in qcom_cpufreq_hw_cpu_online()
480 		dev_err(&pdev->dev, "Failed to set CPU affinity of %s[%d]\n",  in qcom_cpufreq_hw_cpu_online()
481 			data->irq_name, data->throttle_irq);  in qcom_cpufreq_hw_cpu_online()
488 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_offline()
490 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_cpu_offline()
493 	mutex_lock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
494 	data->cancel_throttle = true;  in qcom_cpufreq_hw_cpu_offline()
495 	mutex_unlock(&data->throttle_lock);  in qcom_cpufreq_hw_cpu_offline()
497 	cancel_delayed_work_sync(&data->throttle_work);  in qcom_cpufreq_hw_cpu_offline()
498 	irq_set_affinity_and_hint(data->throttle_irq, NULL);  in qcom_cpufreq_hw_cpu_offline()
499 	disable_irq_nosync(data->throttle_irq);  in qcom_cpufreq_hw_cpu_offline()
506 	if (data->throttle_irq <= 0)  in qcom_cpufreq_hw_lmh_exit()
509 	free_irq(data->throttle_irq, data);  in qcom_cpufreq_hw_lmh_exit()
515 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_cpu_init()
522 	cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
525 		       policy->cpu);  in qcom_cpufreq_hw_cpu_init()
526 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
529 	cpu_np = of_cpu_device_node_get(policy->cpu);  in qcom_cpufreq_hw_cpu_init()
531 		return -EINVAL;  in qcom_cpufreq_hw_cpu_init()
533 	ret = of_parse_phandle_with_args(cpu_np, "qcom,freq-domain",  in qcom_cpufreq_hw_cpu_init()
534 					 "#freq-domain-cells", 0, &args);  in qcom_cpufreq_hw_cpu_init()
543 	if (!(readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_enable) & 0x1)) {  in qcom_cpufreq_hw_cpu_init()
544 		dev_err(dev, "Domain-%d cpufreq hardware not enabled\n", index);  in qcom_cpufreq_hw_cpu_init()
545 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
548 	if (readl_relaxed(data->base + qcom_cpufreq.soc_data->reg_dcvs_ctrl) & 0x1)  in qcom_cpufreq_hw_cpu_init()
549 		data->per_core_dcvs = true;  in qcom_cpufreq_hw_cpu_init()
551 	qcom_get_related_cpus(index, policy->cpus);  in qcom_cpufreq_hw_cpu_init()
553 	policy->driver_data = data;  in qcom_cpufreq_hw_cpu_init()
554 	policy->dvfs_possible_from_any_cpu = true;  in qcom_cpufreq_hw_cpu_init()
558 		dev_err(dev, "Domain-%d failed to read LUT\n", index);  in qcom_cpufreq_hw_cpu_init()
565 		return -ENODEV;  in qcom_cpufreq_hw_cpu_init()
579 	struct device *cpu_dev = get_cpu_device(policy->cpu);  in qcom_cpufreq_hw_cpu_exit()
580 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_hw_cpu_exit()
583 	dev_pm_opp_of_cpumask_remove_table(policy->related_cpus);  in qcom_cpufreq_hw_cpu_exit()
585 	kfree(policy->freq_table);  in qcom_cpufreq_hw_cpu_exit()
591 	struct qcom_cpufreq_data *data = policy->driver_data;  in qcom_cpufreq_ready()
593 	if (data->throttle_irq >= 0)  in qcom_cpufreq_ready()
594 		enable_irq(data->throttle_irq);  in qcom_cpufreq_ready()
616 	.name		= "qcom-cpufreq-hw",
635 	struct device *dev = &pdev->dev;  in qcom_cpufreq_hw_driver_probe()
659 		return -EPROBE_DEFER;  in qcom_cpufreq_hw_driver_probe()
672 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
676 		return -ENODEV;  in qcom_cpufreq_hw_driver_probe()
680 		return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
682 	clk_data->num = num_domains;  in qcom_cpufreq_hw_driver_probe()
687 		void __iomem *base;  in qcom_cpufreq_hw_driver_probe()  local
689 		base = devm_platform_ioremap_resource(pdev, i);  in qcom_cpufreq_hw_driver_probe()
690 		if (IS_ERR(base)) {  in qcom_cpufreq_hw_driver_probe()
692 			return PTR_ERR(base);  in qcom_cpufreq_hw_driver_probe()
695 		data->base = base;  in qcom_cpufreq_hw_driver_probe()
700 			return -ENOMEM;  in qcom_cpufreq_hw_driver_probe()
704 		data->cpu_clk.init = &clk_init;  in qcom_cpufreq_hw_driver_probe()
706 		ret = devm_clk_hw_register(dev, &data->cpu_clk);  in qcom_cpufreq_hw_driver_probe()
713 		clk_data->hws[i] = &data->cpu_clk;  in qcom_cpufreq_hw_driver_probe()
741 		.name = "qcom-cpufreq-hw",
759 MODULE_LICENSE("GPL v2");