1 /* 2 * arch/arm64/kernel/topology.c 3 * 4 * Copyright (C) 2011,2013,2014 Linaro Limited. 5 * 6 * Based on the arm32 version written by Vincent Guittot in turn based on 7 * arch/sh/kernel/topology.c 8 * 9 * This file is subject to the terms and conditions of the GNU General Public 10 * License. See the file "COPYING" in the main directory of this archive 11 * for more details. 12 */ 13 14 #include <linux/acpi.h> 15 #include <linux/arch_topology.h> 16 #include <linux/cacheinfo.h> 17 #include <linux/cpufreq.h> 18 #include <linux/cpu_smt.h> 19 #include <linux/init.h> 20 #include <linux/percpu.h> 21 #include <linux/sched/isolation.h> 22 #include <linux/xarray.h> 23 24 #include <asm/cpu.h> 25 #include <asm/cputype.h> 26 #include <asm/topology.h> 27 28 #ifdef CONFIG_ARM64_AMU_EXTN 29 #define read_corecnt() read_sysreg_s(SYS_AMEVCNTR0_CORE_EL0) 30 #define read_constcnt() read_sysreg_s(SYS_AMEVCNTR0_CONST_EL0) 31 #else 32 #define read_corecnt() (0UL) 33 #define read_constcnt() (0UL) 34 #endif 35 36 #undef pr_fmt 37 #define pr_fmt(fmt) "AMU: " fmt 38 39 /* 40 * Ensure that amu_scale_freq_tick() will return SCHED_CAPACITY_SCALE until 41 * the CPU capacity and its associated frequency have been correctly 42 * initialized. 43 */ 44 static DEFINE_PER_CPU_READ_MOSTLY(unsigned long, arch_max_freq_scale) = 1UL << (2 * SCHED_CAPACITY_SHIFT); 45 static cpumask_var_t amu_fie_cpus; 46 47 struct amu_cntr_sample { 48 u64 arch_const_cycles_prev; 49 u64 arch_core_cycles_prev; 50 unsigned long last_scale_update; 51 }; 52 53 static DEFINE_PER_CPU_SHARED_ALIGNED(struct amu_cntr_sample, cpu_amu_samples); 54 55 void update_freq_counters_refs(void) 56 { 57 struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples); 58 59 amu_sample->arch_core_cycles_prev = read_corecnt(); 60 amu_sample->arch_const_cycles_prev = read_constcnt(); 61 } 62 63 static inline bool freq_counters_valid(int cpu) 64 { 65 struct amu_cntr_sample *amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu); 66 67 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 68 return false; 69 70 if (!cpu_has_amu_feat(cpu)) { 71 pr_debug("CPU%d: counters are not supported.\n", cpu); 72 return false; 73 } 74 75 if (unlikely(!amu_sample->arch_const_cycles_prev || 76 !amu_sample->arch_core_cycles_prev)) { 77 pr_debug("CPU%d: cycle counters are not enabled.\n", cpu); 78 return false; 79 } 80 81 return true; 82 } 83 84 void freq_inv_set_max_ratio(int cpu, u64 max_rate) 85 { 86 u64 ratio, ref_rate = arch_timer_get_rate(); 87 88 if (unlikely(!max_rate || !ref_rate)) { 89 WARN_ONCE(1, "CPU%d: invalid maximum or reference frequency.\n", 90 cpu); 91 return; 92 } 93 94 /* 95 * Pre-compute the fixed ratio between the frequency of the constant 96 * reference counter and the maximum frequency of the CPU. 97 * 98 * ref_rate 99 * arch_max_freq_scale = ---------- * SCHED_CAPACITY_SCALE² 100 * max_rate 101 * 102 * We use a factor of 2 * SCHED_CAPACITY_SHIFT -> SCHED_CAPACITY_SCALE² 103 * in order to ensure a good resolution for arch_max_freq_scale for 104 * very low reference frequencies (down to the KHz range which should 105 * be unlikely). 106 */ 107 ratio = ref_rate << (2 * SCHED_CAPACITY_SHIFT); 108 ratio = div64_u64(ratio, max_rate); 109 if (!ratio) { 110 WARN_ONCE(1, "Reference frequency too low.\n"); 111 return; 112 } 113 114 WRITE_ONCE(per_cpu(arch_max_freq_scale, cpu), (unsigned long)ratio); 115 } 116 117 static void amu_scale_freq_tick(void) 118 { 119 struct amu_cntr_sample *amu_sample = this_cpu_ptr(&cpu_amu_samples); 120 u64 prev_core_cnt, prev_const_cnt; 121 u64 core_cnt, const_cnt, scale; 122 123 prev_const_cnt = amu_sample->arch_const_cycles_prev; 124 prev_core_cnt = amu_sample->arch_core_cycles_prev; 125 126 update_freq_counters_refs(); 127 128 const_cnt = amu_sample->arch_const_cycles_prev; 129 core_cnt = amu_sample->arch_core_cycles_prev; 130 131 /* 132 * This should not happen unless the AMUs have been reset and the 133 * counter values have not been restored - unlikely 134 */ 135 if (unlikely(core_cnt <= prev_core_cnt || 136 const_cnt <= prev_const_cnt)) 137 return; 138 139 /* 140 * /\core arch_max_freq_scale 141 * scale = ------- * -------------------- 142 * /\const SCHED_CAPACITY_SCALE 143 * 144 * See validate_cpu_freq_invariance_counters() for details on 145 * arch_max_freq_scale and the use of SCHED_CAPACITY_SHIFT. 146 */ 147 scale = core_cnt - prev_core_cnt; 148 scale *= this_cpu_read(arch_max_freq_scale); 149 scale = div64_u64(scale >> SCHED_CAPACITY_SHIFT, 150 const_cnt - prev_const_cnt); 151 152 scale = min_t(unsigned long, scale, SCHED_CAPACITY_SCALE); 153 this_cpu_write(arch_freq_scale, (unsigned long)scale); 154 155 amu_sample->last_scale_update = jiffies; 156 } 157 158 static struct scale_freq_data amu_sfd = { 159 .source = SCALE_FREQ_SOURCE_ARCH, 160 .set_freq_scale = amu_scale_freq_tick, 161 }; 162 163 static __always_inline bool amu_fie_cpu_supported(unsigned int cpu) 164 { 165 return cpumask_available(amu_fie_cpus) && 166 cpumask_test_cpu(cpu, amu_fie_cpus); 167 } 168 169 void arch_cpu_idle_enter(void) 170 { 171 unsigned int cpu = smp_processor_id(); 172 173 if (!amu_fie_cpu_supported(cpu)) 174 return; 175 176 /* Kick in AMU update but only if one has not happened already */ 177 if (housekeeping_cpu(cpu, HK_TYPE_TICK) && 178 time_is_before_jiffies(per_cpu(cpu_amu_samples.last_scale_update, cpu))) 179 amu_scale_freq_tick(); 180 } 181 182 #define AMU_SAMPLE_EXP_MS 20 183 184 int arch_freq_get_on_cpu(int cpu) 185 { 186 struct amu_cntr_sample *amu_sample; 187 unsigned int start_cpu = cpu; 188 unsigned long last_update; 189 unsigned int freq = 0; 190 u64 scale; 191 192 if (!amu_fie_cpu_supported(cpu) || !arch_scale_freq_ref(cpu)) 193 return -EOPNOTSUPP; 194 195 while (1) { 196 197 amu_sample = per_cpu_ptr(&cpu_amu_samples, cpu); 198 199 last_update = amu_sample->last_scale_update; 200 201 /* 202 * For those CPUs that are in full dynticks mode, or those that have 203 * not seen tick for a while, try an alternative source for the counters 204 * (and thus freq scale), if available, for given policy: this boils 205 * down to identifying an active cpu within the same freq domain, if any. 206 */ 207 if (!housekeeping_cpu(cpu, HK_TYPE_TICK) || 208 time_is_before_jiffies(last_update + msecs_to_jiffies(AMU_SAMPLE_EXP_MS))) { 209 struct cpufreq_policy *policy = cpufreq_cpu_get(cpu); 210 int ref_cpu; 211 212 if (!policy) 213 return -EINVAL; 214 215 if (!cpumask_intersects(policy->related_cpus, 216 housekeeping_cpumask(HK_TYPE_TICK))) { 217 cpufreq_cpu_put(policy); 218 return -EOPNOTSUPP; 219 } 220 221 for_each_cpu_wrap(ref_cpu, policy->cpus, cpu + 1) { 222 if (ref_cpu == start_cpu) { 223 /* Prevent verifying same CPU twice */ 224 ref_cpu = nr_cpu_ids; 225 break; 226 } 227 if (!idle_cpu(ref_cpu)) 228 break; 229 } 230 231 cpufreq_cpu_put(policy); 232 233 if (ref_cpu >= nr_cpu_ids) 234 /* No alternative to pull info from */ 235 return -EAGAIN; 236 237 cpu = ref_cpu; 238 } else { 239 break; 240 } 241 } 242 /* 243 * Reversed computation to the one used to determine 244 * the arch_freq_scale value 245 * (see amu_scale_freq_tick for details) 246 */ 247 scale = arch_scale_freq_capacity(cpu); 248 freq = scale * arch_scale_freq_ref(cpu); 249 freq >>= SCHED_CAPACITY_SHIFT; 250 return freq; 251 } 252 253 static void amu_fie_setup(const struct cpumask *cpus) 254 { 255 int cpu; 256 257 /* We are already set since the last insmod of cpufreq driver */ 258 if (cpumask_available(amu_fie_cpus) && 259 unlikely(cpumask_subset(cpus, amu_fie_cpus))) 260 return; 261 262 for_each_cpu(cpu, cpus) 263 if (!freq_counters_valid(cpu)) 264 return; 265 266 if (!cpumask_available(amu_fie_cpus) && 267 !zalloc_cpumask_var(&amu_fie_cpus, GFP_KERNEL)) { 268 WARN_ONCE(1, "Failed to allocate FIE cpumask for CPUs[%*pbl]\n", 269 cpumask_pr_args(cpus)); 270 return; 271 } 272 273 cpumask_or(amu_fie_cpus, amu_fie_cpus, cpus); 274 275 topology_set_scale_freq_source(&amu_sfd, amu_fie_cpus); 276 277 pr_debug("CPUs[%*pbl]: counters will be used for FIE.", 278 cpumask_pr_args(cpus)); 279 } 280 281 static int init_amu_fie_callback(struct notifier_block *nb, unsigned long val, 282 void *data) 283 { 284 struct cpufreq_policy *policy = data; 285 286 if (val == CPUFREQ_CREATE_POLICY) 287 amu_fie_setup(policy->related_cpus); 288 289 /* 290 * We don't need to handle CPUFREQ_REMOVE_POLICY event as the AMU 291 * counters don't have any dependency on cpufreq driver once we have 292 * initialized AMU support and enabled invariance. The AMU counters will 293 * keep on working just fine in the absence of the cpufreq driver, and 294 * for the CPUs for which there are no counters available, the last set 295 * value of arch_freq_scale will remain valid as that is the frequency 296 * those CPUs are running at. 297 */ 298 299 return 0; 300 } 301 302 static struct notifier_block init_amu_fie_notifier = { 303 .notifier_call = init_amu_fie_callback, 304 }; 305 306 static int __init init_amu_fie(void) 307 { 308 return cpufreq_register_notifier(&init_amu_fie_notifier, 309 CPUFREQ_POLICY_NOTIFIER); 310 } 311 core_initcall(init_amu_fie); 312 313 #ifdef CONFIG_ACPI_CPPC_LIB 314 #include <acpi/cppc_acpi.h> 315 316 static void cpu_read_corecnt(void *val) 317 { 318 /* 319 * A value of 0 can be returned if the current CPU does not support AMUs 320 * or if the counter is disabled for this CPU. A return value of 0 at 321 * counter read is properly handled as an error case by the users of the 322 * counter. 323 */ 324 *(u64 *)val = read_corecnt(); 325 } 326 327 static void cpu_read_constcnt(void *val) 328 { 329 /* 330 * Return 0 if the current CPU is affected by erratum 2457168. A value 331 * of 0 is also returned if the current CPU does not support AMUs or if 332 * the counter is disabled. A return value of 0 at counter read is 333 * properly handled as an error case by the users of the counter. 334 */ 335 *(u64 *)val = this_cpu_has_cap(ARM64_WORKAROUND_2457168) ? 336 0UL : read_constcnt(); 337 } 338 339 static inline 340 int counters_read_on_cpu(int cpu, smp_call_func_t func, u64 *val) 341 { 342 /* 343 * Abort call on counterless CPU or when interrupts are 344 * disabled - can lead to deadlock in smp sync call. 345 */ 346 if (!cpu_has_amu_feat(cpu)) 347 return -EOPNOTSUPP; 348 349 if (WARN_ON_ONCE(irqs_disabled())) 350 return -EPERM; 351 352 smp_call_function_single(cpu, func, val, 1); 353 354 return 0; 355 } 356 357 /* 358 * Refer to drivers/acpi/cppc_acpi.c for the description of the functions 359 * below. 360 */ 361 bool cpc_ffh_supported(void) 362 { 363 int cpu = get_cpu_with_amu_feat(); 364 365 /* 366 * FFH is considered supported if there is at least one present CPU that 367 * supports AMUs. Using FFH to read core and reference counters for CPUs 368 * that do not support AMUs, have counters disabled or that are affected 369 * by errata, will result in a return value of 0. 370 * 371 * This is done to allow any enabled and valid counters to be read 372 * through FFH, knowing that potentially returning 0 as counter value is 373 * properly handled by the users of these counters. 374 */ 375 if ((cpu >= nr_cpu_ids) || !cpumask_test_cpu(cpu, cpu_present_mask)) 376 return false; 377 378 return true; 379 } 380 381 int cpc_read_ffh(int cpu, struct cpc_reg *reg, u64 *val) 382 { 383 int ret = -EOPNOTSUPP; 384 385 switch ((u64)reg->address) { 386 case 0x0: 387 ret = counters_read_on_cpu(cpu, cpu_read_corecnt, val); 388 break; 389 case 0x1: 390 ret = counters_read_on_cpu(cpu, cpu_read_constcnt, val); 391 break; 392 } 393 394 if (!ret) { 395 *val &= GENMASK_ULL(reg->bit_offset + reg->bit_width - 1, 396 reg->bit_offset); 397 *val >>= reg->bit_offset; 398 } 399 400 return ret; 401 } 402 403 int cpc_write_ffh(int cpunum, struct cpc_reg *reg, u64 val) 404 { 405 return -EOPNOTSUPP; 406 } 407 #endif /* CONFIG_ACPI_CPPC_LIB */ 408