1 // SPDX-License-Identifier: GPL-2.0 2 // Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 3 4 #define pr_fmt(fmt) "mips-gic-timer: " fmt 5 6 #include <linux/clk.h> 7 #include <linux/clockchips.h> 8 #include <linux/cpu.h> 9 #include <linux/delay.h> 10 #include <linux/init.h> 11 #include <linux/interrupt.h> 12 #include <linux/notifier.h> 13 #include <linux/of_irq.h> 14 #include <linux/percpu.h> 15 #include <linux/sched_clock.h> 16 #include <linux/smp.h> 17 #include <linux/time.h> 18 #include <asm/mips-cps.h> 19 20 static DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device); 21 static int gic_timer_irq; 22 static unsigned int gic_frequency; 23 static unsigned int gic_count_width; 24 static bool __read_mostly gic_clock_unstable; 25 static bool __read_mostly gic_use_local_counter = true; 26 static unsigned long *gic_synced_cl_map; 27 static struct work_struct gic_promote_work; 28 29 static void gic_clocksource_unstable(char *reason); 30 31 static u64 notrace gic_read_count_2x32(void) 32 { 33 unsigned int hi, hi2, lo; 34 35 do { 36 hi = read_gic_counter_32h(); 37 lo = read_gic_counter_32l(); 38 hi2 = read_gic_counter_32h(); 39 } while (hi2 != hi); 40 41 return (((u64) hi) << 32) + lo; 42 } 43 44 static u64 notrace gic_read_count_64(void) 45 { 46 return read_gic_counter(); 47 } 48 49 static u64 notrace gic_read_count(void) 50 { 51 if (mips_cm_is64) 52 return gic_read_count_64(); 53 54 return gic_read_count_2x32(); 55 } 56 57 static int gic_next_event(unsigned long delta, struct clock_event_device *evt) 58 { 59 u64 cnt; 60 int res; 61 62 cnt = gic_read_count(); 63 cnt += (u64)delta; 64 write_gic_vl_compare(cnt); 65 66 res = ((int)(gic_read_count() - cnt) >= 0) ? -ETIME : 0; 67 return res; 68 } 69 70 static irqreturn_t gic_compare_interrupt(int irq, void *dev_id) 71 { 72 struct clock_event_device *cd = dev_id; 73 74 write_gic_vl_compare(read_gic_vl_compare()); 75 cd->event_handler(cd); 76 return IRQ_HANDLED; 77 } 78 79 static void gic_clockevent_cpu_init(unsigned int cpu, 80 struct clock_event_device *cd) 81 { 82 cd->name = "MIPS GIC"; 83 cd->features = CLOCK_EVT_FEAT_ONESHOT | 84 CLOCK_EVT_FEAT_PERCPU | 85 CLOCK_EVT_FEAT_C3STOP; 86 87 cd->rating = 350; 88 cd->irq = gic_timer_irq; 89 cd->cpumask = cpumask_of(cpu); 90 cd->set_next_event = gic_next_event; 91 92 clockevents_config_and_register(cd, gic_frequency, 0x300, 0x7fffffff); 93 94 enable_percpu_irq(gic_timer_irq, IRQ_TYPE_NONE); 95 } 96 97 static void gic_clockevent_cpu_exit(struct clock_event_device *cd) 98 { 99 disable_percpu_irq(gic_timer_irq); 100 } 101 102 static void gic_update_frequency(void *data) 103 { 104 unsigned long rate = (unsigned long)data; 105 106 clockevents_update_freq(this_cpu_ptr(&gic_clockevent_device), rate); 107 } 108 109 /* Number of iterations to synchronize the local GIC counter */ 110 #define GIC_SYNC_ITERATIONS 4 111 112 /* Delay in us to check if the local GIC counter is still in sync with cluster 0 */ 113 #define GIC_SYNC_CHECK_DELAY 100 114 115 static void gic_sync_counter_64(unsigned int cluster) 116 { 117 unsigned int config = read_gic_config(); 118 u64 t0, t1, t2; 119 s64 offset = 0; 120 121 mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); 122 123 for (int i = 0; i < GIC_SYNC_ITERATIONS; i++) { 124 write_gic_config(config | GIC_CONFIG_COUNTSTOP); 125 write_gic_counter(read_gic_redir_counter() + offset); 126 write_gic_config(config & ~GIC_CONFIG_COUNTSTOP); 127 128 t0 = read_gic_counter(); 129 t1 = read_gic_redir_counter(); 130 t2 = read_gic_counter(); 131 132 if (time_in_range64(t1, t0, t2)) 133 break; 134 135 /* 136 * Compute the offset to apply to the local counter 137 * so that (t1 - t0) equals (t2 - t1). 138 */ 139 offset += (s64)(2 * t1 - t0 - t2) / 2; 140 } 141 142 mips_cm_unlock_other(); 143 144 if (!time_in_range64(t1, t0, t2)) 145 return; 146 147 udelay(GIC_SYNC_CHECK_DELAY); 148 149 mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); 150 t0 = read_gic_counter(); 151 t1 = read_gic_redir_counter(); 152 t2 = read_gic_counter(); 153 mips_cm_unlock_other(); 154 155 /* If so, mark the cluster as synchronized */ 156 if (time_in_range64(t1, t0, t2) && gic_synced_cl_map) 157 bitmap_set(gic_synced_cl_map, cluster, 1); 158 } 159 160 static int gic_starting_cpu(unsigned int cpu) 161 { 162 unsigned int cluster = cpu_cluster(&cpu_data[cpu]); 163 164 if (read_gic_config() & GIC_CONFIG_COUNTSTOP) { 165 clear_gic_config(GIC_CONFIG_COUNTSTOP); 166 167 if (cluster && mips_cm_is64 && !gic_clock_unstable) 168 gic_sync_counter_64(cluster); 169 170 if (gic_synced_cl_map && 171 bitmap_full(gic_synced_cl_map, mips_cps_numclusters())) 172 schedule_work(&gic_promote_work); 173 } 174 175 gic_clockevent_cpu_init(cpu, this_cpu_ptr(&gic_clockevent_device)); 176 return 0; 177 } 178 179 static int gic_clk_notifier(struct notifier_block *nb, unsigned long action, 180 void *data) 181 { 182 struct clk_notifier_data *cnd = data; 183 184 if (action == POST_RATE_CHANGE) { 185 gic_clocksource_unstable("ref clock rate change"); 186 on_each_cpu(gic_update_frequency, (void *)cnd->new_rate, 1); 187 } 188 189 return NOTIFY_OK; 190 } 191 192 static int gic_dying_cpu(unsigned int cpu) 193 { 194 gic_clockevent_cpu_exit(this_cpu_ptr(&gic_clockevent_device)); 195 return 0; 196 } 197 198 static struct notifier_block gic_clk_nb = { 199 .notifier_call = gic_clk_notifier, 200 }; 201 202 static int gic_clockevent_init(void) 203 { 204 int ret; 205 206 if (!gic_frequency) 207 return -ENXIO; 208 209 ret = request_percpu_irq(gic_timer_irq, gic_compare_interrupt, 210 "timer", &gic_clockevent_device); 211 if (ret < 0) { 212 pr_err("IRQ %d setup failed (%d)\n", gic_timer_irq, ret); 213 return ret; 214 } 215 216 cpuhp_setup_state(CPUHP_AP_MIPS_GIC_TIMER_STARTING, 217 "clockevents/mips/gic/timer:starting", 218 gic_starting_cpu, gic_dying_cpu); 219 return 0; 220 } 221 222 static u64 gic_hpt_read_multicluster(struct clocksource *cs) 223 { 224 unsigned int hi, hi2, lo; 225 u64 count; 226 227 mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); 228 229 if (mips_cm_is64) { 230 count = read_gic_redir_counter(); 231 goto out; 232 } 233 234 hi = read_gic_redir_counter_32h(); 235 while (true) { 236 lo = read_gic_redir_counter_32l(); 237 238 /* If hi didn't change then lo didn't wrap & we're done */ 239 hi2 = read_gic_redir_counter_32h(); 240 if (hi2 == hi) 241 break; 242 243 /* Otherwise, repeat with the latest hi value */ 244 hi = hi2; 245 } 246 247 count = (((u64)hi) << 32) + lo; 248 out: 249 mips_cm_unlock_other(); 250 return count; 251 } 252 253 static u64 gic_hpt_read(struct clocksource *cs) 254 { 255 if (unlikely(!READ_ONCE(gic_use_local_counter))) 256 return gic_hpt_read_multicluster(cs); 257 258 return gic_read_count(); 259 } 260 261 static struct clocksource gic_clocksource = { 262 .name = "GIC", 263 .read = gic_hpt_read, 264 .flags = CLOCK_SOURCE_IS_CONTINUOUS, 265 .vdso_clock_mode = VDSO_CLOCKMODE_GIC, 266 }; 267 268 static void gic_clocksource_unstable(char *reason) 269 { 270 if (gic_clock_unstable) 271 return; 272 273 gic_clock_unstable = true; 274 275 pr_info("GIC timer is unstable due to %s\n", reason); 276 277 clocksource_mark_unstable(&gic_clocksource); 278 } 279 280 static void gic_clocksource_promote(struct work_struct *work) 281 { 282 if (gic_clock_unstable || READ_ONCE(gic_use_local_counter)) 283 return; 284 285 WRITE_ONCE(gic_use_local_counter, true); 286 287 #ifdef CONFIG_GENERIC_GETTIMEOFDAY 288 gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_GIC; 289 #endif 290 291 if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) { 292 sched_clock_register(mips_cm_is64 ? 293 gic_read_count_64 : gic_read_count_2x32, 294 gic_count_width, gic_frequency); 295 } 296 } 297 298 static int __init __gic_clocksource_init(void) 299 { 300 unsigned int numclusters; 301 bool synced = false; 302 int ret; 303 304 /* Set clocksource mask. */ 305 gic_count_width = read_gic_config() & GIC_CONFIG_COUNTBITS; 306 gic_count_width >>= __ffs(GIC_CONFIG_COUNTBITS); 307 gic_count_width *= 4; 308 gic_count_width += 32; 309 gic_clocksource.mask = CLOCKSOURCE_MASK(gic_count_width); 310 311 /* Calculate a somewhat reasonable rating value. */ 312 if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) 313 gic_clocksource.rating = 300; /* Good when frequency is stable */ 314 else 315 gic_clocksource.rating = 200; 316 gic_clocksource.rating += clamp(gic_frequency / 10000000, 0, 99); 317 318 numclusters = mips_cps_numclusters(); 319 if (numclusters > 1) 320 gic_synced_cl_map = bitmap_zalloc(numclusters, GFP_KERNEL); 321 322 /* 323 * Mark cluster 0 as synchronized (with itself), and all clusters 324 * without cores since there is no local GIC counter access on those. 325 */ 326 if (gic_synced_cl_map) { 327 bitmap_set(gic_synced_cl_map, 0, 1); 328 for (unsigned int cl = 0; cl < numclusters; cl++) { 329 if (!mips_cps_numcores(cl)) 330 bitmap_set(gic_synced_cl_map, cl, 1); 331 } 332 synced = bitmap_full(gic_synced_cl_map, numclusters); 333 } 334 335 if (numclusters > 1 && !synced) { 336 WRITE_ONCE(gic_use_local_counter, false); 337 gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_NONE; 338 INIT_WORK(&gic_promote_work, gic_clocksource_promote); 339 } 340 341 ret = clocksource_register_hz(&gic_clocksource, gic_frequency); 342 if (ret < 0) 343 pr_warn("Unable to register clocksource\n"); 344 345 return ret; 346 } 347 348 static int __init gic_clocksource_of_init(struct device_node *node) 349 { 350 struct clk *clk; 351 int ret; 352 353 if (!mips_gic_present() || !node->parent || 354 !of_device_is_compatible(node->parent, "mti,gic")) { 355 pr_warn("No DT definition\n"); 356 return -ENXIO; 357 } 358 359 clk = of_clk_get(node, 0); 360 if (!IS_ERR(clk)) { 361 ret = clk_prepare_enable(clk); 362 if (ret < 0) { 363 pr_err("Failed to enable clock\n"); 364 clk_put(clk); 365 return ret; 366 } 367 368 gic_frequency = clk_get_rate(clk); 369 } else if (of_property_read_u32(node, "clock-frequency", 370 &gic_frequency)) { 371 pr_err("Frequency not specified\n"); 372 return -EINVAL; 373 } 374 gic_timer_irq = irq_of_parse_and_map(node, 0); 375 if (!gic_timer_irq) { 376 pr_err("IRQ not specified\n"); 377 return -EINVAL; 378 } 379 380 ret = __gic_clocksource_init(); 381 if (ret) 382 return ret; 383 384 ret = gic_clockevent_init(); 385 if (!ret && !IS_ERR(clk)) { 386 if (clk_notifier_register(clk, &gic_clk_nb) < 0) 387 pr_warn("Unable to register clock notifier\n"); 388 } 389 390 /* 391 * It's safe to use the MIPS GIC timer as a sched clock source only if 392 * its ticks are stable, which is true on either the platforms with 393 * stable CPU frequency or on the platforms with CM3 and CPU frequency 394 * change performed by the CPC core clocks divider. 395 */ 396 if ((mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) && 397 READ_ONCE(gic_use_local_counter)) { 398 sched_clock_register(mips_cm_is64 ? 399 gic_read_count_64 : gic_read_count_2x32, 400 gic_count_width, gic_frequency); 401 } 402 403 return 0; 404 } 405 TIMER_OF_DECLARE(mips_gic_timer, "mti,gic-timer", 406 gic_clocksource_of_init); 407