xref: /linux/drivers/clocksource/mips-gic-timer.c (revision 486b257357c23743d9729ea715136d105fff461e)
1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2012 MIPS Technologies, Inc.  All rights reserved.
3 
4 #define pr_fmt(fmt) "mips-gic-timer: " fmt
5 
6 #include <linux/clk.h>
7 #include <linux/clockchips.h>
8 #include <linux/cpu.h>
9 #include <linux/delay.h>
10 #include <linux/init.h>
11 #include <linux/interrupt.h>
12 #include <linux/notifier.h>
13 #include <linux/of_irq.h>
14 #include <linux/percpu.h>
15 #include <linux/sched_clock.h>
16 #include <linux/smp.h>
17 #include <linux/time.h>
18 #include <asm/mips-cps.h>
19 
20 static DEFINE_PER_CPU(struct clock_event_device, gic_clockevent_device);
21 static int gic_timer_irq;
22 static unsigned int gic_frequency;
23 static unsigned int gic_count_width;
24 static bool __read_mostly gic_clock_unstable;
25 static bool __read_mostly gic_use_local_counter = true;
26 static unsigned long *gic_synced_cl_map;
27 static struct work_struct gic_promote_work;
28 
29 static void gic_clocksource_unstable(char *reason);
30 
31 static u64 notrace gic_read_count_2x32(void)
32 {
33 	unsigned int hi, hi2, lo;
34 
35 	do {
36 		hi = read_gic_counter_32h();
37 		lo = read_gic_counter_32l();
38 		hi2 = read_gic_counter_32h();
39 	} while (hi2 != hi);
40 
41 	return (((u64) hi) << 32) + lo;
42 }
43 
44 static u64 notrace gic_read_count_64(void)
45 {
46 	return read_gic_counter();
47 }
48 
49 static u64 notrace gic_read_count(void)
50 {
51 	if (mips_cm_is64)
52 		return gic_read_count_64();
53 
54 	return gic_read_count_2x32();
55 }
56 
57 static int gic_next_event(unsigned long delta, struct clock_event_device *evt)
58 {
59 	u64 cnt;
60 	int res;
61 
62 	cnt = gic_read_count();
63 	cnt += (u64)delta;
64 	write_gic_vl_compare(cnt);
65 
66 	res = ((int)(gic_read_count() - cnt) >= 0) ? -ETIME : 0;
67 	return res;
68 }
69 
70 static irqreturn_t gic_compare_interrupt(int irq, void *dev_id)
71 {
72 	struct clock_event_device *cd = dev_id;
73 
74 	write_gic_vl_compare(read_gic_vl_compare());
75 	cd->event_handler(cd);
76 	return IRQ_HANDLED;
77 }
78 
79 static void gic_clockevent_cpu_init(unsigned int cpu,
80 				    struct clock_event_device *cd)
81 {
82 	cd->name		= "MIPS GIC";
83 	cd->features		= CLOCK_EVT_FEAT_ONESHOT |
84 				  CLOCK_EVT_FEAT_PERCPU |
85 				  CLOCK_EVT_FEAT_C3STOP;
86 
87 	cd->rating		= 350;
88 	cd->irq			= gic_timer_irq;
89 	cd->cpumask		= cpumask_of(cpu);
90 	cd->set_next_event	= gic_next_event;
91 
92 	clockevents_config_and_register(cd, gic_frequency, 0x300, 0x7fffffff);
93 
94 	enable_percpu_irq(gic_timer_irq, IRQ_TYPE_NONE);
95 }
96 
97 static void gic_clockevent_cpu_exit(struct clock_event_device *cd)
98 {
99 	disable_percpu_irq(gic_timer_irq);
100 }
101 
102 static void gic_update_frequency(void *data)
103 {
104 	unsigned long rate = (unsigned long)data;
105 
106 	clockevents_update_freq(this_cpu_ptr(&gic_clockevent_device), rate);
107 }
108 
109 /* Number of iterations to synchronize the local GIC counter */
110 #define GIC_SYNC_ITERATIONS 4
111 
112 /* Delay in us to check if the local GIC counter is still in sync with cluster 0 */
113 #define GIC_SYNC_CHECK_DELAY 100
114 
115 static void gic_sync_counter_64(unsigned int cluster)
116 {
117 	unsigned int config = read_gic_config();
118 	u64 t0, t1, t2;
119 	s64 offset = 0;
120 
121 	mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
122 
123 	for (int i = 0; i < GIC_SYNC_ITERATIONS; i++) {
124 		write_gic_config(config | GIC_CONFIG_COUNTSTOP);
125 		write_gic_counter(read_gic_redir_counter() + offset);
126 		write_gic_config(config & ~GIC_CONFIG_COUNTSTOP);
127 
128 		t0 = read_gic_counter();
129 		t1 = read_gic_redir_counter();
130 		t2 = read_gic_counter();
131 
132 		if (time_in_range64(t1, t0, t2))
133 			break;
134 
135 		/*
136 		 * Compute the offset to apply to the local counter
137 		 * so that (t1 - t0) equals (t2 - t1).
138 		 */
139 		offset += (s64)(2 * t1 - t0 - t2) / 2;
140 	}
141 
142 	mips_cm_unlock_other();
143 
144 	if (!time_in_range64(t1, t0, t2))
145 		return;
146 
147 	udelay(GIC_SYNC_CHECK_DELAY);
148 
149 	mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
150 	t0 = read_gic_counter();
151 	t1 = read_gic_redir_counter();
152 	t2 = read_gic_counter();
153 	mips_cm_unlock_other();
154 
155 	/* If so, mark the cluster as synchronized */
156 	if (time_in_range64(t1, t0, t2) && gic_synced_cl_map)
157 		bitmap_set(gic_synced_cl_map, cluster, 1);
158 }
159 
160 static int gic_starting_cpu(unsigned int cpu)
161 {
162 	unsigned int cluster = cpu_cluster(&cpu_data[cpu]);
163 
164 	if (read_gic_config() & GIC_CONFIG_COUNTSTOP) {
165 		clear_gic_config(GIC_CONFIG_COUNTSTOP);
166 
167 		if (cluster && mips_cm_is64 && !gic_clock_unstable)
168 			gic_sync_counter_64(cluster);
169 
170 		if (gic_synced_cl_map &&
171 		    bitmap_full(gic_synced_cl_map, mips_cps_numclusters()))
172 			schedule_work(&gic_promote_work);
173 	}
174 
175 	gic_clockevent_cpu_init(cpu, this_cpu_ptr(&gic_clockevent_device));
176 	return 0;
177 }
178 
179 static int gic_clk_notifier(struct notifier_block *nb, unsigned long action,
180 			    void *data)
181 {
182 	struct clk_notifier_data *cnd = data;
183 
184 	if (action == POST_RATE_CHANGE) {
185 		gic_clocksource_unstable("ref clock rate change");
186 		on_each_cpu(gic_update_frequency, (void *)cnd->new_rate, 1);
187 	}
188 
189 	return NOTIFY_OK;
190 }
191 
192 static int gic_dying_cpu(unsigned int cpu)
193 {
194 	gic_clockevent_cpu_exit(this_cpu_ptr(&gic_clockevent_device));
195 	return 0;
196 }
197 
198 static struct notifier_block gic_clk_nb = {
199 	.notifier_call = gic_clk_notifier,
200 };
201 
202 static int gic_clockevent_init(void)
203 {
204 	int ret;
205 
206 	if (!gic_frequency)
207 		return -ENXIO;
208 
209 	ret = request_percpu_irq(gic_timer_irq, gic_compare_interrupt,
210 				 "timer", &gic_clockevent_device);
211 	if (ret < 0) {
212 		pr_err("IRQ %d setup failed (%d)\n", gic_timer_irq, ret);
213 		return ret;
214 	}
215 
216 	cpuhp_setup_state(CPUHP_AP_MIPS_GIC_TIMER_STARTING,
217 			  "clockevents/mips/gic/timer:starting",
218 			  gic_starting_cpu, gic_dying_cpu);
219 	return 0;
220 }
221 
222 static u64 gic_hpt_read_multicluster(struct clocksource *cs)
223 {
224 	unsigned int hi, hi2, lo;
225 	u64 count;
226 
227 	mips_cm_lock_other(0, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL);
228 
229 	if (mips_cm_is64) {
230 		count = read_gic_redir_counter();
231 		goto out;
232 	}
233 
234 	hi = read_gic_redir_counter_32h();
235 	while (true) {
236 		lo = read_gic_redir_counter_32l();
237 
238 		/* If hi didn't change then lo didn't wrap & we're done */
239 		hi2 = read_gic_redir_counter_32h();
240 		if (hi2 == hi)
241 			break;
242 
243 		/* Otherwise, repeat with the latest hi value */
244 		hi = hi2;
245 	}
246 
247 	count = (((u64)hi) << 32) + lo;
248 out:
249 	mips_cm_unlock_other();
250 	return count;
251 }
252 
253 static u64 gic_hpt_read(struct clocksource *cs)
254 {
255 	if (unlikely(!READ_ONCE(gic_use_local_counter)))
256 		return gic_hpt_read_multicluster(cs);
257 
258 	return gic_read_count();
259 }
260 
261 static struct clocksource gic_clocksource = {
262 	.name			= "GIC",
263 	.read			= gic_hpt_read,
264 	.flags			= CLOCK_SOURCE_IS_CONTINUOUS,
265 	.vdso_clock_mode	= VDSO_CLOCKMODE_GIC,
266 };
267 
268 static void gic_clocksource_unstable(char *reason)
269 {
270 	if (gic_clock_unstable)
271 		return;
272 
273 	gic_clock_unstable = true;
274 
275 	pr_info("GIC timer is unstable due to %s\n", reason);
276 
277 	clocksource_mark_unstable(&gic_clocksource);
278 }
279 
280 static void gic_clocksource_promote(struct work_struct *work)
281 {
282 	if (gic_clock_unstable || READ_ONCE(gic_use_local_counter))
283 		return;
284 
285 	WRITE_ONCE(gic_use_local_counter, true);
286 
287 #ifdef CONFIG_GENERIC_GETTIMEOFDAY
288 	gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_GIC;
289 #endif
290 
291 	if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) {
292 		sched_clock_register(mips_cm_is64 ?
293 				     gic_read_count_64 : gic_read_count_2x32,
294 				     gic_count_width, gic_frequency);
295 	}
296 }
297 
298 static int __init __gic_clocksource_init(void)
299 {
300 	unsigned int numclusters;
301 	bool synced = false;
302 	int ret;
303 
304 	/* Set clocksource mask. */
305 	gic_count_width = read_gic_config() & GIC_CONFIG_COUNTBITS;
306 	gic_count_width >>= __ffs(GIC_CONFIG_COUNTBITS);
307 	gic_count_width *= 4;
308 	gic_count_width += 32;
309 	gic_clocksource.mask = CLOCKSOURCE_MASK(gic_count_width);
310 
311 	/* Calculate a somewhat reasonable rating value. */
312 	if (mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ))
313 		gic_clocksource.rating = 300; /* Good when frequency is stable */
314 	else
315 		gic_clocksource.rating = 200;
316 	gic_clocksource.rating += clamp(gic_frequency / 10000000, 0, 99);
317 
318 	numclusters = mips_cps_numclusters();
319 	if (numclusters > 1)
320 		gic_synced_cl_map = bitmap_zalloc(numclusters, GFP_KERNEL);
321 
322 	/*
323 	 * Mark cluster 0 as synchronized (with itself), and all clusters
324 	 * without cores since there is no local GIC counter access on those.
325 	 */
326 	if (gic_synced_cl_map) {
327 		bitmap_set(gic_synced_cl_map, 0, 1);
328 		for (unsigned int cl = 0; cl < numclusters; cl++) {
329 			if (!mips_cps_numcores(cl))
330 				bitmap_set(gic_synced_cl_map, cl, 1);
331 		}
332 		synced = bitmap_full(gic_synced_cl_map, numclusters);
333 	}
334 
335 	if (numclusters > 1 && !synced) {
336 		WRITE_ONCE(gic_use_local_counter, false);
337 		gic_clocksource.vdso_clock_mode = VDSO_CLOCKMODE_NONE;
338 		INIT_WORK(&gic_promote_work, gic_clocksource_promote);
339 	}
340 
341 	ret = clocksource_register_hz(&gic_clocksource, gic_frequency);
342 	if (ret < 0)
343 		pr_warn("Unable to register clocksource\n");
344 
345 	return ret;
346 }
347 
348 static int __init gic_clocksource_of_init(struct device_node *node)
349 {
350 	struct clk *clk;
351 	int ret;
352 
353 	if (!mips_gic_present() || !node->parent ||
354 	    !of_device_is_compatible(node->parent, "mti,gic")) {
355 		pr_warn("No DT definition\n");
356 		return -ENXIO;
357 	}
358 
359 	clk = of_clk_get(node, 0);
360 	if (!IS_ERR(clk)) {
361 		ret = clk_prepare_enable(clk);
362 		if (ret < 0) {
363 			pr_err("Failed to enable clock\n");
364 			clk_put(clk);
365 			return ret;
366 		}
367 
368 		gic_frequency = clk_get_rate(clk);
369 	} else if (of_property_read_u32(node, "clock-frequency",
370 					&gic_frequency)) {
371 		pr_err("Frequency not specified\n");
372 		return -EINVAL;
373 	}
374 	gic_timer_irq = irq_of_parse_and_map(node, 0);
375 	if (!gic_timer_irq) {
376 		pr_err("IRQ not specified\n");
377 		return -EINVAL;
378 	}
379 
380 	ret = __gic_clocksource_init();
381 	if (ret)
382 		return ret;
383 
384 	ret = gic_clockevent_init();
385 	if (!ret && !IS_ERR(clk)) {
386 		if (clk_notifier_register(clk, &gic_clk_nb) < 0)
387 			pr_warn("Unable to register clock notifier\n");
388 	}
389 
390 	/*
391 	 * It's safe to use the MIPS GIC timer as a sched clock source only if
392 	 * its ticks are stable, which is true on either the platforms with
393 	 * stable CPU frequency or on the platforms with CM3 and CPU frequency
394 	 * change performed by the CPC core clocks divider.
395 	 */
396 	if ((mips_cm_revision() >= CM_REV_CM3 || !IS_ENABLED(CONFIG_CPU_FREQ)) &&
397 	     READ_ONCE(gic_use_local_counter)) {
398 		sched_clock_register(mips_cm_is64 ?
399 				     gic_read_count_64 : gic_read_count_2x32,
400 				     gic_count_width, gic_frequency);
401 	}
402 
403 	return 0;
404 }
405 TIMER_OF_DECLARE(mips_gic_timer, "mti,gic-timer",
406 		       gic_clocksource_of_init);
407