xref: /linux/arch/s390/kernel/vtime.c (revision d2a4a07190f42e4f82805daf58e708400b703f1c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  *    Virtual cpu timer based timer functions.
4  *
5  *    Copyright IBM Corp. 2004, 2012
6  *    Author(s): Jan Glauber <jan.glauber@de.ibm.com>
7  */
8 
9 #include <linux/kernel_stat.h>
10 #include <linux/export.h>
11 #include <linux/kernel.h>
12 #include <linux/timex.h>
13 #include <linux/types.h>
14 #include <linux/time.h>
15 #include <asm/alternative.h>
16 #include <asm/cputime.h>
17 #include <asm/vtimer.h>
18 #include <asm/vtime.h>
19 #include <asm/cpu_mf.h>
20 #include <asm/smp.h>
21 
22 #include "entry.h"
23 
24 static void virt_timer_expire(void);
25 
26 static LIST_HEAD(virt_timer_list);
27 static DEFINE_SPINLOCK(virt_timer_lock);
28 static atomic64_t virt_timer_current;
29 static atomic64_t virt_timer_elapsed;
30 
31 DEFINE_PER_CPU(u64, mt_cycles[8]);
32 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
33 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
34 static DEFINE_PER_CPU(u64, mt_scaling_jiffies);
35 
36 static inline void set_vtimer(u64 expires)
37 {
38 	u64 timer;
39 
40 	asm volatile(
41 		"	stpt	%0\n"	/* Store current cpu timer value */
42 		"	spt	%1"	/* Set new value imm. afterwards */
43 		: "=Q" (timer) : "Q" (expires));
44 	S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer;
45 	S390_lowcore.last_update_timer = expires;
46 }
47 
48 static inline int virt_timer_forward(u64 elapsed)
49 {
50 	BUG_ON(!irqs_disabled());
51 
52 	if (list_empty(&virt_timer_list))
53 		return 0;
54 	elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed);
55 	return elapsed >= atomic64_read(&virt_timer_current);
56 }
57 
58 static void update_mt_scaling(void)
59 {
60 	u64 cycles_new[8], *cycles_old;
61 	u64 delta, fac, mult, div;
62 	int i;
63 
64 	stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
65 	cycles_old = this_cpu_ptr(mt_cycles);
66 	fac = 1;
67 	mult = div = 0;
68 	for (i = 0; i <= smp_cpu_mtid; i++) {
69 		delta = cycles_new[i] - cycles_old[i];
70 		div += delta;
71 		mult *= i + 1;
72 		mult += delta * fac;
73 		fac *= i + 1;
74 	}
75 	div *= fac;
76 	if (div > 0) {
77 		/* Update scaling factor */
78 		__this_cpu_write(mt_scaling_mult, mult);
79 		__this_cpu_write(mt_scaling_div, div);
80 		memcpy(cycles_old, cycles_new,
81 		       sizeof(u64) * (smp_cpu_mtid + 1));
82 	}
83 	__this_cpu_write(mt_scaling_jiffies, jiffies_64);
84 }
85 
86 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
87 {
88 	u64 delta;
89 
90 	delta = new - *tsk_vtime;
91 	*tsk_vtime = new;
92 	return delta;
93 }
94 
95 
96 static inline u64 scale_vtime(u64 vtime)
97 {
98 	u64 mult = __this_cpu_read(mt_scaling_mult);
99 	u64 div = __this_cpu_read(mt_scaling_div);
100 
101 	if (smp_cpu_mtid)
102 		return vtime * mult / div;
103 	return vtime;
104 }
105 
106 static void account_system_index_scaled(struct task_struct *p, u64 cputime,
107 					enum cpu_usage_stat index)
108 {
109 	p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
110 	account_system_index_time(p, cputime_to_nsecs(cputime), index);
111 }
112 
113 /*
114  * Update process times based on virtual cpu times stored by entry.S
115  * to the lowcore fields user_timer, system_timer & steal_clock.
116  */
117 static int do_account_vtime(struct task_struct *tsk)
118 {
119 	u64 timer, clock, user, guest, system, hardirq, softirq;
120 
121 	timer = S390_lowcore.last_update_timer;
122 	clock = S390_lowcore.last_update_clock;
123 	asm volatile(
124 		"	stpt	%0\n"	/* Store current cpu timer value */
125 		"	stckf	%1"	/* Store current tod clock value */
126 		: "=Q" (S390_lowcore.last_update_timer),
127 		  "=Q" (S390_lowcore.last_update_clock)
128 		: : "cc");
129 	clock = S390_lowcore.last_update_clock - clock;
130 	timer -= S390_lowcore.last_update_timer;
131 
132 	if (hardirq_count())
133 		S390_lowcore.hardirq_timer += timer;
134 	else
135 		S390_lowcore.system_timer += timer;
136 
137 	/* Update MT utilization calculation */
138 	if (smp_cpu_mtid &&
139 	    time_after64(jiffies_64, this_cpu_read(mt_scaling_jiffies)))
140 		update_mt_scaling();
141 
142 	/* Calculate cputime delta */
143 	user = update_tsk_timer(&tsk->thread.user_timer,
144 				READ_ONCE(S390_lowcore.user_timer));
145 	guest = update_tsk_timer(&tsk->thread.guest_timer,
146 				 READ_ONCE(S390_lowcore.guest_timer));
147 	system = update_tsk_timer(&tsk->thread.system_timer,
148 				  READ_ONCE(S390_lowcore.system_timer));
149 	hardirq = update_tsk_timer(&tsk->thread.hardirq_timer,
150 				   READ_ONCE(S390_lowcore.hardirq_timer));
151 	softirq = update_tsk_timer(&tsk->thread.softirq_timer,
152 				   READ_ONCE(S390_lowcore.softirq_timer));
153 	S390_lowcore.steal_timer +=
154 		clock - user - guest - system - hardirq - softirq;
155 
156 	/* Push account value */
157 	if (user) {
158 		account_user_time(tsk, cputime_to_nsecs(user));
159 		tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
160 	}
161 
162 	if (guest) {
163 		account_guest_time(tsk, cputime_to_nsecs(guest));
164 		tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
165 	}
166 
167 	if (system)
168 		account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
169 	if (hardirq)
170 		account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
171 	if (softirq)
172 		account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
173 
174 	return virt_timer_forward(user + guest + system + hardirq + softirq);
175 }
176 
177 void vtime_task_switch(struct task_struct *prev)
178 {
179 	do_account_vtime(prev);
180 	prev->thread.user_timer = S390_lowcore.user_timer;
181 	prev->thread.guest_timer = S390_lowcore.guest_timer;
182 	prev->thread.system_timer = S390_lowcore.system_timer;
183 	prev->thread.hardirq_timer = S390_lowcore.hardirq_timer;
184 	prev->thread.softirq_timer = S390_lowcore.softirq_timer;
185 	S390_lowcore.user_timer = current->thread.user_timer;
186 	S390_lowcore.guest_timer = current->thread.guest_timer;
187 	S390_lowcore.system_timer = current->thread.system_timer;
188 	S390_lowcore.hardirq_timer = current->thread.hardirq_timer;
189 	S390_lowcore.softirq_timer = current->thread.softirq_timer;
190 }
191 
192 /*
193  * In s390, accounting pending user time also implies
194  * accounting system time in order to correctly compute
195  * the stolen time accounting.
196  */
197 void vtime_flush(struct task_struct *tsk)
198 {
199 	u64 steal, avg_steal;
200 
201 	if (do_account_vtime(tsk))
202 		virt_timer_expire();
203 
204 	steal = S390_lowcore.steal_timer;
205 	avg_steal = S390_lowcore.avg_steal_timer;
206 	if ((s64) steal > 0) {
207 		S390_lowcore.steal_timer = 0;
208 		account_steal_time(cputime_to_nsecs(steal));
209 		avg_steal += steal;
210 	}
211 	S390_lowcore.avg_steal_timer = avg_steal / 2;
212 }
213 
214 static u64 vtime_delta(void)
215 {
216 	u64 timer = S390_lowcore.last_update_timer;
217 
218 	S390_lowcore.last_update_timer = get_cpu_timer();
219 
220 	return timer - S390_lowcore.last_update_timer;
221 }
222 
223 /*
224  * Update process times based on virtual cpu times stored by entry.S
225  * to the lowcore fields user_timer, system_timer & steal_clock.
226  */
227 void vtime_account_kernel(struct task_struct *tsk)
228 {
229 	u64 delta = vtime_delta();
230 
231 	if (tsk->flags & PF_VCPU)
232 		S390_lowcore.guest_timer += delta;
233 	else
234 		S390_lowcore.system_timer += delta;
235 
236 	virt_timer_forward(delta);
237 }
238 EXPORT_SYMBOL_GPL(vtime_account_kernel);
239 
240 void vtime_account_softirq(struct task_struct *tsk)
241 {
242 	u64 delta = vtime_delta();
243 
244 	S390_lowcore.softirq_timer += delta;
245 
246 	virt_timer_forward(delta);
247 }
248 
249 void vtime_account_hardirq(struct task_struct *tsk)
250 {
251 	u64 delta = vtime_delta();
252 
253 	S390_lowcore.hardirq_timer += delta;
254 
255 	virt_timer_forward(delta);
256 }
257 
258 /*
259  * Sorted add to a list. List is linear searched until first bigger
260  * element is found.
261  */
262 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head)
263 {
264 	struct vtimer_list *tmp;
265 
266 	list_for_each_entry(tmp, head, entry) {
267 		if (tmp->expires > timer->expires) {
268 			list_add_tail(&timer->entry, &tmp->entry);
269 			return;
270 		}
271 	}
272 	list_add_tail(&timer->entry, head);
273 }
274 
275 /*
276  * Handler for expired virtual CPU timer.
277  */
278 static void virt_timer_expire(void)
279 {
280 	struct vtimer_list *timer, *tmp;
281 	unsigned long elapsed;
282 	LIST_HEAD(cb_list);
283 
284 	/* walk timer list, fire all expired timers */
285 	spin_lock(&virt_timer_lock);
286 	elapsed = atomic64_read(&virt_timer_elapsed);
287 	list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) {
288 		if (timer->expires < elapsed)
289 			/* move expired timer to the callback queue */
290 			list_move_tail(&timer->entry, &cb_list);
291 		else
292 			timer->expires -= elapsed;
293 	}
294 	if (!list_empty(&virt_timer_list)) {
295 		timer = list_first_entry(&virt_timer_list,
296 					 struct vtimer_list, entry);
297 		atomic64_set(&virt_timer_current, timer->expires);
298 	}
299 	atomic64_sub(elapsed, &virt_timer_elapsed);
300 	spin_unlock(&virt_timer_lock);
301 
302 	/* Do callbacks and recharge periodic timers */
303 	list_for_each_entry_safe(timer, tmp, &cb_list, entry) {
304 		list_del_init(&timer->entry);
305 		timer->function(timer->data);
306 		if (timer->interval) {
307 			/* Recharge interval timer */
308 			timer->expires = timer->interval +
309 				atomic64_read(&virt_timer_elapsed);
310 			spin_lock(&virt_timer_lock);
311 			list_add_sorted(timer, &virt_timer_list);
312 			spin_unlock(&virt_timer_lock);
313 		}
314 	}
315 }
316 
317 void init_virt_timer(struct vtimer_list *timer)
318 {
319 	timer->function = NULL;
320 	INIT_LIST_HEAD(&timer->entry);
321 }
322 EXPORT_SYMBOL(init_virt_timer);
323 
324 static inline int vtimer_pending(struct vtimer_list *timer)
325 {
326 	return !list_empty(&timer->entry);
327 }
328 
329 static void internal_add_vtimer(struct vtimer_list *timer)
330 {
331 	if (list_empty(&virt_timer_list)) {
332 		/* First timer, just program it. */
333 		atomic64_set(&virt_timer_current, timer->expires);
334 		atomic64_set(&virt_timer_elapsed, 0);
335 		list_add(&timer->entry, &virt_timer_list);
336 	} else {
337 		/* Update timer against current base. */
338 		timer->expires += atomic64_read(&virt_timer_elapsed);
339 		if (likely((s64) timer->expires <
340 			   (s64) atomic64_read(&virt_timer_current)))
341 			/* The new timer expires before the current timer. */
342 			atomic64_set(&virt_timer_current, timer->expires);
343 		/* Insert new timer into the list. */
344 		list_add_sorted(timer, &virt_timer_list);
345 	}
346 }
347 
348 static void __add_vtimer(struct vtimer_list *timer, int periodic)
349 {
350 	unsigned long flags;
351 
352 	timer->interval = periodic ? timer->expires : 0;
353 	spin_lock_irqsave(&virt_timer_lock, flags);
354 	internal_add_vtimer(timer);
355 	spin_unlock_irqrestore(&virt_timer_lock, flags);
356 }
357 
358 /*
359  * add_virt_timer - add a oneshot virtual CPU timer
360  */
361 void add_virt_timer(struct vtimer_list *timer)
362 {
363 	__add_vtimer(timer, 0);
364 }
365 EXPORT_SYMBOL(add_virt_timer);
366 
367 /*
368  * add_virt_timer_int - add an interval virtual CPU timer
369  */
370 void add_virt_timer_periodic(struct vtimer_list *timer)
371 {
372 	__add_vtimer(timer, 1);
373 }
374 EXPORT_SYMBOL(add_virt_timer_periodic);
375 
376 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic)
377 {
378 	unsigned long flags;
379 	int rc;
380 
381 	BUG_ON(!timer->function);
382 
383 	if (timer->expires == expires && vtimer_pending(timer))
384 		return 1;
385 	spin_lock_irqsave(&virt_timer_lock, flags);
386 	rc = vtimer_pending(timer);
387 	if (rc)
388 		list_del_init(&timer->entry);
389 	timer->interval = periodic ? expires : 0;
390 	timer->expires = expires;
391 	internal_add_vtimer(timer);
392 	spin_unlock_irqrestore(&virt_timer_lock, flags);
393 	return rc;
394 }
395 
396 /*
397  * returns whether it has modified a pending timer (1) or not (0)
398  */
399 int mod_virt_timer(struct vtimer_list *timer, u64 expires)
400 {
401 	return __mod_vtimer(timer, expires, 0);
402 }
403 EXPORT_SYMBOL(mod_virt_timer);
404 
405 /*
406  * returns whether it has modified a pending timer (1) or not (0)
407  */
408 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires)
409 {
410 	return __mod_vtimer(timer, expires, 1);
411 }
412 EXPORT_SYMBOL(mod_virt_timer_periodic);
413 
414 /*
415  * Delete a virtual timer.
416  *
417  * returns whether the deleted timer was pending (1) or not (0)
418  */
419 int del_virt_timer(struct vtimer_list *timer)
420 {
421 	unsigned long flags;
422 
423 	if (!vtimer_pending(timer))
424 		return 0;
425 	spin_lock_irqsave(&virt_timer_lock, flags);
426 	list_del_init(&timer->entry);
427 	spin_unlock_irqrestore(&virt_timer_lock, flags);
428 	return 1;
429 }
430 EXPORT_SYMBOL(del_virt_timer);
431 
432 /*
433  * Start the virtual CPU timer on the current CPU.
434  */
435 void vtime_init(void)
436 {
437 	/* set initial cpu timer */
438 	set_vtimer(VTIMER_MAX_SLICE);
439 	/* Setup initial MT scaling values */
440 	if (smp_cpu_mtid) {
441 		__this_cpu_write(mt_scaling_jiffies, jiffies);
442 		__this_cpu_write(mt_scaling_mult, 1);
443 		__this_cpu_write(mt_scaling_div, 1);
444 		stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
445 	}
446 }
447