1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Virtual CPU time accounting
4 *
5 * Copyright IBM Corp. 2004, 2012
6 * Author(s): Jan Glauber <jan.glauber@de.ibm.com>
7 */
8
9 #include <linux/kernel_stat.h>
10 #include <linux/export.h>
11 #include <linux/kernel.h>
12 #include <linux/timex.h>
13 #include <linux/types.h>
14 #include <linux/time.h>
15 #include <asm/alternative.h>
16 #include <asm/cputime.h>
17 #include <asm/vtime.h>
18 #include <asm/cpu_mf.h>
19 #include <asm/idle.h>
20 #include <asm/smp.h>
21
22 #include "entry.h"
23
24 #define CPU_TIMER_MAX 0x7fffffffffffffffUL
25
26 DEFINE_PER_CPU(u64, mt_cycles[8]);
27 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 };
28 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 };
29 static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies);
30
cpu_timer_init(u64 value)31 static inline void cpu_timer_init(u64 value)
32 {
33 struct lowcore *lc = get_lowcore();
34 u64 timer;
35
36 asm volatile(
37 " stpt %0\n" /* Store current cpu timer value */
38 " spt %1" /* Set new value imm. afterwards */
39 : "=Q" (timer) : "Q" (value));
40 lc->system_timer += lc->last_update_timer - timer;
41 lc->last_update_timer = value;
42 }
43
update_mt_scaling(void)44 static void update_mt_scaling(void)
45 {
46 u64 cycles_new[8], *cycles_old;
47 u64 delta, fac, mult, div;
48 int i;
49
50 stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new);
51 cycles_old = this_cpu_ptr(mt_cycles);
52 fac = 1;
53 mult = div = 0;
54 for (i = 0; i <= smp_cpu_mtid; i++) {
55 delta = cycles_new[i] - cycles_old[i];
56 div += delta;
57 mult *= i + 1;
58 mult += delta * fac;
59 fac *= i + 1;
60 }
61 div *= fac;
62 if (div > 0) {
63 /* Update scaling factor */
64 __this_cpu_write(mt_scaling_mult, mult);
65 __this_cpu_write(mt_scaling_div, div);
66 memcpy(cycles_old, cycles_new,
67 sizeof(u64) * (smp_cpu_mtid + 1));
68 }
69 __this_cpu_write(mt_scaling_jiffies, jiffies);
70 }
71
update_tsk_timer(unsigned long * tsk_vtime,u64 new)72 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new)
73 {
74 u64 delta;
75
76 delta = new - *tsk_vtime;
77 *tsk_vtime = new;
78 return delta;
79 }
80
81
scale_vtime(u64 vtime)82 static inline u64 scale_vtime(u64 vtime)
83 {
84 u64 mult = __this_cpu_read(mt_scaling_mult);
85 u64 div = __this_cpu_read(mt_scaling_div);
86
87 if (smp_cpu_mtid)
88 return vtime * mult / div;
89 return vtime;
90 }
91
account_system_index_scaled(struct task_struct * p,u64 cputime,enum cpu_usage_stat index)92 static void account_system_index_scaled(struct task_struct *p, u64 cputime,
93 enum cpu_usage_stat index)
94 {
95 p->stimescaled += cputime_to_nsecs(scale_vtime(cputime));
96 account_system_index_time(p, cputime_to_nsecs(cputime), index);
97 }
98
vtime_reset_last_update(struct lowcore * lc)99 static inline void vtime_reset_last_update(struct lowcore *lc)
100 {
101 asm volatile(
102 " stpt %0\n" /* Store current cpu timer value */
103 " stckf %1" /* Store current tod clock value */
104 : "=Q" (lc->last_update_timer),
105 "=Q" (lc->last_update_clock)
106 : : "cc");
107 }
108
109 /*
110 * Update process times based on virtual cpu times stored by entry.S
111 * to the lowcore fields user_timer, system_timer & steal_clock.
112 */
do_account_vtime(struct task_struct * tsk)113 static void do_account_vtime(struct task_struct *tsk)
114 {
115 u64 timer, clock, user, guest, system, hardirq, softirq;
116 struct lowcore *lc = get_lowcore();
117
118 timer = lc->last_update_timer;
119 clock = lc->last_update_clock;
120
121 vtime_reset_last_update(lc);
122
123 clock = lc->last_update_clock - clock;
124 timer -= lc->last_update_timer;
125
126 if (hardirq_count())
127 lc->hardirq_timer += timer;
128 else
129 lc->system_timer += timer;
130
131 /* Update MT utilization calculation */
132 if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies)))
133 update_mt_scaling();
134
135 /* Calculate cputime delta */
136 user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer);
137 guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer);
138 system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer);
139 hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer);
140 softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer);
141 lc->steal_timer += clock - user - guest - system - hardirq - softirq;
142
143 /* Push account value */
144 if (user) {
145 account_user_time(tsk, cputime_to_nsecs(user));
146 tsk->utimescaled += cputime_to_nsecs(scale_vtime(user));
147 }
148
149 if (guest) {
150 account_guest_time(tsk, cputime_to_nsecs(guest));
151 tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest));
152 }
153
154 if (system)
155 account_system_index_scaled(tsk, system, CPUTIME_SYSTEM);
156 if (hardirq)
157 account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ);
158 if (softirq)
159 account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ);
160 }
161
vtime_task_switch(struct task_struct * prev)162 void vtime_task_switch(struct task_struct *prev)
163 {
164 struct lowcore *lc = get_lowcore();
165
166 do_account_vtime(prev);
167 prev->thread.user_timer = lc->user_timer;
168 prev->thread.guest_timer = lc->guest_timer;
169 prev->thread.system_timer = lc->system_timer;
170 prev->thread.hardirq_timer = lc->hardirq_timer;
171 prev->thread.softirq_timer = lc->softirq_timer;
172 lc->user_timer = current->thread.user_timer;
173 lc->guest_timer = current->thread.guest_timer;
174 lc->system_timer = current->thread.system_timer;
175 lc->hardirq_timer = current->thread.hardirq_timer;
176 lc->softirq_timer = current->thread.softirq_timer;
177 }
178
179 /*
180 * In s390, accounting pending user time also implies
181 * accounting system time in order to correctly compute
182 * the stolen time accounting.
183 */
vtime_flush(struct task_struct * tsk)184 void vtime_flush(struct task_struct *tsk)
185 {
186 struct lowcore *lc = get_lowcore();
187 u64 steal, avg_steal;
188
189 do_account_vtime(tsk);
190
191 steal = lc->steal_timer;
192 avg_steal = lc->avg_steal_timer;
193 if ((s64) steal > 0) {
194 lc->steal_timer = 0;
195 account_steal_time(cputime_to_nsecs(steal));
196 avg_steal += steal;
197 }
198 lc->avg_steal_timer = avg_steal / 2;
199 }
200
vtime_delta(void)201 static u64 vtime_delta(void)
202 {
203 struct lowcore *lc = get_lowcore();
204 u64 timer = lc->last_update_timer;
205
206 lc->last_update_timer = get_cpu_timer();
207 return timer - lc->last_update_timer;
208 }
209
vtime_account_kernel(struct task_struct * tsk)210 void vtime_account_kernel(struct task_struct *tsk)
211 {
212 struct lowcore *lc = get_lowcore();
213 u64 delta = vtime_delta();
214
215 if (tsk->flags & PF_VCPU)
216 lc->guest_timer += delta;
217 else
218 lc->system_timer += delta;
219 }
220 EXPORT_SYMBOL_GPL(vtime_account_kernel);
221
vtime_account_softirq(struct task_struct * tsk)222 void vtime_account_softirq(struct task_struct *tsk)
223 {
224 get_lowcore()->softirq_timer += vtime_delta();
225 }
226
vtime_account_hardirq(struct task_struct * tsk)227 void vtime_account_hardirq(struct task_struct *tsk)
228 {
229 get_lowcore()->hardirq_timer += vtime_delta();
230 }
231
vtime_init(void)232 void vtime_init(void)
233 {
234 cpu_timer_init(CPU_TIMER_MAX);
235 /* Setup initial MT scaling values */
236 if (smp_cpu_mtid) {
237 __this_cpu_write(mt_scaling_jiffies, jiffies);
238 __this_cpu_write(mt_scaling_mult, 1);
239 __this_cpu_write(mt_scaling_div, 1);
240 stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles));
241 }
242 }
243