1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Virtual CPU time accounting 4 * 5 * Copyright IBM Corp. 2004, 2012 6 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 7 */ 8 9 #include <linux/kernel_stat.h> 10 #include <linux/export.h> 11 #include <linux/kernel.h> 12 #include <linux/timex.h> 13 #include <linux/types.h> 14 #include <linux/time.h> 15 #include <asm/alternative.h> 16 #include <asm/cputime.h> 17 #include <asm/vtime.h> 18 #include <asm/cpu_mf.h> 19 #include <asm/idle.h> 20 #include <asm/smp.h> 21 22 #include "entry.h" 23 24 #define CPU_TIMER_MAX 0x7fffffffffffffffUL 25 26 DEFINE_PER_CPU(u64, mt_cycles[8]); 27 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; 28 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; 29 static DEFINE_PER_CPU(unsigned long, mt_scaling_jiffies); 30 31 static inline void cpu_timer_init(u64 value) 32 { 33 struct lowcore *lc = get_lowcore(); 34 u64 timer; 35 36 asm volatile( 37 " stpt %0\n" /* Store current cpu timer value */ 38 " spt %1" /* Set new value imm. afterwards */ 39 : "=Q" (timer) : "Q" (value)); 40 lc->system_timer += lc->last_update_timer - timer; 41 lc->last_update_timer = value; 42 } 43 44 static void update_mt_scaling(void) 45 { 46 u64 cycles_new[8], *cycles_old; 47 u64 delta, fac, mult, div; 48 int i; 49 50 stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new); 51 cycles_old = this_cpu_ptr(mt_cycles); 52 fac = 1; 53 mult = div = 0; 54 for (i = 0; i <= smp_cpu_mtid; i++) { 55 delta = cycles_new[i] - cycles_old[i]; 56 div += delta; 57 mult *= i + 1; 58 mult += delta * fac; 59 fac *= i + 1; 60 } 61 div *= fac; 62 if (div > 0) { 63 /* Update scaling factor */ 64 __this_cpu_write(mt_scaling_mult, mult); 65 __this_cpu_write(mt_scaling_div, div); 66 memcpy(cycles_old, cycles_new, 67 sizeof(u64) * (smp_cpu_mtid + 1)); 68 } 69 __this_cpu_write(mt_scaling_jiffies, jiffies); 70 } 71 72 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) 73 { 74 u64 delta; 75 76 delta = new - *tsk_vtime; 77 *tsk_vtime = new; 78 return delta; 79 } 80 81 82 static inline u64 scale_vtime(u64 vtime) 83 { 84 u64 mult = __this_cpu_read(mt_scaling_mult); 85 u64 div = __this_cpu_read(mt_scaling_div); 86 87 if (smp_cpu_mtid) 88 return vtime * mult / div; 89 return vtime; 90 } 91 92 static void account_system_index_scaled(struct task_struct *p, u64 cputime, 93 enum cpu_usage_stat index) 94 { 95 p->stimescaled += cputime_to_nsecs(scale_vtime(cputime)); 96 account_system_index_time(p, cputime_to_nsecs(cputime), index); 97 } 98 99 static inline void vtime_reset_last_update(struct lowcore *lc) 100 { 101 asm volatile( 102 " stpt %0\n" /* Store current cpu timer value */ 103 " stckf %1" /* Store current tod clock value */ 104 : "=Q" (lc->last_update_timer), 105 "=Q" (lc->last_update_clock) 106 : : "cc"); 107 } 108 109 /* 110 * Update process times based on virtual cpu times stored by entry.S 111 * to the lowcore fields user_timer, system_timer & steal_clock. 112 */ 113 static void do_account_vtime(struct task_struct *tsk) 114 { 115 u64 timer, clock, user, guest, system, hardirq, softirq; 116 struct lowcore *lc = get_lowcore(); 117 118 timer = lc->last_update_timer; 119 clock = lc->last_update_clock; 120 121 vtime_reset_last_update(lc); 122 123 clock = lc->last_update_clock - clock; 124 timer -= lc->last_update_timer; 125 126 if (hardirq_count()) 127 lc->hardirq_timer += timer; 128 else 129 lc->system_timer += timer; 130 131 /* Update MT utilization calculation */ 132 if (smp_cpu_mtid && time_after(jiffies, __this_cpu_read(mt_scaling_jiffies))) 133 update_mt_scaling(); 134 135 /* Calculate cputime delta */ 136 user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer); 137 guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer); 138 system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer); 139 hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer); 140 softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer); 141 lc->steal_timer += clock - user - guest - system - hardirq - softirq; 142 143 /* Push account value */ 144 if (user) { 145 account_user_time(tsk, cputime_to_nsecs(user)); 146 tsk->utimescaled += cputime_to_nsecs(scale_vtime(user)); 147 } 148 149 if (guest) { 150 account_guest_time(tsk, cputime_to_nsecs(guest)); 151 tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest)); 152 } 153 154 if (system) 155 account_system_index_scaled(tsk, system, CPUTIME_SYSTEM); 156 if (hardirq) 157 account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ); 158 if (softirq) 159 account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ); 160 } 161 162 void vtime_task_switch(struct task_struct *prev) 163 { 164 struct lowcore *lc = get_lowcore(); 165 166 do_account_vtime(prev); 167 prev->thread.user_timer = lc->user_timer; 168 prev->thread.guest_timer = lc->guest_timer; 169 prev->thread.system_timer = lc->system_timer; 170 prev->thread.hardirq_timer = lc->hardirq_timer; 171 prev->thread.softirq_timer = lc->softirq_timer; 172 lc->user_timer = current->thread.user_timer; 173 lc->guest_timer = current->thread.guest_timer; 174 lc->system_timer = current->thread.system_timer; 175 lc->hardirq_timer = current->thread.hardirq_timer; 176 lc->softirq_timer = current->thread.softirq_timer; 177 } 178 179 /* 180 * In s390, accounting pending user time also implies 181 * accounting system time in order to correctly compute 182 * the stolen time accounting. 183 */ 184 void vtime_flush(struct task_struct *tsk) 185 { 186 struct lowcore *lc = get_lowcore(); 187 u64 steal, avg_steal; 188 189 do_account_vtime(tsk); 190 191 steal = lc->steal_timer; 192 avg_steal = lc->avg_steal_timer; 193 if ((s64) steal > 0) { 194 lc->steal_timer = 0; 195 account_steal_time(cputime_to_nsecs(steal)); 196 avg_steal += steal; 197 } 198 lc->avg_steal_timer = avg_steal / 2; 199 } 200 201 static u64 vtime_delta(void) 202 { 203 struct lowcore *lc = get_lowcore(); 204 u64 timer = lc->last_update_timer; 205 206 lc->last_update_timer = get_cpu_timer(); 207 return timer - lc->last_update_timer; 208 } 209 210 void vtime_account_kernel(struct task_struct *tsk) 211 { 212 struct lowcore *lc = get_lowcore(); 213 u64 delta = vtime_delta(); 214 215 if (tsk->flags & PF_VCPU) 216 lc->guest_timer += delta; 217 else 218 lc->system_timer += delta; 219 } 220 EXPORT_SYMBOL_GPL(vtime_account_kernel); 221 222 void vtime_account_softirq(struct task_struct *tsk) 223 { 224 get_lowcore()->softirq_timer += vtime_delta(); 225 } 226 227 void vtime_account_hardirq(struct task_struct *tsk) 228 { 229 get_lowcore()->hardirq_timer += vtime_delta(); 230 } 231 232 void vtime_init(void) 233 { 234 cpu_timer_init(CPU_TIMER_MAX); 235 /* Setup initial MT scaling values */ 236 if (smp_cpu_mtid) { 237 __this_cpu_write(mt_scaling_jiffies, jiffies); 238 __this_cpu_write(mt_scaling_mult, 1); 239 __this_cpu_write(mt_scaling_div, 1); 240 stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles)); 241 } 242 } 243