1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/export.h> 10 #include <linux/kernel.h> 11 #include <linux/timex.h> 12 #include <linux/types.h> 13 #include <linux/time.h> 14 15 #include <asm/cputime.h> 16 #include <asm/vtimer.h> 17 #include <asm/vtime.h> 18 #include <asm/cpu_mf.h> 19 #include <asm/smp.h> 20 21 static void virt_timer_expire(void); 22 23 static LIST_HEAD(virt_timer_list); 24 static DEFINE_SPINLOCK(virt_timer_lock); 25 static atomic64_t virt_timer_current; 26 static atomic64_t virt_timer_elapsed; 27 28 static DEFINE_PER_CPU(u64, mt_cycles[32]); 29 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; 30 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; 31 32 static inline u64 get_vtimer(void) 33 { 34 u64 timer; 35 36 asm volatile("stpt %0" : "=m" (timer)); 37 return timer; 38 } 39 40 static inline void set_vtimer(u64 expires) 41 { 42 u64 timer; 43 44 asm volatile( 45 " stpt %0\n" /* Store current cpu timer value */ 46 " spt %1" /* Set new value imm. afterwards */ 47 : "=m" (timer) : "m" (expires)); 48 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 49 S390_lowcore.last_update_timer = expires; 50 } 51 52 static inline int virt_timer_forward(u64 elapsed) 53 { 54 BUG_ON(!irqs_disabled()); 55 56 if (list_empty(&virt_timer_list)) 57 return 0; 58 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 59 return elapsed >= atomic64_read(&virt_timer_current); 60 } 61 62 /* 63 * Update process times based on virtual cpu times stored by entry.S 64 * to the lowcore fields user_timer, system_timer & steal_clock. 65 */ 66 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 67 { 68 struct thread_info *ti = task_thread_info(tsk); 69 u64 timer, clock, user, system, steal; 70 u64 user_scaled, system_scaled; 71 int i; 72 73 timer = S390_lowcore.last_update_timer; 74 clock = S390_lowcore.last_update_clock; 75 asm volatile( 76 " stpt %0\n" /* Store current cpu timer value */ 77 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES 78 " stckf %1" /* Store current tod clock value */ 79 #else 80 " stck %1" /* Store current tod clock value */ 81 #endif 82 : "=m" (S390_lowcore.last_update_timer), 83 "=m" (S390_lowcore.last_update_clock)); 84 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 85 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 86 87 /* Do MT utilization calculation */ 88 if (smp_cpu_mtid) { 89 u64 cycles_new[32], *cycles_old; 90 u64 delta, mult, div; 91 92 cycles_old = this_cpu_ptr(mt_cycles); 93 if (stcctm5(smp_cpu_mtid + 1, cycles_new) < 2) { 94 mult = div = 0; 95 for (i = 0; i <= smp_cpu_mtid; i++) { 96 delta = cycles_new[i] - cycles_old[i]; 97 mult += delta; 98 div += (i + 1) * delta; 99 } 100 if (mult > 0) { 101 /* Update scaling factor */ 102 __this_cpu_write(mt_scaling_mult, mult); 103 __this_cpu_write(mt_scaling_div, div); 104 memcpy(cycles_old, cycles_new, 105 sizeof(u64) * (smp_cpu_mtid + 1)); 106 } 107 } 108 } 109 110 user = S390_lowcore.user_timer - ti->user_timer; 111 S390_lowcore.steal_timer -= user; 112 ti->user_timer = S390_lowcore.user_timer; 113 114 system = S390_lowcore.system_timer - ti->system_timer; 115 S390_lowcore.steal_timer -= system; 116 ti->system_timer = S390_lowcore.system_timer; 117 118 user_scaled = user; 119 system_scaled = system; 120 /* Do MT utilization scaling */ 121 if (smp_cpu_mtid) { 122 u64 mult = __this_cpu_read(mt_scaling_mult); 123 u64 div = __this_cpu_read(mt_scaling_div); 124 125 user_scaled = (user_scaled * mult) / div; 126 system_scaled = (system_scaled * mult) / div; 127 } 128 account_user_time(tsk, user, user_scaled); 129 account_system_time(tsk, hardirq_offset, system, system_scaled); 130 131 steal = S390_lowcore.steal_timer; 132 if ((s64) steal > 0) { 133 S390_lowcore.steal_timer = 0; 134 account_steal_time(steal); 135 } 136 137 return virt_timer_forward(user + system); 138 } 139 140 void vtime_task_switch(struct task_struct *prev) 141 { 142 struct thread_info *ti; 143 144 do_account_vtime(prev, 0); 145 ti = task_thread_info(prev); 146 ti->user_timer = S390_lowcore.user_timer; 147 ti->system_timer = S390_lowcore.system_timer; 148 ti = task_thread_info(current); 149 S390_lowcore.user_timer = ti->user_timer; 150 S390_lowcore.system_timer = ti->system_timer; 151 } 152 153 /* 154 * In s390, accounting pending user time also implies 155 * accounting system time in order to correctly compute 156 * the stolen time accounting. 157 */ 158 void vtime_account_user(struct task_struct *tsk) 159 { 160 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 161 virt_timer_expire(); 162 } 163 164 /* 165 * Update process times based on virtual cpu times stored by entry.S 166 * to the lowcore fields user_timer, system_timer & steal_clock. 167 */ 168 void vtime_account_irq_enter(struct task_struct *tsk) 169 { 170 struct thread_info *ti = task_thread_info(tsk); 171 u64 timer, system, system_scaled; 172 173 timer = S390_lowcore.last_update_timer; 174 S390_lowcore.last_update_timer = get_vtimer(); 175 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 176 177 system = S390_lowcore.system_timer - ti->system_timer; 178 S390_lowcore.steal_timer -= system; 179 ti->system_timer = S390_lowcore.system_timer; 180 system_scaled = system; 181 /* Do MT utilization scaling */ 182 if (smp_cpu_mtid) { 183 u64 mult = __this_cpu_read(mt_scaling_mult); 184 u64 div = __this_cpu_read(mt_scaling_div); 185 186 system_scaled = (system_scaled * mult) / div; 187 } 188 account_system_time(tsk, 0, system, system_scaled); 189 190 virt_timer_forward(system); 191 } 192 EXPORT_SYMBOL_GPL(vtime_account_irq_enter); 193 194 void vtime_account_system(struct task_struct *tsk) 195 __attribute__((alias("vtime_account_irq_enter"))); 196 EXPORT_SYMBOL_GPL(vtime_account_system); 197 198 /* 199 * Sorted add to a list. List is linear searched until first bigger 200 * element is found. 201 */ 202 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 203 { 204 struct vtimer_list *tmp; 205 206 list_for_each_entry(tmp, head, entry) { 207 if (tmp->expires > timer->expires) { 208 list_add_tail(&timer->entry, &tmp->entry); 209 return; 210 } 211 } 212 list_add_tail(&timer->entry, head); 213 } 214 215 /* 216 * Handler for expired virtual CPU timer. 217 */ 218 static void virt_timer_expire(void) 219 { 220 struct vtimer_list *timer, *tmp; 221 unsigned long elapsed; 222 LIST_HEAD(cb_list); 223 224 /* walk timer list, fire all expired timers */ 225 spin_lock(&virt_timer_lock); 226 elapsed = atomic64_read(&virt_timer_elapsed); 227 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 228 if (timer->expires < elapsed) 229 /* move expired timer to the callback queue */ 230 list_move_tail(&timer->entry, &cb_list); 231 else 232 timer->expires -= elapsed; 233 } 234 if (!list_empty(&virt_timer_list)) { 235 timer = list_first_entry(&virt_timer_list, 236 struct vtimer_list, entry); 237 atomic64_set(&virt_timer_current, timer->expires); 238 } 239 atomic64_sub(elapsed, &virt_timer_elapsed); 240 spin_unlock(&virt_timer_lock); 241 242 /* Do callbacks and recharge periodic timers */ 243 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 244 list_del_init(&timer->entry); 245 timer->function(timer->data); 246 if (timer->interval) { 247 /* Recharge interval timer */ 248 timer->expires = timer->interval + 249 atomic64_read(&virt_timer_elapsed); 250 spin_lock(&virt_timer_lock); 251 list_add_sorted(timer, &virt_timer_list); 252 spin_unlock(&virt_timer_lock); 253 } 254 } 255 } 256 257 void init_virt_timer(struct vtimer_list *timer) 258 { 259 timer->function = NULL; 260 INIT_LIST_HEAD(&timer->entry); 261 } 262 EXPORT_SYMBOL(init_virt_timer); 263 264 static inline int vtimer_pending(struct vtimer_list *timer) 265 { 266 return !list_empty(&timer->entry); 267 } 268 269 static void internal_add_vtimer(struct vtimer_list *timer) 270 { 271 if (list_empty(&virt_timer_list)) { 272 /* First timer, just program it. */ 273 atomic64_set(&virt_timer_current, timer->expires); 274 atomic64_set(&virt_timer_elapsed, 0); 275 list_add(&timer->entry, &virt_timer_list); 276 } else { 277 /* Update timer against current base. */ 278 timer->expires += atomic64_read(&virt_timer_elapsed); 279 if (likely((s64) timer->expires < 280 (s64) atomic64_read(&virt_timer_current))) 281 /* The new timer expires before the current timer. */ 282 atomic64_set(&virt_timer_current, timer->expires); 283 /* Insert new timer into the list. */ 284 list_add_sorted(timer, &virt_timer_list); 285 } 286 } 287 288 static void __add_vtimer(struct vtimer_list *timer, int periodic) 289 { 290 unsigned long flags; 291 292 timer->interval = periodic ? timer->expires : 0; 293 spin_lock_irqsave(&virt_timer_lock, flags); 294 internal_add_vtimer(timer); 295 spin_unlock_irqrestore(&virt_timer_lock, flags); 296 } 297 298 /* 299 * add_virt_timer - add an oneshot virtual CPU timer 300 */ 301 void add_virt_timer(struct vtimer_list *timer) 302 { 303 __add_vtimer(timer, 0); 304 } 305 EXPORT_SYMBOL(add_virt_timer); 306 307 /* 308 * add_virt_timer_int - add an interval virtual CPU timer 309 */ 310 void add_virt_timer_periodic(struct vtimer_list *timer) 311 { 312 __add_vtimer(timer, 1); 313 } 314 EXPORT_SYMBOL(add_virt_timer_periodic); 315 316 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 317 { 318 unsigned long flags; 319 int rc; 320 321 BUG_ON(!timer->function); 322 323 if (timer->expires == expires && vtimer_pending(timer)) 324 return 1; 325 spin_lock_irqsave(&virt_timer_lock, flags); 326 rc = vtimer_pending(timer); 327 if (rc) 328 list_del_init(&timer->entry); 329 timer->interval = periodic ? expires : 0; 330 timer->expires = expires; 331 internal_add_vtimer(timer); 332 spin_unlock_irqrestore(&virt_timer_lock, flags); 333 return rc; 334 } 335 336 /* 337 * returns whether it has modified a pending timer (1) or not (0) 338 */ 339 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 340 { 341 return __mod_vtimer(timer, expires, 0); 342 } 343 EXPORT_SYMBOL(mod_virt_timer); 344 345 /* 346 * returns whether it has modified a pending timer (1) or not (0) 347 */ 348 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 349 { 350 return __mod_vtimer(timer, expires, 1); 351 } 352 EXPORT_SYMBOL(mod_virt_timer_periodic); 353 354 /* 355 * Delete a virtual timer. 356 * 357 * returns whether the deleted timer was pending (1) or not (0) 358 */ 359 int del_virt_timer(struct vtimer_list *timer) 360 { 361 unsigned long flags; 362 363 if (!vtimer_pending(timer)) 364 return 0; 365 spin_lock_irqsave(&virt_timer_lock, flags); 366 list_del_init(&timer->entry); 367 spin_unlock_irqrestore(&virt_timer_lock, flags); 368 return 1; 369 } 370 EXPORT_SYMBOL(del_virt_timer); 371 372 /* 373 * Start the virtual CPU timer on the current CPU. 374 */ 375 void vtime_init(void) 376 { 377 /* set initial cpu timer */ 378 set_vtimer(VTIMER_MAX_SLICE); 379 } 380