1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/export.h> 10 #include <linux/kernel.h> 11 #include <linux/timex.h> 12 #include <linux/types.h> 13 #include <linux/time.h> 14 15 #include <asm/cputime.h> 16 #include <asm/vtimer.h> 17 #include <asm/vtime.h> 18 #include <asm/cpu_mf.h> 19 #include <asm/smp.h> 20 21 static void virt_timer_expire(void); 22 23 static LIST_HEAD(virt_timer_list); 24 static DEFINE_SPINLOCK(virt_timer_lock); 25 static atomic64_t virt_timer_current; 26 static atomic64_t virt_timer_elapsed; 27 28 static DEFINE_PER_CPU(u64, mt_cycles[32]); 29 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; 30 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; 31 static DEFINE_PER_CPU(u64, mt_scaling_jiffies); 32 33 static inline u64 get_vtimer(void) 34 { 35 u64 timer; 36 37 asm volatile("stpt %0" : "=m" (timer)); 38 return timer; 39 } 40 41 static inline void set_vtimer(u64 expires) 42 { 43 u64 timer; 44 45 asm volatile( 46 " stpt %0\n" /* Store current cpu timer value */ 47 " spt %1" /* Set new value imm. afterwards */ 48 : "=m" (timer) : "m" (expires)); 49 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 50 S390_lowcore.last_update_timer = expires; 51 } 52 53 static inline int virt_timer_forward(u64 elapsed) 54 { 55 BUG_ON(!irqs_disabled()); 56 57 if (list_empty(&virt_timer_list)) 58 return 0; 59 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 60 return elapsed >= atomic64_read(&virt_timer_current); 61 } 62 63 /* 64 * Update process times based on virtual cpu times stored by entry.S 65 * to the lowcore fields user_timer, system_timer & steal_clock. 66 */ 67 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 68 { 69 struct thread_info *ti = task_thread_info(tsk); 70 u64 timer, clock, user, system, steal; 71 u64 user_scaled, system_scaled; 72 int i; 73 74 timer = S390_lowcore.last_update_timer; 75 clock = S390_lowcore.last_update_clock; 76 asm volatile( 77 " stpt %0\n" /* Store current cpu timer value */ 78 #ifdef CONFIG_HAVE_MARCH_Z9_109_FEATURES 79 " stckf %1" /* Store current tod clock value */ 80 #else 81 " stck %1" /* Store current tod clock value */ 82 #endif 83 : "=m" (S390_lowcore.last_update_timer), 84 "=m" (S390_lowcore.last_update_clock)); 85 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 86 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 87 88 /* Do MT utilization calculation */ 89 if (smp_cpu_mtid && 90 time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies))) { 91 u64 cycles_new[32], *cycles_old; 92 u64 delta, mult, div; 93 94 cycles_old = this_cpu_ptr(mt_cycles); 95 if (stcctm5(smp_cpu_mtid + 1, cycles_new) < 2) { 96 mult = div = 0; 97 for (i = 0; i <= smp_cpu_mtid; i++) { 98 delta = cycles_new[i] - cycles_old[i]; 99 mult += delta; 100 div += (i + 1) * delta; 101 } 102 if (mult > 0) { 103 /* Update scaling factor */ 104 __this_cpu_write(mt_scaling_mult, mult); 105 __this_cpu_write(mt_scaling_div, div); 106 memcpy(cycles_old, cycles_new, 107 sizeof(u64) * (smp_cpu_mtid + 1)); 108 } 109 } 110 __this_cpu_write(mt_scaling_jiffies, jiffies_64); 111 } 112 113 user = S390_lowcore.user_timer - ti->user_timer; 114 S390_lowcore.steal_timer -= user; 115 ti->user_timer = S390_lowcore.user_timer; 116 117 system = S390_lowcore.system_timer - ti->system_timer; 118 S390_lowcore.steal_timer -= system; 119 ti->system_timer = S390_lowcore.system_timer; 120 121 user_scaled = user; 122 system_scaled = system; 123 /* Do MT utilization scaling */ 124 if (smp_cpu_mtid) { 125 u64 mult = __this_cpu_read(mt_scaling_mult); 126 u64 div = __this_cpu_read(mt_scaling_div); 127 128 user_scaled = (user_scaled * mult) / div; 129 system_scaled = (system_scaled * mult) / div; 130 } 131 account_user_time(tsk, user, user_scaled); 132 account_system_time(tsk, hardirq_offset, system, system_scaled); 133 134 steal = S390_lowcore.steal_timer; 135 if ((s64) steal > 0) { 136 S390_lowcore.steal_timer = 0; 137 account_steal_time(steal); 138 } 139 140 return virt_timer_forward(user + system); 141 } 142 143 void vtime_task_switch(struct task_struct *prev) 144 { 145 struct thread_info *ti; 146 147 do_account_vtime(prev, 0); 148 ti = task_thread_info(prev); 149 ti->user_timer = S390_lowcore.user_timer; 150 ti->system_timer = S390_lowcore.system_timer; 151 ti = task_thread_info(current); 152 S390_lowcore.user_timer = ti->user_timer; 153 S390_lowcore.system_timer = ti->system_timer; 154 } 155 156 /* 157 * In s390, accounting pending user time also implies 158 * accounting system time in order to correctly compute 159 * the stolen time accounting. 160 */ 161 void vtime_account_user(struct task_struct *tsk) 162 { 163 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 164 virt_timer_expire(); 165 } 166 167 /* 168 * Update process times based on virtual cpu times stored by entry.S 169 * to the lowcore fields user_timer, system_timer & steal_clock. 170 */ 171 void vtime_account_irq_enter(struct task_struct *tsk) 172 { 173 struct thread_info *ti = task_thread_info(tsk); 174 u64 timer, system, system_scaled; 175 176 timer = S390_lowcore.last_update_timer; 177 S390_lowcore.last_update_timer = get_vtimer(); 178 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 179 180 system = S390_lowcore.system_timer - ti->system_timer; 181 S390_lowcore.steal_timer -= system; 182 ti->system_timer = S390_lowcore.system_timer; 183 system_scaled = system; 184 /* Do MT utilization scaling */ 185 if (smp_cpu_mtid) { 186 u64 mult = __this_cpu_read(mt_scaling_mult); 187 u64 div = __this_cpu_read(mt_scaling_div); 188 189 system_scaled = (system_scaled * mult) / div; 190 } 191 account_system_time(tsk, 0, system, system_scaled); 192 193 virt_timer_forward(system); 194 } 195 EXPORT_SYMBOL_GPL(vtime_account_irq_enter); 196 197 void vtime_account_system(struct task_struct *tsk) 198 __attribute__((alias("vtime_account_irq_enter"))); 199 EXPORT_SYMBOL_GPL(vtime_account_system); 200 201 /* 202 * Sorted add to a list. List is linear searched until first bigger 203 * element is found. 204 */ 205 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 206 { 207 struct vtimer_list *tmp; 208 209 list_for_each_entry(tmp, head, entry) { 210 if (tmp->expires > timer->expires) { 211 list_add_tail(&timer->entry, &tmp->entry); 212 return; 213 } 214 } 215 list_add_tail(&timer->entry, head); 216 } 217 218 /* 219 * Handler for expired virtual CPU timer. 220 */ 221 static void virt_timer_expire(void) 222 { 223 struct vtimer_list *timer, *tmp; 224 unsigned long elapsed; 225 LIST_HEAD(cb_list); 226 227 /* walk timer list, fire all expired timers */ 228 spin_lock(&virt_timer_lock); 229 elapsed = atomic64_read(&virt_timer_elapsed); 230 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 231 if (timer->expires < elapsed) 232 /* move expired timer to the callback queue */ 233 list_move_tail(&timer->entry, &cb_list); 234 else 235 timer->expires -= elapsed; 236 } 237 if (!list_empty(&virt_timer_list)) { 238 timer = list_first_entry(&virt_timer_list, 239 struct vtimer_list, entry); 240 atomic64_set(&virt_timer_current, timer->expires); 241 } 242 atomic64_sub(elapsed, &virt_timer_elapsed); 243 spin_unlock(&virt_timer_lock); 244 245 /* Do callbacks and recharge periodic timers */ 246 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 247 list_del_init(&timer->entry); 248 timer->function(timer->data); 249 if (timer->interval) { 250 /* Recharge interval timer */ 251 timer->expires = timer->interval + 252 atomic64_read(&virt_timer_elapsed); 253 spin_lock(&virt_timer_lock); 254 list_add_sorted(timer, &virt_timer_list); 255 spin_unlock(&virt_timer_lock); 256 } 257 } 258 } 259 260 void init_virt_timer(struct vtimer_list *timer) 261 { 262 timer->function = NULL; 263 INIT_LIST_HEAD(&timer->entry); 264 } 265 EXPORT_SYMBOL(init_virt_timer); 266 267 static inline int vtimer_pending(struct vtimer_list *timer) 268 { 269 return !list_empty(&timer->entry); 270 } 271 272 static void internal_add_vtimer(struct vtimer_list *timer) 273 { 274 if (list_empty(&virt_timer_list)) { 275 /* First timer, just program it. */ 276 atomic64_set(&virt_timer_current, timer->expires); 277 atomic64_set(&virt_timer_elapsed, 0); 278 list_add(&timer->entry, &virt_timer_list); 279 } else { 280 /* Update timer against current base. */ 281 timer->expires += atomic64_read(&virt_timer_elapsed); 282 if (likely((s64) timer->expires < 283 (s64) atomic64_read(&virt_timer_current))) 284 /* The new timer expires before the current timer. */ 285 atomic64_set(&virt_timer_current, timer->expires); 286 /* Insert new timer into the list. */ 287 list_add_sorted(timer, &virt_timer_list); 288 } 289 } 290 291 static void __add_vtimer(struct vtimer_list *timer, int periodic) 292 { 293 unsigned long flags; 294 295 timer->interval = periodic ? timer->expires : 0; 296 spin_lock_irqsave(&virt_timer_lock, flags); 297 internal_add_vtimer(timer); 298 spin_unlock_irqrestore(&virt_timer_lock, flags); 299 } 300 301 /* 302 * add_virt_timer - add an oneshot virtual CPU timer 303 */ 304 void add_virt_timer(struct vtimer_list *timer) 305 { 306 __add_vtimer(timer, 0); 307 } 308 EXPORT_SYMBOL(add_virt_timer); 309 310 /* 311 * add_virt_timer_int - add an interval virtual CPU timer 312 */ 313 void add_virt_timer_periodic(struct vtimer_list *timer) 314 { 315 __add_vtimer(timer, 1); 316 } 317 EXPORT_SYMBOL(add_virt_timer_periodic); 318 319 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 320 { 321 unsigned long flags; 322 int rc; 323 324 BUG_ON(!timer->function); 325 326 if (timer->expires == expires && vtimer_pending(timer)) 327 return 1; 328 spin_lock_irqsave(&virt_timer_lock, flags); 329 rc = vtimer_pending(timer); 330 if (rc) 331 list_del_init(&timer->entry); 332 timer->interval = periodic ? expires : 0; 333 timer->expires = expires; 334 internal_add_vtimer(timer); 335 spin_unlock_irqrestore(&virt_timer_lock, flags); 336 return rc; 337 } 338 339 /* 340 * returns whether it has modified a pending timer (1) or not (0) 341 */ 342 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 343 { 344 return __mod_vtimer(timer, expires, 0); 345 } 346 EXPORT_SYMBOL(mod_virt_timer); 347 348 /* 349 * returns whether it has modified a pending timer (1) or not (0) 350 */ 351 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 352 { 353 return __mod_vtimer(timer, expires, 1); 354 } 355 EXPORT_SYMBOL(mod_virt_timer_periodic); 356 357 /* 358 * Delete a virtual timer. 359 * 360 * returns whether the deleted timer was pending (1) or not (0) 361 */ 362 int del_virt_timer(struct vtimer_list *timer) 363 { 364 unsigned long flags; 365 366 if (!vtimer_pending(timer)) 367 return 0; 368 spin_lock_irqsave(&virt_timer_lock, flags); 369 list_del_init(&timer->entry); 370 spin_unlock_irqrestore(&virt_timer_lock, flags); 371 return 1; 372 } 373 EXPORT_SYMBOL(del_virt_timer); 374 375 /* 376 * Start the virtual CPU timer on the current CPU. 377 */ 378 void vtime_init(void) 379 { 380 /* set initial cpu timer */ 381 set_vtimer(VTIMER_MAX_SLICE); 382 /* Setup initial MT scaling values */ 383 if (smp_cpu_mtid) { 384 __this_cpu_write(mt_scaling_jiffies, jiffies); 385 __this_cpu_write(mt_scaling_mult, 1); 386 __this_cpu_write(mt_scaling_div, 1); 387 stcctm5(smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles)); 388 } 389 } 390