1 /* 2 * Virtual cpu timer based timer functions. 3 * 4 * Copyright IBM Corp. 2004, 2012 5 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 6 */ 7 8 #include <linux/kernel_stat.h> 9 #include <linux/notifier.h> 10 #include <linux/kprobes.h> 11 #include <linux/export.h> 12 #include <linux/kernel.h> 13 #include <linux/timex.h> 14 #include <linux/types.h> 15 #include <linux/time.h> 16 #include <linux/cpu.h> 17 #include <linux/smp.h> 18 19 #include <asm/irq_regs.h> 20 #include <asm/cputime.h> 21 #include <asm/vtimer.h> 22 #include <asm/irq.h> 23 #include "entry.h" 24 25 static void virt_timer_expire(void); 26 27 DEFINE_PER_CPU(struct s390_idle_data, s390_idle); 28 29 static LIST_HEAD(virt_timer_list); 30 static DEFINE_SPINLOCK(virt_timer_lock); 31 static atomic64_t virt_timer_current; 32 static atomic64_t virt_timer_elapsed; 33 34 static inline u64 get_vtimer(void) 35 { 36 u64 timer; 37 38 asm volatile("stpt %0" : "=m" (timer)); 39 return timer; 40 } 41 42 static inline void set_vtimer(u64 expires) 43 { 44 u64 timer; 45 46 asm volatile( 47 " stpt %0\n" /* Store current cpu timer value */ 48 " spt %1" /* Set new value imm. afterwards */ 49 : "=m" (timer) : "m" (expires)); 50 S390_lowcore.system_timer += S390_lowcore.last_update_timer - timer; 51 S390_lowcore.last_update_timer = expires; 52 } 53 54 static inline int virt_timer_forward(u64 elapsed) 55 { 56 BUG_ON(!irqs_disabled()); 57 58 if (list_empty(&virt_timer_list)) 59 return 0; 60 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 61 return elapsed >= atomic64_read(&virt_timer_current); 62 } 63 64 /* 65 * Update process times based on virtual cpu times stored by entry.S 66 * to the lowcore fields user_timer, system_timer & steal_clock. 67 */ 68 static int do_account_vtime(struct task_struct *tsk, int hardirq_offset) 69 { 70 struct thread_info *ti = task_thread_info(tsk); 71 u64 timer, clock, user, system, steal; 72 73 timer = S390_lowcore.last_update_timer; 74 clock = S390_lowcore.last_update_clock; 75 asm volatile( 76 " stpt %0\n" /* Store current cpu timer value */ 77 " stck %1" /* Store current tod clock value */ 78 : "=m" (S390_lowcore.last_update_timer), 79 "=m" (S390_lowcore.last_update_clock)); 80 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 81 S390_lowcore.steal_timer += S390_lowcore.last_update_clock - clock; 82 83 user = S390_lowcore.user_timer - ti->user_timer; 84 S390_lowcore.steal_timer -= user; 85 ti->user_timer = S390_lowcore.user_timer; 86 account_user_time(tsk, user, user); 87 88 system = S390_lowcore.system_timer - ti->system_timer; 89 S390_lowcore.steal_timer -= system; 90 ti->system_timer = S390_lowcore.system_timer; 91 account_system_time(tsk, hardirq_offset, system, system); 92 93 steal = S390_lowcore.steal_timer; 94 if ((s64) steal > 0) { 95 S390_lowcore.steal_timer = 0; 96 account_steal_time(steal); 97 } 98 99 return virt_timer_forward(user + system); 100 } 101 102 void vtime_task_switch(struct task_struct *prev) 103 { 104 struct thread_info *ti; 105 106 do_account_vtime(prev, 0); 107 ti = task_thread_info(prev); 108 ti->user_timer = S390_lowcore.user_timer; 109 ti->system_timer = S390_lowcore.system_timer; 110 ti = task_thread_info(current); 111 S390_lowcore.user_timer = ti->user_timer; 112 S390_lowcore.system_timer = ti->system_timer; 113 } 114 115 /* 116 * In s390, accounting pending user time also implies 117 * accounting system time in order to correctly compute 118 * the stolen time accounting. 119 */ 120 void vtime_account_user(struct task_struct *tsk) 121 { 122 if (do_account_vtime(tsk, HARDIRQ_OFFSET)) 123 virt_timer_expire(); 124 } 125 126 /* 127 * Update process times based on virtual cpu times stored by entry.S 128 * to the lowcore fields user_timer, system_timer & steal_clock. 129 */ 130 void vtime_account(struct task_struct *tsk) 131 { 132 struct thread_info *ti = task_thread_info(tsk); 133 u64 timer, system; 134 135 WARN_ON_ONCE(!irqs_disabled()); 136 137 timer = S390_lowcore.last_update_timer; 138 S390_lowcore.last_update_timer = get_vtimer(); 139 S390_lowcore.system_timer += timer - S390_lowcore.last_update_timer; 140 141 system = S390_lowcore.system_timer - ti->system_timer; 142 S390_lowcore.steal_timer -= system; 143 ti->system_timer = S390_lowcore.system_timer; 144 account_system_time(tsk, 0, system, system); 145 146 virt_timer_forward(system); 147 } 148 EXPORT_SYMBOL_GPL(vtime_account); 149 150 void vtime_account_system(struct task_struct *tsk) 151 __attribute__((alias("vtime_account"))); 152 EXPORT_SYMBOL_GPL(vtime_account_system); 153 154 void __kprobes vtime_stop_cpu(void) 155 { 156 struct s390_idle_data *idle = &__get_cpu_var(s390_idle); 157 unsigned long long idle_time; 158 unsigned long psw_mask; 159 160 trace_hardirqs_on(); 161 /* Don't trace preempt off for idle. */ 162 stop_critical_timings(); 163 164 /* Wait for external, I/O or machine check interrupt. */ 165 psw_mask = psw_kernel_bits | PSW_MASK_WAIT | PSW_MASK_DAT | 166 PSW_MASK_IO | PSW_MASK_EXT | PSW_MASK_MCHECK; 167 idle->nohz_delay = 0; 168 169 /* Call the assembler magic in entry.S */ 170 psw_idle(idle, psw_mask); 171 172 /* Reenable preemption tracer. */ 173 start_critical_timings(); 174 175 /* Account time spent with enabled wait psw loaded as idle time. */ 176 idle->sequence++; 177 smp_wmb(); 178 idle_time = idle->clock_idle_exit - idle->clock_idle_enter; 179 idle->clock_idle_enter = idle->clock_idle_exit = 0ULL; 180 idle->idle_time += idle_time; 181 idle->idle_count++; 182 account_idle_time(idle_time); 183 smp_wmb(); 184 idle->sequence++; 185 } 186 187 cputime64_t s390_get_idle_time(int cpu) 188 { 189 struct s390_idle_data *idle = &per_cpu(s390_idle, cpu); 190 unsigned long long now, idle_enter, idle_exit; 191 unsigned int sequence; 192 193 do { 194 now = get_clock(); 195 sequence = ACCESS_ONCE(idle->sequence); 196 idle_enter = ACCESS_ONCE(idle->clock_idle_enter); 197 idle_exit = ACCESS_ONCE(idle->clock_idle_exit); 198 } while ((sequence & 1) || (idle->sequence != sequence)); 199 return idle_enter ? ((idle_exit ?: now) - idle_enter) : 0; 200 } 201 202 /* 203 * Sorted add to a list. List is linear searched until first bigger 204 * element is found. 205 */ 206 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 207 { 208 struct vtimer_list *tmp; 209 210 list_for_each_entry(tmp, head, entry) { 211 if (tmp->expires > timer->expires) { 212 list_add_tail(&timer->entry, &tmp->entry); 213 return; 214 } 215 } 216 list_add_tail(&timer->entry, head); 217 } 218 219 /* 220 * Handler for expired virtual CPU timer. 221 */ 222 static void virt_timer_expire(void) 223 { 224 struct vtimer_list *timer, *tmp; 225 unsigned long elapsed; 226 LIST_HEAD(cb_list); 227 228 /* walk timer list, fire all expired timers */ 229 spin_lock(&virt_timer_lock); 230 elapsed = atomic64_read(&virt_timer_elapsed); 231 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 232 if (timer->expires < elapsed) 233 /* move expired timer to the callback queue */ 234 list_move_tail(&timer->entry, &cb_list); 235 else 236 timer->expires -= elapsed; 237 } 238 if (!list_empty(&virt_timer_list)) { 239 timer = list_first_entry(&virt_timer_list, 240 struct vtimer_list, entry); 241 atomic64_set(&virt_timer_current, timer->expires); 242 } 243 atomic64_sub(elapsed, &virt_timer_elapsed); 244 spin_unlock(&virt_timer_lock); 245 246 /* Do callbacks and recharge periodic timers */ 247 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 248 list_del_init(&timer->entry); 249 timer->function(timer->data); 250 if (timer->interval) { 251 /* Recharge interval timer */ 252 timer->expires = timer->interval + 253 atomic64_read(&virt_timer_elapsed); 254 spin_lock(&virt_timer_lock); 255 list_add_sorted(timer, &virt_timer_list); 256 spin_unlock(&virt_timer_lock); 257 } 258 } 259 } 260 261 void init_virt_timer(struct vtimer_list *timer) 262 { 263 timer->function = NULL; 264 INIT_LIST_HEAD(&timer->entry); 265 } 266 EXPORT_SYMBOL(init_virt_timer); 267 268 static inline int vtimer_pending(struct vtimer_list *timer) 269 { 270 return !list_empty(&timer->entry); 271 } 272 273 static void internal_add_vtimer(struct vtimer_list *timer) 274 { 275 if (list_empty(&virt_timer_list)) { 276 /* First timer, just program it. */ 277 atomic64_set(&virt_timer_current, timer->expires); 278 atomic64_set(&virt_timer_elapsed, 0); 279 list_add(&timer->entry, &virt_timer_list); 280 } else { 281 /* Update timer against current base. */ 282 timer->expires += atomic64_read(&virt_timer_elapsed); 283 if (likely((s64) timer->expires < 284 (s64) atomic64_read(&virt_timer_current))) 285 /* The new timer expires before the current timer. */ 286 atomic64_set(&virt_timer_current, timer->expires); 287 /* Insert new timer into the list. */ 288 list_add_sorted(timer, &virt_timer_list); 289 } 290 } 291 292 static void __add_vtimer(struct vtimer_list *timer, int periodic) 293 { 294 unsigned long flags; 295 296 timer->interval = periodic ? timer->expires : 0; 297 spin_lock_irqsave(&virt_timer_lock, flags); 298 internal_add_vtimer(timer); 299 spin_unlock_irqrestore(&virt_timer_lock, flags); 300 } 301 302 /* 303 * add_virt_timer - add an oneshot virtual CPU timer 304 */ 305 void add_virt_timer(struct vtimer_list *timer) 306 { 307 __add_vtimer(timer, 0); 308 } 309 EXPORT_SYMBOL(add_virt_timer); 310 311 /* 312 * add_virt_timer_int - add an interval virtual CPU timer 313 */ 314 void add_virt_timer_periodic(struct vtimer_list *timer) 315 { 316 __add_vtimer(timer, 1); 317 } 318 EXPORT_SYMBOL(add_virt_timer_periodic); 319 320 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 321 { 322 unsigned long flags; 323 int rc; 324 325 BUG_ON(!timer->function); 326 327 if (timer->expires == expires && vtimer_pending(timer)) 328 return 1; 329 spin_lock_irqsave(&virt_timer_lock, flags); 330 rc = vtimer_pending(timer); 331 if (rc) 332 list_del_init(&timer->entry); 333 timer->interval = periodic ? expires : 0; 334 timer->expires = expires; 335 internal_add_vtimer(timer); 336 spin_unlock_irqrestore(&virt_timer_lock, flags); 337 return rc; 338 } 339 340 /* 341 * returns whether it has modified a pending timer (1) or not (0) 342 */ 343 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 344 { 345 return __mod_vtimer(timer, expires, 0); 346 } 347 EXPORT_SYMBOL(mod_virt_timer); 348 349 /* 350 * returns whether it has modified a pending timer (1) or not (0) 351 */ 352 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 353 { 354 return __mod_vtimer(timer, expires, 1); 355 } 356 EXPORT_SYMBOL(mod_virt_timer_periodic); 357 358 /* 359 * Delete a virtual timer. 360 * 361 * returns whether the deleted timer was pending (1) or not (0) 362 */ 363 int del_virt_timer(struct vtimer_list *timer) 364 { 365 unsigned long flags; 366 367 if (!vtimer_pending(timer)) 368 return 0; 369 spin_lock_irqsave(&virt_timer_lock, flags); 370 list_del_init(&timer->entry); 371 spin_unlock_irqrestore(&virt_timer_lock, flags); 372 return 1; 373 } 374 EXPORT_SYMBOL(del_virt_timer); 375 376 /* 377 * Start the virtual CPU timer on the current CPU. 378 */ 379 void __cpuinit init_cpu_vtimer(void) 380 { 381 /* set initial cpu timer */ 382 set_vtimer(VTIMER_MAX_SLICE); 383 } 384 385 static int __cpuinit s390_nohz_notify(struct notifier_block *self, 386 unsigned long action, void *hcpu) 387 { 388 struct s390_idle_data *idle; 389 long cpu = (long) hcpu; 390 391 idle = &per_cpu(s390_idle, cpu); 392 switch (action & ~CPU_TASKS_FROZEN) { 393 case CPU_DYING: 394 idle->nohz_delay = 0; 395 default: 396 break; 397 } 398 return NOTIFY_OK; 399 } 400 401 void __init vtime_init(void) 402 { 403 /* Enable cpu timer interrupts on the boot cpu. */ 404 init_cpu_vtimer(); 405 cpu_notifier(s390_nohz_notify, 0); 406 } 407