1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Virtual cpu timer based timer functions. 4 * 5 * Copyright IBM Corp. 2004, 2012 6 * Author(s): Jan Glauber <jan.glauber@de.ibm.com> 7 */ 8 9 #include <linux/kernel_stat.h> 10 #include <linux/export.h> 11 #include <linux/kernel.h> 12 #include <linux/timex.h> 13 #include <linux/types.h> 14 #include <linux/time.h> 15 #include <asm/alternative.h> 16 #include <asm/cputime.h> 17 #include <asm/vtimer.h> 18 #include <asm/vtime.h> 19 #include <asm/cpu_mf.h> 20 #include <asm/idle.h> 21 #include <asm/smp.h> 22 23 #include "entry.h" 24 25 static void virt_timer_expire(void); 26 27 static LIST_HEAD(virt_timer_list); 28 static DEFINE_SPINLOCK(virt_timer_lock); 29 static atomic64_t virt_timer_current; 30 static atomic64_t virt_timer_elapsed; 31 32 DEFINE_PER_CPU(u64, mt_cycles[8]); 33 static DEFINE_PER_CPU(u64, mt_scaling_mult) = { 1 }; 34 static DEFINE_PER_CPU(u64, mt_scaling_div) = { 1 }; 35 static DEFINE_PER_CPU(u64, mt_scaling_jiffies); 36 37 static inline void set_vtimer(u64 expires) 38 { 39 struct lowcore *lc = get_lowcore(); 40 u64 timer; 41 42 asm volatile( 43 " stpt %0\n" /* Store current cpu timer value */ 44 " spt %1" /* Set new value imm. afterwards */ 45 : "=Q" (timer) : "Q" (expires)); 46 lc->system_timer += lc->last_update_timer - timer; 47 lc->last_update_timer = expires; 48 } 49 50 static inline int virt_timer_forward(u64 elapsed) 51 { 52 lockdep_assert_irqs_disabled(); 53 if (list_empty(&virt_timer_list)) 54 return 0; 55 elapsed = atomic64_add_return(elapsed, &virt_timer_elapsed); 56 return elapsed >= atomic64_read(&virt_timer_current); 57 } 58 59 static void update_mt_scaling(void) 60 { 61 u64 cycles_new[8], *cycles_old; 62 u64 delta, fac, mult, div; 63 int i; 64 65 stcctm(MT_DIAG, smp_cpu_mtid + 1, cycles_new); 66 cycles_old = this_cpu_ptr(mt_cycles); 67 fac = 1; 68 mult = div = 0; 69 for (i = 0; i <= smp_cpu_mtid; i++) { 70 delta = cycles_new[i] - cycles_old[i]; 71 div += delta; 72 mult *= i + 1; 73 mult += delta * fac; 74 fac *= i + 1; 75 } 76 div *= fac; 77 if (div > 0) { 78 /* Update scaling factor */ 79 __this_cpu_write(mt_scaling_mult, mult); 80 __this_cpu_write(mt_scaling_div, div); 81 memcpy(cycles_old, cycles_new, 82 sizeof(u64) * (smp_cpu_mtid + 1)); 83 } 84 __this_cpu_write(mt_scaling_jiffies, jiffies_64); 85 } 86 87 static inline u64 update_tsk_timer(unsigned long *tsk_vtime, u64 new) 88 { 89 u64 delta; 90 91 delta = new - *tsk_vtime; 92 *tsk_vtime = new; 93 return delta; 94 } 95 96 97 static inline u64 scale_vtime(u64 vtime) 98 { 99 u64 mult = __this_cpu_read(mt_scaling_mult); 100 u64 div = __this_cpu_read(mt_scaling_div); 101 102 if (smp_cpu_mtid) 103 return vtime * mult / div; 104 return vtime; 105 } 106 107 static void account_system_index_scaled(struct task_struct *p, u64 cputime, 108 enum cpu_usage_stat index) 109 { 110 p->stimescaled += cputime_to_nsecs(scale_vtime(cputime)); 111 account_system_index_time(p, cputime_to_nsecs(cputime), index); 112 } 113 114 static inline void vtime_reset_last_update(struct lowcore *lc) 115 { 116 asm volatile( 117 " stpt %0\n" /* Store current cpu timer value */ 118 " stckf %1" /* Store current tod clock value */ 119 : "=Q" (lc->last_update_timer), 120 "=Q" (lc->last_update_clock) 121 : : "cc"); 122 } 123 124 /* 125 * Update process times based on virtual cpu times stored by entry.S 126 * to the lowcore fields user_timer, system_timer & steal_clock. 127 */ 128 static int do_account_vtime(struct task_struct *tsk) 129 { 130 u64 timer, clock, user, guest, system, hardirq, softirq; 131 struct lowcore *lc = get_lowcore(); 132 133 timer = lc->last_update_timer; 134 clock = lc->last_update_clock; 135 136 vtime_reset_last_update(lc); 137 138 clock = lc->last_update_clock - clock; 139 timer -= lc->last_update_timer; 140 141 if (hardirq_count()) 142 lc->hardirq_timer += timer; 143 else 144 lc->system_timer += timer; 145 146 /* Update MT utilization calculation */ 147 if (smp_cpu_mtid && time_after64(jiffies_64, __this_cpu_read(mt_scaling_jiffies))) 148 update_mt_scaling(); 149 150 /* Calculate cputime delta */ 151 user = update_tsk_timer(&tsk->thread.user_timer, lc->user_timer); 152 guest = update_tsk_timer(&tsk->thread.guest_timer, lc->guest_timer); 153 system = update_tsk_timer(&tsk->thread.system_timer, lc->system_timer); 154 hardirq = update_tsk_timer(&tsk->thread.hardirq_timer, lc->hardirq_timer); 155 softirq = update_tsk_timer(&tsk->thread.softirq_timer, lc->softirq_timer); 156 lc->steal_timer += clock - user - guest - system - hardirq - softirq; 157 158 /* Push account value */ 159 if (user) { 160 account_user_time(tsk, cputime_to_nsecs(user)); 161 tsk->utimescaled += cputime_to_nsecs(scale_vtime(user)); 162 } 163 164 if (guest) { 165 account_guest_time(tsk, cputime_to_nsecs(guest)); 166 tsk->utimescaled += cputime_to_nsecs(scale_vtime(guest)); 167 } 168 169 if (system) 170 account_system_index_scaled(tsk, system, CPUTIME_SYSTEM); 171 if (hardirq) 172 account_system_index_scaled(tsk, hardirq, CPUTIME_IRQ); 173 if (softirq) 174 account_system_index_scaled(tsk, softirq, CPUTIME_SOFTIRQ); 175 176 return virt_timer_forward(user + guest + system + hardirq + softirq); 177 } 178 179 void vtime_task_switch(struct task_struct *prev) 180 { 181 struct lowcore *lc = get_lowcore(); 182 183 do_account_vtime(prev); 184 prev->thread.user_timer = lc->user_timer; 185 prev->thread.guest_timer = lc->guest_timer; 186 prev->thread.system_timer = lc->system_timer; 187 prev->thread.hardirq_timer = lc->hardirq_timer; 188 prev->thread.softirq_timer = lc->softirq_timer; 189 lc->user_timer = current->thread.user_timer; 190 lc->guest_timer = current->thread.guest_timer; 191 lc->system_timer = current->thread.system_timer; 192 lc->hardirq_timer = current->thread.hardirq_timer; 193 lc->softirq_timer = current->thread.softirq_timer; 194 } 195 196 /* 197 * In s390, accounting pending user time also implies 198 * accounting system time in order to correctly compute 199 * the stolen time accounting. 200 */ 201 void vtime_flush(struct task_struct *tsk) 202 { 203 struct lowcore *lc = get_lowcore(); 204 u64 steal, avg_steal; 205 206 if (do_account_vtime(tsk)) 207 virt_timer_expire(); 208 209 steal = lc->steal_timer; 210 avg_steal = lc->avg_steal_timer; 211 if ((s64) steal > 0) { 212 lc->steal_timer = 0; 213 account_steal_time(cputime_to_nsecs(steal)); 214 avg_steal += steal; 215 } 216 lc->avg_steal_timer = avg_steal / 2; 217 } 218 219 static u64 vtime_delta(void) 220 { 221 struct lowcore *lc = get_lowcore(); 222 u64 timer = lc->last_update_timer; 223 224 lc->last_update_timer = get_cpu_timer(); 225 return timer - lc->last_update_timer; 226 } 227 228 void vtime_account_kernel(struct task_struct *tsk) 229 { 230 struct lowcore *lc = get_lowcore(); 231 u64 delta = vtime_delta(); 232 233 if (tsk->flags & PF_VCPU) 234 lc->guest_timer += delta; 235 else 236 lc->system_timer += delta; 237 } 238 EXPORT_SYMBOL_GPL(vtime_account_kernel); 239 240 void vtime_account_softirq(struct task_struct *tsk) 241 { 242 get_lowcore()->softirq_timer += vtime_delta(); 243 } 244 245 void vtime_account_hardirq(struct task_struct *tsk) 246 { 247 get_lowcore()->hardirq_timer += vtime_delta(); 248 } 249 250 /* 251 * Sorted add to a list. List is linear searched until first bigger 252 * element is found. 253 */ 254 static void list_add_sorted(struct vtimer_list *timer, struct list_head *head) 255 { 256 struct vtimer_list *tmp; 257 258 list_for_each_entry(tmp, head, entry) { 259 if (tmp->expires > timer->expires) { 260 list_add_tail(&timer->entry, &tmp->entry); 261 return; 262 } 263 } 264 list_add_tail(&timer->entry, head); 265 } 266 267 /* 268 * Handler for expired virtual CPU timer. 269 */ 270 static void virt_timer_expire(void) 271 { 272 struct vtimer_list *timer, *tmp; 273 unsigned long elapsed; 274 LIST_HEAD(cb_list); 275 276 /* walk timer list, fire all expired timers */ 277 spin_lock(&virt_timer_lock); 278 elapsed = atomic64_read(&virt_timer_elapsed); 279 list_for_each_entry_safe(timer, tmp, &virt_timer_list, entry) { 280 if (timer->expires < elapsed) 281 /* move expired timer to the callback queue */ 282 list_move_tail(&timer->entry, &cb_list); 283 else 284 timer->expires -= elapsed; 285 } 286 if (!list_empty(&virt_timer_list)) { 287 timer = list_first_entry(&virt_timer_list, 288 struct vtimer_list, entry); 289 atomic64_set(&virt_timer_current, timer->expires); 290 } 291 atomic64_sub(elapsed, &virt_timer_elapsed); 292 spin_unlock(&virt_timer_lock); 293 294 /* Do callbacks and recharge periodic timers */ 295 list_for_each_entry_safe(timer, tmp, &cb_list, entry) { 296 list_del_init(&timer->entry); 297 timer->function(timer->data); 298 if (timer->interval) { 299 /* Recharge interval timer */ 300 timer->expires = timer->interval + 301 atomic64_read(&virt_timer_elapsed); 302 spin_lock(&virt_timer_lock); 303 list_add_sorted(timer, &virt_timer_list); 304 spin_unlock(&virt_timer_lock); 305 } 306 } 307 } 308 309 void init_virt_timer(struct vtimer_list *timer) 310 { 311 timer->function = NULL; 312 INIT_LIST_HEAD(&timer->entry); 313 } 314 EXPORT_SYMBOL(init_virt_timer); 315 316 static inline int vtimer_pending(struct vtimer_list *timer) 317 { 318 return !list_empty(&timer->entry); 319 } 320 321 static void internal_add_vtimer(struct vtimer_list *timer) 322 { 323 if (list_empty(&virt_timer_list)) { 324 /* First timer, just program it. */ 325 atomic64_set(&virt_timer_current, timer->expires); 326 atomic64_set(&virt_timer_elapsed, 0); 327 list_add(&timer->entry, &virt_timer_list); 328 } else { 329 /* Update timer against current base. */ 330 timer->expires += atomic64_read(&virt_timer_elapsed); 331 if (likely((s64) timer->expires < 332 (s64) atomic64_read(&virt_timer_current))) 333 /* The new timer expires before the current timer. */ 334 atomic64_set(&virt_timer_current, timer->expires); 335 /* Insert new timer into the list. */ 336 list_add_sorted(timer, &virt_timer_list); 337 } 338 } 339 340 static void __add_vtimer(struct vtimer_list *timer, int periodic) 341 { 342 unsigned long flags; 343 344 timer->interval = periodic ? timer->expires : 0; 345 spin_lock_irqsave(&virt_timer_lock, flags); 346 internal_add_vtimer(timer); 347 spin_unlock_irqrestore(&virt_timer_lock, flags); 348 } 349 350 /* 351 * add_virt_timer - add a oneshot virtual CPU timer 352 */ 353 void add_virt_timer(struct vtimer_list *timer) 354 { 355 __add_vtimer(timer, 0); 356 } 357 EXPORT_SYMBOL(add_virt_timer); 358 359 /* 360 * add_virt_timer_int - add an interval virtual CPU timer 361 */ 362 void add_virt_timer_periodic(struct vtimer_list *timer) 363 { 364 __add_vtimer(timer, 1); 365 } 366 EXPORT_SYMBOL(add_virt_timer_periodic); 367 368 static int __mod_vtimer(struct vtimer_list *timer, u64 expires, int periodic) 369 { 370 unsigned long flags; 371 int rc; 372 373 BUG_ON(!timer->function); 374 375 if (timer->expires == expires && vtimer_pending(timer)) 376 return 1; 377 spin_lock_irqsave(&virt_timer_lock, flags); 378 rc = vtimer_pending(timer); 379 if (rc) 380 list_del_init(&timer->entry); 381 timer->interval = periodic ? expires : 0; 382 timer->expires = expires; 383 internal_add_vtimer(timer); 384 spin_unlock_irqrestore(&virt_timer_lock, flags); 385 return rc; 386 } 387 388 /* 389 * returns whether it has modified a pending timer (1) or not (0) 390 */ 391 int mod_virt_timer(struct vtimer_list *timer, u64 expires) 392 { 393 return __mod_vtimer(timer, expires, 0); 394 } 395 EXPORT_SYMBOL(mod_virt_timer); 396 397 /* 398 * returns whether it has modified a pending timer (1) or not (0) 399 */ 400 int mod_virt_timer_periodic(struct vtimer_list *timer, u64 expires) 401 { 402 return __mod_vtimer(timer, expires, 1); 403 } 404 EXPORT_SYMBOL(mod_virt_timer_periodic); 405 406 /* 407 * Delete a virtual timer. 408 * 409 * returns whether the deleted timer was pending (1) or not (0) 410 */ 411 int del_virt_timer(struct vtimer_list *timer) 412 { 413 unsigned long flags; 414 415 if (!vtimer_pending(timer)) 416 return 0; 417 spin_lock_irqsave(&virt_timer_lock, flags); 418 list_del_init(&timer->entry); 419 spin_unlock_irqrestore(&virt_timer_lock, flags); 420 return 1; 421 } 422 EXPORT_SYMBOL(del_virt_timer); 423 424 /* 425 * Start the virtual CPU timer on the current CPU. 426 */ 427 void vtime_init(void) 428 { 429 /* set initial cpu timer */ 430 set_vtimer(VTIMER_MAX_SLICE); 431 /* Setup initial MT scaling values */ 432 if (smp_cpu_mtid) { 433 __this_cpu_write(mt_scaling_jiffies, jiffies); 434 __this_cpu_write(mt_scaling_mult, 1); 435 __this_cpu_write(mt_scaling_div, 1); 436 stcctm(MT_DIAG, smp_cpu_mtid + 1, this_cpu_ptr(mt_cycles)); 437 } 438 } 439