1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Simple CPU accounting cgroup controller 4 */ 5 #include <linux/sched/clock.h> 6 #include <linux/sched/cputime.h> 7 #include <linux/tsacct_kern.h> 8 #include "sched.h" 9 10 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 11 #include <asm/cputime.h> 12 #endif 13 14 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 15 16 DEFINE_STATIC_KEY_FALSE(sched_clock_irqtime); 17 18 /* 19 * There are no locks covering percpu hardirq/softirq time. 20 * They are only modified in vtime_account, on corresponding CPU 21 * with interrupts disabled. So, writes are safe. 22 * They are read and saved off onto struct rq in update_rq_clock(). 23 * This may result in other CPU reading this CPU's IRQ time and can 24 * race with irq/vtime_account on this CPU. We would either get old 25 * or new value with a side effect of accounting a slice of IRQ time to wrong 26 * task when IRQ is in progress while we read rq->clock. That is a worthy 27 * compromise in place of having locks on each IRQ in account_system_time. 28 */ 29 DEFINE_PER_CPU(struct irqtime, cpu_irqtime); 30 31 void enable_sched_clock_irqtime(void) 32 { 33 static_branch_enable(&sched_clock_irqtime); 34 } 35 36 void disable_sched_clock_irqtime(void) 37 { 38 if (irqtime_enabled()) 39 static_branch_disable(&sched_clock_irqtime); 40 } 41 42 static void irqtime_account_delta(struct irqtime *irqtime, u64 delta, 43 enum cpu_usage_stat idx) 44 { 45 u64 *cpustat = kcpustat_this_cpu->cpustat; 46 47 u64_stats_update_begin(&irqtime->sync); 48 cpustat[idx] += delta; 49 irqtime->total += delta; 50 if (!kcpustat_idle_dyntick()) 51 irqtime->tick_delta += delta; 52 u64_stats_update_end(&irqtime->sync); 53 } 54 55 /* 56 * Called after incrementing preempt_count on {soft,}irq_enter 57 * and before decrementing preempt_count on {soft,}irq_exit. 58 */ 59 void irqtime_account_irq(struct task_struct *curr, unsigned int offset) 60 { 61 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 62 unsigned int pc; 63 s64 delta; 64 int cpu; 65 66 if (!irqtime_enabled()) 67 return; 68 69 cpu = smp_processor_id(); 70 delta = sched_clock_cpu(cpu) - irqtime->irq_start_time; 71 irqtime->irq_start_time += delta; 72 pc = irq_count() - offset; 73 74 /* 75 * We do not account for softirq time from ksoftirqd here. 76 * We want to continue accounting softirq time to ksoftirqd thread 77 * in that case, so as not to confuse scheduler with a special task 78 * that do not consume any time, but still wants to run. 79 */ 80 if (pc & HARDIRQ_MASK) 81 irqtime_account_delta(irqtime, delta, CPUTIME_IRQ); 82 else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd()) 83 irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ); 84 } 85 86 static u64 irqtime_tick_accounted(u64 maxtime) 87 { 88 struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime); 89 u64 delta; 90 91 delta = min(irqtime->tick_delta, maxtime); 92 irqtime->tick_delta -= delta; 93 94 return delta; 95 } 96 97 #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ 98 99 static u64 irqtime_tick_accounted(u64 dummy) 100 { 101 return 0; 102 } 103 104 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ 105 106 static inline void task_group_account_field(struct task_struct *p, int index, 107 u64 tmp) 108 { 109 /* 110 * Since all updates are sure to touch the root cgroup, we 111 * get ourselves ahead and touch it first. If the root cgroup 112 * is the only cgroup, then nothing else should be necessary. 113 * 114 */ 115 __this_cpu_add(kernel_cpustat.cpustat[index], tmp); 116 117 cgroup_account_cputime_field(p, index, tmp); 118 } 119 120 /* 121 * Account user CPU time to a process. 122 * @p: the process that the CPU time gets accounted to 123 * @cputime: the CPU time spent in user space since the last update 124 */ 125 void account_user_time(struct task_struct *p, u64 cputime) 126 { 127 int index; 128 129 /* Add user time to process. */ 130 p->utime += cputime; 131 account_group_user_time(p, cputime); 132 133 index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER; 134 135 /* Add user time to cpustat. */ 136 task_group_account_field(p, index, cputime); 137 138 /* Account for user time used */ 139 acct_account_cputime(p); 140 } 141 142 /* 143 * Account guest CPU time to a process. 144 * @p: the process that the CPU time gets accounted to 145 * @cputime: the CPU time spent in virtual machine since the last update 146 */ 147 void account_guest_time(struct task_struct *p, u64 cputime) 148 { 149 u64 *cpustat = kcpustat_this_cpu->cpustat; 150 151 /* Add guest time to process. */ 152 p->utime += cputime; 153 account_group_user_time(p, cputime); 154 p->gtime += cputime; 155 156 /* Add guest time to cpustat. */ 157 if (task_nice(p) > 0) { 158 task_group_account_field(p, CPUTIME_NICE, cputime); 159 cpustat[CPUTIME_GUEST_NICE] += cputime; 160 } else { 161 task_group_account_field(p, CPUTIME_USER, cputime); 162 cpustat[CPUTIME_GUEST] += cputime; 163 } 164 } 165 166 /* 167 * Account system CPU time to a process and desired cpustat field 168 * @p: the process that the CPU time gets accounted to 169 * @cputime: the CPU time spent in kernel space since the last update 170 * @index: pointer to cpustat field that has to be updated 171 */ 172 void account_system_index_time(struct task_struct *p, 173 u64 cputime, enum cpu_usage_stat index) 174 { 175 /* Add system time to process. */ 176 p->stime += cputime; 177 account_group_system_time(p, cputime); 178 179 /* Add system time to cpustat. */ 180 task_group_account_field(p, index, cputime); 181 182 /* Account for system time used */ 183 acct_account_cputime(p); 184 } 185 186 /* 187 * Account system CPU time to a process. 188 * @p: the process that the CPU time gets accounted to 189 * @hardirq_offset: the offset to subtract from hardirq_count() 190 * @cputime: the CPU time spent in kernel space since the last update 191 */ 192 void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime) 193 { 194 int index; 195 196 if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) { 197 account_guest_time(p, cputime); 198 return; 199 } 200 201 if (hardirq_count() - hardirq_offset) 202 index = CPUTIME_IRQ; 203 else if (in_serving_softirq()) 204 index = CPUTIME_SOFTIRQ; 205 else 206 index = CPUTIME_SYSTEM; 207 208 account_system_index_time(p, cputime, index); 209 } 210 211 /* 212 * Account for involuntary wait time. 213 * @cputime: the CPU time spent in involuntary wait 214 */ 215 void account_steal_time(u64 cputime) 216 { 217 u64 *cpustat = kcpustat_this_cpu->cpustat; 218 219 cpustat[CPUTIME_STEAL] += cputime; 220 } 221 222 /* 223 * Account for idle time. 224 * @cputime: the CPU time spent in idle wait 225 */ 226 void account_idle_time(u64 cputime) 227 { 228 u64 *cpustat = kcpustat_this_cpu->cpustat; 229 230 if (cpu_has_iowait(smp_processor_id())) 231 cpustat[CPUTIME_IOWAIT] += cputime; 232 else 233 cpustat[CPUTIME_IDLE] += cputime; 234 } 235 236 237 #ifdef CONFIG_SCHED_CORE 238 /* 239 * Account for forceidle time due to core scheduling. 240 * 241 * REQUIRES: schedstat is enabled. 242 */ 243 void __account_forceidle_time(struct task_struct *p, u64 delta) 244 { 245 __schedstat_add(p->stats.core_forceidle_sum, delta); 246 247 task_group_account_field(p, CPUTIME_FORCEIDLE, delta); 248 } 249 #endif /* CONFIG_SCHED_CORE */ 250 251 /* 252 * When a guest is interrupted for a longer amount of time, missed clock 253 * ticks are not redelivered later. Due to that, this function may on 254 * occasion account more time than the calling functions think elapsed. 255 */ 256 #ifdef CONFIG_PARAVIRT 257 DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled); 258 259 #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN 260 static u64 native_steal_clock(int cpu) 261 { 262 return 0; 263 } 264 265 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 266 #endif 267 #endif 268 269 static __always_inline u64 steal_account_process_time(u64 maxtime) 270 { 271 #ifdef CONFIG_PARAVIRT 272 if (static_branch_unlikely(¶virt_steal_enabled)) { 273 u64 steal; 274 275 steal = paravirt_steal_clock(smp_processor_id()); 276 steal -= this_rq()->prev_steal_time; 277 steal = min(steal, maxtime); 278 account_steal_time(steal); 279 this_rq()->prev_steal_time += steal; 280 281 return steal; 282 } 283 #endif /* CONFIG_PARAVIRT */ 284 return 0; 285 } 286 287 /* 288 * Account how much elapsed time was spent in steal, IRQ, or softirq time. 289 */ 290 static inline u64 account_other_time(u64 max) 291 { 292 u64 accounted; 293 294 lockdep_assert_irqs_disabled(); 295 296 accounted = steal_account_process_time(max); 297 298 if (accounted < max) 299 accounted += irqtime_tick_accounted(max - accounted); 300 301 return accounted; 302 } 303 304 #ifdef CONFIG_64BIT 305 static inline u64 read_sum_exec_runtime(struct task_struct *t) 306 { 307 return t->se.sum_exec_runtime; 308 } 309 #else /* !CONFIG_64BIT: */ 310 static u64 read_sum_exec_runtime(struct task_struct *t) 311 { 312 u64 ns; 313 struct rq_flags rf; 314 struct rq *rq; 315 316 rq = task_rq_lock(t, &rf); 317 ns = t->se.sum_exec_runtime; 318 task_rq_unlock(rq, t, &rf); 319 320 return ns; 321 } 322 #endif /* !CONFIG_64BIT */ 323 324 /* 325 * Accumulate raw cputime values of dead tasks (sig->[us]time) and live 326 * tasks (sum on group iteration) belonging to @tsk's group. 327 */ 328 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times) 329 { 330 struct signal_struct *sig = tsk->signal; 331 struct task_struct *t; 332 u64 utime, stime; 333 334 /* 335 * Update current task runtime to account pending time since last 336 * scheduler action or thread_group_cputime() call. This thread group 337 * might have other running tasks on different CPUs, but updating 338 * their runtime can affect syscall performance, so we skip account 339 * those pending times and rely only on values updated on tick or 340 * other scheduler action. 341 */ 342 if (same_thread_group(current, tsk)) 343 (void) task_sched_runtime(current); 344 345 guard(rcu)(); 346 scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { 347 times->utime = sig->utime; 348 times->stime = sig->stime; 349 times->sum_exec_runtime = sig->sum_sched_runtime; 350 351 __for_each_thread(sig, t) { 352 task_cputime(t, &utime, &stime); 353 times->utime += utime; 354 times->stime += stime; 355 times->sum_exec_runtime += read_sum_exec_runtime(t); 356 } 357 } 358 } 359 360 #ifdef CONFIG_IRQ_TIME_ACCOUNTING 361 /* 362 * Account a tick to a process and cpustat 363 * @p: the process that the CPU time gets accounted to 364 * @user_tick: is the tick from userspace 365 * @rq: the pointer to rq 366 * 367 * Tick demultiplexing follows the order 368 * - pending hardirq update 369 * - pending softirq update 370 * - user_time 371 * - idle_time 372 * - system time 373 * - check for guest_time 374 * - else account as system_time 375 * 376 * Check for hardirq is done both for system and user time as there is 377 * no timer going off while we are on hardirq and hence we may never get an 378 * opportunity to update it solely in system time. 379 * p->stime and friends are only updated on system time and not on IRQ 380 * softirq as those do not count in task exec_runtime any more. 381 */ 382 static void irqtime_account_process_tick(struct task_struct *p, int user_tick, 383 u64 cputime) 384 { 385 u64 other; 386 387 /* 388 * When returning from idle, many ticks can get accounted at 389 * once, including some ticks of steal, IRQ, and softirq time. 390 * Subtract those ticks from the amount of time accounted to 391 * idle, or potentially user or system time. Due to rounding, 392 * other time can exceed ticks occasionally. 393 */ 394 other = account_other_time(ULONG_MAX); 395 if (other >= cputime) 396 return; 397 398 cputime -= other; 399 400 if (this_cpu_ksoftirqd() == p) { 401 /* 402 * ksoftirqd time do not get accounted in cpu_softirq_time. 403 * So, we have to handle it separately here. 404 * Also, p->stime needs to be updated for ksoftirqd. 405 */ 406 account_system_index_time(p, cputime, CPUTIME_SOFTIRQ); 407 } else if (user_tick) { 408 account_user_time(p, cputime); 409 } else if (p == this_rq()->idle) { 410 account_idle_time(cputime); 411 } else if (p->flags & PF_VCPU) { /* System time or guest time */ 412 account_guest_time(p, cputime); 413 } else { 414 account_system_index_time(p, cputime, CPUTIME_SYSTEM); 415 } 416 } 417 418 #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */ 419 static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick, 420 u64 cputime) { } 421 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */ 422 423 #if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) 424 static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now) 425 { 426 u64 *cpustat = kc->cpustat; 427 u64 delta, steal, steal_delta; 428 int iowait; 429 430 if (!kc->idle_elapse) 431 return; 432 433 iowait = cpu_has_iowait(smp_processor_id()); 434 delta = now - kc->idle_entrytime; 435 steal = steal_account_process_time(delta); 436 437 /* 438 * Record the idle time after substracting the steal time from 439 * previous update sequence. Don't substract the steal time from 440 * the current update sequence to avoid readers moving backward. 441 */ 442 write_seqcount_begin(&kc->idle_sleeptime_seq); 443 steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta); 444 delta -= steal_delta; 445 kc->idle_stealtime[iowait] -= steal_delta; 446 447 if (iowait) 448 cpustat[CPUTIME_IOWAIT] += delta; 449 else 450 cpustat[CPUTIME_IDLE] += delta; 451 452 kc->idle_stealtime[iowait] += steal; 453 kc->idle_entrytime = now; 454 kc->idle_elapse = false; 455 write_seqcount_end(&kc->idle_sleeptime_seq); 456 } 457 458 static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now) 459 { 460 /* Irqtime accounting might have been enabled in the middle of the IRQ */ 461 if (kc->idle_elapse) 462 return; 463 464 write_seqcount_begin(&kc->idle_sleeptime_seq); 465 kc->idle_entrytime = now; 466 kc->idle_elapse = true; 467 write_seqcount_end(&kc->idle_sleeptime_seq); 468 } 469 470 void kcpustat_dyntick_stop(u64 now, u64 tick_start) 471 { 472 struct kernel_cpustat *kc = kcpustat_this_cpu; 473 474 if (!vtime_generic_enabled_this_cpu()) { 475 WARN_ON_ONCE(!kc->idle_dyntick); 476 if (kc->idle_tick_period != tick_start) { 477 kc->idle_tick_period = tick_start; 478 kc->idle_tick_overlap = 0; 479 } 480 tick_start = max(tick_start, kc->idle_dyntick_entry); 481 /* 482 * After a switch from the jiffies clocksource the tick can be 483 * armed more than a tick ahead, so the period may not have 484 * started yet. 485 */ 486 if (now > tick_start) 487 kc->idle_tick_overlap += now - tick_start; 488 kcpustat_idle_stop(kc, now); 489 kc->idle_dyntick = false; 490 vtime_dyntick_stop(); 491 } 492 } 493 494 /* 495 * The first tick after dyntick-idle covers a period that the dyntick-idle 496 * accounting may already have accounted in part. 497 */ 498 static inline u64 kcpustat_tick_overlap(void) 499 { 500 struct kernel_cpustat *kc = kcpustat_this_cpu; 501 u64 overlap = kc->idle_tick_overlap; 502 503 kc->idle_tick_overlap = 0; 504 return overlap; 505 } 506 507 void kcpustat_dyntick_start(u64 now) 508 { 509 struct kernel_cpustat *kc = kcpustat_this_cpu; 510 511 if (!vtime_generic_enabled_this_cpu()) { 512 vtime_dyntick_start(); 513 kc->idle_dyntick = true; 514 kc->idle_dyntick_entry = now; 515 kcpustat_idle_start(kc, now); 516 } 517 } 518 519 void kcpustat_irq_enter(u64 now) 520 { 521 struct kernel_cpustat *kc = kcpustat_this_cpu; 522 523 if (!vtime_generic_enabled_this_cpu() && 524 (irqtime_enabled() || vtime_accounting_enabled_this_cpu())) 525 kcpustat_idle_stop(kc, now); 526 } 527 528 void kcpustat_irq_exit(u64 now) 529 { 530 struct kernel_cpustat *kc = kcpustat_this_cpu; 531 532 /* 533 * Generic vtime already does its own idle accounting. 534 * But irqtime accounting or arch vtime which also accounts IRQs 535 * need to pause nohz accounting. Resume nohz accounting as long 536 * as the irqtime config is enabled to handle case where irqtime 537 * accounting got runtime disabled in the middle of an IRQ. 538 */ 539 if (!vtime_generic_enabled_this_cpu() && 540 (IS_ENABLED(CONFIG_IRQ_TIME_ACCOUNTING) || vtime_accounting_enabled_this_cpu())) 541 kcpustat_idle_start(kc, now); 542 } 543 544 static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, 545 bool compute_delta, u64 now) 546 { 547 struct kernel_cpustat *kc = &kcpustat_cpu(cpu); 548 int iowait = idx == CPUTIME_IOWAIT; 549 u64 *cpustat = kc->cpustat; 550 unsigned int seq; 551 u64 idle; 552 553 do { 554 seq = read_seqcount_begin(&kc->idle_sleeptime_seq); 555 556 idle = cpustat[idx]; 557 558 if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) { 559 u64 delta = now - kc->idle_entrytime; 560 561 delta -= min_t(u64, kc->idle_stealtime[iowait], delta); 562 idle += delta; 563 } 564 } while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq)); 565 566 return idle; 567 } 568 569 u64 kcpustat_field_idle(int cpu) 570 { 571 return kcpustat_field_dyntick(cpu, CPUTIME_IDLE, 572 !cpu_has_iowait(cpu), ktime_get()); 573 } 574 EXPORT_SYMBOL_GPL(kcpustat_field_idle); 575 576 u64 kcpustat_field_iowait(int cpu) 577 { 578 return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT, 579 cpu_has_iowait(cpu), ktime_get()); 580 } 581 EXPORT_SYMBOL_GPL(kcpustat_field_iowait); 582 #else 583 static inline u64 kcpustat_tick_overlap(void) 584 { 585 return 0; 586 } 587 588 static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx, 589 bool compute_delta, ktime_t now) 590 { 591 return kcpustat_cpu(cpu).cpustat[idx]; 592 } 593 #endif /* CONFIG_NO_HZ_COMMON && !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */ 594 595 static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx, 596 bool compute_delta, u64 *last_update_time) 597 { 598 ktime_t now = ktime_get(); 599 u64 res; 600 601 if (vtime_generic_enabled_cpu(cpu)) 602 res = kcpustat_field(idx, cpu); 603 else 604 res = kcpustat_field_dyntick(cpu, idx, compute_delta, now); 605 606 do_div(res, NSEC_PER_USEC); 607 608 if (last_update_time) 609 *last_update_time = ktime_to_us(now); 610 611 return res; 612 } 613 614 /** 615 * get_cpu_idle_time_us - get the total idle time of a CPU 616 * @cpu: CPU number to query 617 * @last_update_time: variable to store update time in. Do not update 618 * counters if NULL. 619 * 620 * Return the cumulative idle time (since boot) for a given 621 * CPU, in microseconds. Note that this is partially broken due to 622 * the counter of iowait tasks that can be remotely updated without 623 * any synchronization. Therefore it is possible to observe backward 624 * values within two consecutive reads. 625 * 626 * This time is measured via accounting rather than sampling, 627 * and is as accurate as ktime_get() is. 628 * 629 * Return: total idle time of the @cpu 630 */ 631 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time) 632 { 633 return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE, 634 !cpu_has_iowait(cpu), last_update_time); 635 } 636 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us); 637 638 /** 639 * get_cpu_iowait_time_us - get the total iowait time of a CPU 640 * @cpu: CPU number to query 641 * @last_update_time: variable to store update time in. Do not update 642 * counters if NULL. 643 * 644 * Return the cumulative iowait time (since boot) for a given 645 * CPU, in microseconds. Note this is partially broken due to 646 * the counter of iowait tasks that can be remotely updated without 647 * any synchronization. Therefore it is possible to observe backward 648 * values within two consecutive reads. 649 * 650 * This time is measured via accounting rather than sampling, 651 * and is as accurate as ktime_get() is. 652 * 653 * Return: total iowait time of @cpu 654 */ 655 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time) 656 { 657 return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT, 658 cpu_has_iowait(cpu), last_update_time); 659 } 660 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us); 661 662 /* 663 * Use precise platform statistics if available: 664 */ 665 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE 666 667 void vtime_account_irq(struct task_struct *tsk, unsigned int offset) 668 { 669 unsigned int pc = irq_count() - offset; 670 671 if (pc & HARDIRQ_OFFSET) { 672 vtime_account_hardirq(tsk); 673 } else if (pc & SOFTIRQ_OFFSET) { 674 vtime_account_softirq(tsk); 675 } else if (!kcpustat_idle_dyntick()) { 676 if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) && 677 is_idle_task(tsk)) { 678 vtime_account_idle(tsk); 679 } else { 680 vtime_account_kernel(tsk); 681 } 682 } else { 683 vtime_reset(); 684 } 685 } 686 687 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, 688 u64 *ut, u64 *st) 689 { 690 *ut = curr->utime; 691 *st = curr->stime; 692 } 693 694 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) 695 { 696 *ut = p->utime; 697 *st = p->stime; 698 } 699 EXPORT_SYMBOL_GPL(task_cputime_adjusted); 700 701 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) 702 { 703 struct task_cputime cputime; 704 705 thread_group_cputime(p, &cputime); 706 707 *ut = cputime.utime; 708 *st = cputime.stime; 709 } 710 711 #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */ 712 713 /* 714 * Account a single tick of CPU time. 715 * @p: the process that the CPU time gets accounted to 716 * @user_tick: indicates if the tick is a user or a system tick 717 */ 718 void account_process_tick(struct task_struct *p, int user_tick) 719 { 720 u64 cputime, steal; 721 722 if (vtime_accounting_enabled_this_cpu()) 723 return; 724 725 if (kcpustat_idle_dyntick()) 726 return; 727 728 cputime = TICK_NSEC - kcpustat_tick_overlap(); 729 730 if (irqtime_enabled()) { 731 irqtime_account_process_tick(p, user_tick, cputime); 732 return; 733 } 734 735 steal = steal_account_process_time(ULONG_MAX); 736 737 if (steal >= cputime) 738 return; 739 740 cputime -= steal; 741 742 if (user_tick) 743 account_user_time(p, cputime); 744 else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET)) 745 account_system_time(p, HARDIRQ_OFFSET, cputime); 746 else 747 account_idle_time(cputime); 748 } 749 750 /* 751 * Adjust tick based cputime random precision against scheduler runtime 752 * accounting. 753 * 754 * Tick based cputime accounting depend on random scheduling timeslices of a 755 * task to be interrupted or not by the timer. Depending on these 756 * circumstances, the number of these interrupts may be over or 757 * under-optimistic, matching the real user and system cputime with a variable 758 * precision. 759 * 760 * Fix this by scaling these tick based values against the total runtime 761 * accounted by the CFS scheduler. 762 * 763 * This code provides the following guarantees: 764 * 765 * stime + utime == rtime 766 * stime_i+1 >= stime_i, utime_i+1 >= utime_i 767 * 768 * Assuming that rtime_i+1 >= rtime_i. 769 */ 770 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev, 771 u64 *ut, u64 *st) 772 { 773 u64 rtime, stime, utime; 774 unsigned long flags; 775 776 /* Serialize concurrent callers such that we can honour our guarantees */ 777 raw_spin_lock_irqsave(&prev->lock, flags); 778 rtime = curr->sum_exec_runtime; 779 780 /* 781 * This is possible under two circumstances: 782 * - rtime isn't monotonic after all (a bug); 783 * - we got reordered by the lock. 784 * 785 * In both cases this acts as a filter such that the rest of the code 786 * can assume it is monotonic regardless of anything else. 787 */ 788 if (prev->stime + prev->utime >= rtime) 789 goto out; 790 791 stime = curr->stime; 792 utime = curr->utime; 793 794 /* 795 * If either stime or utime are 0, assume all runtime is userspace. 796 * Once a task gets some ticks, the monotonicity code at 'update:' 797 * will ensure things converge to the observed ratio. 798 */ 799 if (stime == 0) { 800 utime = rtime; 801 goto update; 802 } 803 804 if (utime == 0) { 805 stime = rtime; 806 goto update; 807 } 808 809 stime = mul_u64_u64_div_u64(stime, rtime, stime + utime); 810 811 update: 812 /* 813 * Make sure stime doesn't go backwards; this preserves monotonicity 814 * for utime because rtime is monotonic. 815 * 816 * utime_i+1 = rtime_i+1 - stime_i 817 * = rtime_i+1 - (rtime_i - utime_i) 818 * = (rtime_i+1 - rtime_i) + utime_i 819 * >= utime_i 820 */ 821 if (stime < prev->stime) 822 stime = prev->stime; 823 utime = rtime - stime; 824 825 /* 826 * Make sure utime doesn't go backwards; this still preserves 827 * monotonicity for stime, analogous argument to above. 828 */ 829 if (utime < prev->utime) { 830 utime = prev->utime; 831 stime = rtime - utime; 832 } 833 834 prev->stime = stime; 835 prev->utime = utime; 836 out: 837 *ut = prev->utime; 838 *st = prev->stime; 839 raw_spin_unlock_irqrestore(&prev->lock, flags); 840 } 841 842 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) 843 { 844 struct task_cputime cputime = { 845 .sum_exec_runtime = p->se.sum_exec_runtime, 846 }; 847 848 if (task_cputime(p, &cputime.utime, &cputime.stime)) 849 cputime.sum_exec_runtime = task_sched_runtime(p); 850 cputime_adjust(&cputime, &p->prev_cputime, ut, st); 851 } 852 EXPORT_SYMBOL_GPL(task_cputime_adjusted); 853 854 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st) 855 { 856 struct task_cputime cputime; 857 858 thread_group_cputime(p, &cputime); 859 cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st); 860 } 861 #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */ 862 863 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN 864 static u64 vtime_delta(struct vtime *vtime) 865 { 866 unsigned long long clock; 867 868 clock = sched_clock(); 869 if (clock < vtime->starttime) 870 return 0; 871 872 return clock - vtime->starttime; 873 } 874 875 static u64 get_vtime_delta(struct vtime *vtime) 876 { 877 u64 delta = vtime_delta(vtime); 878 u64 other; 879 880 /* 881 * Unlike tick based timing, vtime based timing never has lost 882 * ticks, and no need for steal time accounting to make up for 883 * lost ticks. Vtime accounts a rounded version of actual 884 * elapsed time. Limit account_other_time to prevent rounding 885 * errors from causing elapsed vtime to go negative. 886 */ 887 other = account_other_time(delta); 888 WARN_ON_ONCE(vtime->state == VTIME_INACTIVE); 889 vtime->starttime += delta; 890 891 return delta - other; 892 } 893 894 static void vtime_account_system(struct task_struct *tsk, 895 struct vtime *vtime) 896 { 897 vtime->stime += get_vtime_delta(vtime); 898 if (vtime->stime >= TICK_NSEC) { 899 account_system_time(tsk, irq_count(), vtime->stime); 900 vtime->stime = 0; 901 } 902 } 903 904 static void vtime_account_guest(struct task_struct *tsk, 905 struct vtime *vtime) 906 { 907 vtime->gtime += get_vtime_delta(vtime); 908 if (vtime->gtime >= TICK_NSEC) { 909 account_guest_time(tsk, vtime->gtime); 910 vtime->gtime = 0; 911 } 912 } 913 914 static void __vtime_account_kernel(struct task_struct *tsk, 915 struct vtime *vtime) 916 { 917 /* We might have scheduled out from guest path */ 918 if (vtime->state == VTIME_GUEST) 919 vtime_account_guest(tsk, vtime); 920 else 921 vtime_account_system(tsk, vtime); 922 } 923 924 void vtime_account_kernel(struct task_struct *tsk) 925 { 926 struct vtime *vtime = &tsk->vtime; 927 928 if (!vtime_delta(vtime)) 929 return; 930 931 write_seqcount_begin(&vtime->seqcount); 932 __vtime_account_kernel(tsk, vtime); 933 write_seqcount_end(&vtime->seqcount); 934 } 935 936 void vtime_user_enter(struct task_struct *tsk) 937 { 938 struct vtime *vtime = &tsk->vtime; 939 940 write_seqcount_begin(&vtime->seqcount); 941 vtime_account_system(tsk, vtime); 942 vtime->state = VTIME_USER; 943 write_seqcount_end(&vtime->seqcount); 944 } 945 946 void vtime_user_exit(struct task_struct *tsk) 947 { 948 struct vtime *vtime = &tsk->vtime; 949 950 write_seqcount_begin(&vtime->seqcount); 951 vtime->utime += get_vtime_delta(vtime); 952 if (vtime->utime >= TICK_NSEC) { 953 account_user_time(tsk, vtime->utime); 954 vtime->utime = 0; 955 } 956 vtime->state = VTIME_SYS; 957 write_seqcount_end(&vtime->seqcount); 958 } 959 960 void vtime_guest_enter(struct task_struct *tsk) 961 { 962 struct vtime *vtime = &tsk->vtime; 963 /* 964 * The flags must be updated under the lock with 965 * the vtime_starttime flush and update. 966 * That enforces a right ordering and update sequence 967 * synchronization against the reader (task_gtime()) 968 * that can thus safely catch up with a tickless delta. 969 */ 970 write_seqcount_begin(&vtime->seqcount); 971 vtime_account_system(tsk, vtime); 972 tsk->flags |= PF_VCPU; 973 vtime->state = VTIME_GUEST; 974 write_seqcount_end(&vtime->seqcount); 975 } 976 EXPORT_SYMBOL_GPL(vtime_guest_enter); 977 978 void vtime_guest_exit(struct task_struct *tsk) 979 { 980 struct vtime *vtime = &tsk->vtime; 981 982 write_seqcount_begin(&vtime->seqcount); 983 vtime_account_guest(tsk, vtime); 984 tsk->flags &= ~PF_VCPU; 985 vtime->state = VTIME_SYS; 986 write_seqcount_end(&vtime->seqcount); 987 } 988 EXPORT_SYMBOL_GPL(vtime_guest_exit); 989 990 static void __vtime_account_idle(struct vtime *vtime) 991 { 992 account_idle_time(get_vtime_delta(vtime)); 993 } 994 995 void vtime_task_switch_generic(struct task_struct *prev) 996 { 997 struct vtime *vtime = &prev->vtime; 998 999 write_seqcount_begin(&vtime->seqcount); 1000 if (vtime->state == VTIME_IDLE) 1001 __vtime_account_idle(vtime); 1002 else 1003 __vtime_account_kernel(prev, vtime); 1004 vtime->state = VTIME_INACTIVE; 1005 vtime->cpu = -1; 1006 write_seqcount_end(&vtime->seqcount); 1007 1008 vtime = ¤t->vtime; 1009 1010 write_seqcount_begin(&vtime->seqcount); 1011 if (is_idle_task(current)) 1012 vtime->state = VTIME_IDLE; 1013 else if (current->flags & PF_VCPU) 1014 vtime->state = VTIME_GUEST; 1015 else 1016 vtime->state = VTIME_SYS; 1017 vtime->starttime = sched_clock(); 1018 vtime->cpu = smp_processor_id(); 1019 write_seqcount_end(&vtime->seqcount); 1020 } 1021 1022 void vtime_init_idle(struct task_struct *t, int cpu) 1023 { 1024 struct vtime *vtime = &t->vtime; 1025 unsigned long flags; 1026 1027 local_irq_save(flags); 1028 write_seqcount_begin(&vtime->seqcount); 1029 vtime->state = VTIME_IDLE; 1030 vtime->starttime = sched_clock(); 1031 vtime->cpu = cpu; 1032 write_seqcount_end(&vtime->seqcount); 1033 local_irq_restore(flags); 1034 } 1035 1036 u64 task_gtime(struct task_struct *t) 1037 { 1038 struct vtime *vtime = &t->vtime; 1039 unsigned int seq; 1040 u64 gtime; 1041 1042 if (!vtime_accounting_enabled()) 1043 return t->gtime; 1044 1045 do { 1046 seq = read_seqcount_begin(&vtime->seqcount); 1047 1048 gtime = t->gtime; 1049 if (vtime->state == VTIME_GUEST) 1050 gtime += vtime->gtime + vtime_delta(vtime); 1051 1052 } while (read_seqcount_retry(&vtime->seqcount, seq)); 1053 1054 return gtime; 1055 } 1056 1057 /* 1058 * Fetch cputime raw values from fields of task_struct and 1059 * add up the pending nohz execution time since the last 1060 * cputime snapshot. 1061 */ 1062 bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime) 1063 { 1064 struct vtime *vtime = &t->vtime; 1065 unsigned int seq; 1066 u64 delta; 1067 int ret; 1068 1069 if (!vtime_accounting_enabled()) { 1070 *utime = t->utime; 1071 *stime = t->stime; 1072 return false; 1073 } 1074 1075 do { 1076 ret = false; 1077 seq = read_seqcount_begin(&vtime->seqcount); 1078 1079 *utime = t->utime; 1080 *stime = t->stime; 1081 1082 /* Task is sleeping or idle, nothing to add */ 1083 if (vtime->state < VTIME_SYS) 1084 continue; 1085 1086 ret = true; 1087 delta = vtime_delta(vtime); 1088 1089 /* 1090 * Task runs either in user (including guest) or kernel space, 1091 * add pending nohz time to the right place. 1092 */ 1093 if (vtime->state == VTIME_SYS) 1094 *stime += vtime->stime + delta; 1095 else 1096 *utime += vtime->utime + delta; 1097 } while (read_seqcount_retry(&vtime->seqcount, seq)); 1098 1099 return ret; 1100 } 1101 1102 static int vtime_state_fetch(struct vtime *vtime, int cpu) 1103 { 1104 int state = READ_ONCE(vtime->state); 1105 1106 /* 1107 * We raced against a context switch, fetch the 1108 * kcpustat task again. 1109 */ 1110 if (vtime->cpu != cpu && vtime->cpu != -1) 1111 return -EAGAIN; 1112 1113 /* 1114 * Two possible things here: 1115 * 1) We are seeing the scheduling out task (prev) or any past one. 1116 * 2) We are seeing the scheduling in task (next) but it hasn't 1117 * passed though vtime_task_switch() yet so the pending 1118 * cputime of the prev task may not be flushed yet. 1119 * 1120 * Case 1) is ok but 2) is not. So wait for a safe VTIME state. 1121 */ 1122 if (state == VTIME_INACTIVE) 1123 return -EAGAIN; 1124 1125 return state; 1126 } 1127 1128 static u64 kcpustat_user_vtime(struct vtime *vtime) 1129 { 1130 if (vtime->state == VTIME_USER) 1131 return vtime->utime + vtime_delta(vtime); 1132 else if (vtime->state == VTIME_GUEST) 1133 return vtime->gtime + vtime_delta(vtime); 1134 return 0; 1135 } 1136 1137 static int kcpustat_field_vtime(u64 *cpustat, 1138 struct task_struct *tsk, 1139 enum cpu_usage_stat usage, 1140 int cpu, u64 *val) 1141 { 1142 struct vtime *vtime = &tsk->vtime; 1143 unsigned int seq; 1144 1145 do { 1146 int state; 1147 1148 seq = read_seqcount_begin(&vtime->seqcount); 1149 1150 state = vtime_state_fetch(vtime, cpu); 1151 if (state < 0) 1152 return state; 1153 1154 *val = cpustat[usage]; 1155 1156 /* 1157 * Nice VS unnice cputime accounting may be inaccurate if 1158 * the nice value has changed since the last vtime update. 1159 * But proper fix would involve interrupting target on nice 1160 * updates which is a no go on nohz_full (although the scheduler 1161 * may still interrupt the target if rescheduling is needed...) 1162 */ 1163 switch (usage) { 1164 case CPUTIME_SYSTEM: 1165 if (state == VTIME_SYS) 1166 *val += vtime->stime + vtime_delta(vtime); 1167 break; 1168 case CPUTIME_USER: 1169 if (task_nice(tsk) <= 0) 1170 *val += kcpustat_user_vtime(vtime); 1171 break; 1172 case CPUTIME_NICE: 1173 if (task_nice(tsk) > 0) 1174 *val += kcpustat_user_vtime(vtime); 1175 break; 1176 case CPUTIME_GUEST: 1177 if (state == VTIME_GUEST && task_nice(tsk) <= 0) 1178 *val += vtime->gtime + vtime_delta(vtime); 1179 break; 1180 case CPUTIME_GUEST_NICE: 1181 if (state == VTIME_GUEST && task_nice(tsk) > 0) 1182 *val += vtime->gtime + vtime_delta(vtime); 1183 break; 1184 case CPUTIME_IDLE: 1185 if (state == VTIME_IDLE && !cpu_has_iowait(cpu)) 1186 *val += vtime_delta(vtime); 1187 break; 1188 case CPUTIME_IOWAIT: 1189 if (state == VTIME_IDLE && cpu_has_iowait(cpu)) 1190 *val += vtime_delta(vtime); 1191 break; 1192 default: 1193 break; 1194 } 1195 } while (read_seqcount_retry(&vtime->seqcount, seq)); 1196 1197 return 0; 1198 } 1199 1200 u64 kcpustat_field(enum cpu_usage_stat usage, int cpu) 1201 { 1202 u64 *cpustat = kcpustat_cpu(cpu).cpustat; 1203 u64 val = cpustat[usage]; 1204 struct rq *rq; 1205 int err; 1206 1207 if (!vtime_generic_enabled_cpu(cpu)) 1208 return kcpustat_field_default(usage, cpu); 1209 1210 rq = cpu_rq(cpu); 1211 1212 for (;;) { 1213 struct task_struct *curr; 1214 1215 rcu_read_lock(); 1216 curr = rcu_dereference(rq->curr); 1217 if (WARN_ON_ONCE(!curr)) { 1218 rcu_read_unlock(); 1219 return cpustat[usage]; 1220 } 1221 1222 err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val); 1223 rcu_read_unlock(); 1224 1225 if (!err) 1226 return val; 1227 1228 cpu_relax(); 1229 } 1230 } 1231 EXPORT_SYMBOL_GPL(kcpustat_field); 1232 1233 static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst, 1234 const struct kernel_cpustat *src, 1235 struct task_struct *tsk, int cpu) 1236 { 1237 struct vtime *vtime = &tsk->vtime; 1238 unsigned int seq; 1239 1240 do { 1241 u64 *cpustat; 1242 u64 delta; 1243 int state; 1244 1245 seq = read_seqcount_begin(&vtime->seqcount); 1246 1247 state = vtime_state_fetch(vtime, cpu); 1248 if (state < 0) 1249 return state; 1250 1251 *dst = *src; 1252 cpustat = dst->cpustat; 1253 1254 /* Task is sleeping or dead, nothing to add */ 1255 if (state < VTIME_IDLE) 1256 continue; 1257 1258 delta = vtime_delta(vtime); 1259 1260 /* 1261 * Task runs either in user (including guest) or kernel space, 1262 * add pending nohz time to the right place. 1263 */ 1264 switch (state) { 1265 case VTIME_SYS: 1266 cpustat[CPUTIME_SYSTEM] += vtime->stime + delta; 1267 break; 1268 case VTIME_USER: 1269 if (task_nice(tsk) > 0) 1270 cpustat[CPUTIME_NICE] += vtime->utime + delta; 1271 else 1272 cpustat[CPUTIME_USER] += vtime->utime + delta; 1273 break; 1274 case VTIME_GUEST: 1275 if (task_nice(tsk) > 0) { 1276 cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta; 1277 cpustat[CPUTIME_NICE] += vtime->gtime + delta; 1278 } else { 1279 cpustat[CPUTIME_GUEST] += vtime->gtime + delta; 1280 cpustat[CPUTIME_USER] += vtime->gtime + delta; 1281 } 1282 break; 1283 case VTIME_IDLE: 1284 if (cpu_has_iowait(cpu)) 1285 cpustat[CPUTIME_IOWAIT] += delta; 1286 else 1287 cpustat[CPUTIME_IDLE] += delta; 1288 break; 1289 default: 1290 WARN_ON_ONCE(1); 1291 } 1292 } while (read_seqcount_retry(&vtime->seqcount, seq)); 1293 1294 return 0; 1295 } 1296 1297 void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu) 1298 { 1299 const struct kernel_cpustat *src = &kcpustat_cpu(cpu); 1300 struct rq *rq; 1301 int err; 1302 1303 if (!vtime_generic_enabled_cpu(cpu)) { 1304 kcpustat_cpu_fetch_default(dst, cpu); 1305 return; 1306 } 1307 1308 rq = cpu_rq(cpu); 1309 1310 for (;;) { 1311 struct task_struct *curr; 1312 1313 rcu_read_lock(); 1314 curr = rcu_dereference(rq->curr); 1315 if (WARN_ON_ONCE(!curr)) { 1316 rcu_read_unlock(); 1317 kcpustat_cpu_fetch_default(dst, cpu); 1318 return; 1319 } 1320 1321 err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu); 1322 rcu_read_unlock(); 1323 1324 if (!err) 1325 return; 1326 1327 cpu_relax(); 1328 } 1329 } 1330 EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch); 1331 1332 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */ 1333