xref: /linux/kernel/sched/cputime.c (revision b327e63683a2b894b500027375342c25222b58ef)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Simple CPU accounting cgroup controller
4  */
5 #include <linux/sched/clock.h>
6 #include <linux/sched/cputime.h>
7 #include <linux/tsacct_kern.h>
8 #include "sched.h"
9 
10 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
11  #include <asm/cputime.h>
12 #endif
13 
14 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
15 
16 DEFINE_STATIC_KEY_FALSE(sched_clock_irqtime);
17 
18 /*
19  * There are no locks covering percpu hardirq/softirq time.
20  * They are only modified in vtime_account, on corresponding CPU
21  * with interrupts disabled. So, writes are safe.
22  * They are read and saved off onto struct rq in update_rq_clock().
23  * This may result in other CPU reading this CPU's IRQ time and can
24  * race with irq/vtime_account on this CPU. We would either get old
25  * or new value with a side effect of accounting a slice of IRQ time to wrong
26  * task when IRQ is in progress while we read rq->clock. That is a worthy
27  * compromise in place of having locks on each IRQ in account_system_time.
28  */
29 DEFINE_PER_CPU(struct irqtime, cpu_irqtime);
30 
31 void enable_sched_clock_irqtime(void)
32 {
33 	static_branch_enable(&sched_clock_irqtime);
34 }
35 
36 void disable_sched_clock_irqtime(void)
37 {
38 	if (irqtime_enabled())
39 		static_branch_disable(&sched_clock_irqtime);
40 }
41 
42 static void irqtime_account_delta(struct irqtime *irqtime, u64 delta,
43 				  enum cpu_usage_stat idx)
44 {
45 	u64 *cpustat = kcpustat_this_cpu->cpustat;
46 
47 	u64_stats_update_begin(&irqtime->sync);
48 	cpustat[idx] += delta;
49 	irqtime->total += delta;
50 	if (!kcpustat_idle_dyntick())
51 		irqtime->tick_delta += delta;
52 	u64_stats_update_end(&irqtime->sync);
53 }
54 
55 /*
56  * Called after incrementing preempt_count on {soft,}irq_enter
57  * and before decrementing preempt_count on {soft,}irq_exit.
58  */
59 void irqtime_account_irq(struct task_struct *curr, unsigned int offset)
60 {
61 	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
62 	unsigned int pc;
63 	s64 delta;
64 	int cpu;
65 
66 	if (!irqtime_enabled())
67 		return;
68 
69 	cpu = smp_processor_id();
70 	delta = sched_clock_cpu(cpu) - irqtime->irq_start_time;
71 	irqtime->irq_start_time += delta;
72 	pc = irq_count() - offset;
73 
74 	/*
75 	 * We do not account for softirq time from ksoftirqd here.
76 	 * We want to continue accounting softirq time to ksoftirqd thread
77 	 * in that case, so as not to confuse scheduler with a special task
78 	 * that do not consume any time, but still wants to run.
79 	 */
80 	if (pc & HARDIRQ_MASK)
81 		irqtime_account_delta(irqtime, delta, CPUTIME_IRQ);
82 	else if ((pc & SOFTIRQ_OFFSET) && curr != this_cpu_ksoftirqd())
83 		irqtime_account_delta(irqtime, delta, CPUTIME_SOFTIRQ);
84 }
85 
86 static u64 irqtime_tick_accounted(u64 maxtime)
87 {
88 	struct irqtime *irqtime = this_cpu_ptr(&cpu_irqtime);
89 	u64 delta;
90 
91 	delta = min(irqtime->tick_delta, maxtime);
92 	irqtime->tick_delta -= delta;
93 
94 	return delta;
95 }
96 
97 #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
98 
99 static u64 irqtime_tick_accounted(u64 dummy)
100 {
101 	return 0;
102 }
103 
104 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
105 
106 static inline void task_group_account_field(struct task_struct *p, int index,
107 					    u64 tmp)
108 {
109 	/*
110 	 * Since all updates are sure to touch the root cgroup, we
111 	 * get ourselves ahead and touch it first. If the root cgroup
112 	 * is the only cgroup, then nothing else should be necessary.
113 	 *
114 	 */
115 	__this_cpu_add(kernel_cpustat.cpustat[index], tmp);
116 
117 	cgroup_account_cputime_field(p, index, tmp);
118 }
119 
120 /*
121  * Account user CPU time to a process.
122  * @p: the process that the CPU time gets accounted to
123  * @cputime: the CPU time spent in user space since the last update
124  */
125 void account_user_time(struct task_struct *p, u64 cputime)
126 {
127 	int index;
128 
129 	/* Add user time to process. */
130 	p->utime += cputime;
131 	account_group_user_time(p, cputime);
132 
133 	index = (task_nice(p) > 0) ? CPUTIME_NICE : CPUTIME_USER;
134 
135 	/* Add user time to cpustat. */
136 	task_group_account_field(p, index, cputime);
137 
138 	/* Account for user time used */
139 	acct_account_cputime(p);
140 }
141 
142 /*
143  * Account guest CPU time to a process.
144  * @p: the process that the CPU time gets accounted to
145  * @cputime: the CPU time spent in virtual machine since the last update
146  */
147 void account_guest_time(struct task_struct *p, u64 cputime)
148 {
149 	u64 *cpustat = kcpustat_this_cpu->cpustat;
150 
151 	/* Add guest time to process. */
152 	p->utime += cputime;
153 	account_group_user_time(p, cputime);
154 	p->gtime += cputime;
155 
156 	/* Add guest time to cpustat. */
157 	if (task_nice(p) > 0) {
158 		task_group_account_field(p, CPUTIME_NICE, cputime);
159 		cpustat[CPUTIME_GUEST_NICE] += cputime;
160 	} else {
161 		task_group_account_field(p, CPUTIME_USER, cputime);
162 		cpustat[CPUTIME_GUEST] += cputime;
163 	}
164 }
165 
166 /*
167  * Account system CPU time to a process and desired cpustat field
168  * @p: the process that the CPU time gets accounted to
169  * @cputime: the CPU time spent in kernel space since the last update
170  * @index: pointer to cpustat field that has to be updated
171  */
172 void account_system_index_time(struct task_struct *p,
173 			       u64 cputime, enum cpu_usage_stat index)
174 {
175 	/* Add system time to process. */
176 	p->stime += cputime;
177 	account_group_system_time(p, cputime);
178 
179 	/* Add system time to cpustat. */
180 	task_group_account_field(p, index, cputime);
181 
182 	/* Account for system time used */
183 	acct_account_cputime(p);
184 }
185 
186 /*
187  * Account system CPU time to a process.
188  * @p: the process that the CPU time gets accounted to
189  * @hardirq_offset: the offset to subtract from hardirq_count()
190  * @cputime: the CPU time spent in kernel space since the last update
191  */
192 void account_system_time(struct task_struct *p, int hardirq_offset, u64 cputime)
193 {
194 	int index;
195 
196 	if ((p->flags & PF_VCPU) && (irq_count() - hardirq_offset == 0)) {
197 		account_guest_time(p, cputime);
198 		return;
199 	}
200 
201 	if (hardirq_count() - hardirq_offset)
202 		index = CPUTIME_IRQ;
203 	else if (in_serving_softirq())
204 		index = CPUTIME_SOFTIRQ;
205 	else
206 		index = CPUTIME_SYSTEM;
207 
208 	account_system_index_time(p, cputime, index);
209 }
210 
211 /*
212  * Account for involuntary wait time.
213  * @cputime: the CPU time spent in involuntary wait
214  */
215 void account_steal_time(u64 cputime)
216 {
217 	u64 *cpustat = kcpustat_this_cpu->cpustat;
218 
219 	cpustat[CPUTIME_STEAL] += cputime;
220 }
221 
222 /*
223  * Account for idle time.
224  * @cputime: the CPU time spent in idle wait
225  */
226 void account_idle_time(u64 cputime)
227 {
228 	u64 *cpustat = kcpustat_this_cpu->cpustat;
229 
230 	if (cpu_has_iowait(smp_processor_id()))
231 		cpustat[CPUTIME_IOWAIT] += cputime;
232 	else
233 		cpustat[CPUTIME_IDLE] += cputime;
234 }
235 
236 
237 #ifdef CONFIG_SCHED_CORE
238 /*
239  * Account for forceidle time due to core scheduling.
240  *
241  * REQUIRES: schedstat is enabled.
242  */
243 void __account_forceidle_time(struct task_struct *p, u64 delta)
244 {
245 	__schedstat_add(p->stats.core_forceidle_sum, delta);
246 
247 	task_group_account_field(p, CPUTIME_FORCEIDLE, delta);
248 }
249 #endif /* CONFIG_SCHED_CORE */
250 
251 /*
252  * When a guest is interrupted for a longer amount of time, missed clock
253  * ticks are not redelivered later. Due to that, this function may on
254  * occasion account more time than the calling functions think elapsed.
255  */
256 #ifdef CONFIG_PARAVIRT
257 DEFINE_STATIC_KEY_FALSE(paravirt_steal_enabled);
258 
259 #ifdef CONFIG_HAVE_PV_STEAL_CLOCK_GEN
260 static u64 native_steal_clock(int cpu)
261 {
262 	return 0;
263 }
264 
265 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock);
266 #endif
267 #endif
268 
269 static __always_inline u64 steal_account_process_time(u64 maxtime)
270 {
271 #ifdef CONFIG_PARAVIRT
272 	if (static_branch_unlikely(&paravirt_steal_enabled)) {
273 		u64 steal;
274 
275 		steal = paravirt_steal_clock(smp_processor_id());
276 		steal -= this_rq()->prev_steal_time;
277 		steal = min(steal, maxtime);
278 		account_steal_time(steal);
279 		this_rq()->prev_steal_time += steal;
280 
281 		return steal;
282 	}
283 #endif /* CONFIG_PARAVIRT */
284 	return 0;
285 }
286 
287 /*
288  * Account how much elapsed time was spent in steal, IRQ, or softirq time.
289  */
290 static inline u64 account_other_time(u64 max)
291 {
292 	u64 accounted;
293 
294 	lockdep_assert_irqs_disabled();
295 
296 	accounted = steal_account_process_time(max);
297 
298 	if (accounted < max)
299 		accounted += irqtime_tick_accounted(max - accounted);
300 
301 	return accounted;
302 }
303 
304 #ifdef CONFIG_64BIT
305 static inline u64 read_sum_exec_runtime(struct task_struct *t)
306 {
307 	return t->se.sum_exec_runtime;
308 }
309 #else /* !CONFIG_64BIT: */
310 static u64 read_sum_exec_runtime(struct task_struct *t)
311 {
312 	u64 ns;
313 	struct rq_flags rf;
314 	struct rq *rq;
315 
316 	rq = task_rq_lock(t, &rf);
317 	ns = t->se.sum_exec_runtime;
318 	task_rq_unlock(rq, t, &rf);
319 
320 	return ns;
321 }
322 #endif /* !CONFIG_64BIT */
323 
324 /*
325  * Accumulate raw cputime values of dead tasks (sig->[us]time) and live
326  * tasks (sum on group iteration) belonging to @tsk's group.
327  */
328 void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times)
329 {
330 	struct signal_struct *sig = tsk->signal;
331 	struct task_struct *t;
332 	u64 utime, stime;
333 
334 	/*
335 	 * Update current task runtime to account pending time since last
336 	 * scheduler action or thread_group_cputime() call. This thread group
337 	 * might have other running tasks on different CPUs, but updating
338 	 * their runtime can affect syscall performance, so we skip account
339 	 * those pending times and rely only on values updated on tick or
340 	 * other scheduler action.
341 	 */
342 	if (same_thread_group(current, tsk))
343 		(void) task_sched_runtime(current);
344 
345 	guard(rcu)();
346 	scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) {
347 		times->utime = sig->utime;
348 		times->stime = sig->stime;
349 		times->sum_exec_runtime = sig->sum_sched_runtime;
350 
351 		__for_each_thread(sig, t) {
352 			task_cputime(t, &utime, &stime);
353 			times->utime += utime;
354 			times->stime += stime;
355 			times->sum_exec_runtime += read_sum_exec_runtime(t);
356 		}
357 	}
358 }
359 
360 #ifdef CONFIG_IRQ_TIME_ACCOUNTING
361 /*
362  * Account a tick to a process and cpustat
363  * @p: the process that the CPU time gets accounted to
364  * @user_tick: is the tick from userspace
365  * @rq: the pointer to rq
366  *
367  * Tick demultiplexing follows the order
368  * - pending hardirq update
369  * - pending softirq update
370  * - user_time
371  * - idle_time
372  * - system time
373  *   - check for guest_time
374  *   - else account as system_time
375  *
376  * Check for hardirq is done both for system and user time as there is
377  * no timer going off while we are on hardirq and hence we may never get an
378  * opportunity to update it solely in system time.
379  * p->stime and friends are only updated on system time and not on IRQ
380  * softirq as those do not count in task exec_runtime any more.
381  */
382 static void irqtime_account_process_tick(struct task_struct *p, int user_tick,
383 					 u64 cputime)
384 {
385 	u64 other;
386 
387 	/*
388 	 * When returning from idle, many ticks can get accounted at
389 	 * once, including some ticks of steal, IRQ, and softirq time.
390 	 * Subtract those ticks from the amount of time accounted to
391 	 * idle, or potentially user or system time. Due to rounding,
392 	 * other time can exceed ticks occasionally.
393 	 */
394 	other = account_other_time(ULONG_MAX);
395 	if (other >= cputime)
396 		return;
397 
398 	cputime -= other;
399 
400 	if (this_cpu_ksoftirqd() == p) {
401 		/*
402 		 * ksoftirqd time do not get accounted in cpu_softirq_time.
403 		 * So, we have to handle it separately here.
404 		 * Also, p->stime needs to be updated for ksoftirqd.
405 		 */
406 		account_system_index_time(p, cputime, CPUTIME_SOFTIRQ);
407 	} else if (user_tick) {
408 		account_user_time(p, cputime);
409 	} else if (p == this_rq()->idle) {
410 		account_idle_time(cputime);
411 	} else if (p->flags & PF_VCPU) { /* System time or guest time */
412 		account_guest_time(p, cputime);
413 	} else {
414 		account_system_index_time(p, cputime, CPUTIME_SYSTEM);
415 	}
416 }
417 
418 #else /* !CONFIG_IRQ_TIME_ACCOUNTING: */
419 static inline void irqtime_account_process_tick(struct task_struct *p, int user_tick,
420 						u64 cputime) { }
421 #endif /* !CONFIG_IRQ_TIME_ACCOUNTING */
422 
423 #if defined(CONFIG_NO_HZ_COMMON) && !defined(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE)
424 static void kcpustat_idle_stop(struct kernel_cpustat *kc, u64 now)
425 {
426 	u64 *cpustat = kc->cpustat;
427 	u64 delta, steal, steal_delta;
428 	int iowait;
429 
430 	if (!kc->idle_elapse)
431 		return;
432 
433 	iowait = cpu_has_iowait(smp_processor_id());
434 	delta = now - kc->idle_entrytime;
435 	steal = steal_account_process_time(delta);
436 
437 	/*
438 	 * Record the idle time after substracting the steal time from
439 	 * previous update sequence. Don't substract the steal time from
440 	 * the current update sequence to avoid readers moving backward.
441 	 */
442 	write_seqcount_begin(&kc->idle_sleeptime_seq);
443 	steal_delta = min_t(u64, kc->idle_stealtime[iowait], delta);
444 	delta -= steal_delta;
445 	kc->idle_stealtime[iowait] -= steal_delta;
446 
447 	if (iowait)
448 		cpustat[CPUTIME_IOWAIT] += delta;
449 	else
450 		cpustat[CPUTIME_IDLE] += delta;
451 
452 	kc->idle_stealtime[iowait] += steal;
453 	kc->idle_entrytime = now;
454 	kc->idle_elapse = false;
455 	write_seqcount_end(&kc->idle_sleeptime_seq);
456 }
457 
458 static void kcpustat_idle_start(struct kernel_cpustat *kc, u64 now)
459 {
460 	/* Irqtime accounting might have been enabled in the middle of the IRQ */
461 	if (kc->idle_elapse)
462 		return;
463 
464 	write_seqcount_begin(&kc->idle_sleeptime_seq);
465 	kc->idle_entrytime = now;
466 	kc->idle_elapse = true;
467 	write_seqcount_end(&kc->idle_sleeptime_seq);
468 }
469 
470 void kcpustat_dyntick_stop(u64 now, u64 tick_start)
471 {
472 	struct kernel_cpustat *kc = kcpustat_this_cpu;
473 
474 	if (!vtime_generic_enabled_this_cpu()) {
475 		WARN_ON_ONCE(!kc->idle_dyntick);
476 		if (kc->idle_tick_period != tick_start) {
477 			kc->idle_tick_period = tick_start;
478 			kc->idle_tick_overlap = 0;
479 		}
480 		tick_start = max(tick_start, kc->idle_dyntick_entry);
481 		/*
482 		 * After a switch from the jiffies clocksource the tick can be
483 		 * armed more than a tick ahead, so the period may not have
484 		 * started yet.
485 		 */
486 		if (now > tick_start)
487 			kc->idle_tick_overlap += now - tick_start;
488 		kcpustat_idle_stop(kc, now);
489 		kc->idle_dyntick = false;
490 		vtime_dyntick_stop();
491 	}
492 }
493 
494 /*
495  * The first tick after dyntick-idle covers a period that the dyntick-idle
496  * accounting may already have accounted in part.
497  */
498 static inline u64 kcpustat_tick_overlap(void)
499 {
500 	struct kernel_cpustat *kc = kcpustat_this_cpu;
501 	u64 overlap = kc->idle_tick_overlap;
502 
503 	kc->idle_tick_overlap = 0;
504 	return overlap;
505 }
506 
507 void kcpustat_dyntick_start(u64 now)
508 {
509 	struct kernel_cpustat *kc = kcpustat_this_cpu;
510 
511 	if (!vtime_generic_enabled_this_cpu()) {
512 		vtime_dyntick_start();
513 		kc->idle_dyntick = true;
514 		kc->idle_dyntick_entry = now;
515 		kcpustat_idle_start(kc, now);
516 	}
517 }
518 
519 void kcpustat_irq_enter(u64 now)
520 {
521 	struct kernel_cpustat *kc = kcpustat_this_cpu;
522 
523 	if (!vtime_generic_enabled_this_cpu() &&
524 	    (irqtime_enabled() || vtime_accounting_enabled_this_cpu()))
525 		kcpustat_idle_stop(kc, now);
526 }
527 
528 void kcpustat_irq_exit(u64 now)
529 {
530 	struct kernel_cpustat *kc = kcpustat_this_cpu;
531 
532 	/*
533 	 * Generic vtime already does its own idle accounting.
534 	 * But irqtime accounting or arch vtime which also accounts IRQs
535 	 * need to pause nohz accounting. Resume nohz accounting as long
536 	 * as the irqtime config is enabled to handle case where irqtime
537 	 * accounting got runtime disabled in the middle of an IRQ.
538 	 */
539 	if (!vtime_generic_enabled_this_cpu() &&
540 	    (IS_ENABLED(CONFIG_IRQ_TIME_ACCOUNTING) || vtime_accounting_enabled_this_cpu()))
541 		kcpustat_idle_start(kc, now);
542 }
543 
544 static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
545 				  bool compute_delta, u64 now)
546 {
547 	struct kernel_cpustat *kc = &kcpustat_cpu(cpu);
548 	int iowait = idx == CPUTIME_IOWAIT;
549 	u64 *cpustat = kc->cpustat;
550 	unsigned int seq;
551 	u64 idle;
552 
553 	do {
554 		seq = read_seqcount_begin(&kc->idle_sleeptime_seq);
555 
556 		idle = cpustat[idx];
557 
558 		if (kc->idle_elapse && compute_delta && now > kc->idle_entrytime) {
559 			u64 delta = now - kc->idle_entrytime;
560 
561 			delta -= min_t(u64, kc->idle_stealtime[iowait], delta);
562 			idle += delta;
563 		}
564 	} while (read_seqcount_retry(&kc->idle_sleeptime_seq, seq));
565 
566 	return idle;
567 }
568 
569 u64 kcpustat_field_idle(int cpu)
570 {
571 	return kcpustat_field_dyntick(cpu, CPUTIME_IDLE,
572 				      !cpu_has_iowait(cpu), ktime_get());
573 }
574 EXPORT_SYMBOL_GPL(kcpustat_field_idle);
575 
576 u64 kcpustat_field_iowait(int cpu)
577 {
578 	return kcpustat_field_dyntick(cpu, CPUTIME_IOWAIT,
579 				      cpu_has_iowait(cpu), ktime_get());
580 }
581 EXPORT_SYMBOL_GPL(kcpustat_field_iowait);
582 #else
583 static inline u64 kcpustat_tick_overlap(void)
584 {
585 	return 0;
586 }
587 
588 static u64 kcpustat_field_dyntick(int cpu, enum cpu_usage_stat idx,
589 				  bool compute_delta, ktime_t now)
590 {
591 	return kcpustat_cpu(cpu).cpustat[idx];
592 }
593 #endif /* CONFIG_NO_HZ_COMMON && !CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE */
594 
595 static u64 get_cpu_sleep_time_us(int cpu, enum cpu_usage_stat idx,
596 				 bool compute_delta, u64 *last_update_time)
597 {
598 	ktime_t now = ktime_get();
599 	u64 res;
600 
601 	if (vtime_generic_enabled_cpu(cpu))
602 		res = kcpustat_field(idx, cpu);
603 	else
604 		res = kcpustat_field_dyntick(cpu, idx, compute_delta, now);
605 
606 	do_div(res, NSEC_PER_USEC);
607 
608 	if (last_update_time)
609 		*last_update_time = ktime_to_us(now);
610 
611 	return res;
612 }
613 
614 /**
615  * get_cpu_idle_time_us - get the total idle time of a CPU
616  * @cpu: CPU number to query
617  * @last_update_time: variable to store update time in. Do not update
618  * counters if NULL.
619  *
620  * Return the cumulative idle time (since boot) for a given
621  * CPU, in microseconds. Note that this is partially broken due to
622  * the counter of iowait tasks that can be remotely updated without
623  * any synchronization. Therefore it is possible to observe backward
624  * values within two consecutive reads.
625  *
626  * This time is measured via accounting rather than sampling,
627  * and is as accurate as ktime_get() is.
628  *
629  * Return: total idle time of the @cpu
630  */
631 u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
632 {
633 	return get_cpu_sleep_time_us(cpu, CPUTIME_IDLE,
634 				     !cpu_has_iowait(cpu), last_update_time);
635 }
636 EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
637 
638 /**
639  * get_cpu_iowait_time_us - get the total iowait time of a CPU
640  * @cpu: CPU number to query
641  * @last_update_time: variable to store update time in. Do not update
642  * counters if NULL.
643  *
644  * Return the cumulative iowait time (since boot) for a given
645  * CPU, in microseconds. Note this is partially broken due to
646  * the counter of iowait tasks that can be remotely updated without
647  * any synchronization. Therefore it is possible to observe backward
648  * values within two consecutive reads.
649  *
650  * This time is measured via accounting rather than sampling,
651  * and is as accurate as ktime_get() is.
652  *
653  * Return: total iowait time of @cpu
654  */
655 u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
656 {
657 	return get_cpu_sleep_time_us(cpu, CPUTIME_IOWAIT,
658 				     cpu_has_iowait(cpu), last_update_time);
659 }
660 EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
661 
662 /*
663  * Use precise platform statistics if available:
664  */
665 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
666 
667 void vtime_account_irq(struct task_struct *tsk, unsigned int offset)
668 {
669 	unsigned int pc = irq_count() - offset;
670 
671 	if (pc & HARDIRQ_OFFSET) {
672 		vtime_account_hardirq(tsk);
673 	} else if (pc & SOFTIRQ_OFFSET) {
674 		vtime_account_softirq(tsk);
675 	} else if (!kcpustat_idle_dyntick()) {
676 		if (!IS_ENABLED(CONFIG_HAVE_VIRT_CPU_ACCOUNTING_IDLE) &&
677 		    is_idle_task(tsk)) {
678 			vtime_account_idle(tsk);
679 		} else {
680 			vtime_account_kernel(tsk);
681 		}
682 	} else {
683 		vtime_reset();
684 	}
685 }
686 
687 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
688 		    u64 *ut, u64 *st)
689 {
690 	*ut = curr->utime;
691 	*st = curr->stime;
692 }
693 
694 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
695 {
696 	*ut = p->utime;
697 	*st = p->stime;
698 }
699 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
700 
701 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
702 {
703 	struct task_cputime cputime;
704 
705 	thread_group_cputime(p, &cputime);
706 
707 	*ut = cputime.utime;
708 	*st = cputime.stime;
709 }
710 
711 #else /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE: */
712 
713 /*
714  * Account a single tick of CPU time.
715  * @p: the process that the CPU time gets accounted to
716  * @user_tick: indicates if the tick is a user or a system tick
717  */
718 void account_process_tick(struct task_struct *p, int user_tick)
719 {
720 	u64 cputime, steal;
721 
722 	if (vtime_accounting_enabled_this_cpu())
723 		return;
724 
725 	if (kcpustat_idle_dyntick())
726 		return;
727 
728 	cputime = TICK_NSEC - kcpustat_tick_overlap();
729 
730 	if (irqtime_enabled()) {
731 		irqtime_account_process_tick(p, user_tick, cputime);
732 		return;
733 	}
734 
735 	steal = steal_account_process_time(ULONG_MAX);
736 
737 	if (steal >= cputime)
738 		return;
739 
740 	cputime -= steal;
741 
742 	if (user_tick)
743 		account_user_time(p, cputime);
744 	else if ((p != this_rq()->idle) || (irq_count() != HARDIRQ_OFFSET))
745 		account_system_time(p, HARDIRQ_OFFSET, cputime);
746 	else
747 		account_idle_time(cputime);
748 }
749 
750 /*
751  * Adjust tick based cputime random precision against scheduler runtime
752  * accounting.
753  *
754  * Tick based cputime accounting depend on random scheduling timeslices of a
755  * task to be interrupted or not by the timer.  Depending on these
756  * circumstances, the number of these interrupts may be over or
757  * under-optimistic, matching the real user and system cputime with a variable
758  * precision.
759  *
760  * Fix this by scaling these tick based values against the total runtime
761  * accounted by the CFS scheduler.
762  *
763  * This code provides the following guarantees:
764  *
765  *   stime + utime == rtime
766  *   stime_i+1 >= stime_i, utime_i+1 >= utime_i
767  *
768  * Assuming that rtime_i+1 >= rtime_i.
769  */
770 void cputime_adjust(struct task_cputime *curr, struct prev_cputime *prev,
771 		    u64 *ut, u64 *st)
772 {
773 	u64 rtime, stime, utime;
774 	unsigned long flags;
775 
776 	/* Serialize concurrent callers such that we can honour our guarantees */
777 	raw_spin_lock_irqsave(&prev->lock, flags);
778 	rtime = curr->sum_exec_runtime;
779 
780 	/*
781 	 * This is possible under two circumstances:
782 	 *  - rtime isn't monotonic after all (a bug);
783 	 *  - we got reordered by the lock.
784 	 *
785 	 * In both cases this acts as a filter such that the rest of the code
786 	 * can assume it is monotonic regardless of anything else.
787 	 */
788 	if (prev->stime + prev->utime >= rtime)
789 		goto out;
790 
791 	stime = curr->stime;
792 	utime = curr->utime;
793 
794 	/*
795 	 * If either stime or utime are 0, assume all runtime is userspace.
796 	 * Once a task gets some ticks, the monotonicity code at 'update:'
797 	 * will ensure things converge to the observed ratio.
798 	 */
799 	if (stime == 0) {
800 		utime = rtime;
801 		goto update;
802 	}
803 
804 	if (utime == 0) {
805 		stime = rtime;
806 		goto update;
807 	}
808 
809 	stime = mul_u64_u64_div_u64(stime, rtime, stime + utime);
810 
811 update:
812 	/*
813 	 * Make sure stime doesn't go backwards; this preserves monotonicity
814 	 * for utime because rtime is monotonic.
815 	 *
816 	 *  utime_i+1 = rtime_i+1 - stime_i
817 	 *            = rtime_i+1 - (rtime_i - utime_i)
818 	 *            = (rtime_i+1 - rtime_i) + utime_i
819 	 *            >= utime_i
820 	 */
821 	if (stime < prev->stime)
822 		stime = prev->stime;
823 	utime = rtime - stime;
824 
825 	/*
826 	 * Make sure utime doesn't go backwards; this still preserves
827 	 * monotonicity for stime, analogous argument to above.
828 	 */
829 	if (utime < prev->utime) {
830 		utime = prev->utime;
831 		stime = rtime - utime;
832 	}
833 
834 	prev->stime = stime;
835 	prev->utime = utime;
836 out:
837 	*ut = prev->utime;
838 	*st = prev->stime;
839 	raw_spin_unlock_irqrestore(&prev->lock, flags);
840 }
841 
842 void task_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
843 {
844 	struct task_cputime cputime = {
845 		.sum_exec_runtime = p->se.sum_exec_runtime,
846 	};
847 
848 	if (task_cputime(p, &cputime.utime, &cputime.stime))
849 		cputime.sum_exec_runtime = task_sched_runtime(p);
850 	cputime_adjust(&cputime, &p->prev_cputime, ut, st);
851 }
852 EXPORT_SYMBOL_GPL(task_cputime_adjusted);
853 
854 void thread_group_cputime_adjusted(struct task_struct *p, u64 *ut, u64 *st)
855 {
856 	struct task_cputime cputime;
857 
858 	thread_group_cputime(p, &cputime);
859 	cputime_adjust(&cputime, &p->signal->prev_cputime, ut, st);
860 }
861 #endif /* !CONFIG_VIRT_CPU_ACCOUNTING_NATIVE */
862 
863 #ifdef CONFIG_VIRT_CPU_ACCOUNTING_GEN
864 static u64 vtime_delta(struct vtime *vtime)
865 {
866 	unsigned long long clock;
867 
868 	clock = sched_clock();
869 	if (clock < vtime->starttime)
870 		return 0;
871 
872 	return clock - vtime->starttime;
873 }
874 
875 static u64 get_vtime_delta(struct vtime *vtime)
876 {
877 	u64 delta = vtime_delta(vtime);
878 	u64 other;
879 
880 	/*
881 	 * Unlike tick based timing, vtime based timing never has lost
882 	 * ticks, and no need for steal time accounting to make up for
883 	 * lost ticks. Vtime accounts a rounded version of actual
884 	 * elapsed time. Limit account_other_time to prevent rounding
885 	 * errors from causing elapsed vtime to go negative.
886 	 */
887 	other = account_other_time(delta);
888 	WARN_ON_ONCE(vtime->state == VTIME_INACTIVE);
889 	vtime->starttime += delta;
890 
891 	return delta - other;
892 }
893 
894 static void vtime_account_system(struct task_struct *tsk,
895 				 struct vtime *vtime)
896 {
897 	vtime->stime += get_vtime_delta(vtime);
898 	if (vtime->stime >= TICK_NSEC) {
899 		account_system_time(tsk, irq_count(), vtime->stime);
900 		vtime->stime = 0;
901 	}
902 }
903 
904 static void vtime_account_guest(struct task_struct *tsk,
905 				struct vtime *vtime)
906 {
907 	vtime->gtime += get_vtime_delta(vtime);
908 	if (vtime->gtime >= TICK_NSEC) {
909 		account_guest_time(tsk, vtime->gtime);
910 		vtime->gtime = 0;
911 	}
912 }
913 
914 static void __vtime_account_kernel(struct task_struct *tsk,
915 				   struct vtime *vtime)
916 {
917 	/* We might have scheduled out from guest path */
918 	if (vtime->state == VTIME_GUEST)
919 		vtime_account_guest(tsk, vtime);
920 	else
921 		vtime_account_system(tsk, vtime);
922 }
923 
924 void vtime_account_kernel(struct task_struct *tsk)
925 {
926 	struct vtime *vtime = &tsk->vtime;
927 
928 	if (!vtime_delta(vtime))
929 		return;
930 
931 	write_seqcount_begin(&vtime->seqcount);
932 	__vtime_account_kernel(tsk, vtime);
933 	write_seqcount_end(&vtime->seqcount);
934 }
935 
936 void vtime_user_enter(struct task_struct *tsk)
937 {
938 	struct vtime *vtime = &tsk->vtime;
939 
940 	write_seqcount_begin(&vtime->seqcount);
941 	vtime_account_system(tsk, vtime);
942 	vtime->state = VTIME_USER;
943 	write_seqcount_end(&vtime->seqcount);
944 }
945 
946 void vtime_user_exit(struct task_struct *tsk)
947 {
948 	struct vtime *vtime = &tsk->vtime;
949 
950 	write_seqcount_begin(&vtime->seqcount);
951 	vtime->utime += get_vtime_delta(vtime);
952 	if (vtime->utime >= TICK_NSEC) {
953 		account_user_time(tsk, vtime->utime);
954 		vtime->utime = 0;
955 	}
956 	vtime->state = VTIME_SYS;
957 	write_seqcount_end(&vtime->seqcount);
958 }
959 
960 void vtime_guest_enter(struct task_struct *tsk)
961 {
962 	struct vtime *vtime = &tsk->vtime;
963 	/*
964 	 * The flags must be updated under the lock with
965 	 * the vtime_starttime flush and update.
966 	 * That enforces a right ordering and update sequence
967 	 * synchronization against the reader (task_gtime())
968 	 * that can thus safely catch up with a tickless delta.
969 	 */
970 	write_seqcount_begin(&vtime->seqcount);
971 	vtime_account_system(tsk, vtime);
972 	tsk->flags |= PF_VCPU;
973 	vtime->state = VTIME_GUEST;
974 	write_seqcount_end(&vtime->seqcount);
975 }
976 EXPORT_SYMBOL_GPL(vtime_guest_enter);
977 
978 void vtime_guest_exit(struct task_struct *tsk)
979 {
980 	struct vtime *vtime = &tsk->vtime;
981 
982 	write_seqcount_begin(&vtime->seqcount);
983 	vtime_account_guest(tsk, vtime);
984 	tsk->flags &= ~PF_VCPU;
985 	vtime->state = VTIME_SYS;
986 	write_seqcount_end(&vtime->seqcount);
987 }
988 EXPORT_SYMBOL_GPL(vtime_guest_exit);
989 
990 static void __vtime_account_idle(struct vtime *vtime)
991 {
992 	account_idle_time(get_vtime_delta(vtime));
993 }
994 
995 void vtime_task_switch_generic(struct task_struct *prev)
996 {
997 	struct vtime *vtime = &prev->vtime;
998 
999 	write_seqcount_begin(&vtime->seqcount);
1000 	if (vtime->state == VTIME_IDLE)
1001 		__vtime_account_idle(vtime);
1002 	else
1003 		__vtime_account_kernel(prev, vtime);
1004 	vtime->state = VTIME_INACTIVE;
1005 	vtime->cpu = -1;
1006 	write_seqcount_end(&vtime->seqcount);
1007 
1008 	vtime = &current->vtime;
1009 
1010 	write_seqcount_begin(&vtime->seqcount);
1011 	if (is_idle_task(current))
1012 		vtime->state = VTIME_IDLE;
1013 	else if (current->flags & PF_VCPU)
1014 		vtime->state = VTIME_GUEST;
1015 	else
1016 		vtime->state = VTIME_SYS;
1017 	vtime->starttime = sched_clock();
1018 	vtime->cpu = smp_processor_id();
1019 	write_seqcount_end(&vtime->seqcount);
1020 }
1021 
1022 void vtime_init_idle(struct task_struct *t, int cpu)
1023 {
1024 	struct vtime *vtime = &t->vtime;
1025 	unsigned long flags;
1026 
1027 	local_irq_save(flags);
1028 	write_seqcount_begin(&vtime->seqcount);
1029 	vtime->state = VTIME_IDLE;
1030 	vtime->starttime = sched_clock();
1031 	vtime->cpu = cpu;
1032 	write_seqcount_end(&vtime->seqcount);
1033 	local_irq_restore(flags);
1034 }
1035 
1036 u64 task_gtime(struct task_struct *t)
1037 {
1038 	struct vtime *vtime = &t->vtime;
1039 	unsigned int seq;
1040 	u64 gtime;
1041 
1042 	if (!vtime_accounting_enabled())
1043 		return t->gtime;
1044 
1045 	do {
1046 		seq = read_seqcount_begin(&vtime->seqcount);
1047 
1048 		gtime = t->gtime;
1049 		if (vtime->state == VTIME_GUEST)
1050 			gtime += vtime->gtime + vtime_delta(vtime);
1051 
1052 	} while (read_seqcount_retry(&vtime->seqcount, seq));
1053 
1054 	return gtime;
1055 }
1056 
1057 /*
1058  * Fetch cputime raw values from fields of task_struct and
1059  * add up the pending nohz execution time since the last
1060  * cputime snapshot.
1061  */
1062 bool task_cputime(struct task_struct *t, u64 *utime, u64 *stime)
1063 {
1064 	struct vtime *vtime = &t->vtime;
1065 	unsigned int seq;
1066 	u64 delta;
1067 	int ret;
1068 
1069 	if (!vtime_accounting_enabled()) {
1070 		*utime = t->utime;
1071 		*stime = t->stime;
1072 		return false;
1073 	}
1074 
1075 	do {
1076 		ret = false;
1077 		seq = read_seqcount_begin(&vtime->seqcount);
1078 
1079 		*utime = t->utime;
1080 		*stime = t->stime;
1081 
1082 		/* Task is sleeping or idle, nothing to add */
1083 		if (vtime->state < VTIME_SYS)
1084 			continue;
1085 
1086 		ret = true;
1087 		delta = vtime_delta(vtime);
1088 
1089 		/*
1090 		 * Task runs either in user (including guest) or kernel space,
1091 		 * add pending nohz time to the right place.
1092 		 */
1093 		if (vtime->state == VTIME_SYS)
1094 			*stime += vtime->stime + delta;
1095 		else
1096 			*utime += vtime->utime + delta;
1097 	} while (read_seqcount_retry(&vtime->seqcount, seq));
1098 
1099 	return ret;
1100 }
1101 
1102 static int vtime_state_fetch(struct vtime *vtime, int cpu)
1103 {
1104 	int state = READ_ONCE(vtime->state);
1105 
1106 	/*
1107 	 * We raced against a context switch, fetch the
1108 	 * kcpustat task again.
1109 	 */
1110 	if (vtime->cpu != cpu && vtime->cpu != -1)
1111 		return -EAGAIN;
1112 
1113 	/*
1114 	 * Two possible things here:
1115 	 * 1) We are seeing the scheduling out task (prev) or any past one.
1116 	 * 2) We are seeing the scheduling in task (next) but it hasn't
1117 	 *    passed though vtime_task_switch() yet so the pending
1118 	 *    cputime of the prev task may not be flushed yet.
1119 	 *
1120 	 * Case 1) is ok but 2) is not. So wait for a safe VTIME state.
1121 	 */
1122 	if (state == VTIME_INACTIVE)
1123 		return -EAGAIN;
1124 
1125 	return state;
1126 }
1127 
1128 static u64 kcpustat_user_vtime(struct vtime *vtime)
1129 {
1130 	if (vtime->state == VTIME_USER)
1131 		return vtime->utime + vtime_delta(vtime);
1132 	else if (vtime->state == VTIME_GUEST)
1133 		return vtime->gtime + vtime_delta(vtime);
1134 	return 0;
1135 }
1136 
1137 static int kcpustat_field_vtime(u64 *cpustat,
1138 				struct task_struct *tsk,
1139 				enum cpu_usage_stat usage,
1140 				int cpu, u64 *val)
1141 {
1142 	struct vtime *vtime = &tsk->vtime;
1143 	unsigned int seq;
1144 
1145 	do {
1146 		int state;
1147 
1148 		seq = read_seqcount_begin(&vtime->seqcount);
1149 
1150 		state = vtime_state_fetch(vtime, cpu);
1151 		if (state < 0)
1152 			return state;
1153 
1154 		*val = cpustat[usage];
1155 
1156 		/*
1157 		 * Nice VS unnice cputime accounting may be inaccurate if
1158 		 * the nice value has changed since the last vtime update.
1159 		 * But proper fix would involve interrupting target on nice
1160 		 * updates which is a no go on nohz_full (although the scheduler
1161 		 * may still interrupt the target if rescheduling is needed...)
1162 		 */
1163 		switch (usage) {
1164 		case CPUTIME_SYSTEM:
1165 			if (state == VTIME_SYS)
1166 				*val += vtime->stime + vtime_delta(vtime);
1167 			break;
1168 		case CPUTIME_USER:
1169 			if (task_nice(tsk) <= 0)
1170 				*val += kcpustat_user_vtime(vtime);
1171 			break;
1172 		case CPUTIME_NICE:
1173 			if (task_nice(tsk) > 0)
1174 				*val += kcpustat_user_vtime(vtime);
1175 			break;
1176 		case CPUTIME_GUEST:
1177 			if (state == VTIME_GUEST && task_nice(tsk) <= 0)
1178 				*val += vtime->gtime + vtime_delta(vtime);
1179 			break;
1180 		case CPUTIME_GUEST_NICE:
1181 			if (state == VTIME_GUEST && task_nice(tsk) > 0)
1182 				*val += vtime->gtime + vtime_delta(vtime);
1183 			break;
1184 		case CPUTIME_IDLE:
1185 			if (state == VTIME_IDLE && !cpu_has_iowait(cpu))
1186 				*val += vtime_delta(vtime);
1187 			break;
1188 		case CPUTIME_IOWAIT:
1189 			if (state == VTIME_IDLE && cpu_has_iowait(cpu))
1190 				*val += vtime_delta(vtime);
1191 			break;
1192 		default:
1193 			break;
1194 		}
1195 	} while (read_seqcount_retry(&vtime->seqcount, seq));
1196 
1197 	return 0;
1198 }
1199 
1200 u64 kcpustat_field(enum cpu_usage_stat usage, int cpu)
1201 {
1202 	u64 *cpustat = kcpustat_cpu(cpu).cpustat;
1203 	u64 val = cpustat[usage];
1204 	struct rq *rq;
1205 	int err;
1206 
1207 	if (!vtime_generic_enabled_cpu(cpu))
1208 		return kcpustat_field_default(usage, cpu);
1209 
1210 	rq = cpu_rq(cpu);
1211 
1212 	for (;;) {
1213 		struct task_struct *curr;
1214 
1215 		rcu_read_lock();
1216 		curr = rcu_dereference(rq->curr);
1217 		if (WARN_ON_ONCE(!curr)) {
1218 			rcu_read_unlock();
1219 			return cpustat[usage];
1220 		}
1221 
1222 		err = kcpustat_field_vtime(cpustat, curr, usage, cpu, &val);
1223 		rcu_read_unlock();
1224 
1225 		if (!err)
1226 			return val;
1227 
1228 		cpu_relax();
1229 	}
1230 }
1231 EXPORT_SYMBOL_GPL(kcpustat_field);
1232 
1233 static int kcpustat_cpu_fetch_vtime(struct kernel_cpustat *dst,
1234 				    const struct kernel_cpustat *src,
1235 				    struct task_struct *tsk, int cpu)
1236 {
1237 	struct vtime *vtime = &tsk->vtime;
1238 	unsigned int seq;
1239 
1240 	do {
1241 		u64 *cpustat;
1242 		u64 delta;
1243 		int state;
1244 
1245 		seq = read_seqcount_begin(&vtime->seqcount);
1246 
1247 		state = vtime_state_fetch(vtime, cpu);
1248 		if (state < 0)
1249 			return state;
1250 
1251 		*dst = *src;
1252 		cpustat = dst->cpustat;
1253 
1254 		/* Task is sleeping or dead, nothing to add */
1255 		if (state < VTIME_IDLE)
1256 			continue;
1257 
1258 		delta = vtime_delta(vtime);
1259 
1260 		/*
1261 		 * Task runs either in user (including guest) or kernel space,
1262 		 * add pending nohz time to the right place.
1263 		 */
1264 		switch (state) {
1265 		case VTIME_SYS:
1266 			cpustat[CPUTIME_SYSTEM] += vtime->stime + delta;
1267 			break;
1268 		case VTIME_USER:
1269 			if (task_nice(tsk) > 0)
1270 				cpustat[CPUTIME_NICE] += vtime->utime + delta;
1271 			else
1272 				cpustat[CPUTIME_USER] += vtime->utime + delta;
1273 			break;
1274 		case VTIME_GUEST:
1275 			if (task_nice(tsk) > 0) {
1276 				cpustat[CPUTIME_GUEST_NICE] += vtime->gtime + delta;
1277 				cpustat[CPUTIME_NICE] += vtime->gtime + delta;
1278 			} else {
1279 				cpustat[CPUTIME_GUEST] += vtime->gtime + delta;
1280 				cpustat[CPUTIME_USER] += vtime->gtime + delta;
1281 			}
1282 			break;
1283 		case VTIME_IDLE:
1284 			if (cpu_has_iowait(cpu))
1285 				cpustat[CPUTIME_IOWAIT] += delta;
1286 			else
1287 				cpustat[CPUTIME_IDLE] += delta;
1288 			break;
1289 		default:
1290 			WARN_ON_ONCE(1);
1291 		}
1292 	} while (read_seqcount_retry(&vtime->seqcount, seq));
1293 
1294 	return 0;
1295 }
1296 
1297 void kcpustat_cpu_fetch(struct kernel_cpustat *dst, int cpu)
1298 {
1299 	const struct kernel_cpustat *src = &kcpustat_cpu(cpu);
1300 	struct rq *rq;
1301 	int err;
1302 
1303 	if (!vtime_generic_enabled_cpu(cpu)) {
1304 		kcpustat_cpu_fetch_default(dst, cpu);
1305 		return;
1306 	}
1307 
1308 	rq = cpu_rq(cpu);
1309 
1310 	for (;;) {
1311 		struct task_struct *curr;
1312 
1313 		rcu_read_lock();
1314 		curr = rcu_dereference(rq->curr);
1315 		if (WARN_ON_ONCE(!curr)) {
1316 			rcu_read_unlock();
1317 			kcpustat_cpu_fetch_default(dst, cpu);
1318 			return;
1319 		}
1320 
1321 		err = kcpustat_cpu_fetch_vtime(dst, src, curr, cpu);
1322 		rcu_read_unlock();
1323 
1324 		if (!err)
1325 			return;
1326 
1327 		cpu_relax();
1328 	}
1329 }
1330 EXPORT_SYMBOL_GPL(kcpustat_cpu_fetch);
1331 
1332 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_GEN */
1333