13 #include <trace/events/timer.h>
22 static void posix_cpu_timer_rearm(struct k_itimer *timer);
34  * Called after updating RLIMIT_CPU to run cpu timer and update
67 	 * If the encoded PID is 0, then the timer is targeted at current  in pid_for_clock()
113 static inline struct task_struct *cpu_timer_task_rcu(struct k_itimer *timer)  in cpu_timer_task_rcu()  argument
115 	return pid_task(timer->it.cpu.pid, clock_pid_type(timer->it_clock));  in cpu_timer_task_rcu()
122 static u64 bump_cpu_timer(struct k_itimer *timer, u64 now)  in bump_cpu_timer()  argument
124 	u64 delta, incr, expires = timer->it.cpu.node.expires;  in bump_cpu_timer()
127 	if (!timer->it_interval)  in bump_cpu_timer()
133 	incr = timer->it_interval;  in bump_cpu_timer()
144 		timer->it.cpu.node.expires += incr;  in bump_cpu_timer()
145 		timer->it_overrun += 1LL << i;  in bump_cpu_timer()
148 	return timer->it.cpu.node.expires;  in bump_cpu_timer()
268  * timer. That means group cputime accounting is already active. Called
289  * CPU timers armed. Before starting a timer it's required to check whether
307 		 * The POSIX timer interface allows for absolute time expiry  in thread_group_start_cputime()
309 		 * to synchronize the timer to the clock every time we start it.  in thread_group_start_cputime()
382  * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
384  * new timer already all-zeros initialized.
399 	 * If posix timer expiry is handled in task work context then  in posix_cpu_timer_create()
400 	 * timer::it_lock can be taken without disabling interrupts as all  in posix_cpu_timer_create()
402 	 * lock class key otherwise regular posix timer expiry would record  in posix_cpu_timer_create()
416 static struct posix_cputimer_base *timer_base(struct k_itimer *timer,  in timer_base()  argument
419 	int clkidx = CPUCLOCK_WHICH(timer->it_clock);  in timer_base()
421 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in timer_base()
433 static void trigger_base_recalc_expires(struct k_itimer *timer,  in trigger_base_recalc_expires()  argument
436 	struct posix_cputimer_base *base = timer_base(timer, tsk);  in trigger_base_recalc_expires()
442  * Dequeue the timer and reset the base if it was its earliest expiration.
447  * If another timer gets queued between this and the next tick, its
451 static void disarm_timer(struct k_itimer *timer, struct task_struct *p)  in disarm_timer()  argument
453 	struct cpu_timer *ctmr = &timer->it.cpu;  in disarm_timer()
459 	base = timer_base(timer, p);  in disarm_timer()
461 		trigger_base_recalc_expires(timer, p);  in disarm_timer()
466  * Clean up a CPU-clock timer that is about to be destroyed.
467  * This is called from timer deletion with the timer already locked.
468  * If we return TIMER_RETRY, it's necessary to release the timer's lock
469  * and try again.  (This happens when the timer is in the middle of firing.)
471 static int posix_cpu_timer_del(struct k_itimer *timer)  in posix_cpu_timer_del()  argument
473 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_del()
480 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_del()
486 	 * thread timer list entry concurrent read/writes.  in posix_cpu_timer_del()
492 		 * should have removed this timer from the timer queue.  in posix_cpu_timer_del()
496 		if (timer->it.cpu.firing) {  in posix_cpu_timer_del()
498 			 * Prevent signal delivery. The timer cannot be dequeued  in posix_cpu_timer_del()
501 			 * the timer lock. So go back, unlock and retry.  in posix_cpu_timer_del()
503 			timer->it.cpu.firing = false;  in posix_cpu_timer_del()
506 			disarm_timer(timer, p);  in posix_cpu_timer_del()
516 		timer->it_status = POSIX_TIMER_DISARMED;  in posix_cpu_timer_del()
562  * Insert the timer on the appropriate list before any timers that
565 static void arm_timer(struct k_itimer *timer, struct task_struct *p)  in arm_timer()  argument
567 	struct posix_cputimer_base *base = timer_base(timer, p);  in arm_timer()
568 	struct cpu_timer *ctmr = &timer->it.cpu;  in arm_timer()
571 	timer->it_status = POSIX_TIMER_ARMED;  in arm_timer()
576 	 * We are the new earliest-expiring POSIX 1.b timer, hence  in arm_timer()
584 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in arm_timer()
591  * The timer is locked, fire it and arrange for its reload.
593 static void cpu_timer_fire(struct k_itimer *timer)  in cpu_timer_fire()  argument
595 	struct cpu_timer *ctmr = &timer->it.cpu;  in cpu_timer_fire()
597 	timer->it_status = POSIX_TIMER_DISARMED;  in cpu_timer_fire()
602 		 * not a normal timer from sys_timer_create.  in cpu_timer_fire()
604 		wake_up_process(timer->it_process);  in cpu_timer_fire()
607 		posix_timer_queue_signal(timer);  in cpu_timer_fire()
609 		if (!timer->it_interval)  in cpu_timer_fire()
614 static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now);
618  * This is called with the timer locked and interrupts disabled.
619  * If we return TIMER_RETRY, it's necessary to release the timer's lock
620  * and try again.  (This happens when the timer is in the middle of firing.)
622 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,  in posix_cpu_timer_set()  argument
625 	bool sigev_none = timer->it_sigev_notify == SIGEV_NONE;  in posix_cpu_timer_set()
626 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_set()
627 	struct cpu_timer *ctmr = &timer->it.cpu;  in posix_cpu_timer_set()
635 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_set()
665 	/* Retrieve the current expiry time before disarming the timer */  in posix_cpu_timer_set()
668 	if (unlikely(timer->it.cpu.firing)) {  in posix_cpu_timer_set()
670 		 * Prevent signal delivery. The timer cannot be dequeued  in posix_cpu_timer_set()
673 		 * the timer lock. So go back, unlock and retry.  in posix_cpu_timer_set()
675 		timer->it.cpu.firing = false;  in posix_cpu_timer_set()
679 		timer->it_status = POSIX_TIMER_DISARMED;  in posix_cpu_timer_set()
684 	 * and for rearming the timer.  in posix_cpu_timer_set()
686 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_set()
695 			__posix_cpu_timer_get(timer, old, now);  in posix_cpu_timer_set()
698 	/* Retry if the timer expiry is running concurrently */  in posix_cpu_timer_set()
712 	 * Arm the timer if it is not disabled, the new expiry value has  in posix_cpu_timer_set()
713 	 * not yet expired and the timer requires signal delivery.  in posix_cpu_timer_set()
714 	 * SIGEV_NONE timers are never armed. In case the timer is not  in posix_cpu_timer_set()
715 	 * armed, enforce the reevaluation of the timer base so that the  in posix_cpu_timer_set()
720 			arm_timer(timer, p);  in posix_cpu_timer_set()
722 			trigger_base_recalc_expires(timer, p);  in posix_cpu_timer_set()
727 	posix_timer_set_common(timer, new);  in posix_cpu_timer_set()
730 	 * If the new expiry time was already in the past the timer was not  in posix_cpu_timer_set()
735 		cpu_timer_fire(timer);  in posix_cpu_timer_set()
741 static void __posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp, u64 now)  in __posix_cpu_timer_get()  argument
743 	bool sigev_none = timer->it_sigev_notify == SIGEV_NONE;  in __posix_cpu_timer_get()
744 	u64 expires, iv = timer->it_interval;  in __posix_cpu_timer_get()
753 	if (iv && timer->it_status != POSIX_TIMER_ARMED)  in __posix_cpu_timer_get()
754 		expires = bump_cpu_timer(timer, now);  in __posix_cpu_timer_get()
756 		expires = cpu_timer_getexpires(&timer->it.cpu);  in __posix_cpu_timer_get()
761 	 * timer expiry is > @now.  in __posix_cpu_timer_get()
767 		 * A single shot SIGEV_NONE timer must return 0, when it is  in __posix_cpu_timer_get()
777 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)  in posix_cpu_timer_get()  argument
779 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_get()
784 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_get()
785 	if (p && cpu_timer_getexpires(&timer->it.cpu)) {  in posix_cpu_timer_get()
786 		itp->it_interval = ktime_to_timespec64(timer->it_interval);  in posix_cpu_timer_get()
788 		if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_get()
793 		__posix_cpu_timer_get(timer, itp, now);  in posix_cpu_timer_get()
952 	 * RLIMIT_CPU) nothing to check. Also skip the process wide timer  in check_process_timers()
1013  * when the last timer signal was delivered and we have to reload the timer.
1015 static void posix_cpu_timer_rearm(struct k_itimer *timer)  in posix_cpu_timer_rearm()  argument
1017 	clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);  in posix_cpu_timer_rearm()
1024 	p = cpu_timer_task_rcu(timer);  in posix_cpu_timer_rearm()
1028 	/* Protect timer list r/w in arm_timer() */  in posix_cpu_timer_rearm()
1034 	 * Fetch the current sample and update the timer's expiry time.  in posix_cpu_timer_rearm()
1036 	if (CPUCLOCK_PERTHREAD(timer->it_clock))  in posix_cpu_timer_rearm()
1041 	bump_cpu_timer(timer, now);  in posix_cpu_timer_rearm()
1046 	arm_timer(timer, p);  in posix_cpu_timer_rearm()
1081  * true if a timer has expired, else return false.
1104 	 * acquire the sighand lock to handle timer expiry.  in fastpath_timer_check()
1108 	 * the change yet, the timer checks are delayed until the next  in fastpath_timer_check()
1110 	 * timer. This isn't an issue in practice because these types of  in fastpath_timer_check()
1142  * Invoked from the posix-timer core when a cancel operation failed because
1143  * the timer is marked firing. The caller holds rcu_read_lock(), which
1144  * protects the timer and the task which is expiring it from being freed.
1250 	 * newly expired timers, reenable the fast path check in the timer  in posix_cpu_timers_enable_work()
1297 	struct k_itimer *timer, *next;  in handle_posix_cpu_timers()  local
1323 		 * The above timer checks have updated the expiry cache and  in handle_posix_cpu_timers()
1326 		 * consistent. So the next timer interrupt fastpath check  in handle_posix_cpu_timers()
1329 		 * If timer expiry runs in the timer interrupt context then  in handle_posix_cpu_timers()
1335 		 * If timer expiry is deferred to task work context then  in handle_posix_cpu_timers()
1344 		 *   work ignored posix CPU timer handling because the  in handle_posix_cpu_timers()
1347 		 *   ticks have happened since the start of the timer  in handle_posix_cpu_timers()
1355 	 * We must release sighand lock before taking any timer's lock.  in handle_posix_cpu_timers()
1356 	 * There is a potential race with timer deletion here, as the  in handle_posix_cpu_timers()
1358 	 * the firing flag in each timer, so that a deletion attempt  in handle_posix_cpu_timers()
1359 	 * that gets the timer lock before we do will give it up and  in handle_posix_cpu_timers()
1360 	 * spin until we've taken care of that timer below.  in handle_posix_cpu_timers()
1367 	 * each timer's lock before clearing its firing flag, so no  in handle_posix_cpu_timers()
1368 	 * timer call will interfere.  in handle_posix_cpu_timers()
1370 	list_for_each_entry_safe(timer, next, &firing, it.cpu.elist) {  in handle_posix_cpu_timers()
1377 		 * because all other operations on timer::it_lock happen in  in handle_posix_cpu_timers()
1380 		spin_lock(&timer->it_lock);  in handle_posix_cpu_timers()
1381 		list_del_init(&timer->it.cpu.elist);  in handle_posix_cpu_timers()
1382 		cpu_firing = timer->it.cpu.firing;  in handle_posix_cpu_timers()
1383 		timer->it.cpu.firing = false;  in handle_posix_cpu_timers()
1386 		 * timer rearm/delete operation. So don't generate an  in handle_posix_cpu_timers()
1390 			cpu_timer_fire(timer);  in handle_posix_cpu_timers()
1392 		rcu_assign_pointer(timer->it.cpu.handling, NULL);  in handle_posix_cpu_timers()
1393 		spin_unlock(&timer->it_lock);  in handle_posix_cpu_timers()
1398  * This is called from the timer interrupt handler.  The irq handler has
1412 	 * miss timer->it.cpu.firing != 0.  in run_posix_cpu_timers()
1469 	 * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF  in set_process_cpu_timer()
1482 	struct k_itimer timer;  in do_cpu_nanosleep()  local
1487 	 * Set up a temporary timer and then wait for it to go off.  in do_cpu_nanosleep()
1489 	memset(&timer, 0, sizeof timer);  in do_cpu_nanosleep()
1490 	spin_lock_init(&timer.it_lock);  in do_cpu_nanosleep()
1491 	timer.it_clock = which_clock;  in do_cpu_nanosleep()
1492 	timer.it_overrun = -1;  in do_cpu_nanosleep()
1493 	error = posix_cpu_timer_create(&timer);  in do_cpu_nanosleep()
1494 	timer.it_process = current;  in do_cpu_nanosleep()
1495 	timer.it.cpu.nanosleep = true;  in do_cpu_nanosleep()
1504 		spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1505 		error = posix_cpu_timer_set(&timer, flags, &it, NULL);  in do_cpu_nanosleep()
1507 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1512 			if (!cpu_timer_getexpires(&timer.it.cpu)) {  in do_cpu_nanosleep()
1514 				 * Our timer fired and was reset, below  in do_cpu_nanosleep()
1517 				posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1518 				spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1526 			spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1528 			spin_lock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1534 		expires = cpu_timer_getexpires(&timer.it.cpu);  in do_cpu_nanosleep()
1535 		error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);  in do_cpu_nanosleep()
1537 			/* Timer is now unarmed, deletion can not fail. */  in do_cpu_nanosleep()
1538 			posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1541 				posix_cpu_timer_wait_running_nsleep(&timer);  in do_cpu_nanosleep()
1542 				error = posix_cpu_timer_del(&timer);  in do_cpu_nanosleep()
1546 		spin_unlock_irq(&timer.it_lock);  in do_cpu_nanosleep()
1620 static int process_cpu_timer_create(struct k_itimer *timer)  in process_cpu_timer_create()  argument
1622 	timer->it_clock = PROCESS_CLOCK;  in process_cpu_timer_create()
1623 	return posix_cpu_timer_create(timer);  in process_cpu_timer_create()
1640 static int thread_cpu_timer_create(struct k_itimer *timer)  in thread_cpu_timer_create()  argument
1642 	timer->it_clock = THREAD_CLOCK;  in thread_cpu_timer_create()
1643 	return posix_cpu_timer_create(timer);  in thread_cpu_timer_create()