1 // SPDX-License-Identifier: GPL-2.0
3 * Implement CPU time clocks for the POSIX clock interface.
6 #include <linux/sched/signal.h>
7 #include <linux/sched/cputime.h>
8 #include <linux/posix-timers.h>
9 #include <linux/errno.h>
10 #include <linux/math64.h>
11 #include <linux/uaccess.h>
12 #include <linux/kernel_stat.h>
13 #include <trace/events/timer.h>
14 #include <linux/tick.h>
15 #include <linux/workqueue.h>
16 #include <linux/compat.h>
17 #include <linux/sched/deadline.h>
19 #include "posix-timers.h"
21 static void posix_cpu_timer_rearm(struct k_itimer *timer);
23 void posix_cputimers_group_init(struct posix_cputimers *pct, u64 cpu_limit)
25 posix_cputimers_init(pct);
26 if (cpu_limit != RLIM_INFINITY)
27 pct->bases[CPUCLOCK_PROF].nextevt = cpu_limit * NSEC_PER_SEC;
31 * Called after updating RLIMIT_CPU to run cpu timer and update
32 * tsk->signal->posix_cputimers.bases[clock].nextevt expiration cache if
33 * necessary. Needs siglock protection since other code may update the
34 * expiration cache as well.
36 void update_rlimit_cpu(struct task_struct *task, unsigned long rlim_new)
38 u64 nsecs = rlim_new * NSEC_PER_SEC;
40 spin_lock_irq(&task->sighand->siglock);
41 set_process_cpu_timer(task, CPUCLOCK_PROF, &nsecs, NULL);
42 spin_unlock_irq(&task->sighand->siglock);
46 * Functions for validating access to tasks.
48 static struct task_struct *lookup_task(const pid_t pid, bool thread)
50 struct task_struct *p;
53 return thread ? current : current->group_leader;
55 p = find_task_by_vpid(pid);
56 if (!p || p == current)
59 return same_thread_group(p, current) ? p : NULL;
62 return has_group_leader_pid(p) ? p : NULL;
65 static struct task_struct *__get_task_for_clock(const clockid_t clock,
68 const bool thread = !!CPUCLOCK_PERTHREAD(clock);
69 const pid_t pid = CPUCLOCK_PID(clock);
70 struct task_struct *p;
72 if (CPUCLOCK_WHICH(clock) >= CPUCLOCK_MAX)
76 p = lookup_task(pid, thread);
83 static inline struct task_struct *get_task_for_clock(const clockid_t clock)
85 return __get_task_for_clock(clock, true);
88 static inline int validate_clock_permissions(const clockid_t clock)
90 return __get_task_for_clock(clock, false) ? 0 : -EINVAL;
94 * Update expiry time from increment, and increase overrun count,
95 * given the current clock sample.
97 static void bump_cpu_timer(struct k_itimer *timer, u64 now)
102 if (!timer->it_interval)
105 if (now < timer->it.cpu.expires)
108 incr = timer->it_interval;
109 delta = now + incr - timer->it.cpu.expires;
111 /* Don't use (incr*2 < delta), incr*2 might overflow. */
112 for (i = 0; incr < delta - incr; i++)
115 for (; i >= 0; incr >>= 1, i--) {
119 timer->it.cpu.expires += incr;
120 timer->it_overrun += 1LL << i;
125 /* Check whether all cache entries contain U64_MAX, i.e. eternal expiry time */
126 static inline bool expiry_cache_is_inactive(const struct posix_cputimers *pct)
128 return !(~pct->bases[CPUCLOCK_PROF].nextevt |
129 ~pct->bases[CPUCLOCK_VIRT].nextevt |
130 ~pct->bases[CPUCLOCK_SCHED].nextevt);
134 posix_cpu_clock_getres(const clockid_t which_clock, struct timespec64 *tp)
136 int error = validate_clock_permissions(which_clock);
140 tp->tv_nsec = ((NSEC_PER_SEC + HZ - 1) / HZ);
141 if (CPUCLOCK_WHICH(which_clock) == CPUCLOCK_SCHED) {
143 * If sched_clock is using a cycle counter, we
144 * don't have any idea of its true resolution
145 * exported, but it is much more than 1s/HZ.
154 posix_cpu_clock_set(const clockid_t clock, const struct timespec64 *tp)
156 int error = validate_clock_permissions(clock);
159 * You can never reset a CPU clock, but we check for other errors
160 * in the call before failing with EPERM.
162 return error ? : -EPERM;
166 * Sample a per-thread clock for the given task. clkid is validated.
168 static u64 cpu_clock_sample(const clockid_t clkid, struct task_struct *p)
172 if (clkid == CPUCLOCK_SCHED)
173 return task_sched_runtime(p);
175 task_cputime(p, &utime, &stime);
179 return utime + stime;
188 static inline void store_samples(u64 *samples, u64 stime, u64 utime, u64 rtime)
190 samples[CPUCLOCK_PROF] = stime + utime;
191 samples[CPUCLOCK_VIRT] = utime;
192 samples[CPUCLOCK_SCHED] = rtime;
195 static void task_sample_cputime(struct task_struct *p, u64 *samples)
199 task_cputime(p, &utime, &stime);
200 store_samples(samples, stime, utime, p->se.sum_exec_runtime);
203 static void proc_sample_cputime_atomic(struct task_cputime_atomic *at,
206 u64 stime, utime, rtime;
208 utime = atomic64_read(&at->utime);
209 stime = atomic64_read(&at->stime);
210 rtime = atomic64_read(&at->sum_exec_runtime);
211 store_samples(samples, stime, utime, rtime);
215 * Set cputime to sum_cputime if sum_cputime > cputime. Use cmpxchg
216 * to avoid race conditions with concurrent updates to cputime.
218 static inline void __update_gt_cputime(atomic64_t *cputime, u64 sum_cputime)
222 curr_cputime = atomic64_read(cputime);
223 if (sum_cputime > curr_cputime) {
224 if (atomic64_cmpxchg(cputime, curr_cputime, sum_cputime) != curr_cputime)
229 static void update_gt_cputime(struct task_cputime_atomic *cputime_atomic,
230 struct task_cputime *sum)
232 __update_gt_cputime(&cputime_atomic->utime, sum->utime);
233 __update_gt_cputime(&cputime_atomic->stime, sum->stime);
234 __update_gt_cputime(&cputime_atomic->sum_exec_runtime, sum->sum_exec_runtime);
238 * thread_group_sample_cputime - Sample cputime for a given task
239 * @tsk: Task for which cputime needs to be started
240 * @iimes: Storage for time samples
242 * Called from sys_getitimer() to calculate the expiry time of an active
243 * timer. That means group cputime accounting is already active. Called
244 * with task sighand lock held.
246 * Updates @times with an uptodate sample of the thread group cputimes.
248 void thread_group_sample_cputime(struct task_struct *tsk, u64 *samples)
250 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
252 WARN_ON_ONCE(!cputimer->running);
254 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
258 * thread_group_start_cputime - Start cputime and return a sample
259 * @tsk: Task for which cputime needs to be started
260 * @samples: Storage for time samples
262 * The thread group cputime accouting is avoided when there are no posix
263 * CPU timers armed. Before starting a timer it's required to check whether
264 * the time accounting is active. If not, a full update of the atomic
265 * accounting store needs to be done and the accounting enabled.
267 * Updates @times with an uptodate sample of the thread group cputimes.
269 static void thread_group_start_cputime(struct task_struct *tsk, u64 *samples)
271 struct thread_group_cputimer *cputimer = &tsk->signal->cputimer;
273 /* Check if cputimer isn't running. This is accessed without locking. */
274 if (!READ_ONCE(cputimer->running)) {
275 struct task_cputime sum;
278 * The POSIX timer interface allows for absolute time expiry
279 * values through the TIMER_ABSTIME flag, therefore we have
280 * to synchronize the timer to the clock every time we start it.
282 thread_group_cputime(tsk, &sum);
283 update_gt_cputime(&cputimer->cputime_atomic, &sum);
286 * We're setting cputimer->running without a lock. Ensure
287 * this only gets written to in one operation. We set
288 * running after update_gt_cputime() as a small optimization,
289 * but barriers are not required because update_gt_cputime()
290 * can handle concurrent updates.
292 WRITE_ONCE(cputimer->running, true);
294 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
297 static void __thread_group_cputime(struct task_struct *tsk, u64 *samples)
299 struct task_cputime ct;
301 thread_group_cputime(tsk, &ct);
302 store_samples(samples, ct.stime, ct.utime, ct.sum_exec_runtime);
306 * Sample a process (thread group) clock for the given task clkid. If the
307 * group's cputime accounting is already enabled, read the atomic
308 * store. Otherwise a full update is required. Task's sighand lock must be
309 * held to protect the task traversal on a full update. clkid is already
312 static u64 cpu_clock_sample_group(const clockid_t clkid, struct task_struct *p,
315 struct thread_group_cputimer *cputimer = &p->signal->cputimer;
316 u64 samples[CPUCLOCK_MAX];
318 if (!READ_ONCE(cputimer->running)) {
320 thread_group_start_cputime(p, samples);
322 __thread_group_cputime(p, samples);
324 proc_sample_cputime_atomic(&cputimer->cputime_atomic, samples);
327 return samples[clkid];
330 static int posix_cpu_clock_get(const clockid_t clock, struct timespec64 *tp)
332 const clockid_t clkid = CPUCLOCK_WHICH(clock);
333 struct task_struct *tsk;
336 tsk = get_task_for_clock(clock);
340 if (CPUCLOCK_PERTHREAD(clock))
341 t = cpu_clock_sample(clkid, tsk);
343 t = cpu_clock_sample_group(clkid, tsk, false);
344 put_task_struct(tsk);
346 *tp = ns_to_timespec64(t);
351 * Validate the clockid_t for a new CPU-clock timer, and initialize the timer.
352 * This is called from sys_timer_create() and do_cpu_nanosleep() with the
353 * new timer already all-zeros initialized.
355 static int posix_cpu_timer_create(struct k_itimer *new_timer)
357 struct task_struct *p = get_task_for_clock(new_timer->it_clock);
362 new_timer->kclock = &clock_posix_cpu;
363 INIT_LIST_HEAD(&new_timer->it.cpu.entry);
364 new_timer->it.cpu.task = p;
369 * Clean up a CPU-clock timer that is about to be destroyed.
370 * This is called from timer deletion with the timer already locked.
371 * If we return TIMER_RETRY, it's necessary to release the timer's lock
372 * and try again. (This happens when the timer is in the middle of firing.)
374 static int posix_cpu_timer_del(struct k_itimer *timer)
378 struct sighand_struct *sighand;
379 struct task_struct *p = timer->it.cpu.task;
381 if (WARN_ON_ONCE(!p))
385 * Protect against sighand release/switch in exit/exec and process/
386 * thread timer list entry concurrent read/writes.
388 sighand = lock_task_sighand(p, &flags);
389 if (unlikely(sighand == NULL)) {
391 * We raced with the reaping of the task.
392 * The deletion should have cleared us off the list.
394 WARN_ON_ONCE(!list_empty(&timer->it.cpu.entry));
396 if (timer->it.cpu.firing)
399 list_del(&timer->it.cpu.entry);
401 unlock_task_sighand(p, &flags);
410 static void cleanup_timers_list(struct list_head *head)
412 struct cpu_timer_list *timer, *next;
414 list_for_each_entry_safe(timer, next, head, entry)
415 list_del_init(&timer->entry);
419 * Clean out CPU timers which are still armed when a thread exits. The
420 * timers are only removed from the list. No other updates are done. The
421 * corresponding posix timers are still accessible, but cannot be rearmed.
423 * This must be called with the siglock held.
425 static void cleanup_timers(struct posix_cputimers *pct)
427 cleanup_timers_list(&pct->bases[CPUCLOCK_PROF].cpu_timers);
428 cleanup_timers_list(&pct->bases[CPUCLOCK_VIRT].cpu_timers);
429 cleanup_timers_list(&pct->bases[CPUCLOCK_SCHED].cpu_timers);
433 * These are both called with the siglock held, when the current thread
434 * is being reaped. When the final (leader) thread in the group is reaped,
435 * posix_cpu_timers_exit_group will be called after posix_cpu_timers_exit.
437 void posix_cpu_timers_exit(struct task_struct *tsk)
439 cleanup_timers(&tsk->posix_cputimers);
441 void posix_cpu_timers_exit_group(struct task_struct *tsk)
443 cleanup_timers(&tsk->signal->posix_cputimers);
447 * Insert the timer on the appropriate list before any timers that
448 * expire later. This must be called with the sighand lock held.
450 static void arm_timer(struct k_itimer *timer)
452 struct cpu_timer_list *const nt = &timer->it.cpu;
453 int clkidx = CPUCLOCK_WHICH(timer->it_clock);
454 struct task_struct *p = timer->it.cpu.task;
455 u64 newexp = timer->it.cpu.expires;
456 struct posix_cputimer_base *base;
457 struct list_head *head, *listpos;
458 struct cpu_timer_list *next;
460 if (CPUCLOCK_PERTHREAD(timer->it_clock))
461 base = p->posix_cputimers.bases + clkidx;
463 base = p->signal->posix_cputimers.bases + clkidx;
465 listpos = head = &base->cpu_timers;
466 list_for_each_entry(next,head, entry) {
467 if (nt->expires < next->expires)
469 listpos = &next->entry;
471 list_add(&nt->entry, listpos);
477 * We are the new earliest-expiring POSIX 1.b timer, hence
478 * need to update expiration cache. Take into account that
479 * for process timers we share expiration cache with itimers
480 * and RLIMIT_CPU and for thread timers with RLIMIT_RTTIME.
482 if (newexp < base->nextevt)
483 base->nextevt = newexp;
485 if (CPUCLOCK_PERTHREAD(timer->it_clock))
486 tick_dep_set_task(p, TICK_DEP_BIT_POSIX_TIMER);
488 tick_dep_set_signal(p->signal, TICK_DEP_BIT_POSIX_TIMER);
492 * The timer is locked, fire it and arrange for its reload.
494 static void cpu_timer_fire(struct k_itimer *timer)
496 if ((timer->it_sigev_notify & ~SIGEV_THREAD_ID) == SIGEV_NONE) {
498 * User don't want any signal.
500 timer->it.cpu.expires = 0;
501 } else if (unlikely(timer->sigq == NULL)) {
503 * This a special case for clock_nanosleep,
504 * not a normal timer from sys_timer_create.
506 wake_up_process(timer->it_process);
507 timer->it.cpu.expires = 0;
508 } else if (!timer->it_interval) {
510 * One-shot timer. Clear it as soon as it's fired.
512 posix_timer_event(timer, 0);
513 timer->it.cpu.expires = 0;
514 } else if (posix_timer_event(timer, ++timer->it_requeue_pending)) {
516 * The signal did not get queued because the signal
517 * was ignored, so we won't get any callback to
518 * reload the timer. But we need to keep it
519 * ticking in case the signal is deliverable next time.
521 posix_cpu_timer_rearm(timer);
522 ++timer->it_requeue_pending;
527 * Guts of sys_timer_settime for CPU timers.
528 * This is called with the timer locked and interrupts disabled.
529 * If we return TIMER_RETRY, it's necessary to release the timer's lock
530 * and try again. (This happens when the timer is in the middle of firing.)
532 static int posix_cpu_timer_set(struct k_itimer *timer, int timer_flags,
533 struct itimerspec64 *new, struct itimerspec64 *old)
535 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
536 u64 old_expires, new_expires, old_incr, val;
537 struct task_struct *p = timer->it.cpu.task;
538 struct sighand_struct *sighand;
542 if (WARN_ON_ONCE(!p))
546 * Use the to_ktime conversion because that clamps the maximum
547 * value to KTIME_MAX and avoid multiplication overflows.
549 new_expires = ktime_to_ns(timespec64_to_ktime(new->it_value));
552 * Protect against sighand release/switch in exit/exec and p->cpu_timers
553 * and p->signal->cpu_timers read/write in arm_timer()
555 sighand = lock_task_sighand(p, &flags);
557 * If p has just been reaped, we can no
558 * longer get any information about it at all.
560 if (unlikely(sighand == NULL)) {
565 * Disarm any old timer after extracting its expiry time.
569 old_incr = timer->it_interval;
570 old_expires = timer->it.cpu.expires;
571 if (unlikely(timer->it.cpu.firing)) {
572 timer->it.cpu.firing = -1;
575 list_del_init(&timer->it.cpu.entry);
578 * We need to sample the current value to convert the new
579 * value from to relative and absolute, and to convert the
580 * old value from absolute to relative. To set a process
581 * timer, we need a sample to balance the thread expiry
582 * times (in arm_timer). With an absolute time, we must
583 * check if it's already passed. In short, we need a sample.
585 if (CPUCLOCK_PERTHREAD(timer->it_clock))
586 val = cpu_clock_sample(clkid, p);
588 val = cpu_clock_sample_group(clkid, p, true);
591 if (old_expires == 0) {
592 old->it_value.tv_sec = 0;
593 old->it_value.tv_nsec = 0;
596 * Update the timer in case it has
597 * overrun already. If it has,
598 * we'll report it as having overrun
599 * and with the next reloaded timer
600 * already ticking, though we are
601 * swallowing that pending
602 * notification here to install the
605 bump_cpu_timer(timer, val);
606 if (val < timer->it.cpu.expires) {
607 old_expires = timer->it.cpu.expires - val;
608 old->it_value = ns_to_timespec64(old_expires);
610 old->it_value.tv_nsec = 1;
611 old->it_value.tv_sec = 0;
618 * We are colliding with the timer actually firing.
619 * Punt after filling in the timer's old value, and
620 * disable this firing since we are already reporting
621 * it as an overrun (thanks to bump_cpu_timer above).
623 unlock_task_sighand(p, &flags);
627 if (new_expires != 0 && !(timer_flags & TIMER_ABSTIME)) {
632 * Install the new expiry time (or zero).
633 * For a timer with no notification action, we don't actually
634 * arm the timer (we'll just fake it for timer_gettime).
636 timer->it.cpu.expires = new_expires;
637 if (new_expires != 0 && val < new_expires) {
641 unlock_task_sighand(p, &flags);
643 * Install the new reload setting, and
644 * set up the signal and overrun bookkeeping.
646 timer->it_interval = timespec64_to_ktime(new->it_interval);
649 * This acts as a modification timestamp for the timer,
650 * so any automatic reload attempt will punt on seeing
651 * that we have reset the timer manually.
653 timer->it_requeue_pending = (timer->it_requeue_pending + 2) &
655 timer->it_overrun_last = 0;
656 timer->it_overrun = -1;
658 if (new_expires != 0 && !(val < new_expires)) {
660 * The designated time already passed, so we notify
661 * immediately, even if the thread never runs to
662 * accumulate more time on this clock.
664 cpu_timer_fire(timer);
670 old->it_interval = ns_to_timespec64(old_incr);
675 static void posix_cpu_timer_get(struct k_itimer *timer, struct itimerspec64 *itp)
677 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
678 struct task_struct *p = timer->it.cpu.task;
681 if (WARN_ON_ONCE(!p))
685 * Easy part: convert the reload time.
687 itp->it_interval = ktime_to_timespec64(timer->it_interval);
689 if (!timer->it.cpu.expires)
693 * Sample the clock to take the difference with the expiry time.
695 if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
696 now = cpu_clock_sample(clkid, p);
698 struct sighand_struct *sighand;
702 * Protect against sighand release/switch in exit/exec and
703 * also make timer sampling safe if it ends up calling
704 * thread_group_cputime().
706 sighand = lock_task_sighand(p, &flags);
707 if (unlikely(sighand == NULL)) {
709 * The process has been reaped.
710 * We can't even collect a sample any more.
711 * Call the timer disarmed, nothing else to do.
713 timer->it.cpu.expires = 0;
716 now = cpu_clock_sample_group(clkid, p, false);
717 unlock_task_sighand(p, &flags);
721 if (now < timer->it.cpu.expires) {
722 itp->it_value = ns_to_timespec64(timer->it.cpu.expires - now);
725 * The timer should have expired already, but the firing
726 * hasn't taken place yet. Say it's just about to expire.
728 itp->it_value.tv_nsec = 1;
729 itp->it_value.tv_sec = 0;
733 static unsigned long long
734 check_timers_list(struct list_head *timers,
735 struct list_head *firing,
736 unsigned long long curr)
740 while (!list_empty(timers)) {
741 struct cpu_timer_list *t;
743 t = list_first_entry(timers, struct cpu_timer_list, entry);
745 if (!--maxfire || curr < t->expires)
749 list_move_tail(&t->entry, firing);
755 static void collect_posix_cputimers(struct posix_cputimers *pct,
756 u64 *samples, struct list_head *firing)
758 struct posix_cputimer_base *base = pct->bases;
761 for (i = 0; i < CPUCLOCK_MAX; i++, base++) {
762 base->nextevt = check_timers_list(&base->cpu_timers, firing,
767 static inline void check_dl_overrun(struct task_struct *tsk)
769 if (tsk->dl.dl_overrun) {
770 tsk->dl.dl_overrun = 0;
771 __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
776 * Check for any per-thread CPU timers that have fired and move them off
777 * the tsk->cpu_timers[N] list onto the firing list. Here we update the
778 * tsk->it_*_expires values to reflect the remaining thread CPU timers.
780 static void check_thread_timers(struct task_struct *tsk,
781 struct list_head *firing)
783 struct posix_cputimers *pct = &tsk->posix_cputimers;
784 u64 samples[CPUCLOCK_MAX];
788 check_dl_overrun(tsk);
790 if (expiry_cache_is_inactive(pct))
793 task_sample_cputime(tsk, samples);
794 collect_posix_cputimers(pct, samples, firing);
797 * Check for the special case thread timers.
799 soft = task_rlimit(tsk, RLIMIT_RTTIME);
800 if (soft != RLIM_INFINITY) {
801 /* Task RT timeout is accounted in jiffies. RTTIME is usec */
802 unsigned long rtim = tsk->rt.timeout * (USEC_PER_SEC / HZ);
803 unsigned long hard = task_rlimit_max(tsk, RLIMIT_RTTIME);
805 if (hard != RLIM_INFINITY && rtim >= hard) {
807 * At the hard limit, we just die.
808 * No need to calculate anything else now.
810 if (print_fatal_signals) {
811 pr_info("CPU Watchdog Timeout (hard): %s[%d]\n",
812 tsk->comm, task_pid_nr(tsk));
814 __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
819 * At the soft limit, send a SIGXCPU every second.
822 soft += USEC_PER_SEC;
823 tsk->signal->rlim[RLIMIT_RTTIME].rlim_cur =
826 if (print_fatal_signals) {
827 pr_info("RT Watchdog Timeout (soft): %s[%d]\n",
828 tsk->comm, task_pid_nr(tsk));
830 __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
834 if (expiry_cache_is_inactive(pct))
835 tick_dep_clear_task(tsk, TICK_DEP_BIT_POSIX_TIMER);
838 static inline void stop_process_timers(struct signal_struct *sig)
840 struct thread_group_cputimer *cputimer = &sig->cputimer;
842 /* Turn off cputimer->running. This is done without locking. */
843 WRITE_ONCE(cputimer->running, false);
844 tick_dep_clear_signal(sig, TICK_DEP_BIT_POSIX_TIMER);
847 static void check_cpu_itimer(struct task_struct *tsk, struct cpu_itimer *it,
848 u64 *expires, u64 cur_time, int signo)
853 if (cur_time >= it->expires) {
855 it->expires += it->incr;
859 trace_itimer_expire(signo == SIGPROF ?
860 ITIMER_PROF : ITIMER_VIRTUAL,
861 task_tgid(tsk), cur_time);
862 __group_send_sig_info(signo, SEND_SIG_PRIV, tsk);
865 if (it->expires && it->expires < *expires)
866 *expires = it->expires;
870 * Check for any per-thread CPU timers that have fired and move them
871 * off the tsk->*_timers list onto the firing list. Per-thread timers
872 * have already been taken off.
874 static void check_process_timers(struct task_struct *tsk,
875 struct list_head *firing)
877 struct signal_struct *const sig = tsk->signal;
878 struct posix_cputimers *pct = &sig->posix_cputimers;
879 u64 samples[CPUCLOCK_MAX];
883 * If cputimer is not running, then there are no active
884 * process wide timers (POSIX 1.b, itimers, RLIMIT_CPU).
886 if (!READ_ONCE(sig->cputimer.running))
890 * Signify that a thread is checking for process timers.
891 * Write access to this field is protected by the sighand lock.
893 sig->cputimer.checking_timer = true;
896 * Collect the current process totals. Group accounting is active
897 * so the sample can be taken directly.
899 proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic, samples);
900 collect_posix_cputimers(pct, samples, firing);
903 * Check for the special case process timers.
905 check_cpu_itimer(tsk, &sig->it[CPUCLOCK_PROF],
906 &pct->bases[CPUCLOCK_PROF].nextevt,
907 samples[CPUCLOCK_PROF], SIGPROF);
908 check_cpu_itimer(tsk, &sig->it[CPUCLOCK_VIRT],
909 &pct->bases[CPUCLOCK_VIRT].nextevt,
910 samples[CPUCLOCK_VIRT], SIGVTALRM);
912 soft = task_rlimit(tsk, RLIMIT_CPU);
913 if (soft != RLIM_INFINITY) {
914 /* RLIMIT_CPU is in seconds. Samples are nanoseconds */
915 unsigned long hard = task_rlimit_max(tsk, RLIMIT_CPU);
916 u64 ptime = samples[CPUCLOCK_PROF];
917 u64 softns = (u64)soft * NSEC_PER_SEC;
918 u64 hardns = (u64)hard * NSEC_PER_SEC;
920 if (hard != RLIM_INFINITY && ptime >= hardns) {
922 * At the hard limit, we just die.
923 * No need to calculate anything else now.
925 if (print_fatal_signals) {
926 pr_info("RT Watchdog Timeout (hard): %s[%d]\n",
927 tsk->comm, task_pid_nr(tsk));
929 __group_send_sig_info(SIGKILL, SEND_SIG_PRIV, tsk);
932 if (ptime >= softns) {
934 * At the soft limit, send a SIGXCPU every second.
936 if (print_fatal_signals) {
937 pr_info("CPU Watchdog Timeout (soft): %s[%d]\n",
938 tsk->comm, task_pid_nr(tsk));
940 __group_send_sig_info(SIGXCPU, SEND_SIG_PRIV, tsk);
942 sig->rlim[RLIMIT_CPU].rlim_cur = soft + 1;
943 softns += NSEC_PER_SEC;
947 /* Update the expiry cache */
948 if (softns < pct->bases[CPUCLOCK_PROF].nextevt)
949 pct->bases[CPUCLOCK_PROF].nextevt = softns;
952 if (expiry_cache_is_inactive(pct))
953 stop_process_timers(sig);
955 sig->cputimer.checking_timer = false;
959 * This is called from the signal code (via posixtimer_rearm)
960 * when the last timer signal was delivered and we have to reload the timer.
962 static void posix_cpu_timer_rearm(struct k_itimer *timer)
964 clockid_t clkid = CPUCLOCK_WHICH(timer->it_clock);
965 struct task_struct *p = timer->it.cpu.task;
966 struct sighand_struct *sighand;
970 if (WARN_ON_ONCE(!p))
974 * Fetch the current sample and update the timer's expiry time.
976 if (CPUCLOCK_PERTHREAD(timer->it_clock)) {
977 now = cpu_clock_sample(clkid, p);
978 bump_cpu_timer(timer, now);
979 if (unlikely(p->exit_state))
982 /* Protect timer list r/w in arm_timer() */
983 sighand = lock_task_sighand(p, &flags);
988 * Protect arm_timer() and timer sampling in case of call to
989 * thread_group_cputime().
991 sighand = lock_task_sighand(p, &flags);
992 if (unlikely(sighand == NULL)) {
994 * The process has been reaped.
995 * We can't even collect a sample any more.
997 timer->it.cpu.expires = 0;
999 } else if (unlikely(p->exit_state) && thread_group_empty(p)) {
1000 /* If the process is dying, no need to rearm */
1003 now = cpu_clock_sample_group(clkid, p, true);
1004 bump_cpu_timer(timer, now);
1005 /* Leave the sighand locked for the call below. */
1009 * Now re-arm for the new expiry time.
1013 unlock_task_sighand(p, &flags);
1017 * task_cputimers_expired - Check whether posix CPU timers are expired
1019 * @samples: Array of current samples for the CPUCLOCK clocks
1020 * @pct: Pointer to a posix_cputimers container
1022 * Returns true if any member of @samples is greater than the corresponding
1023 * member of @pct->bases[CLK].nextevt. False otherwise
1026 task_cputimers_expired(const u64 *sample, struct posix_cputimers *pct)
1030 for (i = 0; i < CPUCLOCK_MAX; i++) {
1031 if (sample[i] >= pct->bases[i].nextevt)
1038 * fastpath_timer_check - POSIX CPU timers fast path.
1040 * @tsk: The task (thread) being checked.
1042 * Check the task and thread group timers. If both are zero (there are no
1043 * timers set) return false. Otherwise snapshot the task and thread group
1044 * timers and compare them with the corresponding expiration times. Return
1045 * true if a timer has expired, else return false.
1047 static inline bool fastpath_timer_check(struct task_struct *tsk)
1049 struct signal_struct *sig;
1051 if (!expiry_cache_is_inactive(&tsk->posix_cputimers)) {
1052 u64 samples[CPUCLOCK_MAX];
1054 task_sample_cputime(tsk, samples);
1055 if (task_cputimers_expired(samples, &tsk->posix_cputimers))
1061 * Check if thread group timers expired when the cputimer is
1062 * running and no other thread in the group is already checking
1063 * for thread group cputimers. These fields are read without the
1064 * sighand lock. However, this is fine because this is meant to
1065 * be a fastpath heuristic to determine whether we should try to
1066 * acquire the sighand lock to check/handle timers.
1068 * In the worst case scenario, if 'running' or 'checking_timer' gets
1069 * set but the current thread doesn't see the change yet, we'll wait
1070 * until the next thread in the group gets a scheduler interrupt to
1071 * handle the timer. This isn't an issue in practice because these
1072 * types of delays with signals actually getting sent are expected.
1074 if (READ_ONCE(sig->cputimer.running) &&
1075 !READ_ONCE(sig->cputimer.checking_timer)) {
1076 u64 samples[CPUCLOCK_MAX];
1078 proc_sample_cputime_atomic(&sig->cputimer.cputime_atomic,
1081 if (task_cputimers_expired(samples, &sig->posix_cputimers))
1085 if (dl_task(tsk) && tsk->dl.dl_overrun)
1092 * This is called from the timer interrupt handler. The irq handler has
1093 * already updated our counts. We need to check if any timers fire now.
1094 * Interrupts are disabled.
1096 void run_posix_cpu_timers(void)
1098 struct task_struct *tsk = current;
1099 struct k_itimer *timer, *next;
1100 unsigned long flags;
1103 lockdep_assert_irqs_disabled();
1106 * The fast path checks that there are no expired thread or thread
1107 * group timers. If that's so, just return.
1109 if (!fastpath_timer_check(tsk))
1112 if (!lock_task_sighand(tsk, &flags))
1115 * Here we take off tsk->signal->cpu_timers[N] and
1116 * tsk->cpu_timers[N] all the timers that are firing, and
1117 * put them on the firing list.
1119 check_thread_timers(tsk, &firing);
1121 check_process_timers(tsk, &firing);
1124 * We must release these locks before taking any timer's lock.
1125 * There is a potential race with timer deletion here, as the
1126 * siglock now protects our private firing list. We have set
1127 * the firing flag in each timer, so that a deletion attempt
1128 * that gets the timer lock before we do will give it up and
1129 * spin until we've taken care of that timer below.
1131 unlock_task_sighand(tsk, &flags);
1134 * Now that all the timers on our list have the firing flag,
1135 * no one will touch their list entries but us. We'll take
1136 * each timer's lock before clearing its firing flag, so no
1137 * timer call will interfere.
1139 list_for_each_entry_safe(timer, next, &firing, it.cpu.entry) {
1142 spin_lock(&timer->it_lock);
1143 list_del_init(&timer->it.cpu.entry);
1144 cpu_firing = timer->it.cpu.firing;
1145 timer->it.cpu.firing = 0;
1147 * The firing flag is -1 if we collided with a reset
1148 * of the timer, which already reported this
1149 * almost-firing as an overrun. So don't generate an event.
1151 if (likely(cpu_firing >= 0))
1152 cpu_timer_fire(timer);
1153 spin_unlock(&timer->it_lock);
1158 * Set one of the process-wide special case CPU timers or RLIMIT_CPU.
1159 * The tsk->sighand->siglock must be held by the caller.
1161 void set_process_cpu_timer(struct task_struct *tsk, unsigned int clkid,
1162 u64 *newval, u64 *oldval)
1166 if (WARN_ON_ONCE(clkid >= CPUCLOCK_SCHED))
1169 nextevt = &tsk->signal->posix_cputimers.bases[clkid].nextevt;
1170 now = cpu_clock_sample_group(clkid, tsk, true);
1174 * We are setting itimer. The *oldval is absolute and we update
1175 * it to be relative, *newval argument is relative and we update
1176 * it to be absolute.
1179 if (*oldval <= now) {
1180 /* Just about to fire. */
1181 *oldval = TICK_NSEC;
1193 * Update expiration cache if this is the earliest timer. CPUCLOCK_PROF
1194 * expiry cache is also used by RLIMIT_CPU!.
1196 if (*newval < *nextevt)
1199 tick_dep_set_signal(tsk->signal, TICK_DEP_BIT_POSIX_TIMER);
1202 static int do_cpu_nanosleep(const clockid_t which_clock, int flags,
1203 const struct timespec64 *rqtp)
1205 struct itimerspec64 it;
1206 struct k_itimer timer;
1211 * Set up a temporary timer and then wait for it to go off.
1213 memset(&timer, 0, sizeof timer);
1214 spin_lock_init(&timer.it_lock);
1215 timer.it_clock = which_clock;
1216 timer.it_overrun = -1;
1217 error = posix_cpu_timer_create(&timer);
1218 timer.it_process = current;
1220 static struct itimerspec64 zero_it;
1221 struct restart_block *restart;
1223 memset(&it, 0, sizeof(it));
1224 it.it_value = *rqtp;
1226 spin_lock_irq(&timer.it_lock);
1227 error = posix_cpu_timer_set(&timer, flags, &it, NULL);
1229 spin_unlock_irq(&timer.it_lock);
1233 while (!signal_pending(current)) {
1234 if (timer.it.cpu.expires == 0) {
1236 * Our timer fired and was reset, below
1237 * deletion can not fail.
1239 posix_cpu_timer_del(&timer);
1240 spin_unlock_irq(&timer.it_lock);
1245 * Block until cpu_timer_fire (or a signal) wakes us.
1247 __set_current_state(TASK_INTERRUPTIBLE);
1248 spin_unlock_irq(&timer.it_lock);
1250 spin_lock_irq(&timer.it_lock);
1254 * We were interrupted by a signal.
1256 expires = timer.it.cpu.expires;
1257 error = posix_cpu_timer_set(&timer, 0, &zero_it, &it);
1260 * Timer is now unarmed, deletion can not fail.
1262 posix_cpu_timer_del(&timer);
1264 spin_unlock_irq(&timer.it_lock);
1266 while (error == TIMER_RETRY) {
1268 * We need to handle case when timer was or is in the
1269 * middle of firing. In other cases we already freed
1272 spin_lock_irq(&timer.it_lock);
1273 error = posix_cpu_timer_del(&timer);
1274 spin_unlock_irq(&timer.it_lock);
1277 if ((it.it_value.tv_sec | it.it_value.tv_nsec) == 0) {
1279 * It actually did fire already.
1284 error = -ERESTART_RESTARTBLOCK;
1286 * Report back to the user the time still remaining.
1288 restart = ¤t->restart_block;
1289 restart->nanosleep.expires = expires;
1290 if (restart->nanosleep.type != TT_NONE)
1291 error = nanosleep_copyout(restart, &it.it_value);
1297 static long posix_cpu_nsleep_restart(struct restart_block *restart_block);
1299 static int posix_cpu_nsleep(const clockid_t which_clock, int flags,
1300 const struct timespec64 *rqtp)
1302 struct restart_block *restart_block = ¤t->restart_block;
1306 * Diagnose required errors first.
1308 if (CPUCLOCK_PERTHREAD(which_clock) &&
1309 (CPUCLOCK_PID(which_clock) == 0 ||
1310 CPUCLOCK_PID(which_clock) == task_pid_vnr(current)))
1313 error = do_cpu_nanosleep(which_clock, flags, rqtp);
1315 if (error == -ERESTART_RESTARTBLOCK) {
1317 if (flags & TIMER_ABSTIME)
1318 return -ERESTARTNOHAND;
1320 restart_block->fn = posix_cpu_nsleep_restart;
1321 restart_block->nanosleep.clockid = which_clock;
1326 static long posix_cpu_nsleep_restart(struct restart_block *restart_block)
1328 clockid_t which_clock = restart_block->nanosleep.clockid;
1329 struct timespec64 t;
1331 t = ns_to_timespec64(restart_block->nanosleep.expires);
1333 return do_cpu_nanosleep(which_clock, TIMER_ABSTIME, &t);
1336 #define PROCESS_CLOCK make_process_cpuclock(0, CPUCLOCK_SCHED)
1337 #define THREAD_CLOCK make_thread_cpuclock(0, CPUCLOCK_SCHED)
1339 static int process_cpu_clock_getres(const clockid_t which_clock,
1340 struct timespec64 *tp)
1342 return posix_cpu_clock_getres(PROCESS_CLOCK, tp);
1344 static int process_cpu_clock_get(const clockid_t which_clock,
1345 struct timespec64 *tp)
1347 return posix_cpu_clock_get(PROCESS_CLOCK, tp);
1349 static int process_cpu_timer_create(struct k_itimer *timer)
1351 timer->it_clock = PROCESS_CLOCK;
1352 return posix_cpu_timer_create(timer);
1354 static int process_cpu_nsleep(const clockid_t which_clock, int flags,
1355 const struct timespec64 *rqtp)
1357 return posix_cpu_nsleep(PROCESS_CLOCK, flags, rqtp);
1359 static int thread_cpu_clock_getres(const clockid_t which_clock,
1360 struct timespec64 *tp)
1362 return posix_cpu_clock_getres(THREAD_CLOCK, tp);
1364 static int thread_cpu_clock_get(const clockid_t which_clock,
1365 struct timespec64 *tp)
1367 return posix_cpu_clock_get(THREAD_CLOCK, tp);
1369 static int thread_cpu_timer_create(struct k_itimer *timer)
1371 timer->it_clock = THREAD_CLOCK;
1372 return posix_cpu_timer_create(timer);
1375 const struct k_clock clock_posix_cpu = {
1376 .clock_getres = posix_cpu_clock_getres,
1377 .clock_set = posix_cpu_clock_set,
1378 .clock_get = posix_cpu_clock_get,
1379 .timer_create = posix_cpu_timer_create,
1380 .nsleep = posix_cpu_nsleep,
1381 .timer_set = posix_cpu_timer_set,
1382 .timer_del = posix_cpu_timer_del,
1383 .timer_get = posix_cpu_timer_get,
1384 .timer_rearm = posix_cpu_timer_rearm,
1387 const struct k_clock clock_process = {
1388 .clock_getres = process_cpu_clock_getres,
1389 .clock_get = process_cpu_clock_get,
1390 .timer_create = process_cpu_timer_create,
1391 .nsleep = process_cpu_nsleep,
1394 const struct k_clock clock_thread = {
1395 .clock_getres = thread_cpu_clock_getres,
1396 .clock_get = thread_cpu_clock_get,
1397 .timer_create = thread_cpu_timer_create,