1
2
3
4
5
6
7
8
9
10
11#include <linux/cpu.h>
12#include <linux/err.h>
13#include <linux/hrtimer.h>
14#include <linux/interrupt.h>
15#include <linux/kernel_stat.h>
16#include <linux/percpu.h>
17#include <linux/nmi.h>
18#include <linux/profile.h>
19#include <linux/sched/signal.h>
20#include <linux/sched/clock.h>
21#include <linux/sched/stat.h>
22#include <linux/sched/nohz.h>
23#include <linux/module.h>
24#include <linux/irq_work.h>
25#include <linux/posix-timers.h>
26#include <linux/context_tracking.h>
27#include <linux/mm.h>
28
29#include <asm/irq_regs.h>
30
31#include "tick-internal.h"
32
33#include <trace/events/timer.h>
34
35
36
37
38static DEFINE_PER_CPU(struct tick_sched, tick_cpu_sched);
39
40struct tick_sched *tick_get_tick_sched(int cpu)
41{
42 return &per_cpu(tick_cpu_sched, cpu);
43}
44
45#if defined(CONFIG_NO_HZ_COMMON) || defined(CONFIG_HIGH_RES_TIMERS)
46
47
48
49static ktime_t last_jiffies_update;
50
51
52
53
54static void tick_do_update_jiffies64(ktime_t now)
55{
56 unsigned long ticks = 0;
57 ktime_t delta;
58
59
60
61
62
63 delta = ktime_sub(now, READ_ONCE(last_jiffies_update));
64 if (delta < tick_period)
65 return;
66
67
68 raw_spin_lock(&jiffies_lock);
69 write_seqcount_begin(&jiffies_seq);
70
71 delta = ktime_sub(now, last_jiffies_update);
72 if (delta >= tick_period) {
73
74 delta = ktime_sub(delta, tick_period);
75
76 WRITE_ONCE(last_jiffies_update,
77 ktime_add(last_jiffies_update, tick_period));
78
79
80 if (unlikely(delta >= tick_period)) {
81 s64 incr = ktime_to_ns(tick_period);
82
83 ticks = ktime_divns(delta, incr);
84
85
86 WRITE_ONCE(last_jiffies_update,
87 ktime_add_ns(last_jiffies_update,
88 incr * ticks));
89 }
90 do_timer(++ticks);
91
92
93 tick_next_period = ktime_add(last_jiffies_update, tick_period);
94 } else {
95 write_seqcount_end(&jiffies_seq);
96 raw_spin_unlock(&jiffies_lock);
97 return;
98 }
99 write_seqcount_end(&jiffies_seq);
100 raw_spin_unlock(&jiffies_lock);
101 update_wall_time();
102}
103
104
105
106
107static ktime_t tick_init_jiffy_update(void)
108{
109 ktime_t period;
110
111 raw_spin_lock(&jiffies_lock);
112 write_seqcount_begin(&jiffies_seq);
113
114 if (last_jiffies_update == 0)
115 last_jiffies_update = tick_next_period;
116 period = last_jiffies_update;
117 write_seqcount_end(&jiffies_seq);
118 raw_spin_unlock(&jiffies_lock);
119 return period;
120}
121
122static void tick_sched_do_timer(struct tick_sched *ts, ktime_t now)
123{
124 int cpu = smp_processor_id();
125
126#ifdef CONFIG_NO_HZ_COMMON
127
128
129
130
131
132
133
134
135
136
137 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_NONE)) {
138#ifdef CONFIG_NO_HZ_FULL
139 WARN_ON(tick_nohz_full_running);
140#endif
141 tick_do_timer_cpu = cpu;
142 }
143#endif
144
145
146 if (tick_do_timer_cpu == cpu)
147 tick_do_update_jiffies64(now);
148
149 if (ts->inidle)
150 ts->got_idle_tick = 1;
151}
152
153static void tick_sched_handle(struct tick_sched *ts, struct pt_regs *regs)
154{
155#ifdef CONFIG_NO_HZ_COMMON
156
157
158
159
160
161
162
163
164 if (ts->tick_stopped) {
165 touch_softlockup_watchdog_sched();
166 if (is_idle_task(current))
167 ts->idle_jiffies++;
168
169
170
171
172
173 ts->next_tick = 0;
174 }
175#endif
176 update_process_times(user_mode(regs));
177 profile_tick(CPU_PROFILING);
178}
179#endif
180
181#ifdef CONFIG_NO_HZ_FULL
182cpumask_var_t tick_nohz_full_mask;
183bool tick_nohz_full_running;
184EXPORT_SYMBOL_GPL(tick_nohz_full_running);
185static atomic_t tick_dep_mask;
186
187static bool check_tick_dependency(atomic_t *dep)
188{
189 int val = atomic_read(dep);
190
191 if (val & TICK_DEP_MASK_POSIX_TIMER) {
192 trace_tick_stop(0, TICK_DEP_MASK_POSIX_TIMER);
193 return true;
194 }
195
196 if (val & TICK_DEP_MASK_PERF_EVENTS) {
197 trace_tick_stop(0, TICK_DEP_MASK_PERF_EVENTS);
198 return true;
199 }
200
201 if (val & TICK_DEP_MASK_SCHED) {
202 trace_tick_stop(0, TICK_DEP_MASK_SCHED);
203 return true;
204 }
205
206 if (val & TICK_DEP_MASK_CLOCK_UNSTABLE) {
207 trace_tick_stop(0, TICK_DEP_MASK_CLOCK_UNSTABLE);
208 return true;
209 }
210
211 if (val & TICK_DEP_MASK_RCU) {
212 trace_tick_stop(0, TICK_DEP_MASK_RCU);
213 return true;
214 }
215
216 return false;
217}
218
219static bool can_stop_full_tick(int cpu, struct tick_sched *ts)
220{
221 lockdep_assert_irqs_disabled();
222
223 if (unlikely(!cpu_online(cpu)))
224 return false;
225
226 if (check_tick_dependency(&tick_dep_mask))
227 return false;
228
229 if (check_tick_dependency(&ts->tick_dep_mask))
230 return false;
231
232 if (check_tick_dependency(¤t->tick_dep_mask))
233 return false;
234
235 if (check_tick_dependency(¤t->signal->tick_dep_mask))
236 return false;
237
238 return true;
239}
240
241static void nohz_full_kick_func(struct irq_work *work)
242{
243
244}
245
246static DEFINE_PER_CPU(struct irq_work, nohz_full_kick_work) = {
247 .func = nohz_full_kick_func,
248 .flags = ATOMIC_INIT(IRQ_WORK_HARD_IRQ),
249};
250
251
252
253
254
255
256
257static void tick_nohz_full_kick(void)
258{
259 if (!tick_nohz_full_cpu(smp_processor_id()))
260 return;
261
262 irq_work_queue(this_cpu_ptr(&nohz_full_kick_work));
263}
264
265
266
267
268
269void tick_nohz_full_kick_cpu(int cpu)
270{
271 if (!tick_nohz_full_cpu(cpu))
272 return;
273
274 irq_work_queue_on(&per_cpu(nohz_full_kick_work, cpu), cpu);
275}
276
277
278
279
280
281static void tick_nohz_full_kick_all(void)
282{
283 int cpu;
284
285 if (!tick_nohz_full_running)
286 return;
287
288 preempt_disable();
289 for_each_cpu_and(cpu, tick_nohz_full_mask, cpu_online_mask)
290 tick_nohz_full_kick_cpu(cpu);
291 preempt_enable();
292}
293
294static void tick_nohz_dep_set_all(atomic_t *dep,
295 enum tick_dep_bits bit)
296{
297 int prev;
298
299 prev = atomic_fetch_or(BIT(bit), dep);
300 if (!prev)
301 tick_nohz_full_kick_all();
302}
303
304
305
306
307
308void tick_nohz_dep_set(enum tick_dep_bits bit)
309{
310 tick_nohz_dep_set_all(&tick_dep_mask, bit);
311}
312
313void tick_nohz_dep_clear(enum tick_dep_bits bit)
314{
315 atomic_andnot(BIT(bit), &tick_dep_mask);
316}
317
318
319
320
321
322void tick_nohz_dep_set_cpu(int cpu, enum tick_dep_bits bit)
323{
324 int prev;
325 struct tick_sched *ts;
326
327 ts = per_cpu_ptr(&tick_cpu_sched, cpu);
328
329 prev = atomic_fetch_or(BIT(bit), &ts->tick_dep_mask);
330 if (!prev) {
331 preempt_disable();
332
333 if (cpu == smp_processor_id()) {
334 tick_nohz_full_kick();
335 } else {
336
337 if (!WARN_ON_ONCE(in_nmi()))
338 tick_nohz_full_kick_cpu(cpu);
339 }
340 preempt_enable();
341 }
342}
343EXPORT_SYMBOL_GPL(tick_nohz_dep_set_cpu);
344
345void tick_nohz_dep_clear_cpu(int cpu, enum tick_dep_bits bit)
346{
347 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
348
349 atomic_andnot(BIT(bit), &ts->tick_dep_mask);
350}
351EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_cpu);
352
353
354
355
356
357void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit)
358{
359 if (!atomic_fetch_or(BIT(bit), &tsk->tick_dep_mask)) {
360 if (tsk == current) {
361 preempt_disable();
362 tick_nohz_full_kick();
363 preempt_enable();
364 } else {
365
366
367
368
369 tick_nohz_full_kick_all();
370 }
371 }
372}
373EXPORT_SYMBOL_GPL(tick_nohz_dep_set_task);
374
375void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit)
376{
377 atomic_andnot(BIT(bit), &tsk->tick_dep_mask);
378}
379EXPORT_SYMBOL_GPL(tick_nohz_dep_clear_task);
380
381
382
383
384
385void tick_nohz_dep_set_signal(struct signal_struct *sig, enum tick_dep_bits bit)
386{
387 tick_nohz_dep_set_all(&sig->tick_dep_mask, bit);
388}
389
390void tick_nohz_dep_clear_signal(struct signal_struct *sig, enum tick_dep_bits bit)
391{
392 atomic_andnot(BIT(bit), &sig->tick_dep_mask);
393}
394
395
396
397
398
399
400void __tick_nohz_task_switch(void)
401{
402 unsigned long flags;
403 struct tick_sched *ts;
404
405 local_irq_save(flags);
406
407 if (!tick_nohz_full_cpu(smp_processor_id()))
408 goto out;
409
410 ts = this_cpu_ptr(&tick_cpu_sched);
411
412 if (ts->tick_stopped) {
413 if (atomic_read(¤t->tick_dep_mask) ||
414 atomic_read(¤t->signal->tick_dep_mask))
415 tick_nohz_full_kick();
416 }
417out:
418 local_irq_restore(flags);
419}
420
421
422void __init tick_nohz_full_setup(cpumask_var_t cpumask)
423{
424 alloc_bootmem_cpumask_var(&tick_nohz_full_mask);
425 cpumask_copy(tick_nohz_full_mask, cpumask);
426 tick_nohz_full_running = true;
427}
428EXPORT_SYMBOL_GPL(tick_nohz_full_setup);
429
430static int tick_nohz_cpu_down(unsigned int cpu)
431{
432
433
434
435
436
437 if (tick_nohz_full_running && tick_do_timer_cpu == cpu)
438 return -EBUSY;
439 return 0;
440}
441
442void __init tick_nohz_init(void)
443{
444 int cpu, ret;
445
446 if (!tick_nohz_full_running)
447 return;
448
449
450
451
452
453
454 if (!arch_irq_work_has_interrupt()) {
455 pr_warn("NO_HZ: Can't run full dynticks because arch doesn't support irq work self-IPIs\n");
456 cpumask_clear(tick_nohz_full_mask);
457 tick_nohz_full_running = false;
458 return;
459 }
460
461 if (IS_ENABLED(CONFIG_PM_SLEEP_SMP) &&
462 !IS_ENABLED(CONFIG_PM_SLEEP_SMP_NONZERO_CPU)) {
463 cpu = smp_processor_id();
464
465 if (cpumask_test_cpu(cpu, tick_nohz_full_mask)) {
466 pr_warn("NO_HZ: Clearing %d from nohz_full range "
467 "for timekeeping\n", cpu);
468 cpumask_clear_cpu(cpu, tick_nohz_full_mask);
469 }
470 }
471
472 for_each_cpu(cpu, tick_nohz_full_mask)
473 context_tracking_cpu_set(cpu);
474
475 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN,
476 "kernel/nohz:predown", NULL,
477 tick_nohz_cpu_down);
478 WARN_ON(ret < 0);
479 pr_info("NO_HZ: Full dynticks CPUs: %*pbl.\n",
480 cpumask_pr_args(tick_nohz_full_mask));
481}
482#endif
483
484
485
486
487#ifdef CONFIG_NO_HZ_COMMON
488
489
490
491bool tick_nohz_enabled __read_mostly = true;
492unsigned long tick_nohz_active __read_mostly;
493
494
495
496static int __init setup_tick_nohz(char *str)
497{
498 return (kstrtobool(str, &tick_nohz_enabled) == 0);
499}
500
501__setup("nohz=", setup_tick_nohz);
502
503bool tick_nohz_tick_stopped(void)
504{
505 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
506
507 return ts->tick_stopped;
508}
509
510bool tick_nohz_tick_stopped_cpu(int cpu)
511{
512 struct tick_sched *ts = per_cpu_ptr(&tick_cpu_sched, cpu);
513
514 return ts->tick_stopped;
515}
516
517
518
519
520
521
522
523
524
525
526
527static void tick_nohz_update_jiffies(ktime_t now)
528{
529 unsigned long flags;
530
531 __this_cpu_write(tick_cpu_sched.idle_waketime, now);
532
533 local_irq_save(flags);
534 tick_do_update_jiffies64(now);
535 local_irq_restore(flags);
536
537 touch_softlockup_watchdog_sched();
538}
539
540
541
542
543static void
544update_ts_time_stats(int cpu, struct tick_sched *ts, ktime_t now, u64 *last_update_time)
545{
546 ktime_t delta;
547
548 if (ts->idle_active) {
549 delta = ktime_sub(now, ts->idle_entrytime);
550 if (nr_iowait_cpu(cpu) > 0)
551 ts->iowait_sleeptime = ktime_add(ts->iowait_sleeptime, delta);
552 else
553 ts->idle_sleeptime = ktime_add(ts->idle_sleeptime, delta);
554 ts->idle_entrytime = now;
555 }
556
557 if (last_update_time)
558 *last_update_time = ktime_to_us(now);
559
560}
561
562static void tick_nohz_stop_idle(struct tick_sched *ts, ktime_t now)
563{
564 update_ts_time_stats(smp_processor_id(), ts, now, NULL);
565 ts->idle_active = 0;
566
567 sched_clock_idle_wakeup_event();
568}
569
570static void tick_nohz_start_idle(struct tick_sched *ts)
571{
572 ts->idle_entrytime = ktime_get();
573 ts->idle_active = 1;
574 sched_clock_idle_sleep_event();
575}
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591u64 get_cpu_idle_time_us(int cpu, u64 *last_update_time)
592{
593 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
594 ktime_t now, idle;
595
596 if (!tick_nohz_active)
597 return -1;
598
599 now = ktime_get();
600 if (last_update_time) {
601 update_ts_time_stats(cpu, ts, now, last_update_time);
602 idle = ts->idle_sleeptime;
603 } else {
604 if (ts->idle_active && !nr_iowait_cpu(cpu)) {
605 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
606
607 idle = ktime_add(ts->idle_sleeptime, delta);
608 } else {
609 idle = ts->idle_sleeptime;
610 }
611 }
612
613 return ktime_to_us(idle);
614
615}
616EXPORT_SYMBOL_GPL(get_cpu_idle_time_us);
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632u64 get_cpu_iowait_time_us(int cpu, u64 *last_update_time)
633{
634 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
635 ktime_t now, iowait;
636
637 if (!tick_nohz_active)
638 return -1;
639
640 now = ktime_get();
641 if (last_update_time) {
642 update_ts_time_stats(cpu, ts, now, last_update_time);
643 iowait = ts->iowait_sleeptime;
644 } else {
645 if (ts->idle_active && nr_iowait_cpu(cpu) > 0) {
646 ktime_t delta = ktime_sub(now, ts->idle_entrytime);
647
648 iowait = ktime_add(ts->iowait_sleeptime, delta);
649 } else {
650 iowait = ts->iowait_sleeptime;
651 }
652 }
653
654 return ktime_to_us(iowait);
655}
656EXPORT_SYMBOL_GPL(get_cpu_iowait_time_us);
657
658static void tick_nohz_restart(struct tick_sched *ts, ktime_t now)
659{
660 hrtimer_cancel(&ts->sched_timer);
661 hrtimer_set_expires(&ts->sched_timer, ts->last_tick);
662
663
664 hrtimer_forward(&ts->sched_timer, now, tick_period);
665
666 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
667 hrtimer_start_expires(&ts->sched_timer,
668 HRTIMER_MODE_ABS_PINNED_HARD);
669 } else {
670 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
671 }
672
673
674
675
676
677 ts->next_tick = 0;
678}
679
680static inline bool local_timer_softirq_pending(void)
681{
682 return local_softirq_pending() & BIT(TIMER_SOFTIRQ);
683}
684
685static ktime_t tick_nohz_next_event(struct tick_sched *ts, int cpu)
686{
687 u64 basemono, next_tick, next_tmr, next_rcu, delta, expires;
688 unsigned long basejiff;
689 unsigned int seq;
690
691
692 do {
693 seq = read_seqcount_begin(&jiffies_seq);
694 basemono = last_jiffies_update;
695 basejiff = jiffies;
696 } while (read_seqcount_retry(&jiffies_seq, seq));
697 ts->last_jiffies = basejiff;
698 ts->timer_expires_base = basemono;
699
700
701
702
703
704
705
706
707
708
709
710 if (rcu_needs_cpu(basemono, &next_rcu) || arch_needs_cpu() ||
711 irq_work_needs_cpu() || local_timer_softirq_pending()) {
712 next_tick = basemono + TICK_NSEC;
713 } else {
714
715
716
717
718
719
720
721 next_tmr = get_next_timer_interrupt(basejiff, basemono);
722 ts->next_timer = next_tmr;
723
724 next_tick = next_rcu < next_tmr ? next_rcu : next_tmr;
725 }
726
727
728
729
730
731 delta = next_tick - basemono;
732 if (delta <= (u64)TICK_NSEC) {
733
734
735
736
737 timer_clear_idle();
738
739
740
741
742 if (!ts->tick_stopped) {
743 ts->timer_expires = 0;
744 goto out;
745 }
746 }
747
748
749
750
751
752
753 delta = timekeeping_max_deferment();
754 if (cpu != tick_do_timer_cpu &&
755 (tick_do_timer_cpu != TICK_DO_TIMER_NONE || !ts->do_timer_last))
756 delta = KTIME_MAX;
757
758
759 if (delta < (KTIME_MAX - basemono))
760 expires = basemono + delta;
761 else
762 expires = KTIME_MAX;
763
764 ts->timer_expires = min_t(u64, expires, next_tick);
765
766out:
767 return ts->timer_expires;
768}
769
770static void tick_nohz_stop_tick(struct tick_sched *ts, int cpu)
771{
772 struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
773 u64 basemono = ts->timer_expires_base;
774 u64 expires = ts->timer_expires;
775 ktime_t tick = expires;
776
777
778 ts->timer_expires_base = 0;
779
780
781
782
783
784
785
786
787
788 if (cpu == tick_do_timer_cpu) {
789 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
790 ts->do_timer_last = 1;
791 } else if (tick_do_timer_cpu != TICK_DO_TIMER_NONE) {
792 ts->do_timer_last = 0;
793 }
794
795
796 if (ts->tick_stopped && (expires == ts->next_tick)) {
797
798 if (tick == KTIME_MAX || ts->next_tick == hrtimer_get_expires(&ts->sched_timer))
799 return;
800
801 WARN_ON_ONCE(1);
802 printk_once("basemono: %llu ts->next_tick: %llu dev->next_event: %llu timer->active: %d timer->expires: %llu\n",
803 basemono, ts->next_tick, dev->next_event,
804 hrtimer_active(&ts->sched_timer), hrtimer_get_expires(&ts->sched_timer));
805 }
806
807
808
809
810
811
812
813
814 if (!ts->tick_stopped) {
815 calc_load_nohz_start();
816 quiet_vmstat();
817
818 ts->last_tick = hrtimer_get_expires(&ts->sched_timer);
819 ts->tick_stopped = 1;
820 trace_tick_stop(1, TICK_DEP_MASK_NONE);
821 }
822
823 ts->next_tick = tick;
824
825
826
827
828
829 if (unlikely(expires == KTIME_MAX)) {
830 if (ts->nohz_mode == NOHZ_MODE_HIGHRES)
831 hrtimer_cancel(&ts->sched_timer);
832 return;
833 }
834
835 if (ts->nohz_mode == NOHZ_MODE_HIGHRES) {
836 hrtimer_start(&ts->sched_timer, tick,
837 HRTIMER_MODE_ABS_PINNED_HARD);
838 } else {
839 hrtimer_set_expires(&ts->sched_timer, tick);
840 tick_program_event(tick, 1);
841 }
842}
843
844static void tick_nohz_retain_tick(struct tick_sched *ts)
845{
846 ts->timer_expires_base = 0;
847}
848
849#ifdef CONFIG_NO_HZ_FULL
850static void tick_nohz_stop_sched_tick(struct tick_sched *ts, int cpu)
851{
852 if (tick_nohz_next_event(ts, cpu))
853 tick_nohz_stop_tick(ts, cpu);
854 else
855 tick_nohz_retain_tick(ts);
856}
857#endif
858
859static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
860{
861
862 tick_do_update_jiffies64(now);
863
864
865
866
867
868 timer_clear_idle();
869
870 calc_load_nohz_stop();
871 touch_softlockup_watchdog_sched();
872
873
874
875 ts->tick_stopped = 0;
876 ts->idle_exittime = now;
877
878 tick_nohz_restart(ts, now);
879}
880
881static void tick_nohz_full_update_tick(struct tick_sched *ts)
882{
883#ifdef CONFIG_NO_HZ_FULL
884 int cpu = smp_processor_id();
885
886 if (!tick_nohz_full_cpu(cpu))
887 return;
888
889 if (!ts->tick_stopped && ts->nohz_mode == NOHZ_MODE_INACTIVE)
890 return;
891
892 if (can_stop_full_tick(cpu, ts))
893 tick_nohz_stop_sched_tick(ts, cpu);
894 else if (ts->tick_stopped)
895 tick_nohz_restart_sched_tick(ts, ktime_get());
896#endif
897}
898
899static bool can_stop_idle_tick(int cpu, struct tick_sched *ts)
900{
901
902
903
904
905
906
907
908 if (unlikely(!cpu_online(cpu))) {
909 if (cpu == tick_do_timer_cpu)
910 tick_do_timer_cpu = TICK_DO_TIMER_NONE;
911
912
913
914
915 ts->next_tick = 0;
916 return false;
917 }
918
919 if (unlikely(ts->nohz_mode == NOHZ_MODE_INACTIVE))
920 return false;
921
922 if (need_resched())
923 return false;
924
925 if (unlikely(local_softirq_pending())) {
926 static int ratelimit;
927
928 if (ratelimit < 10 &&
929 (local_softirq_pending() & SOFTIRQ_STOP_IDLE_MASK)) {
930 pr_warn("NOHZ: local_softirq_pending %02x\n",
931 (unsigned int) local_softirq_pending());
932 ratelimit++;
933 }
934 return false;
935 }
936
937 if (tick_nohz_full_enabled()) {
938
939
940
941
942 if (tick_do_timer_cpu == cpu)
943 return false;
944
945
946
947
948
949 if (unlikely(tick_do_timer_cpu == TICK_DO_TIMER_BOOT))
950 return false;
951
952
953 if (WARN_ON_ONCE(tick_do_timer_cpu == TICK_DO_TIMER_NONE))
954 return false;
955 }
956
957 return true;
958}
959
960static void __tick_nohz_idle_stop_tick(struct tick_sched *ts)
961{
962 ktime_t expires;
963 int cpu = smp_processor_id();
964
965
966
967
968
969 if (ts->timer_expires_base)
970 expires = ts->timer_expires;
971 else if (can_stop_idle_tick(cpu, ts))
972 expires = tick_nohz_next_event(ts, cpu);
973 else
974 return;
975
976 ts->idle_calls++;
977
978 if (expires > 0LL) {
979 int was_stopped = ts->tick_stopped;
980
981 tick_nohz_stop_tick(ts, cpu);
982
983 ts->idle_sleeps++;
984 ts->idle_expires = expires;
985
986 if (!was_stopped && ts->tick_stopped) {
987 ts->idle_jiffies = ts->last_jiffies;
988 nohz_balance_enter_idle(cpu);
989 }
990 } else {
991 tick_nohz_retain_tick(ts);
992 }
993}
994
995
996
997
998
999
1000void tick_nohz_idle_stop_tick(void)
1001{
1002 __tick_nohz_idle_stop_tick(this_cpu_ptr(&tick_cpu_sched));
1003}
1004
1005void tick_nohz_idle_retain_tick(void)
1006{
1007 tick_nohz_retain_tick(this_cpu_ptr(&tick_cpu_sched));
1008
1009
1010
1011
1012 timer_clear_idle();
1013}
1014
1015
1016
1017
1018
1019
1020void tick_nohz_idle_enter(void)
1021{
1022 struct tick_sched *ts;
1023
1024 lockdep_assert_irqs_enabled();
1025
1026 local_irq_disable();
1027
1028 ts = this_cpu_ptr(&tick_cpu_sched);
1029
1030 WARN_ON_ONCE(ts->timer_expires_base);
1031
1032 ts->inidle = 1;
1033 tick_nohz_start_idle(ts);
1034
1035 local_irq_enable();
1036}
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046void tick_nohz_irq_exit(void)
1047{
1048 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1049
1050 if (ts->inidle)
1051 tick_nohz_start_idle(ts);
1052 else
1053 tick_nohz_full_update_tick(ts);
1054}
1055
1056
1057
1058
1059bool tick_nohz_idle_got_tick(void)
1060{
1061 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1062
1063 if (ts->got_idle_tick) {
1064 ts->got_idle_tick = 0;
1065 return true;
1066 }
1067 return false;
1068}
1069
1070
1071
1072
1073
1074
1075
1076
1077ktime_t tick_nohz_get_next_hrtimer(void)
1078{
1079 return __this_cpu_read(tick_cpu_device.evtdev)->next_event;
1080}
1081
1082
1083
1084
1085
1086
1087
1088ktime_t tick_nohz_get_sleep_length(ktime_t *delta_next)
1089{
1090 struct clock_event_device *dev = __this_cpu_read(tick_cpu_device.evtdev);
1091 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1092 int cpu = smp_processor_id();
1093
1094
1095
1096
1097 ktime_t now = ts->idle_entrytime;
1098 ktime_t next_event;
1099
1100 WARN_ON_ONCE(!ts->inidle);
1101
1102 *delta_next = ktime_sub(dev->next_event, now);
1103
1104 if (!can_stop_idle_tick(cpu, ts))
1105 return *delta_next;
1106
1107 next_event = tick_nohz_next_event(ts, cpu);
1108 if (!next_event)
1109 return *delta_next;
1110
1111
1112
1113
1114
1115 next_event = min_t(u64, next_event,
1116 hrtimer_next_event_without(&ts->sched_timer));
1117
1118 return ktime_sub(next_event, now);
1119}
1120
1121
1122
1123
1124
1125
1126
1127unsigned long tick_nohz_get_idle_calls_cpu(int cpu)
1128{
1129 struct tick_sched *ts = tick_get_tick_sched(cpu);
1130
1131 return ts->idle_calls;
1132}
1133
1134
1135
1136
1137
1138
1139unsigned long tick_nohz_get_idle_calls(void)
1140{
1141 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1142
1143 return ts->idle_calls;
1144}
1145
1146static void tick_nohz_account_idle_ticks(struct tick_sched *ts)
1147{
1148#ifndef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE
1149 unsigned long ticks;
1150
1151 if (vtime_accounting_enabled_this_cpu())
1152 return;
1153
1154
1155
1156
1157
1158 ticks = jiffies - ts->idle_jiffies;
1159
1160
1161
1162 if (ticks && ticks < LONG_MAX)
1163 account_idle_ticks(ticks);
1164#endif
1165}
1166
1167static void __tick_nohz_idle_restart_tick(struct tick_sched *ts, ktime_t now)
1168{
1169 tick_nohz_restart_sched_tick(ts, now);
1170 tick_nohz_account_idle_ticks(ts);
1171}
1172
1173void tick_nohz_idle_restart_tick(void)
1174{
1175 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1176
1177 if (ts->tick_stopped)
1178 __tick_nohz_idle_restart_tick(ts, ktime_get());
1179}
1180
1181
1182
1183
1184
1185
1186
1187
1188void tick_nohz_idle_exit(void)
1189{
1190 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1191 bool idle_active, tick_stopped;
1192 ktime_t now;
1193
1194 local_irq_disable();
1195
1196 WARN_ON_ONCE(!ts->inidle);
1197 WARN_ON_ONCE(ts->timer_expires_base);
1198
1199 ts->inidle = 0;
1200 idle_active = ts->idle_active;
1201 tick_stopped = ts->tick_stopped;
1202
1203 if (idle_active || tick_stopped)
1204 now = ktime_get();
1205
1206 if (idle_active)
1207 tick_nohz_stop_idle(ts, now);
1208
1209 if (tick_stopped)
1210 __tick_nohz_idle_restart_tick(ts, now);
1211
1212 local_irq_enable();
1213}
1214
1215
1216
1217
1218static void tick_nohz_handler(struct clock_event_device *dev)
1219{
1220 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1221 struct pt_regs *regs = get_irq_regs();
1222 ktime_t now = ktime_get();
1223
1224 dev->next_event = KTIME_MAX;
1225
1226 tick_sched_do_timer(ts, now);
1227 tick_sched_handle(ts, regs);
1228
1229
1230 if (unlikely(ts->tick_stopped))
1231 return;
1232
1233 hrtimer_forward(&ts->sched_timer, now, tick_period);
1234 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1235}
1236
1237static inline void tick_nohz_activate(struct tick_sched *ts, int mode)
1238{
1239 if (!tick_nohz_enabled)
1240 return;
1241 ts->nohz_mode = mode;
1242
1243 if (!test_and_set_bit(0, &tick_nohz_active))
1244 timers_update_nohz();
1245}
1246
1247
1248
1249
1250static void tick_nohz_switch_to_nohz(void)
1251{
1252 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1253 ktime_t next;
1254
1255 if (!tick_nohz_enabled)
1256 return;
1257
1258 if (tick_switch_to_oneshot(tick_nohz_handler))
1259 return;
1260
1261
1262
1263
1264
1265 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1266
1267 next = tick_init_jiffy_update();
1268
1269 hrtimer_set_expires(&ts->sched_timer, next);
1270 hrtimer_forward_now(&ts->sched_timer, tick_period);
1271 tick_program_event(hrtimer_get_expires(&ts->sched_timer), 1);
1272 tick_nohz_activate(ts, NOHZ_MODE_LOWRES);
1273}
1274
1275static inline void tick_nohz_irq_enter(void)
1276{
1277 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1278 ktime_t now;
1279
1280 if (!ts->idle_active && !ts->tick_stopped)
1281 return;
1282 now = ktime_get();
1283 if (ts->idle_active)
1284 tick_nohz_stop_idle(ts, now);
1285 if (ts->tick_stopped)
1286 tick_nohz_update_jiffies(now);
1287}
1288
1289#else
1290
1291static inline void tick_nohz_switch_to_nohz(void) { }
1292static inline void tick_nohz_irq_enter(void) { }
1293static inline void tick_nohz_activate(struct tick_sched *ts, int mode) { }
1294
1295#endif
1296
1297
1298
1299
1300void tick_irq_enter(void)
1301{
1302 tick_check_oneshot_broadcast_this_cpu();
1303 tick_nohz_irq_enter();
1304}
1305
1306
1307
1308
1309#ifdef CONFIG_HIGH_RES_TIMERS
1310
1311
1312
1313
1314static enum hrtimer_restart tick_sched_timer(struct hrtimer *timer)
1315{
1316 struct tick_sched *ts =
1317 container_of(timer, struct tick_sched, sched_timer);
1318 struct pt_regs *regs = get_irq_regs();
1319 ktime_t now = ktime_get();
1320
1321 tick_sched_do_timer(ts, now);
1322
1323
1324
1325
1326
1327 if (regs)
1328 tick_sched_handle(ts, regs);
1329 else
1330 ts->next_tick = 0;
1331
1332
1333 if (unlikely(ts->tick_stopped))
1334 return HRTIMER_NORESTART;
1335
1336 hrtimer_forward(timer, now, tick_period);
1337
1338 return HRTIMER_RESTART;
1339}
1340
1341static int sched_skew_tick;
1342
1343static int __init skew_tick(char *str)
1344{
1345 get_option(&str, &sched_skew_tick);
1346
1347 return 0;
1348}
1349early_param("skew_tick", skew_tick);
1350
1351
1352
1353
1354void tick_setup_sched_timer(void)
1355{
1356 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1357 ktime_t now = ktime_get();
1358
1359
1360
1361
1362 hrtimer_init(&ts->sched_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_HARD);
1363 ts->sched_timer.function = tick_sched_timer;
1364
1365
1366 hrtimer_set_expires(&ts->sched_timer, tick_init_jiffy_update());
1367
1368
1369 if (sched_skew_tick) {
1370 u64 offset = ktime_to_ns(tick_period) >> 1;
1371 do_div(offset, num_possible_cpus());
1372 offset *= smp_processor_id();
1373 hrtimer_add_expires_ns(&ts->sched_timer, offset);
1374 }
1375
1376 hrtimer_forward(&ts->sched_timer, now, tick_period);
1377 hrtimer_start_expires(&ts->sched_timer, HRTIMER_MODE_ABS_PINNED_HARD);
1378 tick_nohz_activate(ts, NOHZ_MODE_HIGHRES);
1379}
1380#endif
1381
1382#if defined CONFIG_NO_HZ_COMMON || defined CONFIG_HIGH_RES_TIMERS
1383void tick_cancel_sched_timer(int cpu)
1384{
1385 struct tick_sched *ts = &per_cpu(tick_cpu_sched, cpu);
1386
1387# ifdef CONFIG_HIGH_RES_TIMERS
1388 if (ts->sched_timer.base)
1389 hrtimer_cancel(&ts->sched_timer);
1390# endif
1391
1392 memset(ts, 0, sizeof(*ts));
1393}
1394#endif
1395
1396
1397
1398
1399void tick_clock_notify(void)
1400{
1401 int cpu;
1402
1403 for_each_possible_cpu(cpu)
1404 set_bit(0, &per_cpu(tick_cpu_sched, cpu).check_clocks);
1405}
1406
1407
1408
1409
1410void tick_oneshot_notify(void)
1411{
1412 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1413
1414 set_bit(0, &ts->check_clocks);
1415}
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425int tick_check_oneshot_change(int allow_nohz)
1426{
1427 struct tick_sched *ts = this_cpu_ptr(&tick_cpu_sched);
1428
1429 if (!test_and_clear_bit(0, &ts->check_clocks))
1430 return 0;
1431
1432 if (ts->nohz_mode != NOHZ_MODE_INACTIVE)
1433 return 0;
1434
1435 if (!timekeeping_valid_for_hres() || !tick_is_oneshot_available())
1436 return 0;
1437
1438 if (!allow_nohz)
1439 return 1;
1440
1441 tick_nohz_switch_to_nohz();
1442 return 0;
1443}
1444