1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34#include <linux/cpu.h>
35#include <linux/export.h>
36#include <linux/percpu.h>
37#include <linux/hrtimer.h>
38#include <linux/notifier.h>
39#include <linux/syscalls.h>
40#include <linux/interrupt.h>
41#include <linux/tick.h>
42#include <linux/seq_file.h>
43#include <linux/err.h>
44#include <linux/debugobjects.h>
45#include <linux/sched/signal.h>
46#include <linux/sched/sysctl.h>
47#include <linux/sched/rt.h>
48#include <linux/sched/deadline.h>
49#include <linux/sched/nohz.h>
50#include <linux/sched/debug.h>
51#include <linux/timer.h>
52#include <linux/freezer.h>
53#include <linux/compat.h>
54
55#include <linux/uaccess.h>
56
57#include <trace/events/timer.h>
58
59#include "tick-internal.h"
60
61
62
63
64
65#define MASK_SHIFT (HRTIMER_BASE_MONOTONIC_SOFT)
66#define HRTIMER_ACTIVE_HARD ((1U << MASK_SHIFT) - 1)
67#define HRTIMER_ACTIVE_SOFT (HRTIMER_ACTIVE_HARD << MASK_SHIFT)
68#define HRTIMER_ACTIVE_ALL (HRTIMER_ACTIVE_SOFT | HRTIMER_ACTIVE_HARD)
69
70
71
72
73
74
75
76
77
78DEFINE_PER_CPU(struct hrtimer_cpu_base, hrtimer_bases) =
79{
80 .lock = __RAW_SPIN_LOCK_UNLOCKED(hrtimer_bases.lock),
81 .clock_base =
82 {
83 {
84 .index = HRTIMER_BASE_MONOTONIC,
85 .clockid = CLOCK_MONOTONIC,
86 .get_time = &ktime_get,
87 },
88 {
89 .index = HRTIMER_BASE_REALTIME,
90 .clockid = CLOCK_REALTIME,
91 .get_time = &ktime_get_real,
92 },
93 {
94 .index = HRTIMER_BASE_BOOTTIME,
95 .clockid = CLOCK_BOOTTIME,
96 .get_time = &ktime_get_boottime,
97 },
98 {
99 .index = HRTIMER_BASE_TAI,
100 .clockid = CLOCK_TAI,
101 .get_time = &ktime_get_clocktai,
102 },
103 {
104 .index = HRTIMER_BASE_MONOTONIC_SOFT,
105 .clockid = CLOCK_MONOTONIC,
106 .get_time = &ktime_get,
107 },
108 {
109 .index = HRTIMER_BASE_REALTIME_SOFT,
110 .clockid = CLOCK_REALTIME,
111 .get_time = &ktime_get_real,
112 },
113 {
114 .index = HRTIMER_BASE_BOOTTIME_SOFT,
115 .clockid = CLOCK_BOOTTIME,
116 .get_time = &ktime_get_boottime,
117 },
118 {
119 .index = HRTIMER_BASE_TAI_SOFT,
120 .clockid = CLOCK_TAI,
121 .get_time = &ktime_get_clocktai,
122 },
123 }
124};
125
126static const int hrtimer_clock_to_base_table[MAX_CLOCKS] = {
127
128 [0 ... MAX_CLOCKS - 1] = HRTIMER_MAX_CLOCK_BASES,
129
130 [CLOCK_REALTIME] = HRTIMER_BASE_REALTIME,
131 [CLOCK_MONOTONIC] = HRTIMER_BASE_MONOTONIC,
132 [CLOCK_BOOTTIME] = HRTIMER_BASE_BOOTTIME,
133 [CLOCK_TAI] = HRTIMER_BASE_TAI,
134};
135
136
137
138
139
140#ifdef CONFIG_SMP
141
142
143
144
145
146
147static struct hrtimer_cpu_base migration_cpu_base = {
148 .clock_base = { { .cpu_base = &migration_cpu_base, }, },
149};
150
151#define migration_base migration_cpu_base.clock_base[0]
152
153
154
155
156
157
158
159
160
161
162
163
164
165static
166struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer,
167 unsigned long *flags)
168{
169 struct hrtimer_clock_base *base;
170
171 for (;;) {
172 base = timer->base;
173 if (likely(base != &migration_base)) {
174 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
175 if (likely(base == timer->base))
176 return base;
177
178 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags);
179 }
180 cpu_relax();
181 }
182}
183
184
185
186
187
188
189
190
191
192
193static int
194hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base)
195{
196 ktime_t expires;
197
198 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset);
199 return expires < new_base->cpu_base->expires_next;
200}
201
202static inline
203struct hrtimer_cpu_base *get_target_base(struct hrtimer_cpu_base *base,
204 int pinned)
205{
206#if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
207 if (static_branch_likely(&timers_migration_enabled) && !pinned)
208 return &per_cpu(hrtimer_bases, get_nohz_timer_target());
209#endif
210 return base;
211}
212
213
214
215
216
217
218
219
220
221
222
223
224
225static inline struct hrtimer_clock_base *
226switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base,
227 int pinned)
228{
229 struct hrtimer_cpu_base *new_cpu_base, *this_cpu_base;
230 struct hrtimer_clock_base *new_base;
231 int basenum = base->index;
232
233 this_cpu_base = this_cpu_ptr(&hrtimer_bases);
234 new_cpu_base = get_target_base(this_cpu_base, pinned);
235again:
236 new_base = &new_cpu_base->clock_base[basenum];
237
238 if (base != new_base) {
239
240
241
242
243
244
245
246
247
248 if (unlikely(hrtimer_callback_running(timer)))
249 return base;
250
251
252 timer->base = &migration_base;
253 raw_spin_unlock(&base->cpu_base->lock);
254 raw_spin_lock(&new_base->cpu_base->lock);
255
256 if (new_cpu_base != this_cpu_base &&
257 hrtimer_check_target(timer, new_base)) {
258 raw_spin_unlock(&new_base->cpu_base->lock);
259 raw_spin_lock(&base->cpu_base->lock);
260 new_cpu_base = this_cpu_base;
261 timer->base = base;
262 goto again;
263 }
264 timer->base = new_base;
265 } else {
266 if (new_cpu_base != this_cpu_base &&
267 hrtimer_check_target(timer, new_base)) {
268 new_cpu_base = this_cpu_base;
269 goto again;
270 }
271 }
272 return new_base;
273}
274
275#else
276
277static inline struct hrtimer_clock_base *
278lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
279{
280 struct hrtimer_clock_base *base = timer->base;
281
282 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags);
283
284 return base;
285}
286
287# define switch_hrtimer_base(t, b, p) (b)
288
289#endif
290
291
292
293
294
295#if BITS_PER_LONG < 64
296
297
298
299s64 __ktime_divns(const ktime_t kt, s64 div)
300{
301 int sft = 0;
302 s64 dclc;
303 u64 tmp;
304
305 dclc = ktime_to_ns(kt);
306 tmp = dclc < 0 ? -dclc : dclc;
307
308
309 while (div >> 32) {
310 sft++;
311 div >>= 1;
312 }
313 tmp >>= sft;
314 do_div(tmp, (unsigned long) div);
315 return dclc < 0 ? -tmp : tmp;
316}
317EXPORT_SYMBOL_GPL(__ktime_divns);
318#endif
319
320
321
322
323ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs)
324{
325 ktime_t res = ktime_add_unsafe(lhs, rhs);
326
327
328
329
330
331 if (res < 0 || res < lhs || res < rhs)
332 res = ktime_set(KTIME_SEC_MAX, 0);
333
334 return res;
335}
336
337EXPORT_SYMBOL_GPL(ktime_add_safe);
338
339#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
340
341static struct debug_obj_descr hrtimer_debug_descr;
342
343static void *hrtimer_debug_hint(void *addr)
344{
345 return ((struct hrtimer *) addr)->function;
346}
347
348
349
350
351
352static bool hrtimer_fixup_init(void *addr, enum debug_obj_state state)
353{
354 struct hrtimer *timer = addr;
355
356 switch (state) {
357 case ODEBUG_STATE_ACTIVE:
358 hrtimer_cancel(timer);
359 debug_object_init(timer, &hrtimer_debug_descr);
360 return true;
361 default:
362 return false;
363 }
364}
365
366
367
368
369
370
371static bool hrtimer_fixup_activate(void *addr, enum debug_obj_state state)
372{
373 switch (state) {
374 case ODEBUG_STATE_ACTIVE:
375 WARN_ON(1);
376
377 default:
378 return false;
379 }
380}
381
382
383
384
385
386static bool hrtimer_fixup_free(void *addr, enum debug_obj_state state)
387{
388 struct hrtimer *timer = addr;
389
390 switch (state) {
391 case ODEBUG_STATE_ACTIVE:
392 hrtimer_cancel(timer);
393 debug_object_free(timer, &hrtimer_debug_descr);
394 return true;
395 default:
396 return false;
397 }
398}
399
400static struct debug_obj_descr hrtimer_debug_descr = {
401 .name = "hrtimer",
402 .debug_hint = hrtimer_debug_hint,
403 .fixup_init = hrtimer_fixup_init,
404 .fixup_activate = hrtimer_fixup_activate,
405 .fixup_free = hrtimer_fixup_free,
406};
407
408static inline void debug_hrtimer_init(struct hrtimer *timer)
409{
410 debug_object_init(timer, &hrtimer_debug_descr);
411}
412
413static inline void debug_hrtimer_activate(struct hrtimer *timer,
414 enum hrtimer_mode mode)
415{
416 debug_object_activate(timer, &hrtimer_debug_descr);
417}
418
419static inline void debug_hrtimer_deactivate(struct hrtimer *timer)
420{
421 debug_object_deactivate(timer, &hrtimer_debug_descr);
422}
423
424static inline void debug_hrtimer_free(struct hrtimer *timer)
425{
426 debug_object_free(timer, &hrtimer_debug_descr);
427}
428
429static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
430 enum hrtimer_mode mode);
431
432void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id,
433 enum hrtimer_mode mode)
434{
435 debug_object_init_on_stack(timer, &hrtimer_debug_descr);
436 __hrtimer_init(timer, clock_id, mode);
437}
438EXPORT_SYMBOL_GPL(hrtimer_init_on_stack);
439
440void destroy_hrtimer_on_stack(struct hrtimer *timer)
441{
442 debug_object_free(timer, &hrtimer_debug_descr);
443}
444EXPORT_SYMBOL_GPL(destroy_hrtimer_on_stack);
445
446#else
447
448static inline void debug_hrtimer_init(struct hrtimer *timer) { }
449static inline void debug_hrtimer_activate(struct hrtimer *timer,
450 enum hrtimer_mode mode) { }
451static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { }
452#endif
453
454static inline void
455debug_init(struct hrtimer *timer, clockid_t clockid,
456 enum hrtimer_mode mode)
457{
458 debug_hrtimer_init(timer);
459 trace_hrtimer_init(timer, clockid, mode);
460}
461
462static inline void debug_activate(struct hrtimer *timer,
463 enum hrtimer_mode mode)
464{
465 debug_hrtimer_activate(timer, mode);
466 trace_hrtimer_start(timer, mode);
467}
468
469static inline void debug_deactivate(struct hrtimer *timer)
470{
471 debug_hrtimer_deactivate(timer);
472 trace_hrtimer_cancel(timer);
473}
474
475static struct hrtimer_clock_base *
476__next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active)
477{
478 unsigned int idx;
479
480 if (!*active)
481 return NULL;
482
483 idx = __ffs(*active);
484 *active &= ~(1U << idx);
485
486 return &cpu_base->clock_base[idx];
487}
488
489#define for_each_active_base(base, cpu_base, active) \
490 while ((base = __next_base((cpu_base), &(active))))
491
492static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base,
493 const struct hrtimer *exclude,
494 unsigned int active,
495 ktime_t expires_next)
496{
497 struct hrtimer_clock_base *base;
498 ktime_t expires;
499
500 for_each_active_base(base, cpu_base, active) {
501 struct timerqueue_node *next;
502 struct hrtimer *timer;
503
504 next = timerqueue_getnext(&base->active);
505 timer = container_of(next, struct hrtimer, node);
506 if (timer == exclude) {
507
508 next = timerqueue_iterate_next(next);
509 if (!next)
510 continue;
511
512 timer = container_of(next, struct hrtimer, node);
513 }
514 expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
515 if (expires < expires_next) {
516 expires_next = expires;
517
518
519 if (exclude)
520 continue;
521
522 if (timer->is_soft)
523 cpu_base->softirq_next_timer = timer;
524 else
525 cpu_base->next_timer = timer;
526 }
527 }
528
529
530
531
532
533 if (expires_next < 0)
534 expires_next = 0;
535 return expires_next;
536}
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555static ktime_t
556__hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask)
557{
558 unsigned int active;
559 struct hrtimer *next_timer = NULL;
560 ktime_t expires_next = KTIME_MAX;
561
562 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) {
563 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
564 cpu_base->softirq_next_timer = NULL;
565 expires_next = __hrtimer_next_event_base(cpu_base, NULL,
566 active, KTIME_MAX);
567
568 next_timer = cpu_base->softirq_next_timer;
569 }
570
571 if (active_mask & HRTIMER_ACTIVE_HARD) {
572 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
573 cpu_base->next_timer = next_timer;
574 expires_next = __hrtimer_next_event_base(cpu_base, NULL, active,
575 expires_next);
576 }
577
578 return expires_next;
579}
580
581static inline ktime_t hrtimer_update_base(struct hrtimer_cpu_base *base)
582{
583 ktime_t *offs_real = &base->clock_base[HRTIMER_BASE_REALTIME].offset;
584 ktime_t *offs_boot = &base->clock_base[HRTIMER_BASE_BOOTTIME].offset;
585 ktime_t *offs_tai = &base->clock_base[HRTIMER_BASE_TAI].offset;
586
587 ktime_t now = ktime_get_update_offsets_now(&base->clock_was_set_seq,
588 offs_real, offs_boot, offs_tai);
589
590 base->clock_base[HRTIMER_BASE_REALTIME_SOFT].offset = *offs_real;
591 base->clock_base[HRTIMER_BASE_BOOTTIME_SOFT].offset = *offs_boot;
592 base->clock_base[HRTIMER_BASE_TAI_SOFT].offset = *offs_tai;
593
594 return now;
595}
596
597
598
599
600static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base)
601{
602 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
603 cpu_base->hres_active : 0;
604}
605
606static inline int hrtimer_hres_active(void)
607{
608 return __hrtimer_hres_active(this_cpu_ptr(&hrtimer_bases));
609}
610
611
612
613
614
615
616static void
617hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal)
618{
619 ktime_t expires_next;
620
621
622
623
624 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
625
626 if (cpu_base->next_timer && cpu_base->next_timer->is_soft) {
627
628
629
630
631
632 if (cpu_base->softirq_activated)
633 expires_next = __hrtimer_get_next_event(cpu_base,
634 HRTIMER_ACTIVE_HARD);
635 else
636 cpu_base->softirq_expires_next = expires_next;
637 }
638
639 if (skip_equal && expires_next == cpu_base->expires_next)
640 return;
641
642 cpu_base->expires_next = expires_next;
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661 if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
662 return;
663
664 tick_program_event(cpu_base->expires_next, 1);
665}
666
667
668#ifdef CONFIG_HIGH_RES_TIMERS
669
670
671
672
673static bool hrtimer_hres_enabled __read_mostly = true;
674unsigned int hrtimer_resolution __read_mostly = LOW_RES_NSEC;
675EXPORT_SYMBOL_GPL(hrtimer_resolution);
676
677
678
679
680static int __init setup_hrtimer_hres(char *str)
681{
682 return (kstrtobool(str, &hrtimer_hres_enabled) == 0);
683}
684
685__setup("highres=", setup_hrtimer_hres);
686
687
688
689
690static inline int hrtimer_is_hres_enabled(void)
691{
692 return hrtimer_hres_enabled;
693}
694
695
696
697
698
699
700static void retrigger_next_event(void *arg)
701{
702 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
703
704 if (!__hrtimer_hres_active(base))
705 return;
706
707 raw_spin_lock(&base->lock);
708 hrtimer_update_base(base);
709 hrtimer_force_reprogram(base, 0);
710 raw_spin_unlock(&base->lock);
711}
712
713
714
715
716static void hrtimer_switch_to_hres(void)
717{
718 struct hrtimer_cpu_base *base = this_cpu_ptr(&hrtimer_bases);
719
720 if (tick_init_highres()) {
721 printk(KERN_WARNING "Could not switch to high resolution "
722 "mode on CPU %d\n", base->cpu);
723 return;
724 }
725 base->hres_active = 1;
726 hrtimer_resolution = HIGH_RES_NSEC;
727
728 tick_setup_sched_timer();
729
730 retrigger_next_event(NULL);
731}
732
733static void clock_was_set_work(struct work_struct *work)
734{
735 clock_was_set();
736}
737
738static DECLARE_WORK(hrtimer_work, clock_was_set_work);
739
740
741
742
743
744void clock_was_set_delayed(void)
745{
746 schedule_work(&hrtimer_work);
747}
748
749#else
750
751static inline int hrtimer_is_hres_enabled(void) { return 0; }
752static inline void hrtimer_switch_to_hres(void) { }
753static inline void retrigger_next_event(void *arg) { }
754
755#endif
756
757
758
759
760
761
762
763
764static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram)
765{
766 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
767 struct hrtimer_clock_base *base = timer->base;
768 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset);
769
770 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0);
771
772
773
774
775
776 if (expires < 0)
777 expires = 0;
778
779 if (timer->is_soft) {
780
781
782
783
784
785
786
787 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base;
788
789 if (timer_cpu_base->softirq_activated)
790 return;
791
792 if (!ktime_before(expires, timer_cpu_base->softirq_expires_next))
793 return;
794
795 timer_cpu_base->softirq_next_timer = timer;
796 timer_cpu_base->softirq_expires_next = expires;
797
798 if (!ktime_before(expires, timer_cpu_base->expires_next) ||
799 !reprogram)
800 return;
801 }
802
803
804
805
806
807 if (base->cpu_base != cpu_base)
808 return;
809
810
811
812
813
814
815
816
817 if (cpu_base->in_hrtirq)
818 return;
819
820 if (expires >= cpu_base->expires_next)
821 return;
822
823
824 cpu_base->next_timer = timer;
825 cpu_base->expires_next = expires;
826
827
828
829
830
831
832
833
834
835
836 if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected)
837 return;
838
839
840
841
842
843 tick_program_event(expires, 1);
844}
845
846
847
848
849
850
851
852
853
854
855
856
857void clock_was_set(void)
858{
859#ifdef CONFIG_HIGH_RES_TIMERS
860
861 on_each_cpu(retrigger_next_event, NULL, 1);
862#endif
863 timerfd_clock_was_set();
864}
865
866
867
868
869
870
871
872void hrtimers_resume(void)
873{
874 lockdep_assert_irqs_disabled();
875
876 retrigger_next_event(NULL);
877
878 clock_was_set_delayed();
879}
880
881
882
883
884static inline
885void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags)
886{
887 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags);
888}
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval)
908{
909 u64 orun = 1;
910 ktime_t delta;
911
912 delta = ktime_sub(now, hrtimer_get_expires(timer));
913
914 if (delta < 0)
915 return 0;
916
917 if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED))
918 return 0;
919
920 if (interval < hrtimer_resolution)
921 interval = hrtimer_resolution;
922
923 if (unlikely(delta >= interval)) {
924 s64 incr = ktime_to_ns(interval);
925
926 orun = ktime_divns(delta, incr);
927 hrtimer_add_expires_ns(timer, incr * orun);
928 if (hrtimer_get_expires_tv64(timer) > now)
929 return orun;
930
931
932
933
934 orun++;
935 }
936 hrtimer_add_expires(timer, interval);
937
938 return orun;
939}
940EXPORT_SYMBOL_GPL(hrtimer_forward);
941
942
943
944
945
946
947
948
949
950static int enqueue_hrtimer(struct hrtimer *timer,
951 struct hrtimer_clock_base *base,
952 enum hrtimer_mode mode)
953{
954 debug_activate(timer, mode);
955
956 base->cpu_base->active_bases |= 1 << base->index;
957
958 timer->state = HRTIMER_STATE_ENQUEUED;
959
960 return timerqueue_add(&base->active, &timer->node);
961}
962
963
964
965
966
967
968
969
970
971
972
973static void __remove_hrtimer(struct hrtimer *timer,
974 struct hrtimer_clock_base *base,
975 u8 newstate, int reprogram)
976{
977 struct hrtimer_cpu_base *cpu_base = base->cpu_base;
978 u8 state = timer->state;
979
980 timer->state = newstate;
981 if (!(state & HRTIMER_STATE_ENQUEUED))
982 return;
983
984 if (!timerqueue_del(&base->active, &timer->node))
985 cpu_base->active_bases &= ~(1 << base->index);
986
987
988
989
990
991
992
993
994
995 if (reprogram && timer == cpu_base->next_timer)
996 hrtimer_force_reprogram(cpu_base, 1);
997}
998
999
1000
1001
1002static inline int
1003remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, bool restart)
1004{
1005 if (hrtimer_is_queued(timer)) {
1006 u8 state = timer->state;
1007 int reprogram;
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017 debug_deactivate(timer);
1018 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases);
1019
1020 if (!restart)
1021 state = HRTIMER_STATE_INACTIVE;
1022
1023 __remove_hrtimer(timer, base, state, reprogram);
1024 return 1;
1025 }
1026 return 0;
1027}
1028
1029static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim,
1030 const enum hrtimer_mode mode)
1031{
1032#ifdef CONFIG_TIME_LOW_RES
1033
1034
1035
1036
1037
1038 timer->is_rel = mode & HRTIMER_MODE_REL;
1039 if (timer->is_rel)
1040 tim = ktime_add_safe(tim, hrtimer_resolution);
1041#endif
1042 return tim;
1043}
1044
1045static void
1046hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram)
1047{
1048 ktime_t expires;
1049
1050
1051
1052
1053 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT);
1054
1055
1056
1057
1058
1059
1060 if (expires == KTIME_MAX)
1061 return;
1062
1063
1064
1065
1066
1067 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram);
1068}
1069
1070static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1071 u64 delta_ns, const enum hrtimer_mode mode,
1072 struct hrtimer_clock_base *base)
1073{
1074 struct hrtimer_clock_base *new_base;
1075
1076
1077 remove_hrtimer(timer, base, true);
1078
1079 if (mode & HRTIMER_MODE_REL)
1080 tim = ktime_add_safe(tim, base->get_time());
1081
1082 tim = hrtimer_update_lowres(timer, tim, mode);
1083
1084 hrtimer_set_expires_range_ns(timer, tim, delta_ns);
1085
1086
1087 new_base = switch_hrtimer_base(timer, base, mode & HRTIMER_MODE_PINNED);
1088
1089 return enqueue_hrtimer(timer, new_base, mode);
1090}
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
1102 u64 delta_ns, const enum hrtimer_mode mode)
1103{
1104 struct hrtimer_clock_base *base;
1105 unsigned long flags;
1106
1107
1108
1109
1110
1111 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft);
1112
1113 base = lock_hrtimer_base(timer, &flags);
1114
1115 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base))
1116 hrtimer_reprogram(timer, true);
1117
1118 unlock_hrtimer_base(timer, &flags);
1119}
1120EXPORT_SYMBOL_GPL(hrtimer_start_range_ns);
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132int hrtimer_try_to_cancel(struct hrtimer *timer)
1133{
1134 struct hrtimer_clock_base *base;
1135 unsigned long flags;
1136 int ret = -1;
1137
1138
1139
1140
1141
1142
1143
1144 if (!hrtimer_active(timer))
1145 return 0;
1146
1147 base = lock_hrtimer_base(timer, &flags);
1148
1149 if (!hrtimer_callback_running(timer))
1150 ret = remove_hrtimer(timer, base, false);
1151
1152 unlock_hrtimer_base(timer, &flags);
1153
1154 return ret;
1155
1156}
1157EXPORT_SYMBOL_GPL(hrtimer_try_to_cancel);
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167int hrtimer_cancel(struct hrtimer *timer)
1168{
1169 for (;;) {
1170 int ret = hrtimer_try_to_cancel(timer);
1171
1172 if (ret >= 0)
1173 return ret;
1174 cpu_relax();
1175 }
1176}
1177EXPORT_SYMBOL_GPL(hrtimer_cancel);
1178
1179
1180
1181
1182
1183
1184ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust)
1185{
1186 unsigned long flags;
1187 ktime_t rem;
1188
1189 lock_hrtimer_base(timer, &flags);
1190 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && adjust)
1191 rem = hrtimer_expires_remaining_adjusted(timer);
1192 else
1193 rem = hrtimer_expires_remaining(timer);
1194 unlock_hrtimer_base(timer, &flags);
1195
1196 return rem;
1197}
1198EXPORT_SYMBOL_GPL(__hrtimer_get_remaining);
1199
1200#ifdef CONFIG_NO_HZ_COMMON
1201
1202
1203
1204
1205
1206u64 hrtimer_get_next_event(void)
1207{
1208 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1209 u64 expires = KTIME_MAX;
1210 unsigned long flags;
1211
1212 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1213
1214 if (!__hrtimer_hres_active(cpu_base))
1215 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1216
1217 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1218
1219 return expires;
1220}
1221
1222
1223
1224
1225
1226
1227
1228
1229u64 hrtimer_next_event_without(const struct hrtimer *exclude)
1230{
1231 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1232 u64 expires = KTIME_MAX;
1233 unsigned long flags;
1234
1235 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1236
1237 if (__hrtimer_hres_active(cpu_base)) {
1238 unsigned int active;
1239
1240 if (!cpu_base->softirq_activated) {
1241 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT;
1242 expires = __hrtimer_next_event_base(cpu_base, exclude,
1243 active, KTIME_MAX);
1244 }
1245 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD;
1246 expires = __hrtimer_next_event_base(cpu_base, exclude, active,
1247 expires);
1248 }
1249
1250 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1251
1252 return expires;
1253}
1254#endif
1255
1256static inline int hrtimer_clockid_to_base(clockid_t clock_id)
1257{
1258 if (likely(clock_id < MAX_CLOCKS)) {
1259 int base = hrtimer_clock_to_base_table[clock_id];
1260
1261 if (likely(base != HRTIMER_MAX_CLOCK_BASES))
1262 return base;
1263 }
1264 WARN(1, "Invalid clockid %d. Using MONOTONIC\n", clock_id);
1265 return HRTIMER_BASE_MONOTONIC;
1266}
1267
1268static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1269 enum hrtimer_mode mode)
1270{
1271 bool softtimer = !!(mode & HRTIMER_MODE_SOFT);
1272 int base = softtimer ? HRTIMER_MAX_CLOCK_BASES / 2 : 0;
1273 struct hrtimer_cpu_base *cpu_base;
1274
1275 memset(timer, 0, sizeof(struct hrtimer));
1276
1277 cpu_base = raw_cpu_ptr(&hrtimer_bases);
1278
1279
1280
1281
1282
1283
1284 if (clock_id == CLOCK_REALTIME && mode & HRTIMER_MODE_REL)
1285 clock_id = CLOCK_MONOTONIC;
1286
1287 base += hrtimer_clockid_to_base(clock_id);
1288 timer->is_soft = softtimer;
1289 timer->base = &cpu_base->clock_base[base];
1290 timerqueue_init(&timer->node);
1291}
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305void hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
1306 enum hrtimer_mode mode)
1307{
1308 debug_init(timer, clock_id, mode);
1309 __hrtimer_init(timer, clock_id, mode);
1310}
1311EXPORT_SYMBOL_GPL(hrtimer_init);
1312
1313
1314
1315
1316
1317
1318
1319
1320bool hrtimer_active(const struct hrtimer *timer)
1321{
1322 struct hrtimer_clock_base *base;
1323 unsigned int seq;
1324
1325 do {
1326 base = READ_ONCE(timer->base);
1327 seq = raw_read_seqcount_begin(&base->seq);
1328
1329 if (timer->state != HRTIMER_STATE_INACTIVE ||
1330 base->running == timer)
1331 return true;
1332
1333 } while (read_seqcount_retry(&base->seq, seq) ||
1334 base != READ_ONCE(timer->base));
1335
1336 return false;
1337}
1338EXPORT_SYMBOL_GPL(hrtimer_active);
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base,
1359 struct hrtimer_clock_base *base,
1360 struct hrtimer *timer, ktime_t *now,
1361 unsigned long flags)
1362{
1363 enum hrtimer_restart (*fn)(struct hrtimer *);
1364 int restart;
1365
1366 lockdep_assert_held(&cpu_base->lock);
1367
1368 debug_deactivate(timer);
1369 base->running = timer;
1370
1371
1372
1373
1374
1375
1376
1377
1378 raw_write_seqcount_barrier(&base->seq);
1379
1380 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0);
1381 fn = timer->function;
1382
1383
1384
1385
1386
1387
1388 if (IS_ENABLED(CONFIG_TIME_LOW_RES))
1389 timer->is_rel = false;
1390
1391
1392
1393
1394
1395
1396 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1397 trace_hrtimer_expire_entry(timer, now);
1398 restart = fn(timer);
1399 trace_hrtimer_expire_exit(timer);
1400 raw_spin_lock_irq(&cpu_base->lock);
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411 if (restart != HRTIMER_NORESTART &&
1412 !(timer->state & HRTIMER_STATE_ENQUEUED))
1413 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS);
1414
1415
1416
1417
1418
1419
1420
1421
1422 raw_write_seqcount_barrier(&base->seq);
1423
1424 WARN_ON_ONCE(base->running != timer);
1425 base->running = NULL;
1426}
1427
1428static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now,
1429 unsigned long flags, unsigned int active_mask)
1430{
1431 struct hrtimer_clock_base *base;
1432 unsigned int active = cpu_base->active_bases & active_mask;
1433
1434 for_each_active_base(base, cpu_base, active) {
1435 struct timerqueue_node *node;
1436 ktime_t basenow;
1437
1438 basenow = ktime_add(now, base->offset);
1439
1440 while ((node = timerqueue_getnext(&base->active))) {
1441 struct hrtimer *timer;
1442
1443 timer = container_of(node, struct hrtimer, node);
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457 if (basenow < hrtimer_get_softexpires_tv64(timer))
1458 break;
1459
1460 __run_hrtimer(cpu_base, base, timer, &basenow, flags);
1461 }
1462 }
1463}
1464
1465static __latent_entropy void hrtimer_run_softirq(struct softirq_action *h)
1466{
1467 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1468 unsigned long flags;
1469 ktime_t now;
1470
1471 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1472
1473 now = hrtimer_update_base(cpu_base);
1474 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT);
1475
1476 cpu_base->softirq_activated = 0;
1477 hrtimer_update_softirq_timer(cpu_base, true);
1478
1479 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1480}
1481
1482#ifdef CONFIG_HIGH_RES_TIMERS
1483
1484
1485
1486
1487
1488void hrtimer_interrupt(struct clock_event_device *dev)
1489{
1490 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1491 ktime_t expires_next, now, entry_time, delta;
1492 unsigned long flags;
1493 int retries = 0;
1494
1495 BUG_ON(!cpu_base->hres_active);
1496 cpu_base->nr_events++;
1497 dev->next_event = KTIME_MAX;
1498
1499 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1500 entry_time = now = hrtimer_update_base(cpu_base);
1501retry:
1502 cpu_base->in_hrtirq = 1;
1503
1504
1505
1506
1507
1508
1509
1510 cpu_base->expires_next = KTIME_MAX;
1511
1512 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1513 cpu_base->softirq_expires_next = KTIME_MAX;
1514 cpu_base->softirq_activated = 1;
1515 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1516 }
1517
1518 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1519
1520
1521 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL);
1522
1523
1524
1525
1526 cpu_base->expires_next = expires_next;
1527 cpu_base->in_hrtirq = 0;
1528 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1529
1530
1531 if (!tick_program_event(expires_next, 0)) {
1532 cpu_base->hang_detected = 0;
1533 return;
1534 }
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1550 now = hrtimer_update_base(cpu_base);
1551 cpu_base->nr_retries++;
1552 if (++retries < 3)
1553 goto retry;
1554
1555
1556
1557
1558
1559
1560 cpu_base->nr_hangs++;
1561 cpu_base->hang_detected = 1;
1562 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1563
1564 delta = ktime_sub(now, entry_time);
1565 if ((unsigned int)delta > cpu_base->max_hang_time)
1566 cpu_base->max_hang_time = (unsigned int) delta;
1567
1568
1569
1570
1571 if (delta > 100 * NSEC_PER_MSEC)
1572 expires_next = ktime_add_ns(now, 100 * NSEC_PER_MSEC);
1573 else
1574 expires_next = ktime_add(now, delta);
1575 tick_program_event(expires_next, 1);
1576 printk_once(KERN_WARNING "hrtimer: interrupt took %llu ns\n",
1577 ktime_to_ns(delta));
1578}
1579
1580
1581static inline void __hrtimer_peek_ahead_timers(void)
1582{
1583 struct tick_device *td;
1584
1585 if (!hrtimer_hres_active())
1586 return;
1587
1588 td = this_cpu_ptr(&tick_cpu_device);
1589 if (td && td->evtdev)
1590 hrtimer_interrupt(td->evtdev);
1591}
1592
1593#else
1594
1595static inline void __hrtimer_peek_ahead_timers(void) { }
1596
1597#endif
1598
1599
1600
1601
1602void hrtimer_run_queues(void)
1603{
1604 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases);
1605 unsigned long flags;
1606 ktime_t now;
1607
1608 if (__hrtimer_hres_active(cpu_base))
1609 return;
1610
1611
1612
1613
1614
1615
1616
1617
1618 if (tick_check_oneshot_change(!hrtimer_is_hres_enabled())) {
1619 hrtimer_switch_to_hres();
1620 return;
1621 }
1622
1623 raw_spin_lock_irqsave(&cpu_base->lock, flags);
1624 now = hrtimer_update_base(cpu_base);
1625
1626 if (!ktime_before(now, cpu_base->softirq_expires_next)) {
1627 cpu_base->softirq_expires_next = KTIME_MAX;
1628 cpu_base->softirq_activated = 1;
1629 raise_softirq_irqoff(HRTIMER_SOFTIRQ);
1630 }
1631
1632 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD);
1633 raw_spin_unlock_irqrestore(&cpu_base->lock, flags);
1634}
1635
1636
1637
1638
1639static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer)
1640{
1641 struct hrtimer_sleeper *t =
1642 container_of(timer, struct hrtimer_sleeper, timer);
1643 struct task_struct *task = t->task;
1644
1645 t->task = NULL;
1646 if (task)
1647 wake_up_process(task);
1648
1649 return HRTIMER_NORESTART;
1650}
1651
1652void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *task)
1653{
1654 sl->timer.function = hrtimer_wakeup;
1655 sl->task = task;
1656}
1657EXPORT_SYMBOL_GPL(hrtimer_init_sleeper);
1658
1659int nanosleep_copyout(struct restart_block *restart, struct timespec64 *ts)
1660{
1661 switch(restart->nanosleep.type) {
1662#ifdef CONFIG_COMPAT
1663 case TT_COMPAT:
1664 if (compat_put_timespec64(ts, restart->nanosleep.compat_rmtp))
1665 return -EFAULT;
1666 break;
1667#endif
1668 case TT_NATIVE:
1669 if (put_timespec64(ts, restart->nanosleep.rmtp))
1670 return -EFAULT;
1671 break;
1672 default:
1673 BUG();
1674 }
1675 return -ERESTART_RESTARTBLOCK;
1676}
1677
1678static int __sched do_nanosleep(struct hrtimer_sleeper *t, enum hrtimer_mode mode)
1679{
1680 struct restart_block *restart;
1681
1682 hrtimer_init_sleeper(t, current);
1683
1684 do {
1685 set_current_state(TASK_INTERRUPTIBLE);
1686 hrtimer_start_expires(&t->timer, mode);
1687
1688 if (likely(t->task))
1689 freezable_schedule();
1690
1691 hrtimer_cancel(&t->timer);
1692 mode = HRTIMER_MODE_ABS;
1693
1694 } while (t->task && !signal_pending(current));
1695
1696 __set_current_state(TASK_RUNNING);
1697
1698 if (!t->task)
1699 return 0;
1700
1701 restart = ¤t->restart_block;
1702 if (restart->nanosleep.type != TT_NONE) {
1703 ktime_t rem = hrtimer_expires_remaining(&t->timer);
1704 struct timespec64 rmt;
1705
1706 if (rem <= 0)
1707 return 0;
1708 rmt = ktime_to_timespec64(rem);
1709
1710 return nanosleep_copyout(restart, &rmt);
1711 }
1712 return -ERESTART_RESTARTBLOCK;
1713}
1714
1715static long __sched hrtimer_nanosleep_restart(struct restart_block *restart)
1716{
1717 struct hrtimer_sleeper t;
1718 int ret;
1719
1720 hrtimer_init_on_stack(&t.timer, restart->nanosleep.clockid,
1721 HRTIMER_MODE_ABS);
1722 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires);
1723
1724 ret = do_nanosleep(&t, HRTIMER_MODE_ABS);
1725 destroy_hrtimer_on_stack(&t.timer);
1726 return ret;
1727}
1728
1729long hrtimer_nanosleep(const struct timespec64 *rqtp,
1730 const enum hrtimer_mode mode, const clockid_t clockid)
1731{
1732 struct restart_block *restart;
1733 struct hrtimer_sleeper t;
1734 int ret = 0;
1735 u64 slack;
1736
1737 slack = current->timer_slack_ns;
1738 if (dl_task(current) || rt_task(current))
1739 slack = 0;
1740
1741 hrtimer_init_on_stack(&t.timer, clockid, mode);
1742 hrtimer_set_expires_range_ns(&t.timer, timespec64_to_ktime(*rqtp), slack);
1743 ret = do_nanosleep(&t, mode);
1744 if (ret != -ERESTART_RESTARTBLOCK)
1745 goto out;
1746
1747
1748 if (mode == HRTIMER_MODE_ABS) {
1749 ret = -ERESTARTNOHAND;
1750 goto out;
1751 }
1752
1753 restart = ¤t->restart_block;
1754 restart->fn = hrtimer_nanosleep_restart;
1755 restart->nanosleep.clockid = t.timer.base->clockid;
1756 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer);
1757out:
1758 destroy_hrtimer_on_stack(&t.timer);
1759 return ret;
1760}
1761
1762SYSCALL_DEFINE2(nanosleep, struct timespec __user *, rqtp,
1763 struct timespec __user *, rmtp)
1764{
1765 struct timespec64 tu;
1766
1767 if (get_timespec64(&tu, rqtp))
1768 return -EFAULT;
1769
1770 if (!timespec64_valid(&tu))
1771 return -EINVAL;
1772
1773 current->restart_block.nanosleep.type = rmtp ? TT_NATIVE : TT_NONE;
1774 current->restart_block.nanosleep.rmtp = rmtp;
1775 return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
1776}
1777
1778#ifdef CONFIG_COMPAT
1779
1780COMPAT_SYSCALL_DEFINE2(nanosleep, struct compat_timespec __user *, rqtp,
1781 struct compat_timespec __user *, rmtp)
1782{
1783 struct timespec64 tu;
1784
1785 if (compat_get_timespec64(&tu, rqtp))
1786 return -EFAULT;
1787
1788 if (!timespec64_valid(&tu))
1789 return -EINVAL;
1790
1791 current->restart_block.nanosleep.type = rmtp ? TT_COMPAT : TT_NONE;
1792 current->restart_block.nanosleep.compat_rmtp = rmtp;
1793 return hrtimer_nanosleep(&tu, HRTIMER_MODE_REL, CLOCK_MONOTONIC);
1794}
1795#endif
1796
1797
1798
1799
1800int hrtimers_prepare_cpu(unsigned int cpu)
1801{
1802 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu);
1803 int i;
1804
1805 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1806 cpu_base->clock_base[i].cpu_base = cpu_base;
1807 timerqueue_init_head(&cpu_base->clock_base[i].active);
1808 }
1809
1810 cpu_base->cpu = cpu;
1811 cpu_base->active_bases = 0;
1812 cpu_base->hres_active = 0;
1813 cpu_base->hang_detected = 0;
1814 cpu_base->next_timer = NULL;
1815 cpu_base->softirq_next_timer = NULL;
1816 cpu_base->expires_next = KTIME_MAX;
1817 cpu_base->softirq_expires_next = KTIME_MAX;
1818 return 0;
1819}
1820
1821#ifdef CONFIG_HOTPLUG_CPU
1822
1823static void migrate_hrtimer_list(struct hrtimer_clock_base *old_base,
1824 struct hrtimer_clock_base *new_base)
1825{
1826 struct hrtimer *timer;
1827 struct timerqueue_node *node;
1828
1829 while ((node = timerqueue_getnext(&old_base->active))) {
1830 timer = container_of(node, struct hrtimer, node);
1831 BUG_ON(hrtimer_callback_running(timer));
1832 debug_deactivate(timer);
1833
1834
1835
1836
1837
1838
1839 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0);
1840 timer->base = new_base;
1841
1842
1843
1844
1845
1846
1847
1848
1849 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS);
1850 }
1851}
1852
1853int hrtimers_dead_cpu(unsigned int scpu)
1854{
1855 struct hrtimer_cpu_base *old_base, *new_base;
1856 int i;
1857
1858 BUG_ON(cpu_online(scpu));
1859 tick_cancel_sched_timer(scpu);
1860
1861
1862
1863
1864
1865
1866 local_bh_disable();
1867 local_irq_disable();
1868 old_base = &per_cpu(hrtimer_bases, scpu);
1869 new_base = this_cpu_ptr(&hrtimer_bases);
1870
1871
1872
1873
1874 raw_spin_lock(&new_base->lock);
1875 raw_spin_lock_nested(&old_base->lock, SINGLE_DEPTH_NESTING);
1876
1877 for (i = 0; i < HRTIMER_MAX_CLOCK_BASES; i++) {
1878 migrate_hrtimer_list(&old_base->clock_base[i],
1879 &new_base->clock_base[i]);
1880 }
1881
1882
1883
1884
1885
1886 hrtimer_update_softirq_timer(new_base, false);
1887
1888 raw_spin_unlock(&old_base->lock);
1889 raw_spin_unlock(&new_base->lock);
1890
1891
1892 __hrtimer_peek_ahead_timers();
1893 local_irq_enable();
1894 local_bh_enable();
1895 return 0;
1896}
1897
1898#endif
1899
1900void __init hrtimers_init(void)
1901{
1902 hrtimers_prepare_cpu(smp_processor_id());
1903 open_softirq(HRTIMER_SOFTIRQ, hrtimer_run_softirq);
1904}
1905
1906
1907
1908
1909
1910
1911
1912
1913int __sched
1914schedule_hrtimeout_range_clock(ktime_t *expires, u64 delta,
1915 const enum hrtimer_mode mode, clockid_t clock_id)
1916{
1917 struct hrtimer_sleeper t;
1918
1919
1920
1921
1922
1923 if (expires && *expires == 0) {
1924 __set_current_state(TASK_RUNNING);
1925 return 0;
1926 }
1927
1928
1929
1930
1931 if (!expires) {
1932 schedule();
1933 return -EINTR;
1934 }
1935
1936 hrtimer_init_on_stack(&t.timer, clock_id, mode);
1937 hrtimer_set_expires_range_ns(&t.timer, *expires, delta);
1938
1939 hrtimer_init_sleeper(&t, current);
1940
1941 hrtimer_start_expires(&t.timer, mode);
1942
1943 if (likely(t.task))
1944 schedule();
1945
1946 hrtimer_cancel(&t.timer);
1947 destroy_hrtimer_on_stack(&t.timer);
1948
1949 __set_current_state(TASK_RUNNING);
1950
1951 return !t.task ? 0 : -EINTR;
1952}
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986int __sched schedule_hrtimeout_range(ktime_t *expires, u64 delta,
1987 const enum hrtimer_mode mode)
1988{
1989 return schedule_hrtimeout_range_clock(expires, delta, mode,
1990 CLOCK_MONOTONIC);
1991}
1992EXPORT_SYMBOL_GPL(schedule_hrtimeout_range);
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020int __sched schedule_hrtimeout(ktime_t *expires,
2021 const enum hrtimer_mode mode)
2022{
2023 return schedule_hrtimeout_range(expires, 0, mode);
2024}
2025EXPORT_SYMBOL_GPL(schedule_hrtimeout);
2026