1
2
3
4
5
6
7
8
9
10
11
12#ifndef _LINUX_HRTIMER_H
13#define _LINUX_HRTIMER_H
14
15#include <linux/hrtimer_defs.h>
16#include <linux/rbtree.h>
17#include <linux/init.h>
18#include <linux/list.h>
19#include <linux/percpu.h>
20#include <linux/seqlock.h>
21#include <linux/timer.h>
22#include <linux/timerqueue.h>
23
24struct hrtimer_clock_base;
25struct hrtimer_cpu_base;
26
27
28
29
30
31
32
33
34
35
36
37
38
39enum hrtimer_mode {
40 HRTIMER_MODE_ABS = 0x00,
41 HRTIMER_MODE_REL = 0x01,
42 HRTIMER_MODE_PINNED = 0x02,
43 HRTIMER_MODE_SOFT = 0x04,
44 HRTIMER_MODE_HARD = 0x08,
45
46 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
47 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
48
49 HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
50 HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
51
52 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
53 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
54
55 HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
56 HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,
57
58 HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
59 HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
60};
61
62
63
64
65enum hrtimer_restart {
66 HRTIMER_NORESTART,
67 HRTIMER_RESTART,
68};
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94#define HRTIMER_STATE_INACTIVE 0x00
95#define HRTIMER_STATE_ENQUEUED 0x01
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118struct hrtimer {
119 struct timerqueue_node node;
120 ktime_t _softexpires;
121 enum hrtimer_restart (*function)(struct hrtimer *);
122 struct hrtimer_clock_base *base;
123 u8 state;
124 u8 is_rel;
125 u8 is_soft;
126 u8 is_hard;
127};
128
129
130
131
132
133
134
135
136struct hrtimer_sleeper {
137 struct hrtimer timer;
138 struct task_struct *task;
139};
140
141#ifdef CONFIG_64BIT
142# define __hrtimer_clock_base_align ____cacheline_aligned
143#else
144# define __hrtimer_clock_base_align
145#endif
146
147
148
149
150
151
152
153
154
155
156
157
158
159struct hrtimer_clock_base {
160 struct hrtimer_cpu_base *cpu_base;
161 unsigned int index;
162 clockid_t clockid;
163 seqcount_raw_spinlock_t seq;
164 struct hrtimer *running;
165 struct timerqueue_head active;
166 ktime_t (*get_time)(void);
167 ktime_t offset;
168} __hrtimer_clock_base_align;
169
170enum hrtimer_base_type {
171 HRTIMER_BASE_MONOTONIC,
172 HRTIMER_BASE_REALTIME,
173 HRTIMER_BASE_BOOTTIME,
174 HRTIMER_BASE_TAI,
175 HRTIMER_BASE_MONOTONIC_SOFT,
176 HRTIMER_BASE_REALTIME_SOFT,
177 HRTIMER_BASE_BOOTTIME_SOFT,
178 HRTIMER_BASE_TAI_SOFT,
179 HRTIMER_MAX_CLOCK_BASES,
180};
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214struct hrtimer_cpu_base {
215 raw_spinlock_t lock;
216 unsigned int cpu;
217 unsigned int active_bases;
218 unsigned int clock_was_set_seq;
219 unsigned int hres_active : 1,
220 in_hrtirq : 1,
221 hang_detected : 1,
222 softirq_activated : 1;
223#ifdef CONFIG_HIGH_RES_TIMERS
224 unsigned int nr_events;
225 unsigned short nr_retries;
226 unsigned short nr_hangs;
227 unsigned int max_hang_time;
228#endif
229#ifdef CONFIG_PREEMPT_RT
230 spinlock_t softirq_expiry_lock;
231 atomic_t timer_waiters;
232#endif
233 ktime_t expires_next;
234 struct hrtimer *next_timer;
235 ktime_t softirq_expires_next;
236 struct hrtimer *softirq_next_timer;
237 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
238} ____cacheline_aligned;
239
240static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
241{
242 timer->node.expires = time;
243 timer->_softexpires = time;
244}
245
246static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
247{
248 timer->_softexpires = time;
249 timer->node.expires = ktime_add_safe(time, delta);
250}
251
252static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
253{
254 timer->_softexpires = time;
255 timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
256}
257
258static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
259{
260 timer->node.expires = tv64;
261 timer->_softexpires = tv64;
262}
263
264static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
265{
266 timer->node.expires = ktime_add_safe(timer->node.expires, time);
267 timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
268}
269
270static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
271{
272 timer->node.expires = ktime_add_ns(timer->node.expires, ns);
273 timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
274}
275
276static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
277{
278 return timer->node.expires;
279}
280
281static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
282{
283 return timer->_softexpires;
284}
285
286static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
287{
288 return timer->node.expires;
289}
290static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
291{
292 return timer->_softexpires;
293}
294
295static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
296{
297 return ktime_to_ns(timer->node.expires);
298}
299
300static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
301{
302 return ktime_sub(timer->node.expires, timer->base->get_time());
303}
304
305static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
306{
307 return timer->base->get_time();
308}
309
310static inline int hrtimer_is_hres_active(struct hrtimer *timer)
311{
312 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
313 timer->base->cpu_base->hres_active : 0;
314}
315
316#ifdef CONFIG_HIGH_RES_TIMERS
317struct clock_event_device;
318
319extern void hrtimer_interrupt(struct clock_event_device *dev);
320
321extern void clock_was_set_delayed(void);
322
323extern unsigned int hrtimer_resolution;
324
325#else
326
327#define hrtimer_resolution (unsigned int)LOW_RES_NSEC
328
329static inline void clock_was_set_delayed(void) { }
330
331#endif
332
333static inline ktime_t
334__hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
335{
336 ktime_t rem = ktime_sub(timer->node.expires, now);
337
338
339
340
341
342 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
343 rem -= hrtimer_resolution;
344 return rem;
345}
346
347static inline ktime_t
348hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
349{
350 return __hrtimer_expires_remaining_adjusted(timer,
351 timer->base->get_time());
352}
353
354extern void clock_was_set(void);
355#ifdef CONFIG_TIMERFD
356extern void timerfd_clock_was_set(void);
357#else
358static inline void timerfd_clock_was_set(void) { }
359#endif
360extern void hrtimers_resume(void);
361
362DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
363
364#ifdef CONFIG_PREEMPT_RT
365void hrtimer_cancel_wait_running(const struct hrtimer *timer);
366#else
367static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
368{
369 cpu_relax();
370}
371#endif
372
373
374
375
376extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
377 enum hrtimer_mode mode);
378extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
379 enum hrtimer_mode mode);
380
381#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
382extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
383 enum hrtimer_mode mode);
384extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
385 clockid_t clock_id,
386 enum hrtimer_mode mode);
387
388extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
389#else
390static inline void hrtimer_init_on_stack(struct hrtimer *timer,
391 clockid_t which_clock,
392 enum hrtimer_mode mode)
393{
394 hrtimer_init(timer, which_clock, mode);
395}
396
397static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
398 clockid_t clock_id,
399 enum hrtimer_mode mode)
400{
401 hrtimer_init_sleeper(sl, clock_id, mode);
402}
403
404static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
405#endif
406
407
408extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
409 u64 range_ns, const enum hrtimer_mode mode);
410
411
412
413
414
415
416
417
418
419static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
420 const enum hrtimer_mode mode)
421{
422 hrtimer_start_range_ns(timer, tim, 0, mode);
423}
424
425extern int hrtimer_cancel(struct hrtimer *timer);
426extern int hrtimer_try_to_cancel(struct hrtimer *timer);
427
428static inline void hrtimer_start_expires(struct hrtimer *timer,
429 enum hrtimer_mode mode)
430{
431 u64 delta;
432 ktime_t soft, hard;
433 soft = hrtimer_get_softexpires(timer);
434 hard = hrtimer_get_expires(timer);
435 delta = ktime_to_ns(ktime_sub(hard, soft));
436 hrtimer_start_range_ns(timer, soft, delta, mode);
437}
438
439void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
440 enum hrtimer_mode mode);
441
442static inline void hrtimer_restart(struct hrtimer *timer)
443{
444 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
445}
446
447
448extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
449
450
451
452
453
454static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
455{
456 return __hrtimer_get_remaining(timer, false);
457}
458
459extern u64 hrtimer_get_next_event(void);
460extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
461
462extern bool hrtimer_active(const struct hrtimer *timer);
463
464
465
466
467
468
469
470
471
472static inline bool hrtimer_is_queued(struct hrtimer *timer)
473{
474
475 return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
476}
477
478
479
480
481
482static inline int hrtimer_callback_running(struct hrtimer *timer)
483{
484 return timer->base->running == timer;
485}
486
487
488extern u64
489hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507static inline u64 hrtimer_forward_now(struct hrtimer *timer,
508 ktime_t interval)
509{
510 return hrtimer_forward(timer, timer->base->get_time(), interval);
511}
512
513
514
515extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
516extern long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
517 const clockid_t clockid);
518
519extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
520 const enum hrtimer_mode mode);
521extern int schedule_hrtimeout_range_clock(ktime_t *expires,
522 u64 delta,
523 const enum hrtimer_mode mode,
524 clockid_t clock_id);
525extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
526
527
528extern void hrtimer_run_queues(void);
529
530
531extern void __init hrtimers_init(void);
532
533
534extern void sysrq_timer_list_show(void);
535
536int hrtimers_prepare_cpu(unsigned int cpu);
537#ifdef CONFIG_HOTPLUG_CPU
538int hrtimers_dead_cpu(unsigned int cpu);
539#else
540#define hrtimers_dead_cpu NULL
541#endif
542
543#endif
544