1
2
3
4
5
6
7
8
9
10
11
12#ifndef _LINUX_HRTIMER_H
13#define _LINUX_HRTIMER_H
14
15#include <linux/hrtimer_defs.h>
16#include <linux/rbtree.h>
17#include <linux/init.h>
18#include <linux/list.h>
19#include <linux/percpu.h>
20#include <linux/seqlock.h>
21#include <linux/timer.h>
22#include <linux/timerqueue.h>
23
24struct hrtimer_clock_base;
25struct hrtimer_cpu_base;
26
27
28
29
30
31
32
33
34
35
36
37
38
39enum hrtimer_mode {
40 HRTIMER_MODE_ABS = 0x00,
41 HRTIMER_MODE_REL = 0x01,
42 HRTIMER_MODE_PINNED = 0x02,
43 HRTIMER_MODE_SOFT = 0x04,
44 HRTIMER_MODE_HARD = 0x08,
45
46 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
47 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
48
49 HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
50 HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
51
52 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
53 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
54
55 HRTIMER_MODE_ABS_HARD = HRTIMER_MODE_ABS | HRTIMER_MODE_HARD,
56 HRTIMER_MODE_REL_HARD = HRTIMER_MODE_REL | HRTIMER_MODE_HARD,
57
58 HRTIMER_MODE_ABS_PINNED_HARD = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_HARD,
59 HRTIMER_MODE_REL_PINNED_HARD = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_HARD,
60};
61
62
63
64
65enum hrtimer_restart {
66 HRTIMER_NORESTART,
67 HRTIMER_RESTART,
68};
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94#define HRTIMER_STATE_INACTIVE 0x00
95#define HRTIMER_STATE_ENQUEUED 0x01
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118struct hrtimer {
119 struct timerqueue_node node;
120 ktime_t _softexpires;
121 enum hrtimer_restart (*function)(struct hrtimer *);
122 struct hrtimer_clock_base *base;
123 u8 state;
124 u8 is_rel;
125 u8 is_soft;
126 u8 is_hard;
127};
128
129
130
131
132
133
134
135
136struct hrtimer_sleeper {
137 struct hrtimer timer;
138 struct task_struct *task;
139};
140
141#ifdef CONFIG_64BIT
142# define __hrtimer_clock_base_align ____cacheline_aligned
143#else
144# define __hrtimer_clock_base_align
145#endif
146
147
148
149
150
151
152
153
154
155
156
157
158
159struct hrtimer_clock_base {
160 struct hrtimer_cpu_base *cpu_base;
161 unsigned int index;
162 clockid_t clockid;
163 seqcount_raw_spinlock_t seq;
164 struct hrtimer *running;
165 struct timerqueue_head active;
166 ktime_t (*get_time)(void);
167 ktime_t offset;
168} __hrtimer_clock_base_align;
169
170enum hrtimer_base_type {
171 HRTIMER_BASE_MONOTONIC,
172 HRTIMER_BASE_REALTIME,
173 HRTIMER_BASE_BOOTTIME,
174 HRTIMER_BASE_TAI,
175 HRTIMER_BASE_MONOTONIC_SOFT,
176 HRTIMER_BASE_REALTIME_SOFT,
177 HRTIMER_BASE_BOOTTIME_SOFT,
178 HRTIMER_BASE_TAI_SOFT,
179 HRTIMER_MAX_CLOCK_BASES,
180};
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214struct hrtimer_cpu_base {
215 raw_spinlock_t lock;
216 unsigned int cpu;
217 unsigned int active_bases;
218 unsigned int clock_was_set_seq;
219 unsigned int hres_active : 1,
220 in_hrtirq : 1,
221 hang_detected : 1,
222 softirq_activated : 1;
223#ifdef CONFIG_HIGH_RES_TIMERS
224 unsigned int nr_events;
225 unsigned short nr_retries;
226 unsigned short nr_hangs;
227 unsigned int max_hang_time;
228#endif
229#ifdef CONFIG_PREEMPT_RT
230 spinlock_t softirq_expiry_lock;
231 atomic_t timer_waiters;
232#endif
233 ktime_t expires_next;
234 struct hrtimer *next_timer;
235 ktime_t softirq_expires_next;
236 struct hrtimer *softirq_next_timer;
237 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
238} ____cacheline_aligned;
239
240static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
241{
242 timer->node.expires = time;
243 timer->_softexpires = time;
244}
245
246static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
247{
248 timer->_softexpires = time;
249 timer->node.expires = ktime_add_safe(time, delta);
250}
251
252static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
253{
254 timer->_softexpires = time;
255 timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
256}
257
258static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
259{
260 timer->node.expires = tv64;
261 timer->_softexpires = tv64;
262}
263
264static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
265{
266 timer->node.expires = ktime_add_safe(timer->node.expires, time);
267 timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
268}
269
270static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
271{
272 timer->node.expires = ktime_add_ns(timer->node.expires, ns);
273 timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
274}
275
276static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
277{
278 return timer->node.expires;
279}
280
281static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
282{
283 return timer->_softexpires;
284}
285
286static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
287{
288 return timer->node.expires;
289}
290static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
291{
292 return timer->_softexpires;
293}
294
295static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
296{
297 return ktime_to_ns(timer->node.expires);
298}
299
300static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
301{
302 return ktime_sub(timer->node.expires, timer->base->get_time());
303}
304
305static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
306{
307 return timer->base->get_time();
308}
309
310static inline int hrtimer_is_hres_active(struct hrtimer *timer)
311{
312 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
313 timer->base->cpu_base->hres_active : 0;
314}
315
316#ifdef CONFIG_HIGH_RES_TIMERS
317struct clock_event_device;
318
319extern void hrtimer_interrupt(struct clock_event_device *dev);
320
321extern void clock_was_set_delayed(void);
322
323extern unsigned int hrtimer_resolution;
324
325#else
326
327#define hrtimer_resolution (unsigned int)LOW_RES_NSEC
328
329static inline void clock_was_set_delayed(void) { }
330
331#endif
332
333static inline ktime_t
334__hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
335{
336 ktime_t rem = ktime_sub(timer->node.expires, now);
337
338
339
340
341
342 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
343 rem -= hrtimer_resolution;
344 return rem;
345}
346
347static inline ktime_t
348hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
349{
350 return __hrtimer_expires_remaining_adjusted(timer,
351 timer->base->get_time());
352}
353
354extern void clock_was_set(void);
355#ifdef CONFIG_TIMERFD
356extern void timerfd_clock_was_set(void);
357#else
358static inline void timerfd_clock_was_set(void) { }
359#endif
360extern void hrtimers_resume(void);
361
362DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
363
364#ifdef CONFIG_PREEMPT_RT
365void hrtimer_cancel_wait_running(const struct hrtimer *timer);
366#else
367static inline void hrtimer_cancel_wait_running(struct hrtimer *timer)
368{
369 cpu_relax();
370}
371#endif
372
373
374
375
376extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
377 enum hrtimer_mode mode);
378extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, clockid_t clock_id,
379 enum hrtimer_mode mode);
380
381#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
382extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
383 enum hrtimer_mode mode);
384extern void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
385 clockid_t clock_id,
386 enum hrtimer_mode mode);
387
388extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
389#else
390static inline void hrtimer_init_on_stack(struct hrtimer *timer,
391 clockid_t which_clock,
392 enum hrtimer_mode mode)
393{
394 hrtimer_init(timer, which_clock, mode);
395}
396
397static inline void hrtimer_init_sleeper_on_stack(struct hrtimer_sleeper *sl,
398 clockid_t clock_id,
399 enum hrtimer_mode mode)
400{
401 hrtimer_init_sleeper(sl, clock_id, mode);
402}
403
404static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
405#endif
406
407
408extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
409 u64 range_ns, const enum hrtimer_mode mode);
410
411
412
413
414
415
416
417
418
419static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
420 const enum hrtimer_mode mode)
421{
422 hrtimer_start_range_ns(timer, tim, 0, mode);
423}
424
425extern int hrtimer_cancel(struct hrtimer *timer);
426extern int hrtimer_try_to_cancel(struct hrtimer *timer);
427
428static inline void hrtimer_start_expires(struct hrtimer *timer,
429 enum hrtimer_mode mode)
430{
431 u64 delta;
432 ktime_t soft, hard;
433 soft = hrtimer_get_softexpires(timer);
434 hard = hrtimer_get_expires(timer);
435 delta = ktime_to_ns(ktime_sub(hard, soft));
436 hrtimer_start_range_ns(timer, soft, delta, mode);
437}
438
439void hrtimer_sleeper_start_expires(struct hrtimer_sleeper *sl,
440 enum hrtimer_mode mode);
441
442static inline void hrtimer_restart(struct hrtimer *timer)
443{
444 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
445}
446
447
448extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
449
450static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
451{
452 return __hrtimer_get_remaining(timer, false);
453}
454
455extern u64 hrtimer_get_next_event(void);
456extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
457
458extern bool hrtimer_active(const struct hrtimer *timer);
459
460
461
462
463
464
465
466
467
468static inline bool hrtimer_is_queued(struct hrtimer *timer)
469{
470
471 return !!(READ_ONCE(timer->state) & HRTIMER_STATE_ENQUEUED);
472}
473
474
475
476
477
478static inline int hrtimer_callback_running(struct hrtimer *timer)
479{
480 return timer->base->running == timer;
481}
482
483
484extern u64
485hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503static inline u64 hrtimer_forward_now(struct hrtimer *timer,
504 ktime_t interval)
505{
506 return hrtimer_forward(timer, timer->base->get_time(), interval);
507}
508
509
510
511extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
512extern long hrtimer_nanosleep(ktime_t rqtp, const enum hrtimer_mode mode,
513 const clockid_t clockid);
514
515extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
516 const enum hrtimer_mode mode);
517extern int schedule_hrtimeout_range_clock(ktime_t *expires,
518 u64 delta,
519 const enum hrtimer_mode mode,
520 clockid_t clock_id);
521extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
522
523
524extern void hrtimer_run_queues(void);
525
526
527extern void __init hrtimers_init(void);
528
529
530extern void sysrq_timer_list_show(void);
531
532int hrtimers_prepare_cpu(unsigned int cpu);
533#ifdef CONFIG_HOTPLUG_CPU
534int hrtimers_dead_cpu(unsigned int cpu);
535#else
536#define hrtimers_dead_cpu NULL
537#endif
538
539#endif
540