1
2
3
4
5
6
7
8
9
10
11
12#ifndef _LINUX_HRTIMER_H
13#define _LINUX_HRTIMER_H
14
15#include <linux/rbtree.h>
16#include <linux/ktime.h>
17#include <linux/init.h>
18#include <linux/list.h>
19#include <linux/percpu.h>
20#include <linux/timer.h>
21#include <linux/timerqueue.h>
22
23struct hrtimer_clock_base;
24struct hrtimer_cpu_base;
25
26
27
28
29
30
31
32
33
34
35
36enum hrtimer_mode {
37 HRTIMER_MODE_ABS = 0x00,
38 HRTIMER_MODE_REL = 0x01,
39 HRTIMER_MODE_PINNED = 0x02,
40 HRTIMER_MODE_SOFT = 0x04,
41
42 HRTIMER_MODE_ABS_PINNED = HRTIMER_MODE_ABS | HRTIMER_MODE_PINNED,
43 HRTIMER_MODE_REL_PINNED = HRTIMER_MODE_REL | HRTIMER_MODE_PINNED,
44
45 HRTIMER_MODE_ABS_SOFT = HRTIMER_MODE_ABS | HRTIMER_MODE_SOFT,
46 HRTIMER_MODE_REL_SOFT = HRTIMER_MODE_REL | HRTIMER_MODE_SOFT,
47
48 HRTIMER_MODE_ABS_PINNED_SOFT = HRTIMER_MODE_ABS_PINNED | HRTIMER_MODE_SOFT,
49 HRTIMER_MODE_REL_PINNED_SOFT = HRTIMER_MODE_REL_PINNED | HRTIMER_MODE_SOFT,
50
51};
52
53
54
55
56enum hrtimer_restart {
57 HRTIMER_NORESTART,
58 HRTIMER_RESTART,
59};
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85#define HRTIMER_STATE_INACTIVE 0x00
86#define HRTIMER_STATE_ENQUEUED 0x01
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107struct hrtimer {
108 struct timerqueue_node node;
109 ktime_t _softexpires;
110 enum hrtimer_restart (*function)(struct hrtimer *);
111 struct hrtimer_clock_base *base;
112 u8 state;
113 u8 is_rel;
114 u8 is_soft;
115};
116
117
118
119
120
121
122
123
124struct hrtimer_sleeper {
125 struct hrtimer timer;
126 struct task_struct *task;
127};
128
129#ifdef CONFIG_64BIT
130# define __hrtimer_clock_base_align ____cacheline_aligned
131#else
132# define __hrtimer_clock_base_align
133#endif
134
135
136
137
138
139
140
141
142
143
144
145
146
147struct hrtimer_clock_base {
148 struct hrtimer_cpu_base *cpu_base;
149 unsigned int index;
150 clockid_t clockid;
151 seqcount_t seq;
152 struct hrtimer *running;
153 struct timerqueue_head active;
154 ktime_t (*get_time)(void);
155 ktime_t offset;
156} __hrtimer_clock_base_align;
157
158enum hrtimer_base_type {
159 HRTIMER_BASE_MONOTONIC,
160 HRTIMER_BASE_REALTIME,
161 HRTIMER_BASE_BOOTTIME,
162 HRTIMER_BASE_TAI,
163 HRTIMER_BASE_MONOTONIC_SOFT,
164 HRTIMER_BASE_REALTIME_SOFT,
165 HRTIMER_BASE_BOOTTIME_SOFT,
166 HRTIMER_BASE_TAI_SOFT,
167 HRTIMER_MAX_CLOCK_BASES,
168};
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198struct hrtimer_cpu_base {
199 raw_spinlock_t lock;
200 unsigned int cpu;
201 unsigned int active_bases;
202 unsigned int clock_was_set_seq;
203 unsigned int hres_active : 1,
204 in_hrtirq : 1,
205 hang_detected : 1,
206 softirq_activated : 1;
207#ifdef CONFIG_HIGH_RES_TIMERS
208 unsigned int nr_events;
209 unsigned short nr_retries;
210 unsigned short nr_hangs;
211 unsigned int max_hang_time;
212#endif
213 ktime_t expires_next;
214 struct hrtimer *next_timer;
215 ktime_t softirq_expires_next;
216 struct hrtimer *softirq_next_timer;
217 struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES];
218} ____cacheline_aligned;
219
220static inline void hrtimer_set_expires(struct hrtimer *timer, ktime_t time)
221{
222 timer->node.expires = time;
223 timer->_softexpires = time;
224}
225
226static inline void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta)
227{
228 timer->_softexpires = time;
229 timer->node.expires = ktime_add_safe(time, delta);
230}
231
232static inline void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, u64 delta)
233{
234 timer->_softexpires = time;
235 timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta));
236}
237
238static inline void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64)
239{
240 timer->node.expires = tv64;
241 timer->_softexpires = tv64;
242}
243
244static inline void hrtimer_add_expires(struct hrtimer *timer, ktime_t time)
245{
246 timer->node.expires = ktime_add_safe(timer->node.expires, time);
247 timer->_softexpires = ktime_add_safe(timer->_softexpires, time);
248}
249
250static inline void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns)
251{
252 timer->node.expires = ktime_add_ns(timer->node.expires, ns);
253 timer->_softexpires = ktime_add_ns(timer->_softexpires, ns);
254}
255
256static inline ktime_t hrtimer_get_expires(const struct hrtimer *timer)
257{
258 return timer->node.expires;
259}
260
261static inline ktime_t hrtimer_get_softexpires(const struct hrtimer *timer)
262{
263 return timer->_softexpires;
264}
265
266static inline s64 hrtimer_get_expires_tv64(const struct hrtimer *timer)
267{
268 return timer->node.expires;
269}
270static inline s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer)
271{
272 return timer->_softexpires;
273}
274
275static inline s64 hrtimer_get_expires_ns(const struct hrtimer *timer)
276{
277 return ktime_to_ns(timer->node.expires);
278}
279
280static inline ktime_t hrtimer_expires_remaining(const struct hrtimer *timer)
281{
282 return ktime_sub(timer->node.expires, timer->base->get_time());
283}
284
285static inline ktime_t hrtimer_cb_get_time(struct hrtimer *timer)
286{
287 return timer->base->get_time();
288}
289
290static inline int hrtimer_is_hres_active(struct hrtimer *timer)
291{
292 return IS_ENABLED(CONFIG_HIGH_RES_TIMERS) ?
293 timer->base->cpu_base->hres_active : 0;
294}
295
296#ifdef CONFIG_HIGH_RES_TIMERS
297struct clock_event_device;
298
299extern void hrtimer_interrupt(struct clock_event_device *dev);
300
301
302
303
304
305
306
307# define HIGH_RES_NSEC 1
308# define KTIME_HIGH_RES (HIGH_RES_NSEC)
309# define MONOTONIC_RES_NSEC HIGH_RES_NSEC
310# define KTIME_MONOTONIC_RES KTIME_HIGH_RES
311
312extern void clock_was_set_delayed(void);
313
314extern unsigned int hrtimer_resolution;
315
316#else
317
318# define MONOTONIC_RES_NSEC LOW_RES_NSEC
319# define KTIME_MONOTONIC_RES KTIME_LOW_RES
320
321#define hrtimer_resolution (unsigned int)LOW_RES_NSEC
322
323static inline void clock_was_set_delayed(void) { }
324
325#endif
326
327static inline ktime_t
328__hrtimer_expires_remaining_adjusted(const struct hrtimer *timer, ktime_t now)
329{
330 ktime_t rem = ktime_sub(timer->node.expires, now);
331
332
333
334
335
336 if (IS_ENABLED(CONFIG_TIME_LOW_RES) && timer->is_rel)
337 rem -= hrtimer_resolution;
338 return rem;
339}
340
341static inline ktime_t
342hrtimer_expires_remaining_adjusted(const struct hrtimer *timer)
343{
344 return __hrtimer_expires_remaining_adjusted(timer,
345 timer->base->get_time());
346}
347
348extern void clock_was_set(void);
349#ifdef CONFIG_TIMERFD
350extern void timerfd_clock_was_set(void);
351#else
352static inline void timerfd_clock_was_set(void) { }
353#endif
354extern void hrtimers_resume(void);
355
356DECLARE_PER_CPU(struct tick_device, tick_cpu_device);
357
358
359
360
361
362extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock,
363 enum hrtimer_mode mode);
364
365#ifdef CONFIG_DEBUG_OBJECTS_TIMERS
366extern void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock,
367 enum hrtimer_mode mode);
368
369extern void destroy_hrtimer_on_stack(struct hrtimer *timer);
370#else
371static inline void hrtimer_init_on_stack(struct hrtimer *timer,
372 clockid_t which_clock,
373 enum hrtimer_mode mode)
374{
375 hrtimer_init(timer, which_clock, mode);
376}
377static inline void destroy_hrtimer_on_stack(struct hrtimer *timer) { }
378#endif
379
380
381extern void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim,
382 u64 range_ns, const enum hrtimer_mode mode);
383
384
385
386
387
388
389
390
391
392static inline void hrtimer_start(struct hrtimer *timer, ktime_t tim,
393 const enum hrtimer_mode mode)
394{
395 hrtimer_start_range_ns(timer, tim, 0, mode);
396}
397
398extern int hrtimer_cancel(struct hrtimer *timer);
399extern int hrtimer_try_to_cancel(struct hrtimer *timer);
400
401static inline void hrtimer_start_expires(struct hrtimer *timer,
402 enum hrtimer_mode mode)
403{
404 u64 delta;
405 ktime_t soft, hard;
406 soft = hrtimer_get_softexpires(timer);
407 hard = hrtimer_get_expires(timer);
408 delta = ktime_to_ns(ktime_sub(hard, soft));
409 hrtimer_start_range_ns(timer, soft, delta, mode);
410}
411
412static inline void hrtimer_restart(struct hrtimer *timer)
413{
414 hrtimer_start_expires(timer, HRTIMER_MODE_ABS);
415}
416
417
418extern ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust);
419
420static inline ktime_t hrtimer_get_remaining(const struct hrtimer *timer)
421{
422 return __hrtimer_get_remaining(timer, false);
423}
424
425extern u64 hrtimer_get_next_event(void);
426extern u64 hrtimer_next_event_without(const struct hrtimer *exclude);
427
428extern bool hrtimer_active(const struct hrtimer *timer);
429
430
431
432
433static inline int hrtimer_is_queued(struct hrtimer *timer)
434{
435 return timer->state & HRTIMER_STATE_ENQUEUED;
436}
437
438
439
440
441
442static inline int hrtimer_callback_running(struct hrtimer *timer)
443{
444 return timer->base->running == timer;
445}
446
447
448extern u64
449hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval);
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467static inline u64 hrtimer_forward_now(struct hrtimer *timer,
468 ktime_t interval)
469{
470 return hrtimer_forward(timer, timer->base->get_time(), interval);
471}
472
473
474
475extern int nanosleep_copyout(struct restart_block *, struct timespec64 *);
476extern long hrtimer_nanosleep(const struct timespec64 *rqtp,
477 const enum hrtimer_mode mode,
478 const clockid_t clockid);
479
480extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl,
481 struct task_struct *tsk);
482
483extern int schedule_hrtimeout_range(ktime_t *expires, u64 delta,
484 const enum hrtimer_mode mode);
485extern int schedule_hrtimeout_range_clock(ktime_t *expires,
486 u64 delta,
487 const enum hrtimer_mode mode,
488 clockid_t clock_id);
489extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode);
490
491
492extern void hrtimer_run_queues(void);
493
494
495extern void __init hrtimers_init(void);
496
497
498extern void sysrq_timer_list_show(void);
499
500int hrtimers_prepare_cpu(unsigned int cpu);
501#ifdef CONFIG_HOTPLUG_CPU
502int hrtimers_dead_cpu(unsigned int cpu);
503#else
504#define hrtimers_dead_cpu NULL
505#endif
506
507#endif
508