1
2
3
4
5
6
7
8
9
10
11#include <linux/mutex.h>
12#include <linux/module.h>
13#include <linux/sched.h>
14#include <linux/sched/idle.h>
15#include <linux/cpuidle.h>
16#include <linux/cpumask.h>
17#include <linux/tick.h>
18#include <linux/cpu.h>
19
20#include "cpuidle.h"
21
22DEFINE_SPINLOCK(cpuidle_driver_lock);
23
24#ifdef CONFIG_CPU_IDLE_MULTIPLE_DRIVERS
25
26static DEFINE_PER_CPU(struct cpuidle_driver *, cpuidle_drivers);
27
28
29
30
31
32
33
34
35static struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu)
36{
37 return per_cpu(cpuidle_drivers, cpu);
38}
39
40
41
42
43
44
45
46
47
48static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv)
49{
50 int cpu;
51
52 for_each_cpu(cpu, drv->cpumask) {
53
54 if (drv != __cpuidle_get_cpu_driver(cpu))
55 continue;
56
57 per_cpu(cpuidle_drivers, cpu) = NULL;
58 }
59}
60
61
62
63
64
65
66
67
68
69
70static inline int __cpuidle_set_driver(struct cpuidle_driver *drv)
71{
72 int cpu;
73
74 for_each_cpu(cpu, drv->cpumask) {
75
76 if (__cpuidle_get_cpu_driver(cpu)) {
77 __cpuidle_unset_driver(drv);
78 return -EBUSY;
79 }
80
81 per_cpu(cpuidle_drivers, cpu) = drv;
82 }
83
84 return 0;
85}
86
87#else
88
89static struct cpuidle_driver *cpuidle_curr_driver;
90
91
92
93
94
95
96
97
98static inline struct cpuidle_driver *__cpuidle_get_cpu_driver(int cpu)
99{
100 return cpuidle_curr_driver;
101}
102
103
104
105
106
107
108
109static inline int __cpuidle_set_driver(struct cpuidle_driver *drv)
110{
111 if (cpuidle_curr_driver)
112 return -EBUSY;
113
114 cpuidle_curr_driver = drv;
115
116 return 0;
117}
118
119
120
121
122
123
124
125
126static inline void __cpuidle_unset_driver(struct cpuidle_driver *drv)
127{
128 if (drv == cpuidle_curr_driver)
129 cpuidle_curr_driver = NULL;
130}
131
132#endif
133
134
135
136
137
138
139
140
141
142
143static void cpuidle_setup_broadcast_timer(void *arg)
144{
145 if (arg)
146 tick_broadcast_enable();
147 else
148 tick_broadcast_disable();
149}
150
151
152
153
154
155static void __cpuidle_driver_init(struct cpuidle_driver *drv)
156{
157 int i;
158
159 drv->refcnt = 0;
160
161
162
163
164
165
166 if (!drv->cpumask)
167 drv->cpumask = (struct cpumask *)cpu_possible_mask;
168
169
170
171
172
173
174 for (i = drv->state_count - 1; i >= 0 ; i--) {
175 if (drv->states[i].flags & CPUIDLE_FLAG_TIMER_STOP) {
176 drv->bctimer = 1;
177 break;
178 }
179 }
180}
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195static int __cpuidle_register_driver(struct cpuidle_driver *drv)
196{
197 int ret;
198
199 if (!drv || !drv->state_count)
200 return -EINVAL;
201
202 ret = cpuidle_coupled_state_verify(drv);
203 if (ret)
204 return ret;
205
206 if (cpuidle_disabled())
207 return -ENODEV;
208
209 __cpuidle_driver_init(drv);
210
211 ret = __cpuidle_set_driver(drv);
212 if (ret)
213 return ret;
214
215 if (drv->bctimer)
216 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer,
217 (void *)1, 1);
218
219 return 0;
220}
221
222
223
224
225
226
227
228
229
230
231static void __cpuidle_unregister_driver(struct cpuidle_driver *drv)
232{
233 if (WARN_ON(drv->refcnt > 0))
234 return;
235
236 if (drv->bctimer) {
237 drv->bctimer = 0;
238 on_each_cpu_mask(drv->cpumask, cpuidle_setup_broadcast_timer,
239 NULL, 1);
240 }
241
242 __cpuidle_unset_driver(drv);
243}
244
245
246
247
248
249
250
251
252
253
254
255int cpuidle_register_driver(struct cpuidle_driver *drv)
256{
257 int ret;
258
259 spin_lock(&cpuidle_driver_lock);
260 ret = __cpuidle_register_driver(drv);
261 spin_unlock(&cpuidle_driver_lock);
262
263 return ret;
264}
265EXPORT_SYMBOL_GPL(cpuidle_register_driver);
266
267
268
269
270
271
272
273
274
275void cpuidle_unregister_driver(struct cpuidle_driver *drv)
276{
277 spin_lock(&cpuidle_driver_lock);
278 __cpuidle_unregister_driver(drv);
279 spin_unlock(&cpuidle_driver_lock);
280}
281EXPORT_SYMBOL_GPL(cpuidle_unregister_driver);
282
283
284
285
286
287
288struct cpuidle_driver *cpuidle_get_driver(void)
289{
290 struct cpuidle_driver *drv;
291 int cpu;
292
293 cpu = get_cpu();
294 drv = __cpuidle_get_cpu_driver(cpu);
295 put_cpu();
296
297 return drv;
298}
299EXPORT_SYMBOL_GPL(cpuidle_get_driver);
300
301
302
303
304
305
306
307
308struct cpuidle_driver *cpuidle_get_cpu_driver(struct cpuidle_device *dev)
309{
310 if (!dev)
311 return NULL;
312
313 return __cpuidle_get_cpu_driver(dev->cpu);
314}
315EXPORT_SYMBOL_GPL(cpuidle_get_cpu_driver);
316
317
318
319
320
321
322
323
324
325struct cpuidle_driver *cpuidle_driver_ref(void)
326{
327 struct cpuidle_driver *drv;
328
329 spin_lock(&cpuidle_driver_lock);
330
331 drv = cpuidle_get_driver();
332 if (drv)
333 drv->refcnt++;
334
335 spin_unlock(&cpuidle_driver_lock);
336 return drv;
337}
338
339
340
341
342
343
344
345void cpuidle_driver_unref(void)
346{
347 struct cpuidle_driver *drv;
348
349 spin_lock(&cpuidle_driver_lock);
350
351 drv = cpuidle_get_driver();
352 if (drv && !WARN_ON(drv->refcnt <= 0))
353 drv->refcnt--;
354
355 spin_unlock(&cpuidle_driver_lock);
356}
357