1
2
3
4
5
6
7
8
9
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/notifier.h>
16#include <linux/pm_qos.h>
17#include <linux/cpu.h>
18#include <linux/cpuidle.h>
19#include <linux/ktime.h>
20#include <linux/hrtimer.h>
21#include <linux/module.h>
22#include <trace/events/power.h>
23
24#include "cpuidle.h"
25
26DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
27DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
28
29DEFINE_MUTEX(cpuidle_lock);
30LIST_HEAD(cpuidle_detected_devices);
31
32static int enabled_devices;
33static int off __read_mostly;
34static int initialized __read_mostly;
35
36int cpuidle_disabled(void)
37{
38 return off;
39}
40void disable_cpuidle(void)
41{
42 off = 1;
43}
44
45static int __cpuidle_register_device(struct cpuidle_device *dev);
46
47
48
49
50
51
52int cpuidle_play_dead(void)
53{
54 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
55 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
56 int i;
57
58 if (!drv)
59 return -ENODEV;
60
61
62 for (i = drv->state_count - 1; i >= CPUIDLE_DRIVER_STATE_START; i--)
63 if (drv->states[i].enter_dead)
64 return drv->states[i].enter_dead(dev, i);
65
66 return -ENODEV;
67}
68
69
70
71
72
73
74
75int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
76 int index)
77{
78 int entered_state;
79
80 struct cpuidle_state *target_state = &drv->states[index];
81 ktime_t time_start, time_end;
82 s64 diff;
83
84 time_start = ktime_get();
85
86 entered_state = target_state->enter(dev, drv, index);
87
88 time_end = ktime_get();
89
90 local_irq_enable();
91
92 diff = ktime_to_us(ktime_sub(time_end, time_start));
93 if (diff > INT_MAX)
94 diff = INT_MAX;
95
96 dev->last_residency = (int) diff;
97
98 if (entered_state >= 0) {
99
100
101
102
103 dev->states_usage[entered_state].time += dev->last_residency;
104 dev->states_usage[entered_state].usage++;
105 } else {
106 dev->last_residency = 0;
107 }
108
109 return entered_state;
110}
111
112
113
114
115
116
117
118int cpuidle_idle_call(void)
119{
120 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
121 struct cpuidle_driver *drv;
122 int next_state, entered_state;
123
124 if (off)
125 return -ENODEV;
126
127 if (!initialized)
128 return -ENODEV;
129
130
131 if (!dev || !dev->enabled)
132 return -EBUSY;
133
134 drv = cpuidle_get_cpu_driver(dev);
135
136
137 next_state = cpuidle_curr_governor->select(drv, dev);
138 if (need_resched()) {
139 dev->last_residency = 0;
140
141 if (cpuidle_curr_governor->reflect)
142 cpuidle_curr_governor->reflect(dev, next_state);
143 local_irq_enable();
144 return 0;
145 }
146
147 trace_cpu_idle_rcuidle(next_state, dev->cpu);
148
149 if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
150 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_ENTER,
151 &dev->cpu);
152
153 if (cpuidle_state_is_coupled(dev, drv, next_state))
154 entered_state = cpuidle_enter_state_coupled(dev, drv,
155 next_state);
156 else
157 entered_state = cpuidle_enter_state(dev, drv, next_state);
158
159 if (drv->states[next_state].flags & CPUIDLE_FLAG_TIMER_STOP)
160 clockevents_notify(CLOCK_EVT_NOTIFY_BROADCAST_EXIT,
161 &dev->cpu);
162
163 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
164
165
166 if (cpuidle_curr_governor->reflect)
167 cpuidle_curr_governor->reflect(dev, entered_state);
168
169 return 0;
170}
171
172
173
174
175void cpuidle_install_idle_handler(void)
176{
177 if (enabled_devices) {
178
179 smp_wmb();
180 initialized = 1;
181 }
182}
183
184
185
186
187void cpuidle_uninstall_idle_handler(void)
188{
189 if (enabled_devices) {
190 initialized = 0;
191 kick_all_cpus_sync();
192 }
193}
194
195
196
197
198void cpuidle_pause_and_lock(void)
199{
200 mutex_lock(&cpuidle_lock);
201 cpuidle_uninstall_idle_handler();
202}
203
204EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
205
206
207
208
209void cpuidle_resume_and_unlock(void)
210{
211 cpuidle_install_idle_handler();
212 mutex_unlock(&cpuidle_lock);
213}
214
215EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
216
217
218void cpuidle_pause(void)
219{
220 mutex_lock(&cpuidle_lock);
221 cpuidle_uninstall_idle_handler();
222 mutex_unlock(&cpuidle_lock);
223}
224
225
226void cpuidle_resume(void)
227{
228 mutex_lock(&cpuidle_lock);
229 cpuidle_install_idle_handler();
230 mutex_unlock(&cpuidle_lock);
231}
232
233#ifdef CONFIG_ARCH_HAS_CPU_RELAX
234static int poll_idle(struct cpuidle_device *dev,
235 struct cpuidle_driver *drv, int index)
236{
237 ktime_t t1, t2;
238 s64 diff;
239
240 t1 = ktime_get();
241 local_irq_enable();
242 while (!need_resched())
243 cpu_relax();
244
245 t2 = ktime_get();
246 diff = ktime_to_us(ktime_sub(t2, t1));
247 if (diff > INT_MAX)
248 diff = INT_MAX;
249
250 dev->last_residency = (int) diff;
251
252 return index;
253}
254
255static void poll_idle_init(struct cpuidle_driver *drv)
256{
257 struct cpuidle_state *state = &drv->states[0];
258
259 snprintf(state->name, CPUIDLE_NAME_LEN, "POLL");
260 snprintf(state->desc, CPUIDLE_DESC_LEN, "CPUIDLE CORE POLL IDLE");
261 state->exit_latency = 0;
262 state->target_residency = 0;
263 state->power_usage = -1;
264 state->flags = 0;
265 state->enter = poll_idle;
266 state->disabled = false;
267}
268#else
269static void poll_idle_init(struct cpuidle_driver *drv) {}
270#endif
271
272
273
274
275
276
277
278
279int cpuidle_enable_device(struct cpuidle_device *dev)
280{
281 int ret, i;
282 struct cpuidle_driver *drv;
283
284 if (!dev)
285 return -EINVAL;
286
287 if (dev->enabled)
288 return 0;
289
290 drv = cpuidle_get_cpu_driver(dev);
291
292 if (!drv || !cpuidle_curr_governor)
293 return -EIO;
294
295 if (!dev->state_count)
296 dev->state_count = drv->state_count;
297
298 if (dev->registered == 0) {
299 ret = __cpuidle_register_device(dev);
300 if (ret)
301 return ret;
302 }
303
304 poll_idle_init(drv);
305
306 ret = cpuidle_add_device_sysfs(dev);
307 if (ret)
308 return ret;
309
310 if (cpuidle_curr_governor->enable &&
311 (ret = cpuidle_curr_governor->enable(drv, dev)))
312 goto fail_sysfs;
313
314 for (i = 0; i < dev->state_count; i++) {
315 dev->states_usage[i].usage = 0;
316 dev->states_usage[i].time = 0;
317 }
318 dev->last_residency = 0;
319
320 smp_wmb();
321
322 dev->enabled = 1;
323
324 enabled_devices++;
325 return 0;
326
327fail_sysfs:
328 cpuidle_remove_device_sysfs(dev);
329
330 return ret;
331}
332
333EXPORT_SYMBOL_GPL(cpuidle_enable_device);
334
335
336
337
338
339
340
341
342void cpuidle_disable_device(struct cpuidle_device *dev)
343{
344 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
345
346 if (!dev || !dev->enabled)
347 return;
348
349 if (!drv || !cpuidle_curr_governor)
350 return;
351
352 dev->enabled = 0;
353
354 if (cpuidle_curr_governor->disable)
355 cpuidle_curr_governor->disable(drv, dev);
356
357 cpuidle_remove_device_sysfs(dev);
358 enabled_devices--;
359}
360
361EXPORT_SYMBOL_GPL(cpuidle_disable_device);
362
363
364
365
366
367
368
369
370static int __cpuidle_register_device(struct cpuidle_device *dev)
371{
372 int ret;
373 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
374
375 if (!try_module_get(drv->owner))
376 return -EINVAL;
377
378 per_cpu(cpuidle_devices, dev->cpu) = dev;
379 list_add(&dev->device_list, &cpuidle_detected_devices);
380 ret = cpuidle_add_sysfs(dev);
381 if (ret)
382 goto err_sysfs;
383
384 ret = cpuidle_coupled_register_device(dev);
385 if (ret)
386 goto err_coupled;
387
388 dev->registered = 1;
389 return 0;
390
391err_coupled:
392 cpuidle_remove_sysfs(dev);
393err_sysfs:
394 list_del(&dev->device_list);
395 per_cpu(cpuidle_devices, dev->cpu) = NULL;
396 module_put(drv->owner);
397 return ret;
398}
399
400
401
402
403
404int cpuidle_register_device(struct cpuidle_device *dev)
405{
406 int ret;
407
408 if (!dev)
409 return -EINVAL;
410
411 mutex_lock(&cpuidle_lock);
412
413 if ((ret = __cpuidle_register_device(dev))) {
414 mutex_unlock(&cpuidle_lock);
415 return ret;
416 }
417
418 cpuidle_enable_device(dev);
419 cpuidle_install_idle_handler();
420
421 mutex_unlock(&cpuidle_lock);
422
423 return 0;
424
425}
426
427EXPORT_SYMBOL_GPL(cpuidle_register_device);
428
429
430
431
432
433void cpuidle_unregister_device(struct cpuidle_device *dev)
434{
435 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
436
437 if (dev->registered == 0)
438 return;
439
440 cpuidle_pause_and_lock();
441
442 cpuidle_disable_device(dev);
443
444 cpuidle_remove_sysfs(dev);
445 list_del(&dev->device_list);
446 per_cpu(cpuidle_devices, dev->cpu) = NULL;
447
448 cpuidle_coupled_unregister_device(dev);
449
450 cpuidle_resume_and_unlock();
451
452 module_put(drv->owner);
453}
454
455EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
456
457
458
459
460
461
462
463
464void cpuidle_unregister(struct cpuidle_driver *drv)
465{
466 int cpu;
467 struct cpuidle_device *device;
468
469 for_each_cpu(cpu, drv->cpumask) {
470 device = &per_cpu(cpuidle_dev, cpu);
471 cpuidle_unregister_device(device);
472 }
473
474 cpuidle_unregister_driver(drv);
475}
476EXPORT_SYMBOL_GPL(cpuidle_unregister);
477
478
479
480
481
482
483
484
485
486
487
488
489int cpuidle_register(struct cpuidle_driver *drv,
490 const struct cpumask *const coupled_cpus)
491{
492 int ret, cpu;
493 struct cpuidle_device *device;
494
495 ret = cpuidle_register_driver(drv);
496 if (ret) {
497 pr_err("failed to register cpuidle driver\n");
498 return ret;
499 }
500
501 for_each_cpu(cpu, drv->cpumask) {
502 device = &per_cpu(cpuidle_dev, cpu);
503 device->cpu = cpu;
504
505#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
506
507
508
509
510
511 if (coupled_cpus)
512 device->coupled_cpus = *coupled_cpus;
513#endif
514 ret = cpuidle_register_device(device);
515 if (!ret)
516 continue;
517
518 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
519
520 cpuidle_unregister(drv);
521 break;
522 }
523
524 return ret;
525}
526EXPORT_SYMBOL_GPL(cpuidle_register);
527
528#ifdef CONFIG_SMP
529
530static void smp_callback(void *v)
531{
532
533}
534
535
536
537
538
539
540
541static int cpuidle_latency_notify(struct notifier_block *b,
542 unsigned long l, void *v)
543{
544 smp_call_function(smp_callback, NULL, 1);
545 return NOTIFY_OK;
546}
547
548static struct notifier_block cpuidle_latency_notifier = {
549 .notifier_call = cpuidle_latency_notify,
550};
551
552static inline void latency_notifier_init(struct notifier_block *n)
553{
554 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
555}
556
557#else
558
559#define latency_notifier_init(x) do { } while (0)
560
561#endif
562
563
564
565
566static int __init cpuidle_init(void)
567{
568 int ret;
569
570 if (cpuidle_disabled())
571 return -ENODEV;
572
573 ret = cpuidle_add_interface(cpu_subsys.dev_root);
574 if (ret)
575 return ret;
576
577 latency_notifier_init(&cpuidle_latency_notifier);
578
579 return 0;
580}
581
582module_param(off, int, 0444);
583core_initcall(cpuidle_init);
584