1
2
3
4
5
6
7
8
9
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/sched/clock.h>
16#include <linux/notifier.h>
17#include <linux/pm_qos.h>
18#include <linux/cpu.h>
19#include <linux/cpuidle.h>
20#include <linux/ktime.h>
21#include <linux/hrtimer.h>
22#include <linux/module.h>
23#include <linux/suspend.h>
24#include <linux/tick.h>
25#include <trace/events/power.h>
26
27#include "cpuidle.h"
28
29DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
30DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
31
32DEFINE_MUTEX(cpuidle_lock);
33LIST_HEAD(cpuidle_detected_devices);
34
35static int enabled_devices;
36static int off __read_mostly;
37static int initialized __read_mostly;
38
39int cpuidle_disabled(void)
40{
41 return off;
42}
43void disable_cpuidle(void)
44{
45 off = 1;
46}
47
48bool cpuidle_not_available(struct cpuidle_driver *drv,
49 struct cpuidle_device *dev)
50{
51 return off || !initialized || !drv || !dev || !dev->enabled;
52}
53
54
55
56
57
58
59int cpuidle_play_dead(void)
60{
61 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
62 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
63 int i;
64
65 if (!drv)
66 return -ENODEV;
67
68
69 for (i = drv->state_count - 1; i >= 0; i--)
70 if (drv->states[i].enter_dead)
71 return drv->states[i].enter_dead(dev, i);
72
73 return -ENODEV;
74}
75
76static int find_deepest_state(struct cpuidle_driver *drv,
77 struct cpuidle_device *dev,
78 unsigned int max_latency,
79 unsigned int forbidden_flags,
80 bool s2idle)
81{
82 unsigned int latency_req = 0;
83 int i, ret = 0;
84
85 for (i = 1; i < drv->state_count; i++) {
86 struct cpuidle_state *s = &drv->states[i];
87 struct cpuidle_state_usage *su = &dev->states_usage[i];
88
89 if (s->disabled || su->disable || s->exit_latency <= latency_req
90 || s->exit_latency > max_latency
91 || (s->flags & forbidden_flags)
92 || (s2idle && !s->enter_s2idle))
93 continue;
94
95 latency_req = s->exit_latency;
96 ret = i;
97 }
98 return ret;
99}
100
101
102
103
104
105
106
107
108void cpuidle_use_deepest_state(bool enable)
109{
110 struct cpuidle_device *dev;
111
112 preempt_disable();
113 dev = cpuidle_get_device();
114 if (dev)
115 dev->use_deepest_state = enable;
116 preempt_enable();
117}
118
119
120
121
122
123
124int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
125 struct cpuidle_device *dev)
126{
127 return find_deepest_state(drv, dev, UINT_MAX, 0, false);
128}
129
130#ifdef CONFIG_SUSPEND
131static void enter_s2idle_proper(struct cpuidle_driver *drv,
132 struct cpuidle_device *dev, int index)
133{
134 ktime_t time_start, time_end;
135
136 time_start = ns_to_ktime(local_clock());
137
138
139
140
141
142
143 RCU_NONIDLE(tick_freeze());
144
145
146
147
148
149 stop_critical_timings();
150 drv->states[index].enter_s2idle(dev, drv, index);
151 WARN_ON(!irqs_disabled());
152
153
154
155
156
157 RCU_NONIDLE(tick_unfreeze());
158 start_critical_timings();
159
160 time_end = ns_to_ktime(local_clock());
161
162 dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);
163 dev->states_usage[index].s2idle_usage++;
164}
165
166
167
168
169
170
171
172
173
174int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
175{
176 int index;
177
178
179
180
181
182
183 index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
184 if (index > 0)
185 enter_s2idle_proper(drv, dev, index);
186
187 return index;
188}
189#endif
190
191
192
193
194
195
196
197int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
198 int index)
199{
200 int entered_state;
201
202 struct cpuidle_state *target_state = &drv->states[index];
203 bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
204 ktime_t time_start, time_end;
205 s64 diff;
206
207
208
209
210
211
212 if (broadcast && tick_broadcast_enter()) {
213 index = find_deepest_state(drv, dev, target_state->exit_latency,
214 CPUIDLE_FLAG_TIMER_STOP, false);
215 if (index < 0) {
216 default_idle_call();
217 return -EBUSY;
218 }
219 target_state = &drv->states[index];
220 broadcast = false;
221 }
222
223
224 sched_idle_set_state(target_state);
225
226 trace_cpu_idle_rcuidle(index, dev->cpu);
227 time_start = ns_to_ktime(local_clock());
228
229 stop_critical_timings();
230 entered_state = target_state->enter(dev, drv, index);
231 start_critical_timings();
232
233 sched_clock_idle_wakeup_event();
234 time_end = ns_to_ktime(local_clock());
235 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
236
237
238 sched_idle_set_state(NULL);
239
240 if (broadcast) {
241 if (WARN_ON_ONCE(!irqs_disabled()))
242 local_irq_disable();
243
244 tick_broadcast_exit();
245 }
246
247 if (!cpuidle_state_is_coupled(drv, index))
248 local_irq_enable();
249
250 diff = ktime_us_delta(time_end, time_start);
251 if (diff > INT_MAX)
252 diff = INT_MAX;
253
254 dev->last_residency = (int) diff;
255
256 if (entered_state >= 0) {
257
258
259
260
261 dev->states_usage[entered_state].time += dev->last_residency;
262 dev->states_usage[entered_state].usage++;
263 } else {
264 dev->last_residency = 0;
265 }
266
267 return entered_state;
268}
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
284 bool *stop_tick)
285{
286 return cpuidle_curr_governor->select(drv, dev, stop_tick);
287}
288
289
290
291
292
293
294
295
296
297
298
299int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
300 int index)
301{
302 if (cpuidle_state_is_coupled(drv, index))
303 return cpuidle_enter_state_coupled(dev, drv, index);
304 return cpuidle_enter_state(dev, drv, index);
305}
306
307
308
309
310
311
312
313
314
315void cpuidle_reflect(struct cpuidle_device *dev, int index)
316{
317 if (cpuidle_curr_governor->reflect && index >= 0)
318 cpuidle_curr_governor->reflect(dev, index);
319}
320
321
322
323
324void cpuidle_install_idle_handler(void)
325{
326 if (enabled_devices) {
327
328 smp_wmb();
329 initialized = 1;
330 }
331}
332
333
334
335
336void cpuidle_uninstall_idle_handler(void)
337{
338 if (enabled_devices) {
339 initialized = 0;
340 wake_up_all_idle_cpus();
341 }
342
343
344
345
346
347 synchronize_rcu();
348}
349
350
351
352
353void cpuidle_pause_and_lock(void)
354{
355 mutex_lock(&cpuidle_lock);
356 cpuidle_uninstall_idle_handler();
357}
358
359EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
360
361
362
363
364void cpuidle_resume_and_unlock(void)
365{
366 cpuidle_install_idle_handler();
367 mutex_unlock(&cpuidle_lock);
368}
369
370EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
371
372
373void cpuidle_pause(void)
374{
375 mutex_lock(&cpuidle_lock);
376 cpuidle_uninstall_idle_handler();
377 mutex_unlock(&cpuidle_lock);
378}
379
380
381void cpuidle_resume(void)
382{
383 mutex_lock(&cpuidle_lock);
384 cpuidle_install_idle_handler();
385 mutex_unlock(&cpuidle_lock);
386}
387
388
389
390
391
392
393
394
395int cpuidle_enable_device(struct cpuidle_device *dev)
396{
397 int ret;
398 struct cpuidle_driver *drv;
399
400 if (!dev)
401 return -EINVAL;
402
403 if (dev->enabled)
404 return 0;
405
406 if (!cpuidle_curr_governor)
407 return -EIO;
408
409 drv = cpuidle_get_cpu_driver(dev);
410
411 if (!drv)
412 return -EIO;
413
414 if (!dev->registered)
415 return -EINVAL;
416
417 ret = cpuidle_add_device_sysfs(dev);
418 if (ret)
419 return ret;
420
421 if (cpuidle_curr_governor->enable) {
422 ret = cpuidle_curr_governor->enable(drv, dev);
423 if (ret)
424 goto fail_sysfs;
425 }
426
427 smp_wmb();
428
429 dev->enabled = 1;
430
431 enabled_devices++;
432 return 0;
433
434fail_sysfs:
435 cpuidle_remove_device_sysfs(dev);
436
437 return ret;
438}
439
440EXPORT_SYMBOL_GPL(cpuidle_enable_device);
441
442
443
444
445
446
447
448
449void cpuidle_disable_device(struct cpuidle_device *dev)
450{
451 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
452
453 if (!dev || !dev->enabled)
454 return;
455
456 if (!drv || !cpuidle_curr_governor)
457 return;
458
459 dev->enabled = 0;
460
461 if (cpuidle_curr_governor->disable)
462 cpuidle_curr_governor->disable(drv, dev);
463
464 cpuidle_remove_device_sysfs(dev);
465 enabled_devices--;
466}
467
468EXPORT_SYMBOL_GPL(cpuidle_disable_device);
469
470static void __cpuidle_unregister_device(struct cpuidle_device *dev)
471{
472 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
473
474 list_del(&dev->device_list);
475 per_cpu(cpuidle_devices, dev->cpu) = NULL;
476 module_put(drv->owner);
477
478 dev->registered = 0;
479}
480
481static void __cpuidle_device_init(struct cpuidle_device *dev)
482{
483 memset(dev->states_usage, 0, sizeof(dev->states_usage));
484 dev->last_residency = 0;
485}
486
487
488
489
490
491
492
493
494static int __cpuidle_register_device(struct cpuidle_device *dev)
495{
496 int ret;
497 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
498
499 if (!try_module_get(drv->owner))
500 return -EINVAL;
501
502 per_cpu(cpuidle_devices, dev->cpu) = dev;
503 list_add(&dev->device_list, &cpuidle_detected_devices);
504
505 ret = cpuidle_coupled_register_device(dev);
506 if (ret)
507 __cpuidle_unregister_device(dev);
508 else
509 dev->registered = 1;
510
511 return ret;
512}
513
514
515
516
517
518int cpuidle_register_device(struct cpuidle_device *dev)
519{
520 int ret = -EBUSY;
521
522 if (!dev)
523 return -EINVAL;
524
525 mutex_lock(&cpuidle_lock);
526
527 if (dev->registered)
528 goto out_unlock;
529
530 __cpuidle_device_init(dev);
531
532 ret = __cpuidle_register_device(dev);
533 if (ret)
534 goto out_unlock;
535
536 ret = cpuidle_add_sysfs(dev);
537 if (ret)
538 goto out_unregister;
539
540 ret = cpuidle_enable_device(dev);
541 if (ret)
542 goto out_sysfs;
543
544 cpuidle_install_idle_handler();
545
546out_unlock:
547 mutex_unlock(&cpuidle_lock);
548
549 return ret;
550
551out_sysfs:
552 cpuidle_remove_sysfs(dev);
553out_unregister:
554 __cpuidle_unregister_device(dev);
555 goto out_unlock;
556}
557
558EXPORT_SYMBOL_GPL(cpuidle_register_device);
559
560
561
562
563
564void cpuidle_unregister_device(struct cpuidle_device *dev)
565{
566 if (!dev || dev->registered == 0)
567 return;
568
569 cpuidle_pause_and_lock();
570
571 cpuidle_disable_device(dev);
572
573 cpuidle_remove_sysfs(dev);
574
575 __cpuidle_unregister_device(dev);
576
577 cpuidle_coupled_unregister_device(dev);
578
579 cpuidle_resume_and_unlock();
580}
581
582EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
583
584
585
586
587
588
589
590
591void cpuidle_unregister(struct cpuidle_driver *drv)
592{
593 int cpu;
594 struct cpuidle_device *device;
595
596 for_each_cpu(cpu, drv->cpumask) {
597 device = &per_cpu(cpuidle_dev, cpu);
598 cpuidle_unregister_device(device);
599 }
600
601 cpuidle_unregister_driver(drv);
602}
603EXPORT_SYMBOL_GPL(cpuidle_unregister);
604
605
606
607
608
609
610
611
612
613
614
615
616int cpuidle_register(struct cpuidle_driver *drv,
617 const struct cpumask *const coupled_cpus)
618{
619 int ret, cpu;
620 struct cpuidle_device *device;
621
622 ret = cpuidle_register_driver(drv);
623 if (ret) {
624 pr_err("failed to register cpuidle driver\n");
625 return ret;
626 }
627
628 for_each_cpu(cpu, drv->cpumask) {
629 device = &per_cpu(cpuidle_dev, cpu);
630 device->cpu = cpu;
631
632#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
633
634
635
636
637
638 if (coupled_cpus)
639 device->coupled_cpus = *coupled_cpus;
640#endif
641 ret = cpuidle_register_device(device);
642 if (!ret)
643 continue;
644
645 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
646
647 cpuidle_unregister(drv);
648 break;
649 }
650
651 return ret;
652}
653EXPORT_SYMBOL_GPL(cpuidle_register);
654
655#ifdef CONFIG_SMP
656
657
658
659
660
661
662
663static int cpuidle_latency_notify(struct notifier_block *b,
664 unsigned long l, void *v)
665{
666 wake_up_all_idle_cpus();
667 return NOTIFY_OK;
668}
669
670static struct notifier_block cpuidle_latency_notifier = {
671 .notifier_call = cpuidle_latency_notify,
672};
673
674static inline void latency_notifier_init(struct notifier_block *n)
675{
676 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
677}
678
679#else
680
681#define latency_notifier_init(x) do { } while (0)
682
683#endif
684
685
686
687
688static int __init cpuidle_init(void)
689{
690 int ret;
691
692 if (cpuidle_disabled())
693 return -ENODEV;
694
695 ret = cpuidle_add_interface(cpu_subsys.dev_root);
696 if (ret)
697 return ret;
698
699 latency_notifier_init(&cpuidle_latency_notifier);
700
701 return 0;
702}
703
704module_param(off, int, 0444);
705core_initcall(cpuidle_init);
706