1
2
3
4
5
6
7
8
9
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/sched/clock.h>
16#include <linux/notifier.h>
17#include <linux/pm_qos.h>
18#include <linux/cpu.h>
19#include <linux/cpuidle.h>
20#include <linux/ktime.h>
21#include <linux/hrtimer.h>
22#include <linux/module.h>
23#include <linux/suspend.h>
24#include <linux/tick.h>
25#include <trace/events/power.h>
26
27#include "cpuidle.h"
28
29DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
30DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
31
32DEFINE_MUTEX(cpuidle_lock);
33LIST_HEAD(cpuidle_detected_devices);
34
35static int enabled_devices;
36static int off __read_mostly;
37static int initialized __read_mostly;
38
39int cpuidle_disabled(void)
40{
41 return off;
42}
43void disable_cpuidle(void)
44{
45 off = 1;
46}
47
48bool cpuidle_not_available(struct cpuidle_driver *drv,
49 struct cpuidle_device *dev)
50{
51 return off || !initialized || !drv || !dev || !dev->enabled;
52}
53
54
55
56
57
58
59int cpuidle_play_dead(void)
60{
61 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
62 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
63 int i;
64
65 if (!drv)
66 return -ENODEV;
67
68
69 for (i = drv->state_count - 1; i >= 0; i--)
70 if (drv->states[i].enter_dead)
71 return drv->states[i].enter_dead(dev, i);
72
73 return -ENODEV;
74}
75
76static int find_deepest_state(struct cpuidle_driver *drv,
77 struct cpuidle_device *dev,
78 unsigned int max_latency,
79 unsigned int forbidden_flags,
80 bool s2idle)
81{
82 unsigned int latency_req = 0;
83 int i, ret = 0;
84
85 for (i = 1; i < drv->state_count; i++) {
86 struct cpuidle_state *s = &drv->states[i];
87 struct cpuidle_state_usage *su = &dev->states_usage[i];
88
89 if (s->disabled || su->disable || s->exit_latency <= latency_req
90 || s->exit_latency > max_latency
91 || (s->flags & forbidden_flags)
92 || (s2idle && !s->enter_s2idle))
93 continue;
94
95 latency_req = s->exit_latency;
96 ret = i;
97 }
98 return ret;
99}
100
101
102
103
104
105
106
107
108void cpuidle_use_deepest_state(bool enable)
109{
110 struct cpuidle_device *dev;
111
112 preempt_disable();
113 dev = cpuidle_get_device();
114 if (dev)
115 dev->use_deepest_state = enable;
116 preempt_enable();
117}
118
119
120
121
122
123
124int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
125 struct cpuidle_device *dev)
126{
127 return find_deepest_state(drv, dev, UINT_MAX, 0, false);
128}
129
130#ifdef CONFIG_SUSPEND
131static void enter_s2idle_proper(struct cpuidle_driver *drv,
132 struct cpuidle_device *dev, int index)
133{
134 ktime_t time_start, time_end;
135
136 time_start = ns_to_ktime(local_clock());
137
138
139
140
141
142
143 RCU_NONIDLE(tick_freeze());
144
145
146
147
148
149 stop_critical_timings();
150 drv->states[index].enter_s2idle(dev, drv, index);
151 WARN_ON(!irqs_disabled());
152
153
154
155
156
157 RCU_NONIDLE(tick_unfreeze());
158 start_critical_timings();
159
160 time_end = ns_to_ktime(local_clock());
161
162 dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);
163 dev->states_usage[index].s2idle_usage++;
164}
165
166
167
168
169
170
171
172
173
174int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
175{
176 int index;
177
178
179
180
181
182
183 index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
184 if (index > 0)
185 enter_s2idle_proper(drv, dev, index);
186
187 return index;
188}
189#endif
190
191
192
193
194
195
196
197int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
198 int index)
199{
200 int entered_state;
201
202 struct cpuidle_state *target_state = &drv->states[index];
203 bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
204 ktime_t time_start, time_end;
205
206
207
208
209
210
211 if (broadcast && tick_broadcast_enter()) {
212 index = find_deepest_state(drv, dev, target_state->exit_latency,
213 CPUIDLE_FLAG_TIMER_STOP, false);
214 if (index < 0) {
215 default_idle_call();
216 return -EBUSY;
217 }
218 target_state = &drv->states[index];
219 broadcast = false;
220 }
221
222
223 sched_idle_set_state(target_state);
224
225 trace_cpu_idle_rcuidle(index, dev->cpu);
226 time_start = ns_to_ktime(local_clock());
227
228 stop_critical_timings();
229 entered_state = target_state->enter(dev, drv, index);
230 start_critical_timings();
231
232 sched_clock_idle_wakeup_event();
233 time_end = ns_to_ktime(local_clock());
234 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
235
236
237 sched_idle_set_state(NULL);
238
239 if (broadcast) {
240 if (WARN_ON_ONCE(!irqs_disabled()))
241 local_irq_disable();
242
243 tick_broadcast_exit();
244 }
245
246 if (!cpuidle_state_is_coupled(drv, index))
247 local_irq_enable();
248
249 if (entered_state >= 0) {
250 s64 diff, delay = drv->states[entered_state].exit_latency;
251 int i;
252
253
254
255
256
257
258 diff = ktime_us_delta(time_end, time_start);
259 if (diff > INT_MAX)
260 diff = INT_MAX;
261
262 dev->last_residency = (int)diff;
263 dev->states_usage[entered_state].time += dev->last_residency;
264 dev->states_usage[entered_state].usage++;
265
266 if (diff < drv->states[entered_state].target_residency) {
267 for (i = entered_state - 1; i >= 0; i--) {
268 if (drv->states[i].disabled ||
269 dev->states_usage[i].disable)
270 continue;
271
272
273 dev->states_usage[entered_state].above++;
274 break;
275 }
276 } else if (diff > delay) {
277 for (i = entered_state + 1; i < drv->state_count; i++) {
278 if (drv->states[i].disabled ||
279 dev->states_usage[i].disable)
280 continue;
281
282
283
284
285
286 if (diff - delay >= drv->states[i].target_residency)
287 dev->states_usage[entered_state].below++;
288
289 break;
290 }
291 }
292 } else {
293 dev->last_residency = 0;
294 }
295
296 return entered_state;
297}
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
313 bool *stop_tick)
314{
315 return cpuidle_curr_governor->select(drv, dev, stop_tick);
316}
317
318
319
320
321
322
323
324
325
326
327
328int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
329 int index)
330{
331 int ret = 0;
332
333
334
335
336
337
338
339 WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());
340
341 if (cpuidle_state_is_coupled(drv, index))
342 ret = cpuidle_enter_state_coupled(dev, drv, index);
343 else
344 ret = cpuidle_enter_state(dev, drv, index);
345
346 WRITE_ONCE(dev->next_hrtimer, 0);
347 return ret;
348}
349
350
351
352
353
354
355
356
357
358void cpuidle_reflect(struct cpuidle_device *dev, int index)
359{
360 if (cpuidle_curr_governor->reflect && index >= 0)
361 cpuidle_curr_governor->reflect(dev, index);
362}
363
364
365
366
367
368
369
370
371
372u64 cpuidle_poll_time(struct cpuidle_driver *drv,
373 struct cpuidle_device *dev)
374{
375 int i;
376 u64 limit_ns;
377
378 if (dev->poll_limit_ns)
379 return dev->poll_limit_ns;
380
381 limit_ns = TICK_NSEC;
382 for (i = 1; i < drv->state_count; i++) {
383 if (drv->states[i].disabled || dev->states_usage[i].disable)
384 continue;
385
386 limit_ns = (u64)drv->states[i].target_residency * NSEC_PER_USEC;
387 }
388
389 dev->poll_limit_ns = limit_ns;
390
391 return dev->poll_limit_ns;
392}
393
394
395
396
397void cpuidle_install_idle_handler(void)
398{
399 if (enabled_devices) {
400
401 smp_wmb();
402 initialized = 1;
403 }
404}
405
406
407
408
409void cpuidle_uninstall_idle_handler(void)
410{
411 if (enabled_devices) {
412 initialized = 0;
413 wake_up_all_idle_cpus();
414 }
415
416
417
418
419
420 synchronize_rcu();
421}
422
423
424
425
426void cpuidle_pause_and_lock(void)
427{
428 mutex_lock(&cpuidle_lock);
429 cpuidle_uninstall_idle_handler();
430}
431
432EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
433
434
435
436
437void cpuidle_resume_and_unlock(void)
438{
439 cpuidle_install_idle_handler();
440 mutex_unlock(&cpuidle_lock);
441}
442
443EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
444
445
446void cpuidle_pause(void)
447{
448 mutex_lock(&cpuidle_lock);
449 cpuidle_uninstall_idle_handler();
450 mutex_unlock(&cpuidle_lock);
451}
452
453
454void cpuidle_resume(void)
455{
456 mutex_lock(&cpuidle_lock);
457 cpuidle_install_idle_handler();
458 mutex_unlock(&cpuidle_lock);
459}
460
461
462
463
464
465
466
467
468int cpuidle_enable_device(struct cpuidle_device *dev)
469{
470 int ret;
471 struct cpuidle_driver *drv;
472
473 if (!dev)
474 return -EINVAL;
475
476 if (dev->enabled)
477 return 0;
478
479 if (!cpuidle_curr_governor)
480 return -EIO;
481
482 drv = cpuidle_get_cpu_driver(dev);
483
484 if (!drv)
485 return -EIO;
486
487 if (!dev->registered)
488 return -EINVAL;
489
490 ret = cpuidle_add_device_sysfs(dev);
491 if (ret)
492 return ret;
493
494 if (cpuidle_curr_governor->enable) {
495 ret = cpuidle_curr_governor->enable(drv, dev);
496 if (ret)
497 goto fail_sysfs;
498 }
499
500 smp_wmb();
501
502 dev->enabled = 1;
503
504 enabled_devices++;
505 return 0;
506
507fail_sysfs:
508 cpuidle_remove_device_sysfs(dev);
509
510 return ret;
511}
512
513EXPORT_SYMBOL_GPL(cpuidle_enable_device);
514
515
516
517
518
519
520
521
522void cpuidle_disable_device(struct cpuidle_device *dev)
523{
524 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
525
526 if (!dev || !dev->enabled)
527 return;
528
529 if (!drv || !cpuidle_curr_governor)
530 return;
531
532 dev->enabled = 0;
533
534 if (cpuidle_curr_governor->disable)
535 cpuidle_curr_governor->disable(drv, dev);
536
537 cpuidle_remove_device_sysfs(dev);
538 enabled_devices--;
539}
540
541EXPORT_SYMBOL_GPL(cpuidle_disable_device);
542
543static void __cpuidle_unregister_device(struct cpuidle_device *dev)
544{
545 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
546
547 list_del(&dev->device_list);
548 per_cpu(cpuidle_devices, dev->cpu) = NULL;
549 module_put(drv->owner);
550
551 dev->registered = 0;
552}
553
554static void __cpuidle_device_init(struct cpuidle_device *dev)
555{
556 memset(dev->states_usage, 0, sizeof(dev->states_usage));
557 dev->last_residency = 0;
558 dev->next_hrtimer = 0;
559}
560
561
562
563
564
565
566
567
568static int __cpuidle_register_device(struct cpuidle_device *dev)
569{
570 int ret;
571 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
572
573 if (!try_module_get(drv->owner))
574 return -EINVAL;
575
576 per_cpu(cpuidle_devices, dev->cpu) = dev;
577 list_add(&dev->device_list, &cpuidle_detected_devices);
578
579 ret = cpuidle_coupled_register_device(dev);
580 if (ret)
581 __cpuidle_unregister_device(dev);
582 else
583 dev->registered = 1;
584
585 return ret;
586}
587
588
589
590
591
592int cpuidle_register_device(struct cpuidle_device *dev)
593{
594 int ret = -EBUSY;
595
596 if (!dev)
597 return -EINVAL;
598
599 mutex_lock(&cpuidle_lock);
600
601 if (dev->registered)
602 goto out_unlock;
603
604 __cpuidle_device_init(dev);
605
606 ret = __cpuidle_register_device(dev);
607 if (ret)
608 goto out_unlock;
609
610 ret = cpuidle_add_sysfs(dev);
611 if (ret)
612 goto out_unregister;
613
614 ret = cpuidle_enable_device(dev);
615 if (ret)
616 goto out_sysfs;
617
618 cpuidle_install_idle_handler();
619
620out_unlock:
621 mutex_unlock(&cpuidle_lock);
622
623 return ret;
624
625out_sysfs:
626 cpuidle_remove_sysfs(dev);
627out_unregister:
628 __cpuidle_unregister_device(dev);
629 goto out_unlock;
630}
631
632EXPORT_SYMBOL_GPL(cpuidle_register_device);
633
634
635
636
637
638void cpuidle_unregister_device(struct cpuidle_device *dev)
639{
640 if (!dev || dev->registered == 0)
641 return;
642
643 cpuidle_pause_and_lock();
644
645 cpuidle_disable_device(dev);
646
647 cpuidle_remove_sysfs(dev);
648
649 __cpuidle_unregister_device(dev);
650
651 cpuidle_coupled_unregister_device(dev);
652
653 cpuidle_resume_and_unlock();
654}
655
656EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
657
658
659
660
661
662
663
664
665void cpuidle_unregister(struct cpuidle_driver *drv)
666{
667 int cpu;
668 struct cpuidle_device *device;
669
670 for_each_cpu(cpu, drv->cpumask) {
671 device = &per_cpu(cpuidle_dev, cpu);
672 cpuidle_unregister_device(device);
673 }
674
675 cpuidle_unregister_driver(drv);
676}
677EXPORT_SYMBOL_GPL(cpuidle_unregister);
678
679
680
681
682
683
684
685
686
687
688
689
690int cpuidle_register(struct cpuidle_driver *drv,
691 const struct cpumask *const coupled_cpus)
692{
693 int ret, cpu;
694 struct cpuidle_device *device;
695
696 ret = cpuidle_register_driver(drv);
697 if (ret) {
698 pr_err("failed to register cpuidle driver\n");
699 return ret;
700 }
701
702 for_each_cpu(cpu, drv->cpumask) {
703 device = &per_cpu(cpuidle_dev, cpu);
704 device->cpu = cpu;
705
706#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
707
708
709
710
711
712 if (coupled_cpus)
713 device->coupled_cpus = *coupled_cpus;
714#endif
715 ret = cpuidle_register_device(device);
716 if (!ret)
717 continue;
718
719 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
720
721 cpuidle_unregister(drv);
722 break;
723 }
724
725 return ret;
726}
727EXPORT_SYMBOL_GPL(cpuidle_register);
728
729#ifdef CONFIG_SMP
730
731
732
733
734
735
736
737static int cpuidle_latency_notify(struct notifier_block *b,
738 unsigned long l, void *v)
739{
740 wake_up_all_idle_cpus();
741 return NOTIFY_OK;
742}
743
744static struct notifier_block cpuidle_latency_notifier = {
745 .notifier_call = cpuidle_latency_notify,
746};
747
748static inline void latency_notifier_init(struct notifier_block *n)
749{
750 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
751}
752
753#else
754
755#define latency_notifier_init(x) do { } while (0)
756
757#endif
758
759
760
761
762static int __init cpuidle_init(void)
763{
764 int ret;
765
766 if (cpuidle_disabled())
767 return -ENODEV;
768
769 ret = cpuidle_add_interface(cpu_subsys.dev_root);
770 if (ret)
771 return ret;
772
773 latency_notifier_init(&cpuidle_latency_notifier);
774
775 return 0;
776}
777
778module_param(off, int, 0444);
779module_param_string(governor, param_governor, CPUIDLE_NAME_LEN, 0444);
780core_initcall(cpuidle_init);
781