1
2
3
4
5
6
7
8
9
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/sched/clock.h>
16#include <linux/notifier.h>
17#include <linux/pm_qos.h>
18#include <linux/cpu.h>
19#include <linux/cpuidle.h>
20#include <linux/ktime.h>
21#include <linux/hrtimer.h>
22#include <linux/module.h>
23#include <linux/suspend.h>
24#include <linux/tick.h>
25#include <trace/events/power.h>
26
27#include "cpuidle.h"
28
29DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
30DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
31
32DEFINE_MUTEX(cpuidle_lock);
33LIST_HEAD(cpuidle_detected_devices);
34
35static int enabled_devices;
36static int off __read_mostly;
37static int initialized __read_mostly;
38
39int cpuidle_disabled(void)
40{
41 return off;
42}
43void disable_cpuidle(void)
44{
45 off = 1;
46}
47
48bool cpuidle_not_available(struct cpuidle_driver *drv,
49 struct cpuidle_device *dev)
50{
51 return off || !initialized || !drv || !dev || !dev->enabled;
52}
53
54
55
56
57
58
59int cpuidle_play_dead(void)
60{
61 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
62 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
63 int i;
64
65 if (!drv)
66 return -ENODEV;
67
68
69 for (i = drv->state_count - 1; i >= 0; i--)
70 if (drv->states[i].enter_dead)
71 return drv->states[i].enter_dead(dev, i);
72
73 return -ENODEV;
74}
75
76static int find_deepest_state(struct cpuidle_driver *drv,
77 struct cpuidle_device *dev,
78 unsigned int max_latency,
79 unsigned int forbidden_flags,
80 bool s2idle)
81{
82 unsigned int latency_req = 0;
83 int i, ret = 0;
84
85 for (i = 1; i < drv->state_count; i++) {
86 struct cpuidle_state *s = &drv->states[i];
87 struct cpuidle_state_usage *su = &dev->states_usage[i];
88
89 if (s->disabled || su->disable || s->exit_latency <= latency_req
90 || s->exit_latency > max_latency
91 || (s->flags & forbidden_flags)
92 || (s2idle && !s->enter_s2idle))
93 continue;
94
95 latency_req = s->exit_latency;
96 ret = i;
97 }
98 return ret;
99}
100
101
102
103
104
105
106
107
108void cpuidle_use_deepest_state(bool enable)
109{
110 struct cpuidle_device *dev;
111
112 preempt_disable();
113 dev = cpuidle_get_device();
114 if (dev)
115 dev->use_deepest_state = enable;
116 preempt_enable();
117}
118
119
120
121
122
123
124int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
125 struct cpuidle_device *dev)
126{
127 return find_deepest_state(drv, dev, UINT_MAX, 0, false);
128}
129
130#ifdef CONFIG_SUSPEND
131static void enter_s2idle_proper(struct cpuidle_driver *drv,
132 struct cpuidle_device *dev, int index)
133{
134 ktime_t time_start, time_end;
135
136 time_start = ns_to_ktime(local_clock());
137
138
139
140
141
142
143 RCU_NONIDLE(tick_freeze());
144
145
146
147
148
149 stop_critical_timings();
150 drv->states[index].enter_s2idle(dev, drv, index);
151 WARN_ON(!irqs_disabled());
152
153
154
155
156
157 RCU_NONIDLE(tick_unfreeze());
158 start_critical_timings();
159
160 time_end = ns_to_ktime(local_clock());
161
162 dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);
163 dev->states_usage[index].s2idle_usage++;
164}
165
166
167
168
169
170
171
172
173
174int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
175{
176 int index;
177
178
179
180
181
182
183 index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
184 if (index > 0)
185 enter_s2idle_proper(drv, dev, index);
186
187 return index;
188}
189#endif
190
191
192
193
194
195
196
197int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
198 int index)
199{
200 int entered_state;
201
202 struct cpuidle_state *target_state = &drv->states[index];
203 bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
204 ktime_t time_start, time_end;
205
206
207
208
209
210
211 if (broadcast && tick_broadcast_enter()) {
212 index = find_deepest_state(drv, dev, target_state->exit_latency,
213 CPUIDLE_FLAG_TIMER_STOP, false);
214 if (index < 0) {
215 default_idle_call();
216 return -EBUSY;
217 }
218 target_state = &drv->states[index];
219 broadcast = false;
220 }
221
222
223 sched_idle_set_state(target_state);
224
225 trace_cpu_idle_rcuidle(index, dev->cpu);
226 time_start = ns_to_ktime(local_clock());
227
228 stop_critical_timings();
229 entered_state = target_state->enter(dev, drv, index);
230 start_critical_timings();
231
232 sched_clock_idle_wakeup_event();
233 time_end = ns_to_ktime(local_clock());
234 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
235
236
237 sched_idle_set_state(NULL);
238
239 if (broadcast) {
240 if (WARN_ON_ONCE(!irqs_disabled()))
241 local_irq_disable();
242
243 tick_broadcast_exit();
244 }
245
246 if (!cpuidle_state_is_coupled(drv, index))
247 local_irq_enable();
248
249 if (entered_state >= 0) {
250 s64 diff, delay = drv->states[entered_state].exit_latency;
251 int i;
252
253
254
255
256
257
258 diff = ktime_us_delta(time_end, time_start);
259 if (diff > INT_MAX)
260 diff = INT_MAX;
261
262 dev->last_residency = (int)diff;
263 dev->states_usage[entered_state].time += dev->last_residency;
264 dev->states_usage[entered_state].usage++;
265
266 if (diff < drv->states[entered_state].target_residency) {
267 for (i = entered_state - 1; i >= 0; i--) {
268 if (drv->states[i].disabled ||
269 dev->states_usage[i].disable)
270 continue;
271
272
273 dev->states_usage[entered_state].above++;
274 break;
275 }
276 } else if (diff > delay) {
277 for (i = entered_state + 1; i < drv->state_count; i++) {
278 if (drv->states[i].disabled ||
279 dev->states_usage[i].disable)
280 continue;
281
282
283
284
285
286 if (diff - delay >= drv->states[i].target_residency)
287 dev->states_usage[entered_state].below++;
288
289 break;
290 }
291 }
292 } else {
293 dev->last_residency = 0;
294 }
295
296 return entered_state;
297}
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev,
313 bool *stop_tick)
314{
315 return cpuidle_curr_governor->select(drv, dev, stop_tick);
316}
317
318
319
320
321
322
323
324
325
326
327
328int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
329 int index)
330{
331 int ret = 0;
332
333
334
335
336
337
338
339 WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());
340
341 if (cpuidle_state_is_coupled(drv, index))
342 ret = cpuidle_enter_state_coupled(dev, drv, index);
343 else
344 ret = cpuidle_enter_state(dev, drv, index);
345
346 WRITE_ONCE(dev->next_hrtimer, 0);
347 return ret;
348}
349
350
351
352
353
354
355
356
357
358void cpuidle_reflect(struct cpuidle_device *dev, int index)
359{
360 if (cpuidle_curr_governor->reflect && index >= 0)
361 cpuidle_curr_governor->reflect(dev, index);
362}
363
364
365
366
367void cpuidle_install_idle_handler(void)
368{
369 if (enabled_devices) {
370
371 smp_wmb();
372 initialized = 1;
373 }
374}
375
376
377
378
379void cpuidle_uninstall_idle_handler(void)
380{
381 if (enabled_devices) {
382 initialized = 0;
383 wake_up_all_idle_cpus();
384 }
385
386
387
388
389
390 synchronize_rcu();
391}
392
393
394
395
396void cpuidle_pause_and_lock(void)
397{
398 mutex_lock(&cpuidle_lock);
399 cpuidle_uninstall_idle_handler();
400}
401
402EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
403
404
405
406
407void cpuidle_resume_and_unlock(void)
408{
409 cpuidle_install_idle_handler();
410 mutex_unlock(&cpuidle_lock);
411}
412
413EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
414
415
416void cpuidle_pause(void)
417{
418 mutex_lock(&cpuidle_lock);
419 cpuidle_uninstall_idle_handler();
420 mutex_unlock(&cpuidle_lock);
421}
422
423
424void cpuidle_resume(void)
425{
426 mutex_lock(&cpuidle_lock);
427 cpuidle_install_idle_handler();
428 mutex_unlock(&cpuidle_lock);
429}
430
431
432
433
434
435
436
437
438int cpuidle_enable_device(struct cpuidle_device *dev)
439{
440 int ret;
441 struct cpuidle_driver *drv;
442
443 if (!dev)
444 return -EINVAL;
445
446 if (dev->enabled)
447 return 0;
448
449 if (!cpuidle_curr_governor)
450 return -EIO;
451
452 drv = cpuidle_get_cpu_driver(dev);
453
454 if (!drv)
455 return -EIO;
456
457 if (!dev->registered)
458 return -EINVAL;
459
460 ret = cpuidle_add_device_sysfs(dev);
461 if (ret)
462 return ret;
463
464 if (cpuidle_curr_governor->enable) {
465 ret = cpuidle_curr_governor->enable(drv, dev);
466 if (ret)
467 goto fail_sysfs;
468 }
469
470 smp_wmb();
471
472 dev->enabled = 1;
473
474 enabled_devices++;
475 return 0;
476
477fail_sysfs:
478 cpuidle_remove_device_sysfs(dev);
479
480 return ret;
481}
482
483EXPORT_SYMBOL_GPL(cpuidle_enable_device);
484
485
486
487
488
489
490
491
492void cpuidle_disable_device(struct cpuidle_device *dev)
493{
494 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
495
496 if (!dev || !dev->enabled)
497 return;
498
499 if (!drv || !cpuidle_curr_governor)
500 return;
501
502 dev->enabled = 0;
503
504 if (cpuidle_curr_governor->disable)
505 cpuidle_curr_governor->disable(drv, dev);
506
507 cpuidle_remove_device_sysfs(dev);
508 enabled_devices--;
509}
510
511EXPORT_SYMBOL_GPL(cpuidle_disable_device);
512
513static void __cpuidle_unregister_device(struct cpuidle_device *dev)
514{
515 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
516
517 list_del(&dev->device_list);
518 per_cpu(cpuidle_devices, dev->cpu) = NULL;
519 module_put(drv->owner);
520
521 dev->registered = 0;
522}
523
524static void __cpuidle_device_init(struct cpuidle_device *dev)
525{
526 memset(dev->states_usage, 0, sizeof(dev->states_usage));
527 dev->last_residency = 0;
528 dev->next_hrtimer = 0;
529}
530
531
532
533
534
535
536
537
538static int __cpuidle_register_device(struct cpuidle_device *dev)
539{
540 int ret;
541 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
542
543 if (!try_module_get(drv->owner))
544 return -EINVAL;
545
546 per_cpu(cpuidle_devices, dev->cpu) = dev;
547 list_add(&dev->device_list, &cpuidle_detected_devices);
548
549 ret = cpuidle_coupled_register_device(dev);
550 if (ret)
551 __cpuidle_unregister_device(dev);
552 else
553 dev->registered = 1;
554
555 return ret;
556}
557
558
559
560
561
562int cpuidle_register_device(struct cpuidle_device *dev)
563{
564 int ret = -EBUSY;
565
566 if (!dev)
567 return -EINVAL;
568
569 mutex_lock(&cpuidle_lock);
570
571 if (dev->registered)
572 goto out_unlock;
573
574 __cpuidle_device_init(dev);
575
576 ret = __cpuidle_register_device(dev);
577 if (ret)
578 goto out_unlock;
579
580 ret = cpuidle_add_sysfs(dev);
581 if (ret)
582 goto out_unregister;
583
584 ret = cpuidle_enable_device(dev);
585 if (ret)
586 goto out_sysfs;
587
588 cpuidle_install_idle_handler();
589
590out_unlock:
591 mutex_unlock(&cpuidle_lock);
592
593 return ret;
594
595out_sysfs:
596 cpuidle_remove_sysfs(dev);
597out_unregister:
598 __cpuidle_unregister_device(dev);
599 goto out_unlock;
600}
601
602EXPORT_SYMBOL_GPL(cpuidle_register_device);
603
604
605
606
607
608void cpuidle_unregister_device(struct cpuidle_device *dev)
609{
610 if (!dev || dev->registered == 0)
611 return;
612
613 cpuidle_pause_and_lock();
614
615 cpuidle_disable_device(dev);
616
617 cpuidle_remove_sysfs(dev);
618
619 __cpuidle_unregister_device(dev);
620
621 cpuidle_coupled_unregister_device(dev);
622
623 cpuidle_resume_and_unlock();
624}
625
626EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
627
628
629
630
631
632
633
634
635void cpuidle_unregister(struct cpuidle_driver *drv)
636{
637 int cpu;
638 struct cpuidle_device *device;
639
640 for_each_cpu(cpu, drv->cpumask) {
641 device = &per_cpu(cpuidle_dev, cpu);
642 cpuidle_unregister_device(device);
643 }
644
645 cpuidle_unregister_driver(drv);
646}
647EXPORT_SYMBOL_GPL(cpuidle_unregister);
648
649
650
651
652
653
654
655
656
657
658
659
660int cpuidle_register(struct cpuidle_driver *drv,
661 const struct cpumask *const coupled_cpus)
662{
663 int ret, cpu;
664 struct cpuidle_device *device;
665
666 ret = cpuidle_register_driver(drv);
667 if (ret) {
668 pr_err("failed to register cpuidle driver\n");
669 return ret;
670 }
671
672 for_each_cpu(cpu, drv->cpumask) {
673 device = &per_cpu(cpuidle_dev, cpu);
674 device->cpu = cpu;
675
676#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
677
678
679
680
681
682 if (coupled_cpus)
683 device->coupled_cpus = *coupled_cpus;
684#endif
685 ret = cpuidle_register_device(device);
686 if (!ret)
687 continue;
688
689 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
690
691 cpuidle_unregister(drv);
692 break;
693 }
694
695 return ret;
696}
697EXPORT_SYMBOL_GPL(cpuidle_register);
698
699#ifdef CONFIG_SMP
700
701
702
703
704
705
706
707static int cpuidle_latency_notify(struct notifier_block *b,
708 unsigned long l, void *v)
709{
710 wake_up_all_idle_cpus();
711 return NOTIFY_OK;
712}
713
714static struct notifier_block cpuidle_latency_notifier = {
715 .notifier_call = cpuidle_latency_notify,
716};
717
718static inline void latency_notifier_init(struct notifier_block *n)
719{
720 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
721}
722
723#else
724
725#define latency_notifier_init(x) do { } while (0)
726
727#endif
728
729
730
731
732static int __init cpuidle_init(void)
733{
734 int ret;
735
736 if (cpuidle_disabled())
737 return -ENODEV;
738
739 ret = cpuidle_add_interface(cpu_subsys.dev_root);
740 if (ret)
741 return ret;
742
743 latency_notifier_init(&cpuidle_latency_notifier);
744
745 return 0;
746}
747
748module_param(off, int, 0444);
749module_param_string(governor, param_governor, CPUIDLE_NAME_LEN, 0444);
750core_initcall(cpuidle_init);
751