1
2
3
4
5
6
7
8
9
10
11#include <linux/clockchips.h>
12#include <linux/kernel.h>
13#include <linux/mutex.h>
14#include <linux/sched.h>
15#include <linux/sched/clock.h>
16#include <linux/notifier.h>
17#include <linux/pm_qos.h>
18#include <linux/cpu.h>
19#include <linux/cpuidle.h>
20#include <linux/ktime.h>
21#include <linux/hrtimer.h>
22#include <linux/module.h>
23#include <linux/suspend.h>
24#include <linux/tick.h>
25#include <trace/events/power.h>
26
27#include "cpuidle.h"
28
29DEFINE_PER_CPU(struct cpuidle_device *, cpuidle_devices);
30DEFINE_PER_CPU(struct cpuidle_device, cpuidle_dev);
31
32DEFINE_MUTEX(cpuidle_lock);
33LIST_HEAD(cpuidle_detected_devices);
34
35static int enabled_devices;
36static int off __read_mostly;
37static int initialized __read_mostly;
38
39int cpuidle_disabled(void)
40{
41 return off;
42}
43void disable_cpuidle(void)
44{
45 off = 1;
46}
47
48bool cpuidle_not_available(struct cpuidle_driver *drv,
49 struct cpuidle_device *dev)
50{
51 return off || !initialized || !drv || !dev || !dev->enabled;
52}
53
54
55
56
57
58
59int cpuidle_play_dead(void)
60{
61 struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices);
62 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
63 int i;
64
65 if (!drv)
66 return -ENODEV;
67
68
69 for (i = drv->state_count - 1; i >= 0; i--)
70 if (drv->states[i].enter_dead)
71 return drv->states[i].enter_dead(dev, i);
72
73 return -ENODEV;
74}
75
76static int find_deepest_state(struct cpuidle_driver *drv,
77 struct cpuidle_device *dev,
78 unsigned int max_latency,
79 unsigned int forbidden_flags,
80 bool s2idle)
81{
82 unsigned int latency_req = 0;
83 int i, ret = 0;
84
85 for (i = 1; i < drv->state_count; i++) {
86 struct cpuidle_state *s = &drv->states[i];
87 struct cpuidle_state_usage *su = &dev->states_usage[i];
88
89 if (s->disabled || su->disable || s->exit_latency <= latency_req
90 || s->exit_latency > max_latency
91 || (s->flags & forbidden_flags)
92 || (s2idle && !s->enter_s2idle))
93 continue;
94
95 latency_req = s->exit_latency;
96 ret = i;
97 }
98 return ret;
99}
100
101
102
103
104
105
106
107
108void cpuidle_use_deepest_state(bool enable)
109{
110 struct cpuidle_device *dev;
111
112 preempt_disable();
113 dev = cpuidle_get_device();
114 if (dev)
115 dev->use_deepest_state = enable;
116 preempt_enable();
117}
118
119
120
121
122
123
124int cpuidle_find_deepest_state(struct cpuidle_driver *drv,
125 struct cpuidle_device *dev)
126{
127 return find_deepest_state(drv, dev, UINT_MAX, 0, false);
128}
129
130#ifdef CONFIG_SUSPEND
131static void enter_s2idle_proper(struct cpuidle_driver *drv,
132 struct cpuidle_device *dev, int index)
133{
134
135
136
137
138
139 RCU_NONIDLE(tick_freeze());
140
141
142
143
144
145 stop_critical_timings();
146 drv->states[index].enter_s2idle(dev, drv, index);
147 WARN_ON(!irqs_disabled());
148
149
150
151
152
153 RCU_NONIDLE(tick_unfreeze());
154 start_critical_timings();
155}
156
157
158
159
160
161
162
163
164
165int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev)
166{
167 int index;
168
169
170
171
172
173
174 index = find_deepest_state(drv, dev, UINT_MAX, 0, true);
175 if (index > 0)
176 enter_s2idle_proper(drv, dev, index);
177
178 return index;
179}
180#endif
181
182
183
184
185
186
187
188int cpuidle_enter_state(struct cpuidle_device *dev, struct cpuidle_driver *drv,
189 int index)
190{
191 int entered_state;
192
193 struct cpuidle_state *target_state = &drv->states[index];
194 bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);
195 ktime_t time_start, time_end;
196 s64 diff;
197
198
199
200
201
202
203 if (broadcast && tick_broadcast_enter()) {
204 index = find_deepest_state(drv, dev, target_state->exit_latency,
205 CPUIDLE_FLAG_TIMER_STOP, false);
206 if (index < 0) {
207 default_idle_call();
208 return -EBUSY;
209 }
210 target_state = &drv->states[index];
211 broadcast = false;
212 }
213
214
215 sched_idle_set_state(target_state);
216
217 trace_cpu_idle_rcuidle(index, dev->cpu);
218 time_start = ns_to_ktime(local_clock());
219
220 stop_critical_timings();
221 entered_state = target_state->enter(dev, drv, index);
222 start_critical_timings();
223
224 sched_clock_idle_wakeup_event();
225 time_end = ns_to_ktime(local_clock());
226 trace_cpu_idle_rcuidle(PWR_EVENT_EXIT, dev->cpu);
227
228
229 sched_idle_set_state(NULL);
230
231 if (broadcast) {
232 if (WARN_ON_ONCE(!irqs_disabled()))
233 local_irq_disable();
234
235 tick_broadcast_exit();
236 }
237
238 if (!cpuidle_state_is_coupled(drv, index))
239 local_irq_enable();
240
241 diff = ktime_us_delta(time_end, time_start);
242 if (diff > INT_MAX)
243 diff = INT_MAX;
244
245 dev->last_residency = (int) diff;
246
247 if (entered_state >= 0) {
248
249
250
251
252 dev->states_usage[entered_state].time += dev->last_residency;
253 dev->states_usage[entered_state].usage++;
254 } else {
255 dev->last_residency = 0;
256 }
257
258 return entered_state;
259}
260
261
262
263
264
265
266
267
268
269int cpuidle_select(struct cpuidle_driver *drv, struct cpuidle_device *dev)
270{
271 return cpuidle_curr_governor->select(drv, dev);
272}
273
274
275
276
277
278
279
280
281
282
283
284int cpuidle_enter(struct cpuidle_driver *drv, struct cpuidle_device *dev,
285 int index)
286{
287 if (cpuidle_state_is_coupled(drv, index))
288 return cpuidle_enter_state_coupled(dev, drv, index);
289 return cpuidle_enter_state(dev, drv, index);
290}
291
292
293
294
295
296
297
298
299
300void cpuidle_reflect(struct cpuidle_device *dev, int index)
301{
302 if (cpuidle_curr_governor->reflect && index >= 0)
303 cpuidle_curr_governor->reflect(dev, index);
304}
305
306
307
308
309void cpuidle_install_idle_handler(void)
310{
311 if (enabled_devices) {
312
313 smp_wmb();
314 initialized = 1;
315 }
316}
317
318
319
320
321void cpuidle_uninstall_idle_handler(void)
322{
323 if (enabled_devices) {
324 initialized = 0;
325 wake_up_all_idle_cpus();
326 }
327
328
329
330
331
332 synchronize_rcu();
333}
334
335
336
337
338void cpuidle_pause_and_lock(void)
339{
340 mutex_lock(&cpuidle_lock);
341 cpuidle_uninstall_idle_handler();
342}
343
344EXPORT_SYMBOL_GPL(cpuidle_pause_and_lock);
345
346
347
348
349void cpuidle_resume_and_unlock(void)
350{
351 cpuidle_install_idle_handler();
352 mutex_unlock(&cpuidle_lock);
353}
354
355EXPORT_SYMBOL_GPL(cpuidle_resume_and_unlock);
356
357
358void cpuidle_pause(void)
359{
360 mutex_lock(&cpuidle_lock);
361 cpuidle_uninstall_idle_handler();
362 mutex_unlock(&cpuidle_lock);
363}
364
365
366void cpuidle_resume(void)
367{
368 mutex_lock(&cpuidle_lock);
369 cpuidle_install_idle_handler();
370 mutex_unlock(&cpuidle_lock);
371}
372
373
374
375
376
377
378
379
380int cpuidle_enable_device(struct cpuidle_device *dev)
381{
382 int ret;
383 struct cpuidle_driver *drv;
384
385 if (!dev)
386 return -EINVAL;
387
388 if (dev->enabled)
389 return 0;
390
391 if (!cpuidle_curr_governor)
392 return -EIO;
393
394 drv = cpuidle_get_cpu_driver(dev);
395
396 if (!drv)
397 return -EIO;
398
399 if (!dev->registered)
400 return -EINVAL;
401
402 ret = cpuidle_add_device_sysfs(dev);
403 if (ret)
404 return ret;
405
406 if (cpuidle_curr_governor->enable) {
407 ret = cpuidle_curr_governor->enable(drv, dev);
408 if (ret)
409 goto fail_sysfs;
410 }
411
412 smp_wmb();
413
414 dev->enabled = 1;
415
416 enabled_devices++;
417 return 0;
418
419fail_sysfs:
420 cpuidle_remove_device_sysfs(dev);
421
422 return ret;
423}
424
425EXPORT_SYMBOL_GPL(cpuidle_enable_device);
426
427
428
429
430
431
432
433
434void cpuidle_disable_device(struct cpuidle_device *dev)
435{
436 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
437
438 if (!dev || !dev->enabled)
439 return;
440
441 if (!drv || !cpuidle_curr_governor)
442 return;
443
444 dev->enabled = 0;
445
446 if (cpuidle_curr_governor->disable)
447 cpuidle_curr_governor->disable(drv, dev);
448
449 cpuidle_remove_device_sysfs(dev);
450 enabled_devices--;
451}
452
453EXPORT_SYMBOL_GPL(cpuidle_disable_device);
454
455static void __cpuidle_unregister_device(struct cpuidle_device *dev)
456{
457 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
458
459 list_del(&dev->device_list);
460 per_cpu(cpuidle_devices, dev->cpu) = NULL;
461 module_put(drv->owner);
462
463 dev->registered = 0;
464}
465
466static void __cpuidle_device_init(struct cpuidle_device *dev)
467{
468 memset(dev->states_usage, 0, sizeof(dev->states_usage));
469 dev->last_residency = 0;
470}
471
472
473
474
475
476
477
478
479static int __cpuidle_register_device(struct cpuidle_device *dev)
480{
481 int ret;
482 struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev);
483
484 if (!try_module_get(drv->owner))
485 return -EINVAL;
486
487 per_cpu(cpuidle_devices, dev->cpu) = dev;
488 list_add(&dev->device_list, &cpuidle_detected_devices);
489
490 ret = cpuidle_coupled_register_device(dev);
491 if (ret)
492 __cpuidle_unregister_device(dev);
493 else
494 dev->registered = 1;
495
496 return ret;
497}
498
499
500
501
502
503int cpuidle_register_device(struct cpuidle_device *dev)
504{
505 int ret = -EBUSY;
506
507 if (!dev)
508 return -EINVAL;
509
510 mutex_lock(&cpuidle_lock);
511
512 if (dev->registered)
513 goto out_unlock;
514
515 __cpuidle_device_init(dev);
516
517 ret = __cpuidle_register_device(dev);
518 if (ret)
519 goto out_unlock;
520
521 ret = cpuidle_add_sysfs(dev);
522 if (ret)
523 goto out_unregister;
524
525 ret = cpuidle_enable_device(dev);
526 if (ret)
527 goto out_sysfs;
528
529 cpuidle_install_idle_handler();
530
531out_unlock:
532 mutex_unlock(&cpuidle_lock);
533
534 return ret;
535
536out_sysfs:
537 cpuidle_remove_sysfs(dev);
538out_unregister:
539 __cpuidle_unregister_device(dev);
540 goto out_unlock;
541}
542
543EXPORT_SYMBOL_GPL(cpuidle_register_device);
544
545
546
547
548
549void cpuidle_unregister_device(struct cpuidle_device *dev)
550{
551 if (!dev || dev->registered == 0)
552 return;
553
554 cpuidle_pause_and_lock();
555
556 cpuidle_disable_device(dev);
557
558 cpuidle_remove_sysfs(dev);
559
560 __cpuidle_unregister_device(dev);
561
562 cpuidle_coupled_unregister_device(dev);
563
564 cpuidle_resume_and_unlock();
565}
566
567EXPORT_SYMBOL_GPL(cpuidle_unregister_device);
568
569
570
571
572
573
574
575
576void cpuidle_unregister(struct cpuidle_driver *drv)
577{
578 int cpu;
579 struct cpuidle_device *device;
580
581 for_each_cpu(cpu, drv->cpumask) {
582 device = &per_cpu(cpuidle_dev, cpu);
583 cpuidle_unregister_device(device);
584 }
585
586 cpuidle_unregister_driver(drv);
587}
588EXPORT_SYMBOL_GPL(cpuidle_unregister);
589
590
591
592
593
594
595
596
597
598
599
600
601int cpuidle_register(struct cpuidle_driver *drv,
602 const struct cpumask *const coupled_cpus)
603{
604 int ret, cpu;
605 struct cpuidle_device *device;
606
607 ret = cpuidle_register_driver(drv);
608 if (ret) {
609 pr_err("failed to register cpuidle driver\n");
610 return ret;
611 }
612
613 for_each_cpu(cpu, drv->cpumask) {
614 device = &per_cpu(cpuidle_dev, cpu);
615 device->cpu = cpu;
616
617#ifdef CONFIG_ARCH_NEEDS_CPU_IDLE_COUPLED
618
619
620
621
622
623 if (coupled_cpus)
624 device->coupled_cpus = *coupled_cpus;
625#endif
626 ret = cpuidle_register_device(device);
627 if (!ret)
628 continue;
629
630 pr_err("Failed to register cpuidle device for cpu%d\n", cpu);
631
632 cpuidle_unregister(drv);
633 break;
634 }
635
636 return ret;
637}
638EXPORT_SYMBOL_GPL(cpuidle_register);
639
640#ifdef CONFIG_SMP
641
642
643
644
645
646
647
648static int cpuidle_latency_notify(struct notifier_block *b,
649 unsigned long l, void *v)
650{
651 wake_up_all_idle_cpus();
652 return NOTIFY_OK;
653}
654
655static struct notifier_block cpuidle_latency_notifier = {
656 .notifier_call = cpuidle_latency_notify,
657};
658
659static inline void latency_notifier_init(struct notifier_block *n)
660{
661 pm_qos_add_notifier(PM_QOS_CPU_DMA_LATENCY, n);
662}
663
664#else
665
666#define latency_notifier_init(x) do { } while (0)
667
668#endif
669
670
671
672
673static int __init cpuidle_init(void)
674{
675 int ret;
676
677 if (cpuidle_disabled())
678 return -ENODEV;
679
680 ret = cpuidle_add_interface(cpu_subsys.dev_root);
681 if (ret)
682 return ret;
683
684 latency_notifier_init(&cpuidle_latency_notifier);
685
686 return 0;
687}
688
689module_param(off, int, 0444);
690core_initcall(cpuidle_init);
691