1
2
3
4
5
6
7
8#include <linux/sched/mm.h>
9#include <linux/ktime.h>
10#include <linux/hrtimer.h>
11#include <linux/export.h>
12#include <linux/pm_runtime.h>
13#include <linux/pm_wakeirq.h>
14#include <trace/events/rpm.h>
15
16#include "../base.h"
17#include "power.h"
18
19typedef int (*pm_callback_t)(struct device *);
20
21static pm_callback_t __rpm_get_callback(struct device *dev, size_t cb_offset)
22{
23 pm_callback_t cb;
24 const struct dev_pm_ops *ops;
25
26 if (dev->pm_domain)
27 ops = &dev->pm_domain->ops;
28 else if (dev->type && dev->type->pm)
29 ops = dev->type->pm;
30 else if (dev->class && dev->class->pm)
31 ops = dev->class->pm;
32 else if (dev->bus && dev->bus->pm)
33 ops = dev->bus->pm;
34 else
35 ops = NULL;
36
37 if (ops)
38 cb = *(pm_callback_t *)((void *)ops + cb_offset);
39 else
40 cb = NULL;
41
42 if (!cb && dev->driver && dev->driver->pm)
43 cb = *(pm_callback_t *)((void *)dev->driver->pm + cb_offset);
44
45 return cb;
46}
47
48#define RPM_GET_CALLBACK(dev, callback) \
49 __rpm_get_callback(dev, offsetof(struct dev_pm_ops, callback))
50
51static int rpm_resume(struct device *dev, int rpmflags);
52static int rpm_suspend(struct device *dev, int rpmflags);
53
54
55
56
57
58
59
60
61
62
63
64
65static void update_pm_runtime_accounting(struct device *dev)
66{
67 u64 now, last, delta;
68
69 if (dev->power.disable_depth > 0)
70 return;
71
72 last = dev->power.accounting_timestamp;
73
74 now = ktime_get_mono_fast_ns();
75 dev->power.accounting_timestamp = now;
76
77
78
79
80
81
82 if (now < last)
83 return;
84
85 delta = now - last;
86
87 if (dev->power.runtime_status == RPM_SUSPENDED)
88 dev->power.suspended_time += delta;
89 else
90 dev->power.active_time += delta;
91}
92
93static void __update_runtime_status(struct device *dev, enum rpm_status status)
94{
95 update_pm_runtime_accounting(dev);
96 dev->power.runtime_status = status;
97}
98
99static u64 rpm_get_accounted_time(struct device *dev, bool suspended)
100{
101 u64 time;
102 unsigned long flags;
103
104 spin_lock_irqsave(&dev->power.lock, flags);
105
106 update_pm_runtime_accounting(dev);
107 time = suspended ? dev->power.suspended_time : dev->power.active_time;
108
109 spin_unlock_irqrestore(&dev->power.lock, flags);
110
111 return time;
112}
113
114u64 pm_runtime_active_time(struct device *dev)
115{
116 return rpm_get_accounted_time(dev, false);
117}
118
119u64 pm_runtime_suspended_time(struct device *dev)
120{
121 return rpm_get_accounted_time(dev, true);
122}
123EXPORT_SYMBOL_GPL(pm_runtime_suspended_time);
124
125
126
127
128
129static void pm_runtime_deactivate_timer(struct device *dev)
130{
131 if (dev->power.timer_expires > 0) {
132 hrtimer_try_to_cancel(&dev->power.suspend_timer);
133 dev->power.timer_expires = 0;
134 }
135}
136
137
138
139
140
141static void pm_runtime_cancel_pending(struct device *dev)
142{
143 pm_runtime_deactivate_timer(dev);
144
145
146
147
148 dev->power.request = RPM_REQ_NONE;
149}
150
151
152
153
154
155
156
157
158
159
160
161
162
163u64 pm_runtime_autosuspend_expiration(struct device *dev)
164{
165 int autosuspend_delay;
166 u64 expires;
167
168 if (!dev->power.use_autosuspend)
169 return 0;
170
171 autosuspend_delay = READ_ONCE(dev->power.autosuspend_delay);
172 if (autosuspend_delay < 0)
173 return 0;
174
175 expires = READ_ONCE(dev->power.last_busy);
176 expires += (u64)autosuspend_delay * NSEC_PER_MSEC;
177 if (expires > ktime_get_mono_fast_ns())
178 return expires;
179
180 return 0;
181}
182EXPORT_SYMBOL_GPL(pm_runtime_autosuspend_expiration);
183
184static int dev_memalloc_noio(struct device *dev, void *data)
185{
186 return dev->power.memalloc_noio;
187}
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217void pm_runtime_set_memalloc_noio(struct device *dev, bool enable)
218{
219 static DEFINE_MUTEX(dev_hotplug_mutex);
220
221 mutex_lock(&dev_hotplug_mutex);
222 for (;;) {
223 bool enabled;
224
225
226 spin_lock_irq(&dev->power.lock);
227 enabled = dev->power.memalloc_noio;
228 dev->power.memalloc_noio = enable;
229 spin_unlock_irq(&dev->power.lock);
230
231
232
233
234
235 if (enabled && enable)
236 break;
237
238 dev = dev->parent;
239
240
241
242
243
244
245 if (!dev || (!enable &&
246 device_for_each_child(dev, NULL,
247 dev_memalloc_noio)))
248 break;
249 }
250 mutex_unlock(&dev_hotplug_mutex);
251}
252EXPORT_SYMBOL_GPL(pm_runtime_set_memalloc_noio);
253
254
255
256
257
258static int rpm_check_suspend_allowed(struct device *dev)
259{
260 int retval = 0;
261
262 if (dev->power.runtime_error)
263 retval = -EINVAL;
264 else if (dev->power.disable_depth > 0)
265 retval = -EACCES;
266 else if (atomic_read(&dev->power.usage_count) > 0)
267 retval = -EAGAIN;
268 else if (!dev->power.ignore_children &&
269 atomic_read(&dev->power.child_count))
270 retval = -EBUSY;
271
272
273 else if ((dev->power.deferred_resume
274 && dev->power.runtime_status == RPM_SUSPENDING)
275 || (dev->power.request_pending
276 && dev->power.request == RPM_REQ_RESUME))
277 retval = -EAGAIN;
278 else if (__dev_pm_qos_resume_latency(dev) == 0)
279 retval = -EPERM;
280 else if (dev->power.runtime_status == RPM_SUSPENDED)
281 retval = 1;
282
283 return retval;
284}
285
286static int rpm_get_suppliers(struct device *dev)
287{
288 struct device_link *link;
289
290 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
291 device_links_read_lock_held()) {
292 int retval;
293
294 if (!(link->flags & DL_FLAG_PM_RUNTIME))
295 continue;
296
297 retval = pm_runtime_get_sync(link->supplier);
298
299 if (retval < 0 && retval != -EACCES) {
300 pm_runtime_put_noidle(link->supplier);
301 return retval;
302 }
303 refcount_inc(&link->rpm_active);
304 }
305 return 0;
306}
307
308static void __rpm_put_suppliers(struct device *dev, bool try_to_suspend)
309{
310 struct device_link *link;
311
312 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
313 device_links_read_lock_held()) {
314
315 while (refcount_dec_not_one(&link->rpm_active))
316 pm_runtime_put_noidle(link->supplier);
317
318 if (try_to_suspend)
319 pm_request_idle(link->supplier);
320 }
321}
322
323static void rpm_put_suppliers(struct device *dev)
324{
325 __rpm_put_suppliers(dev, true);
326}
327
328static void rpm_suspend_suppliers(struct device *dev)
329{
330 struct device_link *link;
331 int idx = device_links_read_lock();
332
333 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
334 device_links_read_lock_held())
335 pm_request_idle(link->supplier);
336
337 device_links_read_unlock(idx);
338}
339
340
341
342
343
344
345static int __rpm_callback(int (*cb)(struct device *), struct device *dev)
346 __releases(&dev->power.lock) __acquires(&dev->power.lock)
347{
348 int retval = 0, idx;
349 bool use_links = dev->power.links_count > 0;
350
351 if (dev->power.irq_safe) {
352 spin_unlock(&dev->power.lock);
353 } else {
354 spin_unlock_irq(&dev->power.lock);
355
356
357
358
359
360
361
362
363 if (use_links && dev->power.runtime_status == RPM_RESUMING) {
364 idx = device_links_read_lock();
365
366 retval = rpm_get_suppliers(dev);
367 if (retval) {
368 rpm_put_suppliers(dev);
369 goto fail;
370 }
371
372 device_links_read_unlock(idx);
373 }
374 }
375
376 if (cb)
377 retval = cb(dev);
378
379 if (dev->power.irq_safe) {
380 spin_lock(&dev->power.lock);
381 } else {
382
383
384
385
386
387
388
389 if (use_links
390 && ((dev->power.runtime_status == RPM_SUSPENDING && !retval)
391 || (dev->power.runtime_status == RPM_RESUMING && retval))) {
392 idx = device_links_read_lock();
393
394 __rpm_put_suppliers(dev, false);
395
396fail:
397 device_links_read_unlock(idx);
398 }
399
400 spin_lock_irq(&dev->power.lock);
401 }
402
403 return retval;
404}
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419static int rpm_idle(struct device *dev, int rpmflags)
420{
421 int (*callback)(struct device *);
422 int retval;
423
424 trace_rpm_idle_rcuidle(dev, rpmflags);
425 retval = rpm_check_suspend_allowed(dev);
426 if (retval < 0)
427 ;
428
429
430 else if (dev->power.runtime_status != RPM_ACTIVE)
431 retval = -EAGAIN;
432
433
434
435
436
437 else if (dev->power.request_pending &&
438 dev->power.request > RPM_REQ_IDLE)
439 retval = -EAGAIN;
440
441
442 else if (dev->power.idle_notification)
443 retval = -EINPROGRESS;
444 if (retval)
445 goto out;
446
447
448 dev->power.request = RPM_REQ_NONE;
449
450 callback = RPM_GET_CALLBACK(dev, runtime_idle);
451
452
453 if (!callback || dev->power.no_callbacks)
454 goto out;
455
456
457 if (rpmflags & RPM_ASYNC) {
458 dev->power.request = RPM_REQ_IDLE;
459 if (!dev->power.request_pending) {
460 dev->power.request_pending = true;
461 queue_work(pm_wq, &dev->power.work);
462 }
463 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, 0);
464 return 0;
465 }
466
467 dev->power.idle_notification = true;
468
469 retval = __rpm_callback(callback, dev);
470
471 dev->power.idle_notification = false;
472 wake_up_all(&dev->power.wait_queue);
473
474 out:
475 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
476 return retval ? retval : rpm_suspend(dev, rpmflags | RPM_AUTO);
477}
478
479
480
481
482
483
484static int rpm_callback(int (*cb)(struct device *), struct device *dev)
485{
486 int retval;
487
488 if (dev->power.memalloc_noio) {
489 unsigned int noio_flag;
490
491
492
493
494
495
496
497
498
499
500 noio_flag = memalloc_noio_save();
501 retval = __rpm_callback(cb, dev);
502 memalloc_noio_restore(noio_flag);
503 } else {
504 retval = __rpm_callback(cb, dev);
505 }
506
507 dev->power.runtime_error = retval;
508 return retval != -EACCES ? retval : -EIO;
509}
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532static int rpm_suspend(struct device *dev, int rpmflags)
533 __releases(&dev->power.lock) __acquires(&dev->power.lock)
534{
535 int (*callback)(struct device *);
536 struct device *parent = NULL;
537 int retval;
538
539 trace_rpm_suspend_rcuidle(dev, rpmflags);
540
541 repeat:
542 retval = rpm_check_suspend_allowed(dev);
543 if (retval < 0)
544 goto out;
545
546
547 if (dev->power.runtime_status == RPM_RESUMING && !(rpmflags & RPM_ASYNC))
548 retval = -EAGAIN;
549 if (retval)
550 goto out;
551
552
553 if ((rpmflags & RPM_AUTO)
554 && dev->power.runtime_status != RPM_SUSPENDING) {
555 u64 expires = pm_runtime_autosuspend_expiration(dev);
556
557 if (expires != 0) {
558
559 dev->power.request = RPM_REQ_NONE;
560
561
562
563
564
565
566
567
568 if (!(dev->power.timer_expires &&
569 dev->power.timer_expires <= expires)) {
570
571
572
573
574 u64 slack = (u64)READ_ONCE(dev->power.autosuspend_delay) *
575 (NSEC_PER_MSEC >> 2);
576
577 dev->power.timer_expires = expires;
578 hrtimer_start_range_ns(&dev->power.suspend_timer,
579 ns_to_ktime(expires),
580 slack,
581 HRTIMER_MODE_ABS);
582 }
583 dev->power.timer_autosuspends = 1;
584 goto out;
585 }
586 }
587
588
589 pm_runtime_cancel_pending(dev);
590
591 if (dev->power.runtime_status == RPM_SUSPENDING) {
592 DEFINE_WAIT(wait);
593
594 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
595 retval = -EINPROGRESS;
596 goto out;
597 }
598
599 if (dev->power.irq_safe) {
600 spin_unlock(&dev->power.lock);
601
602 cpu_relax();
603
604 spin_lock(&dev->power.lock);
605 goto repeat;
606 }
607
608
609 for (;;) {
610 prepare_to_wait(&dev->power.wait_queue, &wait,
611 TASK_UNINTERRUPTIBLE);
612 if (dev->power.runtime_status != RPM_SUSPENDING)
613 break;
614
615 spin_unlock_irq(&dev->power.lock);
616
617 schedule();
618
619 spin_lock_irq(&dev->power.lock);
620 }
621 finish_wait(&dev->power.wait_queue, &wait);
622 goto repeat;
623 }
624
625 if (dev->power.no_callbacks)
626 goto no_callback;
627
628
629 if (rpmflags & RPM_ASYNC) {
630 dev->power.request = (rpmflags & RPM_AUTO) ?
631 RPM_REQ_AUTOSUSPEND : RPM_REQ_SUSPEND;
632 if (!dev->power.request_pending) {
633 dev->power.request_pending = true;
634 queue_work(pm_wq, &dev->power.work);
635 }
636 goto out;
637 }
638
639 __update_runtime_status(dev, RPM_SUSPENDING);
640
641 callback = RPM_GET_CALLBACK(dev, runtime_suspend);
642
643 dev_pm_enable_wake_irq_check(dev, true);
644 retval = rpm_callback(callback, dev);
645 if (retval)
646 goto fail;
647
648 no_callback:
649 __update_runtime_status(dev, RPM_SUSPENDED);
650 pm_runtime_deactivate_timer(dev);
651
652 if (dev->parent) {
653 parent = dev->parent;
654 atomic_add_unless(&parent->power.child_count, -1, 0);
655 }
656 wake_up_all(&dev->power.wait_queue);
657
658 if (dev->power.deferred_resume) {
659 dev->power.deferred_resume = false;
660 rpm_resume(dev, 0);
661 retval = -EAGAIN;
662 goto out;
663 }
664
665 if (dev->power.irq_safe)
666 goto out;
667
668
669 if (parent && !parent->power.ignore_children) {
670 spin_unlock(&dev->power.lock);
671
672 spin_lock(&parent->power.lock);
673 rpm_idle(parent, RPM_ASYNC);
674 spin_unlock(&parent->power.lock);
675
676 spin_lock(&dev->power.lock);
677 }
678
679 if (dev->power.links_count > 0) {
680 spin_unlock_irq(&dev->power.lock);
681
682 rpm_suspend_suppliers(dev);
683
684 spin_lock_irq(&dev->power.lock);
685 }
686
687 out:
688 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
689
690 return retval;
691
692 fail:
693 dev_pm_disable_wake_irq_check(dev);
694 __update_runtime_status(dev, RPM_ACTIVE);
695 dev->power.deferred_resume = false;
696 wake_up_all(&dev->power.wait_queue);
697
698 if (retval == -EAGAIN || retval == -EBUSY) {
699 dev->power.runtime_error = 0;
700
701
702
703
704
705
706
707 if ((rpmflags & RPM_AUTO) &&
708 pm_runtime_autosuspend_expiration(dev) != 0)
709 goto repeat;
710 } else {
711 pm_runtime_cancel_pending(dev);
712 }
713 goto out;
714}
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733static int rpm_resume(struct device *dev, int rpmflags)
734 __releases(&dev->power.lock) __acquires(&dev->power.lock)
735{
736 int (*callback)(struct device *);
737 struct device *parent = NULL;
738 int retval = 0;
739
740 trace_rpm_resume_rcuidle(dev, rpmflags);
741
742 repeat:
743 if (dev->power.runtime_error)
744 retval = -EINVAL;
745 else if (dev->power.disable_depth == 1 && dev->power.is_suspended
746 && dev->power.runtime_status == RPM_ACTIVE)
747 retval = 1;
748 else if (dev->power.disable_depth > 0)
749 retval = -EACCES;
750 if (retval)
751 goto out;
752
753
754
755
756
757
758
759 dev->power.request = RPM_REQ_NONE;
760 if (!dev->power.timer_autosuspends)
761 pm_runtime_deactivate_timer(dev);
762
763 if (dev->power.runtime_status == RPM_ACTIVE) {
764 retval = 1;
765 goto out;
766 }
767
768 if (dev->power.runtime_status == RPM_RESUMING
769 || dev->power.runtime_status == RPM_SUSPENDING) {
770 DEFINE_WAIT(wait);
771
772 if (rpmflags & (RPM_ASYNC | RPM_NOWAIT)) {
773 if (dev->power.runtime_status == RPM_SUSPENDING)
774 dev->power.deferred_resume = true;
775 else
776 retval = -EINPROGRESS;
777 goto out;
778 }
779
780 if (dev->power.irq_safe) {
781 spin_unlock(&dev->power.lock);
782
783 cpu_relax();
784
785 spin_lock(&dev->power.lock);
786 goto repeat;
787 }
788
789
790 for (;;) {
791 prepare_to_wait(&dev->power.wait_queue, &wait,
792 TASK_UNINTERRUPTIBLE);
793 if (dev->power.runtime_status != RPM_RESUMING
794 && dev->power.runtime_status != RPM_SUSPENDING)
795 break;
796
797 spin_unlock_irq(&dev->power.lock);
798
799 schedule();
800
801 spin_lock_irq(&dev->power.lock);
802 }
803 finish_wait(&dev->power.wait_queue, &wait);
804 goto repeat;
805 }
806
807
808
809
810
811
812 if (dev->power.no_callbacks && !parent && dev->parent) {
813 spin_lock_nested(&dev->parent->power.lock, SINGLE_DEPTH_NESTING);
814 if (dev->parent->power.disable_depth > 0
815 || dev->parent->power.ignore_children
816 || dev->parent->power.runtime_status == RPM_ACTIVE) {
817 atomic_inc(&dev->parent->power.child_count);
818 spin_unlock(&dev->parent->power.lock);
819 retval = 1;
820 goto no_callback;
821 }
822 spin_unlock(&dev->parent->power.lock);
823 }
824
825
826 if (rpmflags & RPM_ASYNC) {
827 dev->power.request = RPM_REQ_RESUME;
828 if (!dev->power.request_pending) {
829 dev->power.request_pending = true;
830 queue_work(pm_wq, &dev->power.work);
831 }
832 retval = 0;
833 goto out;
834 }
835
836 if (!parent && dev->parent) {
837
838
839
840
841
842 parent = dev->parent;
843 if (dev->power.irq_safe)
844 goto skip_parent;
845 spin_unlock(&dev->power.lock);
846
847 pm_runtime_get_noresume(parent);
848
849 spin_lock(&parent->power.lock);
850
851
852
853
854 if (!parent->power.disable_depth
855 && !parent->power.ignore_children) {
856 rpm_resume(parent, 0);
857 if (parent->power.runtime_status != RPM_ACTIVE)
858 retval = -EBUSY;
859 }
860 spin_unlock(&parent->power.lock);
861
862 spin_lock(&dev->power.lock);
863 if (retval)
864 goto out;
865 goto repeat;
866 }
867 skip_parent:
868
869 if (dev->power.no_callbacks)
870 goto no_callback;
871
872 __update_runtime_status(dev, RPM_RESUMING);
873
874 callback = RPM_GET_CALLBACK(dev, runtime_resume);
875
876 dev_pm_disable_wake_irq_check(dev);
877 retval = rpm_callback(callback, dev);
878 if (retval) {
879 __update_runtime_status(dev, RPM_SUSPENDED);
880 pm_runtime_cancel_pending(dev);
881 dev_pm_enable_wake_irq_check(dev, false);
882 } else {
883 no_callback:
884 __update_runtime_status(dev, RPM_ACTIVE);
885 pm_runtime_mark_last_busy(dev);
886 if (parent)
887 atomic_inc(&parent->power.child_count);
888 }
889 wake_up_all(&dev->power.wait_queue);
890
891 if (retval >= 0)
892 rpm_idle(dev, RPM_ASYNC);
893
894 out:
895 if (parent && !dev->power.irq_safe) {
896 spin_unlock_irq(&dev->power.lock);
897
898 pm_runtime_put(parent);
899
900 spin_lock_irq(&dev->power.lock);
901 }
902
903 trace_rpm_return_int_rcuidle(dev, _THIS_IP_, retval);
904
905 return retval;
906}
907
908
909
910
911
912
913
914
915static void pm_runtime_work(struct work_struct *work)
916{
917 struct device *dev = container_of(work, struct device, power.work);
918 enum rpm_request req;
919
920 spin_lock_irq(&dev->power.lock);
921
922 if (!dev->power.request_pending)
923 goto out;
924
925 req = dev->power.request;
926 dev->power.request = RPM_REQ_NONE;
927 dev->power.request_pending = false;
928
929 switch (req) {
930 case RPM_REQ_NONE:
931 break;
932 case RPM_REQ_IDLE:
933 rpm_idle(dev, RPM_NOWAIT);
934 break;
935 case RPM_REQ_SUSPEND:
936 rpm_suspend(dev, RPM_NOWAIT);
937 break;
938 case RPM_REQ_AUTOSUSPEND:
939 rpm_suspend(dev, RPM_NOWAIT | RPM_AUTO);
940 break;
941 case RPM_REQ_RESUME:
942 rpm_resume(dev, RPM_NOWAIT);
943 break;
944 }
945
946 out:
947 spin_unlock_irq(&dev->power.lock);
948}
949
950
951
952
953
954
955
956static enum hrtimer_restart pm_suspend_timer_fn(struct hrtimer *timer)
957{
958 struct device *dev = container_of(timer, struct device, power.suspend_timer);
959 unsigned long flags;
960 u64 expires;
961
962 spin_lock_irqsave(&dev->power.lock, flags);
963
964 expires = dev->power.timer_expires;
965
966
967
968
969 if (expires > 0 && expires < ktime_get_mono_fast_ns()) {
970 dev->power.timer_expires = 0;
971 rpm_suspend(dev, dev->power.timer_autosuspends ?
972 (RPM_ASYNC | RPM_AUTO) : RPM_ASYNC);
973 }
974
975 spin_unlock_irqrestore(&dev->power.lock, flags);
976
977 return HRTIMER_NORESTART;
978}
979
980
981
982
983
984
985int pm_schedule_suspend(struct device *dev, unsigned int delay)
986{
987 unsigned long flags;
988 u64 expires;
989 int retval;
990
991 spin_lock_irqsave(&dev->power.lock, flags);
992
993 if (!delay) {
994 retval = rpm_suspend(dev, RPM_ASYNC);
995 goto out;
996 }
997
998 retval = rpm_check_suspend_allowed(dev);
999 if (retval)
1000 goto out;
1001
1002
1003 pm_runtime_cancel_pending(dev);
1004
1005 expires = ktime_get_mono_fast_ns() + (u64)delay * NSEC_PER_MSEC;
1006 dev->power.timer_expires = expires;
1007 dev->power.timer_autosuspends = 0;
1008 hrtimer_start(&dev->power.suspend_timer, expires, HRTIMER_MODE_ABS);
1009
1010 out:
1011 spin_unlock_irqrestore(&dev->power.lock, flags);
1012
1013 return retval;
1014}
1015EXPORT_SYMBOL_GPL(pm_schedule_suspend);
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029int __pm_runtime_idle(struct device *dev, int rpmflags)
1030{
1031 unsigned long flags;
1032 int retval;
1033
1034 if (rpmflags & RPM_GET_PUT) {
1035 if (!atomic_dec_and_test(&dev->power.usage_count)) {
1036 trace_rpm_usage_rcuidle(dev, rpmflags);
1037 return 0;
1038 }
1039 }
1040
1041 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1042
1043 spin_lock_irqsave(&dev->power.lock, flags);
1044 retval = rpm_idle(dev, rpmflags);
1045 spin_unlock_irqrestore(&dev->power.lock, flags);
1046
1047 return retval;
1048}
1049EXPORT_SYMBOL_GPL(__pm_runtime_idle);
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063int __pm_runtime_suspend(struct device *dev, int rpmflags)
1064{
1065 unsigned long flags;
1066 int retval;
1067
1068 if (rpmflags & RPM_GET_PUT) {
1069 if (!atomic_dec_and_test(&dev->power.usage_count)) {
1070 trace_rpm_usage_rcuidle(dev, rpmflags);
1071 return 0;
1072 }
1073 }
1074
1075 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe);
1076
1077 spin_lock_irqsave(&dev->power.lock, flags);
1078 retval = rpm_suspend(dev, rpmflags);
1079 spin_unlock_irqrestore(&dev->power.lock, flags);
1080
1081 return retval;
1082}
1083EXPORT_SYMBOL_GPL(__pm_runtime_suspend);
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096int __pm_runtime_resume(struct device *dev, int rpmflags)
1097{
1098 unsigned long flags;
1099 int retval;
1100
1101 might_sleep_if(!(rpmflags & RPM_ASYNC) && !dev->power.irq_safe &&
1102 dev->power.runtime_status != RPM_ACTIVE);
1103
1104 if (rpmflags & RPM_GET_PUT)
1105 atomic_inc(&dev->power.usage_count);
1106
1107 spin_lock_irqsave(&dev->power.lock, flags);
1108 retval = rpm_resume(dev, rpmflags);
1109 spin_unlock_irqrestore(&dev->power.lock, flags);
1110
1111 return retval;
1112}
1113EXPORT_SYMBOL_GPL(__pm_runtime_resume);
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137int pm_runtime_get_if_active(struct device *dev, bool ign_usage_count)
1138{
1139 unsigned long flags;
1140 int retval;
1141
1142 spin_lock_irqsave(&dev->power.lock, flags);
1143 if (dev->power.disable_depth > 0) {
1144 retval = -EINVAL;
1145 } else if (dev->power.runtime_status != RPM_ACTIVE) {
1146 retval = 0;
1147 } else if (ign_usage_count) {
1148 retval = 1;
1149 atomic_inc(&dev->power.usage_count);
1150 } else {
1151 retval = atomic_inc_not_zero(&dev->power.usage_count);
1152 }
1153 trace_rpm_usage_rcuidle(dev, 0);
1154 spin_unlock_irqrestore(&dev->power.lock, flags);
1155
1156 return retval;
1157}
1158EXPORT_SYMBOL_GPL(pm_runtime_get_if_active);
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184int __pm_runtime_set_status(struct device *dev, unsigned int status)
1185{
1186 struct device *parent = dev->parent;
1187 bool notify_parent = false;
1188 int error = 0;
1189
1190 if (status != RPM_ACTIVE && status != RPM_SUSPENDED)
1191 return -EINVAL;
1192
1193 spin_lock_irq(&dev->power.lock);
1194
1195
1196
1197
1198
1199 if (dev->power.runtime_error || dev->power.disable_depth)
1200 dev->power.disable_depth++;
1201 else
1202 error = -EAGAIN;
1203
1204 spin_unlock_irq(&dev->power.lock);
1205
1206 if (error)
1207 return error;
1208
1209
1210
1211
1212
1213
1214
1215 if (status == RPM_ACTIVE) {
1216 int idx = device_links_read_lock();
1217
1218 error = rpm_get_suppliers(dev);
1219 if (error)
1220 status = RPM_SUSPENDED;
1221
1222 device_links_read_unlock(idx);
1223 }
1224
1225 spin_lock_irq(&dev->power.lock);
1226
1227 if (dev->power.runtime_status == status || !parent)
1228 goto out_set;
1229
1230 if (status == RPM_SUSPENDED) {
1231 atomic_add_unless(&parent->power.child_count, -1, 0);
1232 notify_parent = !parent->power.ignore_children;
1233 } else {
1234 spin_lock_nested(&parent->power.lock, SINGLE_DEPTH_NESTING);
1235
1236
1237
1238
1239
1240
1241 if (!parent->power.disable_depth
1242 && !parent->power.ignore_children
1243 && parent->power.runtime_status != RPM_ACTIVE) {
1244 dev_err(dev, "runtime PM trying to activate child device %s but parent (%s) is not active\n",
1245 dev_name(dev),
1246 dev_name(parent));
1247 error = -EBUSY;
1248 } else if (dev->power.runtime_status == RPM_SUSPENDED) {
1249 atomic_inc(&parent->power.child_count);
1250 }
1251
1252 spin_unlock(&parent->power.lock);
1253
1254 if (error) {
1255 status = RPM_SUSPENDED;
1256 goto out;
1257 }
1258 }
1259
1260 out_set:
1261 __update_runtime_status(dev, status);
1262 if (!error)
1263 dev->power.runtime_error = 0;
1264
1265 out:
1266 spin_unlock_irq(&dev->power.lock);
1267
1268 if (notify_parent)
1269 pm_request_idle(parent);
1270
1271 if (status == RPM_SUSPENDED) {
1272 int idx = device_links_read_lock();
1273
1274 rpm_put_suppliers(dev);
1275
1276 device_links_read_unlock(idx);
1277 }
1278
1279 pm_runtime_enable(dev);
1280
1281 return error;
1282}
1283EXPORT_SYMBOL_GPL(__pm_runtime_set_status);
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294static void __pm_runtime_barrier(struct device *dev)
1295{
1296 pm_runtime_deactivate_timer(dev);
1297
1298 if (dev->power.request_pending) {
1299 dev->power.request = RPM_REQ_NONE;
1300 spin_unlock_irq(&dev->power.lock);
1301
1302 cancel_work_sync(&dev->power.work);
1303
1304 spin_lock_irq(&dev->power.lock);
1305 dev->power.request_pending = false;
1306 }
1307
1308 if (dev->power.runtime_status == RPM_SUSPENDING
1309 || dev->power.runtime_status == RPM_RESUMING
1310 || dev->power.idle_notification) {
1311 DEFINE_WAIT(wait);
1312
1313
1314 for (;;) {
1315 prepare_to_wait(&dev->power.wait_queue, &wait,
1316 TASK_UNINTERRUPTIBLE);
1317 if (dev->power.runtime_status != RPM_SUSPENDING
1318 && dev->power.runtime_status != RPM_RESUMING
1319 && !dev->power.idle_notification)
1320 break;
1321 spin_unlock_irq(&dev->power.lock);
1322
1323 schedule();
1324
1325 spin_lock_irq(&dev->power.lock);
1326 }
1327 finish_wait(&dev->power.wait_queue, &wait);
1328 }
1329}
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345int pm_runtime_barrier(struct device *dev)
1346{
1347 int retval = 0;
1348
1349 pm_runtime_get_noresume(dev);
1350 spin_lock_irq(&dev->power.lock);
1351
1352 if (dev->power.request_pending
1353 && dev->power.request == RPM_REQ_RESUME) {
1354 rpm_resume(dev, 0);
1355 retval = 1;
1356 }
1357
1358 __pm_runtime_barrier(dev);
1359
1360 spin_unlock_irq(&dev->power.lock);
1361 pm_runtime_put_noidle(dev);
1362
1363 return retval;
1364}
1365EXPORT_SYMBOL_GPL(pm_runtime_barrier);
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381void __pm_runtime_disable(struct device *dev, bool check_resume)
1382{
1383 spin_lock_irq(&dev->power.lock);
1384
1385 if (dev->power.disable_depth > 0) {
1386 dev->power.disable_depth++;
1387 goto out;
1388 }
1389
1390
1391
1392
1393
1394
1395 if (check_resume && dev->power.request_pending
1396 && dev->power.request == RPM_REQ_RESUME) {
1397
1398
1399
1400
1401 pm_runtime_get_noresume(dev);
1402
1403 rpm_resume(dev, 0);
1404
1405 pm_runtime_put_noidle(dev);
1406 }
1407
1408
1409 update_pm_runtime_accounting(dev);
1410
1411 if (!dev->power.disable_depth++)
1412 __pm_runtime_barrier(dev);
1413
1414 out:
1415 spin_unlock_irq(&dev->power.lock);
1416}
1417EXPORT_SYMBOL_GPL(__pm_runtime_disable);
1418
1419
1420
1421
1422
1423void pm_runtime_enable(struct device *dev)
1424{
1425 unsigned long flags;
1426
1427 spin_lock_irqsave(&dev->power.lock, flags);
1428
1429 if (dev->power.disable_depth > 0) {
1430 dev->power.disable_depth--;
1431
1432
1433 if (!dev->power.disable_depth)
1434 dev->power.accounting_timestamp = ktime_get_mono_fast_ns();
1435 } else {
1436 dev_warn(dev, "Unbalanced %s!\n", __func__);
1437 }
1438
1439 WARN(!dev->power.disable_depth &&
1440 dev->power.runtime_status == RPM_SUSPENDED &&
1441 !dev->power.ignore_children &&
1442 atomic_read(&dev->power.child_count) > 0,
1443 "Enabling runtime PM for inactive device (%s) with active children\n",
1444 dev_name(dev));
1445
1446 spin_unlock_irqrestore(&dev->power.lock, flags);
1447}
1448EXPORT_SYMBOL_GPL(pm_runtime_enable);
1449
1450static void pm_runtime_disable_action(void *data)
1451{
1452 pm_runtime_disable(data);
1453}
1454
1455
1456
1457
1458
1459int devm_pm_runtime_enable(struct device *dev)
1460{
1461 pm_runtime_enable(dev);
1462
1463 return devm_add_action_or_reset(dev, pm_runtime_disable_action, dev);
1464}
1465EXPORT_SYMBOL_GPL(devm_pm_runtime_enable);
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475void pm_runtime_forbid(struct device *dev)
1476{
1477 spin_lock_irq(&dev->power.lock);
1478 if (!dev->power.runtime_auto)
1479 goto out;
1480
1481 dev->power.runtime_auto = false;
1482 atomic_inc(&dev->power.usage_count);
1483 rpm_resume(dev, 0);
1484
1485 out:
1486 spin_unlock_irq(&dev->power.lock);
1487}
1488EXPORT_SYMBOL_GPL(pm_runtime_forbid);
1489
1490
1491
1492
1493
1494
1495
1496void pm_runtime_allow(struct device *dev)
1497{
1498 spin_lock_irq(&dev->power.lock);
1499 if (dev->power.runtime_auto)
1500 goto out;
1501
1502 dev->power.runtime_auto = true;
1503 if (atomic_dec_and_test(&dev->power.usage_count))
1504 rpm_idle(dev, RPM_AUTO | RPM_ASYNC);
1505 else
1506 trace_rpm_usage_rcuidle(dev, RPM_AUTO | RPM_ASYNC);
1507
1508 out:
1509 spin_unlock_irq(&dev->power.lock);
1510}
1511EXPORT_SYMBOL_GPL(pm_runtime_allow);
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521void pm_runtime_no_callbacks(struct device *dev)
1522{
1523 spin_lock_irq(&dev->power.lock);
1524 dev->power.no_callbacks = 1;
1525 spin_unlock_irq(&dev->power.lock);
1526 if (device_is_registered(dev))
1527 rpm_sysfs_remove(dev);
1528}
1529EXPORT_SYMBOL_GPL(pm_runtime_no_callbacks);
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542void pm_runtime_irq_safe(struct device *dev)
1543{
1544 if (dev->parent)
1545 pm_runtime_get_sync(dev->parent);
1546 spin_lock_irq(&dev->power.lock);
1547 dev->power.irq_safe = 1;
1548 spin_unlock_irq(&dev->power.lock);
1549}
1550EXPORT_SYMBOL_GPL(pm_runtime_irq_safe);
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563static void update_autosuspend(struct device *dev, int old_delay, int old_use)
1564{
1565 int delay = dev->power.autosuspend_delay;
1566
1567
1568 if (dev->power.use_autosuspend && delay < 0) {
1569
1570
1571 if (!old_use || old_delay >= 0) {
1572 atomic_inc(&dev->power.usage_count);
1573 rpm_resume(dev, 0);
1574 } else {
1575 trace_rpm_usage_rcuidle(dev, 0);
1576 }
1577 }
1578
1579
1580 else {
1581
1582
1583 if (old_use && old_delay < 0)
1584 atomic_dec(&dev->power.usage_count);
1585
1586
1587 rpm_idle(dev, RPM_AUTO);
1588 }
1589}
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600void pm_runtime_set_autosuspend_delay(struct device *dev, int delay)
1601{
1602 int old_delay, old_use;
1603
1604 spin_lock_irq(&dev->power.lock);
1605 old_delay = dev->power.autosuspend_delay;
1606 old_use = dev->power.use_autosuspend;
1607 dev->power.autosuspend_delay = delay;
1608 update_autosuspend(dev, old_delay, old_use);
1609 spin_unlock_irq(&dev->power.lock);
1610}
1611EXPORT_SYMBOL_GPL(pm_runtime_set_autosuspend_delay);
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621void __pm_runtime_use_autosuspend(struct device *dev, bool use)
1622{
1623 int old_delay, old_use;
1624
1625 spin_lock_irq(&dev->power.lock);
1626 old_delay = dev->power.autosuspend_delay;
1627 old_use = dev->power.use_autosuspend;
1628 dev->power.use_autosuspend = use;
1629 update_autosuspend(dev, old_delay, old_use);
1630 spin_unlock_irq(&dev->power.lock);
1631}
1632EXPORT_SYMBOL_GPL(__pm_runtime_use_autosuspend);
1633
1634
1635
1636
1637
1638void pm_runtime_init(struct device *dev)
1639{
1640 dev->power.runtime_status = RPM_SUSPENDED;
1641 dev->power.idle_notification = false;
1642
1643 dev->power.disable_depth = 1;
1644 atomic_set(&dev->power.usage_count, 0);
1645
1646 dev->power.runtime_error = 0;
1647
1648 atomic_set(&dev->power.child_count, 0);
1649 pm_suspend_ignore_children(dev, false);
1650 dev->power.runtime_auto = true;
1651
1652 dev->power.request_pending = false;
1653 dev->power.request = RPM_REQ_NONE;
1654 dev->power.deferred_resume = false;
1655 dev->power.needs_force_resume = 0;
1656 INIT_WORK(&dev->power.work, pm_runtime_work);
1657
1658 dev->power.timer_expires = 0;
1659 hrtimer_init(&dev->power.suspend_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1660 dev->power.suspend_timer.function = pm_suspend_timer_fn;
1661
1662 init_waitqueue_head(&dev->power.wait_queue);
1663}
1664
1665
1666
1667
1668
1669void pm_runtime_reinit(struct device *dev)
1670{
1671 if (!pm_runtime_enabled(dev)) {
1672 if (dev->power.runtime_status == RPM_ACTIVE)
1673 pm_runtime_set_suspended(dev);
1674 if (dev->power.irq_safe) {
1675 spin_lock_irq(&dev->power.lock);
1676 dev->power.irq_safe = 0;
1677 spin_unlock_irq(&dev->power.lock);
1678 if (dev->parent)
1679 pm_runtime_put(dev->parent);
1680 }
1681 }
1682}
1683
1684
1685
1686
1687
1688void pm_runtime_remove(struct device *dev)
1689{
1690 __pm_runtime_disable(dev, false);
1691 pm_runtime_reinit(dev);
1692}
1693
1694
1695
1696
1697
1698void pm_runtime_get_suppliers(struct device *dev)
1699{
1700 struct device_link *link;
1701 int idx;
1702
1703 idx = device_links_read_lock();
1704
1705 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1706 device_links_read_lock_held())
1707 if (link->flags & DL_FLAG_PM_RUNTIME) {
1708 link->supplier_preactivated = true;
1709 pm_runtime_get_sync(link->supplier);
1710 refcount_inc(&link->rpm_active);
1711 }
1712
1713 device_links_read_unlock(idx);
1714}
1715
1716
1717
1718
1719
1720void pm_runtime_put_suppliers(struct device *dev)
1721{
1722 struct device_link *link;
1723 unsigned long flags;
1724 bool put;
1725 int idx;
1726
1727 idx = device_links_read_lock();
1728
1729 list_for_each_entry_rcu(link, &dev->links.suppliers, c_node,
1730 device_links_read_lock_held())
1731 if (link->supplier_preactivated) {
1732 link->supplier_preactivated = false;
1733 spin_lock_irqsave(&dev->power.lock, flags);
1734 put = pm_runtime_status_suspended(dev) &&
1735 refcount_dec_not_one(&link->rpm_active);
1736 spin_unlock_irqrestore(&dev->power.lock, flags);
1737 if (put)
1738 pm_runtime_put(link->supplier);
1739 }
1740
1741 device_links_read_unlock(idx);
1742}
1743
1744void pm_runtime_new_link(struct device *dev)
1745{
1746 spin_lock_irq(&dev->power.lock);
1747 dev->power.links_count++;
1748 spin_unlock_irq(&dev->power.lock);
1749}
1750
1751static void pm_runtime_drop_link_count(struct device *dev)
1752{
1753 spin_lock_irq(&dev->power.lock);
1754 WARN_ON(dev->power.links_count == 0);
1755 dev->power.links_count--;
1756 spin_unlock_irq(&dev->power.lock);
1757}
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767void pm_runtime_drop_link(struct device_link *link)
1768{
1769 if (!(link->flags & DL_FLAG_PM_RUNTIME))
1770 return;
1771
1772 pm_runtime_drop_link_count(link->consumer);
1773
1774 while (refcount_dec_not_one(&link->rpm_active))
1775 pm_runtime_put(link->supplier);
1776}
1777
1778static bool pm_runtime_need_not_resume(struct device *dev)
1779{
1780 return atomic_read(&dev->power.usage_count) <= 1 &&
1781 (atomic_read(&dev->power.child_count) == 0 ||
1782 dev->power.ignore_children);
1783}
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802int pm_runtime_force_suspend(struct device *dev)
1803{
1804 int (*callback)(struct device *);
1805 int ret;
1806
1807 pm_runtime_disable(dev);
1808 if (pm_runtime_status_suspended(dev))
1809 return 0;
1810
1811 callback = RPM_GET_CALLBACK(dev, runtime_suspend);
1812
1813 ret = callback ? callback(dev) : 0;
1814 if (ret)
1815 goto err;
1816
1817
1818
1819
1820
1821
1822
1823 if (pm_runtime_need_not_resume(dev)) {
1824 pm_runtime_set_suspended(dev);
1825 } else {
1826 __update_runtime_status(dev, RPM_SUSPENDED);
1827 dev->power.needs_force_resume = 1;
1828 }
1829
1830 return 0;
1831
1832err:
1833 pm_runtime_enable(dev);
1834 return ret;
1835}
1836EXPORT_SYMBOL_GPL(pm_runtime_force_suspend);
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850int pm_runtime_force_resume(struct device *dev)
1851{
1852 int (*callback)(struct device *);
1853 int ret = 0;
1854
1855 if (!pm_runtime_status_suspended(dev) || !dev->power.needs_force_resume)
1856 goto out;
1857
1858
1859
1860
1861
1862 __update_runtime_status(dev, RPM_ACTIVE);
1863
1864 callback = RPM_GET_CALLBACK(dev, runtime_resume);
1865
1866 ret = callback ? callback(dev) : 0;
1867 if (ret) {
1868 pm_runtime_set_suspended(dev);
1869 goto out;
1870 }
1871
1872 pm_runtime_mark_last_busy(dev);
1873out:
1874 dev->power.needs_force_resume = 0;
1875 pm_runtime_enable(dev);
1876 return ret;
1877}
1878EXPORT_SYMBOL_GPL(pm_runtime_force_resume);
1879