1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27#include <linux/export.h>
28#include <linux/kernel.h>
29#include <linux/sched.h>
30#include <linux/init.h>
31#include <linux/signal.h>
32#include <linux/completion.h>
33#include <linux/workqueue.h>
34#include <linux/slab.h>
35#include <linux/cpu.h>
36#include <linux/notifier.h>
37#include <linux/kthread.h>
38#include <linux/hardirq.h>
39#include <linux/mempolicy.h>
40#include <linux/freezer.h>
41#include <linux/kallsyms.h>
42#include <linux/debug_locks.h>
43#include <linux/lockdep.h>
44#include <linux/idr.h>
45#include <linux/jhash.h>
46#include <linux/hashtable.h>
47#include <linux/rculist.h>
48#include <linux/nodemask.h>
49#include <linux/moduleparam.h>
50#include <linux/uaccess.h>
51
52#include "workqueue_internal.h"
53
54enum {
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71 POOL_MANAGE_WORKERS = 1 << 0,
72 POOL_DISASSOCIATED = 1 << 2,
73 POOL_FREEZING = 1 << 3,
74
75
76 WORKER_STARTED = 1 << 0,
77 WORKER_DIE = 1 << 1,
78 WORKER_IDLE = 1 << 2,
79 WORKER_PREP = 1 << 3,
80 WORKER_CPU_INTENSIVE = 1 << 6,
81 WORKER_UNBOUND = 1 << 7,
82 WORKER_REBOUND = 1 << 8,
83
84 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
85 WORKER_UNBOUND | WORKER_REBOUND,
86
87 NR_STD_WORKER_POOLS = 2,
88
89 UNBOUND_POOL_HASH_ORDER = 6,
90 BUSY_WORKER_HASH_ORDER = 6,
91
92 MAX_IDLE_WORKERS_RATIO = 4,
93 IDLE_WORKER_TIMEOUT = 300 * HZ,
94
95 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
96
97
98 MAYDAY_INTERVAL = HZ / 10,
99 CREATE_COOLDOWN = HZ,
100
101
102
103
104
105 RESCUER_NICE_LEVEL = -20,
106 HIGHPRI_NICE_LEVEL = -20,
107
108 WQ_NAME_LEN = 24,
109};
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143struct worker_pool {
144 spinlock_t lock;
145 int cpu;
146 int node;
147 int id;
148 unsigned int flags;
149
150 struct list_head worklist;
151 int nr_workers;
152
153
154 int nr_idle;
155
156 struct list_head idle_list;
157 struct timer_list idle_timer;
158 struct timer_list mayday_timer;
159
160
161 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
162
163
164
165 struct mutex manager_arb;
166 struct mutex manager_mutex;
167 struct idr worker_idr;
168
169 struct workqueue_attrs *attrs;
170 struct hlist_node hash_node;
171 int refcnt;
172
173
174
175
176
177
178 atomic_t nr_running ____cacheline_aligned_in_smp;
179
180
181
182
183
184 struct rcu_head rcu;
185} ____cacheline_aligned_in_smp;
186
187
188
189
190
191
192
193struct pool_workqueue {
194 struct worker_pool *pool;
195 struct workqueue_struct *wq;
196 int work_color;
197 int flush_color;
198 int refcnt;
199 int nr_in_flight[WORK_NR_COLORS];
200
201 int nr_active;
202 int max_active;
203 struct list_head delayed_works;
204 struct list_head pwqs_node;
205 struct list_head mayday_node;
206
207
208
209
210
211
212
213 struct work_struct unbound_release_work;
214 struct rcu_head rcu;
215} __aligned(1 << WORK_STRUCT_FLAG_BITS);
216
217
218
219
220struct wq_flusher {
221 struct list_head list;
222 int flush_color;
223 struct completion done;
224};
225
226struct wq_device;
227
228
229
230
231
232struct workqueue_struct {
233 struct list_head pwqs;
234 struct list_head list;
235
236 struct mutex mutex;
237 int work_color;
238 int flush_color;
239 atomic_t nr_pwqs_to_flush;
240 struct wq_flusher *first_flusher;
241 struct list_head flusher_queue;
242 struct list_head flusher_overflow;
243
244 struct list_head maydays;
245 struct worker *rescuer;
246
247 int nr_drainers;
248 int saved_max_active;
249
250 struct workqueue_attrs *unbound_attrs;
251 struct pool_workqueue *dfl_pwq;
252
253#ifdef CONFIG_SYSFS
254 struct wq_device *wq_dev;
255#endif
256#ifdef CONFIG_LOCKDEP
257 struct lockdep_map lockdep_map;
258#endif
259 char name[WQ_NAME_LEN];
260
261
262 unsigned int flags ____cacheline_aligned;
263 struct pool_workqueue __percpu *cpu_pwqs;
264 struct pool_workqueue __rcu *numa_pwq_tbl[];
265};
266
267static struct kmem_cache *pwq_cache;
268
269static int wq_numa_tbl_len;
270static cpumask_var_t *wq_numa_possible_cpumask;
271
272
273static bool wq_disable_numa;
274module_param_named(disable_numa, wq_disable_numa, bool, 0444);
275
276
277#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
278static bool wq_power_efficient = true;
279#else
280static bool wq_power_efficient;
281#endif
282
283module_param_named(power_efficient, wq_power_efficient, bool, 0444);
284
285static bool wq_numa_enabled;
286
287
288static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
289
290static DEFINE_MUTEX(wq_pool_mutex);
291static DEFINE_SPINLOCK(wq_mayday_lock);
292
293static LIST_HEAD(workqueues);
294static bool workqueue_freezing;
295
296
297static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
298 cpu_worker_pools);
299
300static DEFINE_IDR(worker_pool_idr);
301
302
303static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
304
305
306static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
307
308struct workqueue_struct *system_wq __read_mostly;
309EXPORT_SYMBOL(system_wq);
310struct workqueue_struct *system_highpri_wq __read_mostly;
311EXPORT_SYMBOL_GPL(system_highpri_wq);
312struct workqueue_struct *system_long_wq __read_mostly;
313EXPORT_SYMBOL_GPL(system_long_wq);
314struct workqueue_struct *system_unbound_wq __read_mostly;
315EXPORT_SYMBOL_GPL(system_unbound_wq);
316struct workqueue_struct *system_freezable_wq __read_mostly;
317EXPORT_SYMBOL_GPL(system_freezable_wq);
318struct workqueue_struct *system_power_efficient_wq __read_mostly;
319EXPORT_SYMBOL_GPL(system_power_efficient_wq);
320struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
321EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
322
323static int worker_thread(void *__worker);
324static void copy_workqueue_attrs(struct workqueue_attrs *to,
325 const struct workqueue_attrs *from);
326
327#define CREATE_TRACE_POINTS
328#include <trace/events/workqueue.h>
329
330#define assert_rcu_or_pool_mutex() \
331 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
332 lockdep_is_held(&wq_pool_mutex), \
333 "sched RCU or wq_pool_mutex should be held")
334
335#define assert_rcu_or_wq_mutex(wq) \
336 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
337 lockdep_is_held(&wq->mutex), \
338 "sched RCU or wq->mutex should be held")
339
340#ifdef CONFIG_LOCKDEP
341#define assert_manager_or_pool_lock(pool) \
342 WARN_ONCE(debug_locks && \
343 !lockdep_is_held(&(pool)->manager_mutex) && \
344 !lockdep_is_held(&(pool)->lock), \
345 "pool->manager_mutex or ->lock should be held")
346#else
347#define assert_manager_or_pool_lock(pool) do { } while (0)
348#endif
349
350#define for_each_cpu_worker_pool(pool, cpu) \
351 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
352 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
353 (pool)++)
354
355
356
357
358
359
360
361
362
363
364
365
366
367#define for_each_pool(pool, pi) \
368 idr_for_each_entry(&worker_pool_idr, pool, pi) \
369 if (({ assert_rcu_or_pool_mutex(); false; })) { } \
370 else
371
372
373
374
375
376
377
378
379
380
381
382
383#define for_each_pool_worker(worker, wi, pool) \
384 idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
385 if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
386 else
387
388
389
390
391
392
393
394
395
396
397
398
399
400#define for_each_pwq(pwq, wq) \
401 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
402 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
403 else
404
405#ifdef CONFIG_DEBUG_OBJECTS_WORK
406
407static struct debug_obj_descr work_debug_descr;
408
409static void *work_debug_hint(void *addr)
410{
411 return ((struct work_struct *) addr)->func;
412}
413
414
415
416
417
418static int work_fixup_init(void *addr, enum debug_obj_state state)
419{
420 struct work_struct *work = addr;
421
422 switch (state) {
423 case ODEBUG_STATE_ACTIVE:
424 cancel_work_sync(work);
425 debug_object_init(work, &work_debug_descr);
426 return 1;
427 default:
428 return 0;
429 }
430}
431
432
433
434
435
436
437static int work_fixup_activate(void *addr, enum debug_obj_state state)
438{
439 struct work_struct *work = addr;
440
441 switch (state) {
442
443 case ODEBUG_STATE_NOTAVAILABLE:
444
445
446
447
448
449 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
450 debug_object_init(work, &work_debug_descr);
451 debug_object_activate(work, &work_debug_descr);
452 return 0;
453 }
454 WARN_ON_ONCE(1);
455 return 0;
456
457 case ODEBUG_STATE_ACTIVE:
458 WARN_ON(1);
459
460 default:
461 return 0;
462 }
463}
464
465
466
467
468
469static int work_fixup_free(void *addr, enum debug_obj_state state)
470{
471 struct work_struct *work = addr;
472
473 switch (state) {
474 case ODEBUG_STATE_ACTIVE:
475 cancel_work_sync(work);
476 debug_object_free(work, &work_debug_descr);
477 return 1;
478 default:
479 return 0;
480 }
481}
482
483static struct debug_obj_descr work_debug_descr = {
484 .name = "work_struct",
485 .debug_hint = work_debug_hint,
486 .fixup_init = work_fixup_init,
487 .fixup_activate = work_fixup_activate,
488 .fixup_free = work_fixup_free,
489};
490
491static inline void debug_work_activate(struct work_struct *work)
492{
493 debug_object_activate(work, &work_debug_descr);
494}
495
496static inline void debug_work_deactivate(struct work_struct *work)
497{
498 debug_object_deactivate(work, &work_debug_descr);
499}
500
501void __init_work(struct work_struct *work, int onstack)
502{
503 if (onstack)
504 debug_object_init_on_stack(work, &work_debug_descr);
505 else
506 debug_object_init(work, &work_debug_descr);
507}
508EXPORT_SYMBOL_GPL(__init_work);
509
510void destroy_work_on_stack(struct work_struct *work)
511{
512 debug_object_free(work, &work_debug_descr);
513}
514EXPORT_SYMBOL_GPL(destroy_work_on_stack);
515
516#else
517static inline void debug_work_activate(struct work_struct *work) { }
518static inline void debug_work_deactivate(struct work_struct *work) { }
519#endif
520
521
522static int worker_pool_assign_id(struct worker_pool *pool)
523{
524 int ret;
525
526 lockdep_assert_held(&wq_pool_mutex);
527
528 ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
529 if (ret >= 0) {
530 pool->id = ret;
531 return 0;
532 }
533 return ret;
534}
535
536
537
538
539
540
541
542
543
544
545
546
547static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
548 int node)
549{
550 assert_rcu_or_wq_mutex(wq);
551 return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
552}
553
554static unsigned int work_color_to_flags(int color)
555{
556 return color << WORK_STRUCT_COLOR_SHIFT;
557}
558
559static int get_work_color(struct work_struct *work)
560{
561 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
562 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
563}
564
565static int work_next_color(int color)
566{
567 return (color + 1) % WORK_NR_COLORS;
568}
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590static inline void set_work_data(struct work_struct *work, unsigned long data,
591 unsigned long flags)
592{
593 WARN_ON_ONCE(!work_pending(work));
594 atomic_long_set(&work->data, data | flags | work_static(work));
595}
596
597static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
598 unsigned long extra_flags)
599{
600 set_work_data(work, (unsigned long)pwq,
601 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
602}
603
604static void set_work_pool_and_keep_pending(struct work_struct *work,
605 int pool_id)
606{
607 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
608 WORK_STRUCT_PENDING);
609}
610
611static void set_work_pool_and_clear_pending(struct work_struct *work,
612 int pool_id)
613{
614
615
616
617
618
619
620 smp_wmb();
621 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
622}
623
624static void clear_work_data(struct work_struct *work)
625{
626 smp_wmb();
627 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
628}
629
630static struct pool_workqueue *get_work_pwq(struct work_struct *work)
631{
632 unsigned long data = atomic_long_read(&work->data);
633
634 if (data & WORK_STRUCT_PWQ)
635 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
636 else
637 return NULL;
638}
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655static struct worker_pool *get_work_pool(struct work_struct *work)
656{
657 unsigned long data = atomic_long_read(&work->data);
658 int pool_id;
659
660 assert_rcu_or_pool_mutex();
661
662 if (data & WORK_STRUCT_PWQ)
663 return ((struct pool_workqueue *)
664 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
665
666 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
667 if (pool_id == WORK_OFFQ_POOL_NONE)
668 return NULL;
669
670 return idr_find(&worker_pool_idr, pool_id);
671}
672
673
674
675
676
677
678
679
680static int get_work_pool_id(struct work_struct *work)
681{
682 unsigned long data = atomic_long_read(&work->data);
683
684 if (data & WORK_STRUCT_PWQ)
685 return ((struct pool_workqueue *)
686 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
687
688 return data >> WORK_OFFQ_POOL_SHIFT;
689}
690
691static void mark_work_canceling(struct work_struct *work)
692{
693 unsigned long pool_id = get_work_pool_id(work);
694
695 pool_id <<= WORK_OFFQ_POOL_SHIFT;
696 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
697}
698
699static bool work_is_canceling(struct work_struct *work)
700{
701 unsigned long data = atomic_long_read(&work->data);
702
703 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
704}
705
706
707
708
709
710
711
712static bool __need_more_worker(struct worker_pool *pool)
713{
714 return !atomic_read(&pool->nr_running);
715}
716
717
718
719
720
721
722
723
724
725static bool need_more_worker(struct worker_pool *pool)
726{
727 return !list_empty(&pool->worklist) && __need_more_worker(pool);
728}
729
730
731static bool may_start_working(struct worker_pool *pool)
732{
733 return pool->nr_idle;
734}
735
736
737static bool keep_working(struct worker_pool *pool)
738{
739 return !list_empty(&pool->worklist) &&
740 atomic_read(&pool->nr_running) <= 1;
741}
742
743
744static bool need_to_create_worker(struct worker_pool *pool)
745{
746 return need_more_worker(pool) && !may_start_working(pool);
747}
748
749
750static bool need_to_manage_workers(struct worker_pool *pool)
751{
752 return need_to_create_worker(pool) ||
753 (pool->flags & POOL_MANAGE_WORKERS);
754}
755
756
757static bool too_many_workers(struct worker_pool *pool)
758{
759 bool managing = mutex_is_locked(&pool->manager_arb);
760 int nr_idle = pool->nr_idle + managing;
761 int nr_busy = pool->nr_workers - nr_idle;
762
763
764
765
766
767 if (list_empty(&pool->idle_list))
768 return false;
769
770 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
771}
772
773
774
775
776
777
778static struct worker *first_worker(struct worker_pool *pool)
779{
780 if (unlikely(list_empty(&pool->idle_list)))
781 return NULL;
782
783 return list_first_entry(&pool->idle_list, struct worker, entry);
784}
785
786
787
788
789
790
791
792
793
794
795static void wake_up_worker(struct worker_pool *pool)
796{
797 struct worker *worker = first_worker(pool);
798
799 if (likely(worker))
800 wake_up_process(worker->task);
801}
802
803
804
805
806
807
808
809
810
811
812
813
814void wq_worker_waking_up(struct task_struct *task, int cpu)
815{
816 struct worker *worker = kthread_data(task);
817
818 if (!(worker->flags & WORKER_NOT_RUNNING)) {
819 WARN_ON_ONCE(worker->pool->cpu != cpu);
820 atomic_inc(&worker->pool->nr_running);
821 }
822}
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
840{
841 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
842 struct worker_pool *pool;
843
844
845
846
847
848
849 if (worker->flags & WORKER_NOT_RUNNING)
850 return NULL;
851
852 pool = worker->pool;
853
854
855 if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
856 return NULL;
857
858
859
860
861
862
863
864
865
866
867
868
869 if (atomic_dec_and_test(&pool->nr_running) &&
870 !list_empty(&pool->worklist))
871 to_wakeup = first_worker(pool);
872 return to_wakeup ? to_wakeup->task : NULL;
873}
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888static inline void worker_set_flags(struct worker *worker, unsigned int flags,
889 bool wakeup)
890{
891 struct worker_pool *pool = worker->pool;
892
893 WARN_ON_ONCE(worker->task != current);
894
895
896
897
898
899
900 if ((flags & WORKER_NOT_RUNNING) &&
901 !(worker->flags & WORKER_NOT_RUNNING)) {
902 if (wakeup) {
903 if (atomic_dec_and_test(&pool->nr_running) &&
904 !list_empty(&pool->worklist))
905 wake_up_worker(pool);
906 } else
907 atomic_dec(&pool->nr_running);
908 }
909
910 worker->flags |= flags;
911}
912
913
914
915
916
917
918
919
920
921
922
923static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
924{
925 struct worker_pool *pool = worker->pool;
926 unsigned int oflags = worker->flags;
927
928 WARN_ON_ONCE(worker->task != current);
929
930 worker->flags &= ~flags;
931
932
933
934
935
936
937 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
938 if (!(worker->flags & WORKER_NOT_RUNNING))
939 atomic_inc(&pool->nr_running);
940}
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975static struct worker *find_worker_executing_work(struct worker_pool *pool,
976 struct work_struct *work)
977{
978 struct worker *worker;
979
980 hash_for_each_possible(pool->busy_hash, worker, hentry,
981 (unsigned long)work)
982 if (worker->current_work == work &&
983 worker->current_func == work->func)
984 return worker;
985
986 return NULL;
987}
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006static void move_linked_works(struct work_struct *work, struct list_head *head,
1007 struct work_struct **nextp)
1008{
1009 struct work_struct *n;
1010
1011
1012
1013
1014
1015 list_for_each_entry_safe_from(work, n, NULL, entry) {
1016 list_move_tail(&work->entry, head);
1017 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1018 break;
1019 }
1020
1021
1022
1023
1024
1025
1026 if (nextp)
1027 *nextp = n;
1028}
1029
1030
1031
1032
1033
1034
1035
1036
1037static void get_pwq(struct pool_workqueue *pwq)
1038{
1039 lockdep_assert_held(&pwq->pool->lock);
1040 WARN_ON_ONCE(pwq->refcnt <= 0);
1041 pwq->refcnt++;
1042}
1043
1044
1045
1046
1047
1048
1049
1050
1051static void put_pwq(struct pool_workqueue *pwq)
1052{
1053 lockdep_assert_held(&pwq->pool->lock);
1054 if (likely(--pwq->refcnt))
1055 return;
1056 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1057 return;
1058
1059
1060
1061
1062
1063
1064
1065
1066 schedule_work(&pwq->unbound_release_work);
1067}
1068
1069
1070
1071
1072
1073
1074
1075static void put_pwq_unlocked(struct pool_workqueue *pwq)
1076{
1077 if (pwq) {
1078
1079
1080
1081
1082 spin_lock_irq(&pwq->pool->lock);
1083 put_pwq(pwq);
1084 spin_unlock_irq(&pwq->pool->lock);
1085 }
1086}
1087
1088static void pwq_activate_delayed_work(struct work_struct *work)
1089{
1090 struct pool_workqueue *pwq = get_work_pwq(work);
1091
1092 trace_workqueue_activate_work(work);
1093 move_linked_works(work, &pwq->pool->worklist, NULL);
1094 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1095 pwq->nr_active++;
1096}
1097
1098static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1099{
1100 struct work_struct *work = list_first_entry(&pwq->delayed_works,
1101 struct work_struct, entry);
1102
1103 pwq_activate_delayed_work(work);
1104}
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1118{
1119
1120 if (color == WORK_NO_COLOR)
1121 goto out_put;
1122
1123 pwq->nr_in_flight[color]--;
1124
1125 pwq->nr_active--;
1126 if (!list_empty(&pwq->delayed_works)) {
1127
1128 if (pwq->nr_active < pwq->max_active)
1129 pwq_activate_first_delayed(pwq);
1130 }
1131
1132
1133 if (likely(pwq->flush_color != color))
1134 goto out_put;
1135
1136
1137 if (pwq->nr_in_flight[color])
1138 goto out_put;
1139
1140
1141 pwq->flush_color = -1;
1142
1143
1144
1145
1146
1147 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1148 complete(&pwq->wq->first_flusher->done);
1149out_put:
1150 put_pwq(pwq);
1151}
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1181 unsigned long *flags)
1182{
1183 struct worker_pool *pool;
1184 struct pool_workqueue *pwq;
1185
1186 local_irq_save(*flags);
1187
1188
1189 if (is_dwork) {
1190 struct delayed_work *dwork = to_delayed_work(work);
1191
1192
1193
1194
1195
1196
1197 if (likely(del_timer(&dwork->timer)))
1198 return 1;
1199 }
1200
1201
1202 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1203 return 0;
1204
1205
1206
1207
1208
1209 pool = get_work_pool(work);
1210 if (!pool)
1211 goto fail;
1212
1213 spin_lock(&pool->lock);
1214
1215
1216
1217
1218
1219
1220
1221
1222 pwq = get_work_pwq(work);
1223 if (pwq && pwq->pool == pool) {
1224 debug_work_deactivate(work);
1225
1226
1227
1228
1229
1230
1231
1232
1233 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1234 pwq_activate_delayed_work(work);
1235
1236 list_del_init(&work->entry);
1237 pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1238
1239
1240 set_work_pool_and_keep_pending(work, pool->id);
1241
1242 spin_unlock(&pool->lock);
1243 return 1;
1244 }
1245 spin_unlock(&pool->lock);
1246fail:
1247 local_irq_restore(*flags);
1248 if (work_is_canceling(work))
1249 return -ENOENT;
1250 cpu_relax();
1251 return -EAGAIN;
1252}
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1268 struct list_head *head, unsigned int extra_flags)
1269{
1270 struct worker_pool *pool = pwq->pool;
1271
1272
1273 set_work_pwq(work, pwq, extra_flags);
1274 list_add_tail(&work->entry, head);
1275 get_pwq(pwq);
1276
1277
1278
1279
1280
1281
1282 smp_mb();
1283
1284 if (__need_more_worker(pool))
1285 wake_up_worker(pool);
1286}
1287
1288
1289
1290
1291
1292static bool is_chained_work(struct workqueue_struct *wq)
1293{
1294 struct worker *worker;
1295
1296 worker = current_wq_worker();
1297
1298
1299
1300
1301 return worker && worker->current_pwq->wq == wq;
1302}
1303
1304static void __queue_work(int cpu, struct workqueue_struct *wq,
1305 struct work_struct *work)
1306{
1307 struct pool_workqueue *pwq;
1308 struct worker_pool *last_pool;
1309 struct list_head *worklist;
1310 unsigned int work_flags;
1311 unsigned int req_cpu = cpu;
1312
1313
1314
1315
1316
1317
1318
1319 WARN_ON_ONCE(!irqs_disabled());
1320
1321 debug_work_activate(work);
1322
1323
1324 if (unlikely(wq->flags & __WQ_DRAINING) &&
1325 WARN_ON_ONCE(!is_chained_work(wq)))
1326 return;
1327retry:
1328 if (req_cpu == WORK_CPU_UNBOUND)
1329 cpu = raw_smp_processor_id();
1330
1331
1332 if (!(wq->flags & WQ_UNBOUND))
1333 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1334 else
1335 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1336
1337
1338
1339
1340
1341
1342 last_pool = get_work_pool(work);
1343 if (last_pool && last_pool != pwq->pool) {
1344 struct worker *worker;
1345
1346 spin_lock(&last_pool->lock);
1347
1348 worker = find_worker_executing_work(last_pool, work);
1349
1350 if (worker && worker->current_pwq->wq == wq) {
1351 pwq = worker->current_pwq;
1352 } else {
1353
1354 spin_unlock(&last_pool->lock);
1355 spin_lock(&pwq->pool->lock);
1356 }
1357 } else {
1358 spin_lock(&pwq->pool->lock);
1359 }
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369 if (unlikely(!pwq->refcnt)) {
1370 if (wq->flags & WQ_UNBOUND) {
1371 spin_unlock(&pwq->pool->lock);
1372 cpu_relax();
1373 goto retry;
1374 }
1375
1376 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1377 wq->name, cpu);
1378 }
1379
1380
1381 trace_workqueue_queue_work(req_cpu, pwq, work);
1382
1383 if (WARN_ON(!list_empty(&work->entry))) {
1384 spin_unlock(&pwq->pool->lock);
1385 return;
1386 }
1387
1388 pwq->nr_in_flight[pwq->work_color]++;
1389 work_flags = work_color_to_flags(pwq->work_color);
1390
1391 if (likely(pwq->nr_active < pwq->max_active)) {
1392 trace_workqueue_activate_work(work);
1393 pwq->nr_active++;
1394 worklist = &pwq->pool->worklist;
1395 } else {
1396 work_flags |= WORK_STRUCT_DELAYED;
1397 worklist = &pwq->delayed_works;
1398 }
1399
1400 insert_work(pwq, work, worklist, work_flags);
1401
1402 spin_unlock(&pwq->pool->lock);
1403}
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416bool queue_work_on(int cpu, struct workqueue_struct *wq,
1417 struct work_struct *work)
1418{
1419 bool ret = false;
1420 unsigned long flags;
1421
1422 local_irq_save(flags);
1423
1424 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1425 __queue_work(cpu, wq, work);
1426 ret = true;
1427 }
1428
1429 local_irq_restore(flags);
1430 return ret;
1431}
1432EXPORT_SYMBOL(queue_work_on);
1433
1434void delayed_work_timer_fn(unsigned long __data)
1435{
1436 struct delayed_work *dwork = (struct delayed_work *)__data;
1437
1438
1439 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1440}
1441EXPORT_SYMBOL(delayed_work_timer_fn);
1442
1443static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1444 struct delayed_work *dwork, unsigned long delay)
1445{
1446 struct timer_list *timer = &dwork->timer;
1447 struct work_struct *work = &dwork->work;
1448
1449 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1450 timer->data != (unsigned long)dwork);
1451 WARN_ON_ONCE(timer_pending(timer));
1452 WARN_ON_ONCE(!list_empty(&work->entry));
1453
1454
1455
1456
1457
1458
1459
1460 if (!delay) {
1461 __queue_work(cpu, wq, &dwork->work);
1462 return;
1463 }
1464
1465 timer_stats_timer_set_start_info(&dwork->timer);
1466
1467 dwork->wq = wq;
1468 dwork->cpu = cpu;
1469 timer->expires = jiffies + delay;
1470
1471 if (unlikely(cpu != WORK_CPU_UNBOUND))
1472 add_timer_on(timer, cpu);
1473 else
1474 add_timer(timer);
1475}
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1489 struct delayed_work *dwork, unsigned long delay)
1490{
1491 struct work_struct *work = &dwork->work;
1492 bool ret = false;
1493 unsigned long flags;
1494
1495
1496 local_irq_save(flags);
1497
1498 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1499 __queue_delayed_work(cpu, wq, dwork, delay);
1500 ret = true;
1501 }
1502
1503 local_irq_restore(flags);
1504 return ret;
1505}
1506EXPORT_SYMBOL(queue_delayed_work_on);
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1527 struct delayed_work *dwork, unsigned long delay)
1528{
1529 unsigned long flags;
1530 int ret;
1531
1532 do {
1533 ret = try_to_grab_pending(&dwork->work, true, &flags);
1534 } while (unlikely(ret == -EAGAIN));
1535
1536 if (likely(ret >= 0)) {
1537 __queue_delayed_work(cpu, wq, dwork, delay);
1538 local_irq_restore(flags);
1539 }
1540
1541
1542 return ret;
1543}
1544EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556static void worker_enter_idle(struct worker *worker)
1557{
1558 struct worker_pool *pool = worker->pool;
1559
1560 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1561 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1562 (worker->hentry.next || worker->hentry.pprev)))
1563 return;
1564
1565
1566 worker->flags |= WORKER_IDLE;
1567 pool->nr_idle++;
1568 worker->last_active = jiffies;
1569
1570
1571 list_add(&worker->entry, &pool->idle_list);
1572
1573 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1574 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1575
1576
1577
1578
1579
1580
1581
1582 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1583 pool->nr_workers == pool->nr_idle &&
1584 atomic_read(&pool->nr_running));
1585}
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596static void worker_leave_idle(struct worker *worker)
1597{
1598 struct worker_pool *pool = worker->pool;
1599
1600 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1601 return;
1602 worker_clr_flags(worker, WORKER_IDLE);
1603 pool->nr_idle--;
1604 list_del_init(&worker->entry);
1605}
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1640__acquires(&pool->lock)
1641{
1642 while (true) {
1643
1644
1645
1646
1647
1648
1649 if (!(pool->flags & POOL_DISASSOCIATED))
1650 set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1651
1652 spin_lock_irq(&pool->lock);
1653 if (pool->flags & POOL_DISASSOCIATED)
1654 return false;
1655 if (task_cpu(current) == pool->cpu &&
1656 cpumask_equal(¤t->cpus_allowed, pool->attrs->cpumask))
1657 return true;
1658 spin_unlock_irq(&pool->lock);
1659
1660
1661
1662
1663
1664
1665
1666 cpu_relax();
1667 cond_resched();
1668 }
1669}
1670
1671static struct worker *alloc_worker(void)
1672{
1673 struct worker *worker;
1674
1675 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1676 if (worker) {
1677 INIT_LIST_HEAD(&worker->entry);
1678 INIT_LIST_HEAD(&worker->scheduled);
1679
1680 worker->flags = WORKER_PREP;
1681 }
1682 return worker;
1683}
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699static struct worker *create_worker(struct worker_pool *pool)
1700{
1701 struct worker *worker = NULL;
1702 int id = -1;
1703 char id_buf[16];
1704
1705 lockdep_assert_held(&pool->manager_mutex);
1706
1707
1708
1709
1710
1711 idr_preload(GFP_KERNEL);
1712 spin_lock_irq(&pool->lock);
1713
1714 id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1715
1716 spin_unlock_irq(&pool->lock);
1717 idr_preload_end();
1718 if (id < 0)
1719 goto fail;
1720
1721 worker = alloc_worker();
1722 if (!worker)
1723 goto fail;
1724
1725 worker->pool = pool;
1726 worker->id = id;
1727
1728 if (pool->cpu >= 0)
1729 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1730 pool->attrs->nice < 0 ? "H" : "");
1731 else
1732 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1733
1734 worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1735 "kworker/%s", id_buf);
1736 if (IS_ERR(worker->task))
1737 goto fail;
1738
1739
1740
1741
1742
1743 set_user_nice(worker->task, pool->attrs->nice);
1744 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1745
1746
1747 worker->task->flags |= PF_NO_SETAFFINITY;
1748
1749
1750
1751
1752
1753
1754 if (pool->flags & POOL_DISASSOCIATED)
1755 worker->flags |= WORKER_UNBOUND;
1756
1757
1758 spin_lock_irq(&pool->lock);
1759 idr_replace(&pool->worker_idr, worker, worker->id);
1760 spin_unlock_irq(&pool->lock);
1761
1762 return worker;
1763
1764fail:
1765 if (id >= 0) {
1766 spin_lock_irq(&pool->lock);
1767 idr_remove(&pool->worker_idr, id);
1768 spin_unlock_irq(&pool->lock);
1769 }
1770 kfree(worker);
1771 return NULL;
1772}
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783static void start_worker(struct worker *worker)
1784{
1785 worker->flags |= WORKER_STARTED;
1786 worker->pool->nr_workers++;
1787 worker_enter_idle(worker);
1788 wake_up_process(worker->task);
1789}
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799static int create_and_start_worker(struct worker_pool *pool)
1800{
1801 struct worker *worker;
1802
1803 mutex_lock(&pool->manager_mutex);
1804
1805 worker = create_worker(pool);
1806 if (worker) {
1807 spin_lock_irq(&pool->lock);
1808 start_worker(worker);
1809 spin_unlock_irq(&pool->lock);
1810 }
1811
1812 mutex_unlock(&pool->manager_mutex);
1813
1814 return worker ? 0 : -ENOMEM;
1815}
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826static void destroy_worker(struct worker *worker)
1827{
1828 struct worker_pool *pool = worker->pool;
1829
1830 lockdep_assert_held(&pool->manager_mutex);
1831 lockdep_assert_held(&pool->lock);
1832
1833
1834 if (WARN_ON(worker->current_work) ||
1835 WARN_ON(!list_empty(&worker->scheduled)))
1836 return;
1837
1838 if (worker->flags & WORKER_STARTED)
1839 pool->nr_workers--;
1840 if (worker->flags & WORKER_IDLE)
1841 pool->nr_idle--;
1842
1843 list_del_init(&worker->entry);
1844 worker->flags |= WORKER_DIE;
1845
1846 idr_remove(&pool->worker_idr, worker->id);
1847
1848 spin_unlock_irq(&pool->lock);
1849
1850 kthread_stop(worker->task);
1851 kfree(worker);
1852
1853 spin_lock_irq(&pool->lock);
1854}
1855
1856static void idle_worker_timeout(unsigned long __pool)
1857{
1858 struct worker_pool *pool = (void *)__pool;
1859
1860 spin_lock_irq(&pool->lock);
1861
1862 if (too_many_workers(pool)) {
1863 struct worker *worker;
1864 unsigned long expires;
1865
1866
1867 worker = list_entry(pool->idle_list.prev, struct worker, entry);
1868 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1869
1870 if (time_before(jiffies, expires))
1871 mod_timer(&pool->idle_timer, expires);
1872 else {
1873
1874 pool->flags |= POOL_MANAGE_WORKERS;
1875 wake_up_worker(pool);
1876 }
1877 }
1878
1879 spin_unlock_irq(&pool->lock);
1880}
1881
1882static void send_mayday(struct work_struct *work)
1883{
1884 struct pool_workqueue *pwq = get_work_pwq(work);
1885 struct workqueue_struct *wq = pwq->wq;
1886
1887 lockdep_assert_held(&wq_mayday_lock);
1888
1889 if (!wq->rescuer)
1890 return;
1891
1892
1893 if (list_empty(&pwq->mayday_node)) {
1894 list_add_tail(&pwq->mayday_node, &wq->maydays);
1895 wake_up_process(wq->rescuer->task);
1896 }
1897}
1898
1899static void pool_mayday_timeout(unsigned long __pool)
1900{
1901 struct worker_pool *pool = (void *)__pool;
1902 struct work_struct *work;
1903
1904 spin_lock_irq(&wq_mayday_lock);
1905 spin_lock(&pool->lock);
1906
1907 if (need_to_create_worker(pool)) {
1908
1909
1910
1911
1912
1913
1914 list_for_each_entry(work, &pool->worklist, entry)
1915 send_mayday(work);
1916 }
1917
1918 spin_unlock(&pool->lock);
1919 spin_unlock_irq(&wq_mayday_lock);
1920
1921 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1922}
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946static bool maybe_create_worker(struct worker_pool *pool)
1947__releases(&pool->lock)
1948__acquires(&pool->lock)
1949{
1950 if (!need_to_create_worker(pool))
1951 return false;
1952restart:
1953 spin_unlock_irq(&pool->lock);
1954
1955
1956 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1957
1958 while (true) {
1959 struct worker *worker;
1960
1961 worker = create_worker(pool);
1962 if (worker) {
1963 del_timer_sync(&pool->mayday_timer);
1964 spin_lock_irq(&pool->lock);
1965 start_worker(worker);
1966 if (WARN_ON_ONCE(need_to_create_worker(pool)))
1967 goto restart;
1968 return true;
1969 }
1970
1971 if (!need_to_create_worker(pool))
1972 break;
1973
1974 __set_current_state(TASK_INTERRUPTIBLE);
1975 schedule_timeout(CREATE_COOLDOWN);
1976
1977 if (!need_to_create_worker(pool))
1978 break;
1979 }
1980
1981 del_timer_sync(&pool->mayday_timer);
1982 spin_lock_irq(&pool->lock);
1983 if (need_to_create_worker(pool))
1984 goto restart;
1985 return true;
1986}
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003static bool maybe_destroy_workers(struct worker_pool *pool)
2004{
2005 bool ret = false;
2006
2007 while (too_many_workers(pool)) {
2008 struct worker *worker;
2009 unsigned long expires;
2010
2011 worker = list_entry(pool->idle_list.prev, struct worker, entry);
2012 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2013
2014 if (time_before(jiffies, expires)) {
2015 mod_timer(&pool->idle_timer, expires);
2016 break;
2017 }
2018
2019 destroy_worker(worker);
2020 ret = true;
2021 }
2022
2023 return ret;
2024}
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049static bool manage_workers(struct worker *worker)
2050{
2051 struct worker_pool *pool = worker->pool;
2052 bool ret = false;
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075 if (!mutex_trylock(&pool->manager_arb))
2076 return ret;
2077
2078
2079
2080
2081
2082 if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2083 spin_unlock_irq(&pool->lock);
2084 mutex_lock(&pool->manager_mutex);
2085 spin_lock_irq(&pool->lock);
2086 ret = true;
2087 }
2088
2089 pool->flags &= ~POOL_MANAGE_WORKERS;
2090
2091
2092
2093
2094
2095 ret |= maybe_destroy_workers(pool);
2096 ret |= maybe_create_worker(pool);
2097
2098 mutex_unlock(&pool->manager_mutex);
2099 mutex_unlock(&pool->manager_arb);
2100 return ret;
2101}
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117static void process_one_work(struct worker *worker, struct work_struct *work)
2118__releases(&pool->lock)
2119__acquires(&pool->lock)
2120{
2121 struct pool_workqueue *pwq = get_work_pwq(work);
2122 struct worker_pool *pool = worker->pool;
2123 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2124 int work_color;
2125 struct worker *collision;
2126#ifdef CONFIG_LOCKDEP
2127
2128
2129
2130
2131
2132
2133
2134 struct lockdep_map lockdep_map;
2135
2136 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2137#endif
2138
2139
2140
2141
2142
2143 WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2144 !(pool->flags & POOL_DISASSOCIATED) &&
2145 raw_smp_processor_id() != pool->cpu);
2146
2147
2148
2149
2150
2151
2152
2153 collision = find_worker_executing_work(pool, work);
2154 if (unlikely(collision)) {
2155 move_linked_works(work, &collision->scheduled, NULL);
2156 return;
2157 }
2158
2159
2160 debug_work_deactivate(work);
2161 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2162 worker->current_work = work;
2163 worker->current_func = work->func;
2164 worker->current_pwq = pwq;
2165 work_color = get_work_color(work);
2166
2167 list_del_init(&work->entry);
2168
2169
2170
2171
2172
2173 if (unlikely(cpu_intensive))
2174 worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2175
2176
2177
2178
2179
2180 if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2181 wake_up_worker(pool);
2182
2183
2184
2185
2186
2187
2188
2189 set_work_pool_and_clear_pending(work, pool->id);
2190
2191 spin_unlock_irq(&pool->lock);
2192
2193 lock_map_acquire_read(&pwq->wq->lockdep_map);
2194 lock_map_acquire(&lockdep_map);
2195 trace_workqueue_execute_start(work);
2196 worker->current_func(work);
2197
2198
2199
2200
2201 trace_workqueue_execute_end(work);
2202 lock_map_release(&lockdep_map);
2203 lock_map_release(&pwq->wq->lockdep_map);
2204
2205 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2206 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2207 " last function: %pf\n",
2208 current->comm, preempt_count(), task_pid_nr(current),
2209 worker->current_func);
2210 debug_show_held_locks(current);
2211 dump_stack();
2212 }
2213
2214
2215
2216
2217
2218
2219
2220
2221 cond_resched();
2222
2223 spin_lock_irq(&pool->lock);
2224
2225
2226 if (unlikely(cpu_intensive))
2227 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2228
2229
2230 hash_del(&worker->hentry);
2231 worker->current_work = NULL;
2232 worker->current_func = NULL;
2233 worker->current_pwq = NULL;
2234 worker->desc_valid = false;
2235 pwq_dec_nr_in_flight(pwq, work_color);
2236}
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250static void process_scheduled_works(struct worker *worker)
2251{
2252 while (!list_empty(&worker->scheduled)) {
2253 struct work_struct *work = list_first_entry(&worker->scheduled,
2254 struct work_struct, entry);
2255 process_one_work(worker, work);
2256 }
2257}
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271static int worker_thread(void *__worker)
2272{
2273 struct worker *worker = __worker;
2274 struct worker_pool *pool = worker->pool;
2275
2276
2277 worker->task->flags |= PF_WQ_WORKER;
2278woke_up:
2279 spin_lock_irq(&pool->lock);
2280
2281
2282 if (unlikely(worker->flags & WORKER_DIE)) {
2283 spin_unlock_irq(&pool->lock);
2284 WARN_ON_ONCE(!list_empty(&worker->entry));
2285 worker->task->flags &= ~PF_WQ_WORKER;
2286 return 0;
2287 }
2288
2289 worker_leave_idle(worker);
2290recheck:
2291
2292 if (!need_more_worker(pool))
2293 goto sleep;
2294
2295
2296 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2297 goto recheck;
2298
2299
2300
2301
2302
2303
2304 WARN_ON_ONCE(!list_empty(&worker->scheduled));
2305
2306
2307
2308
2309
2310
2311
2312
2313 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2314
2315 do {
2316 struct work_struct *work =
2317 list_first_entry(&pool->worklist,
2318 struct work_struct, entry);
2319
2320 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2321
2322 process_one_work(worker, work);
2323 if (unlikely(!list_empty(&worker->scheduled)))
2324 process_scheduled_works(worker);
2325 } else {
2326 move_linked_works(work, &worker->scheduled, NULL);
2327 process_scheduled_works(worker);
2328 }
2329 } while (keep_working(pool));
2330
2331 worker_set_flags(worker, WORKER_PREP, false);
2332sleep:
2333 if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2334 goto recheck;
2335
2336
2337
2338
2339
2340
2341
2342
2343 worker_enter_idle(worker);
2344 __set_current_state(TASK_INTERRUPTIBLE);
2345 spin_unlock_irq(&pool->lock);
2346 schedule();
2347 goto woke_up;
2348}
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371static int rescuer_thread(void *__rescuer)
2372{
2373 struct worker *rescuer = __rescuer;
2374 struct workqueue_struct *wq = rescuer->rescue_wq;
2375 struct list_head *scheduled = &rescuer->scheduled;
2376
2377 set_user_nice(current, RESCUER_NICE_LEVEL);
2378
2379
2380
2381
2382
2383 rescuer->task->flags |= PF_WQ_WORKER;
2384repeat:
2385 set_current_state(TASK_INTERRUPTIBLE);
2386
2387 if (kthread_should_stop()) {
2388 __set_current_state(TASK_RUNNING);
2389 rescuer->task->flags &= ~PF_WQ_WORKER;
2390 return 0;
2391 }
2392
2393
2394 spin_lock_irq(&wq_mayday_lock);
2395
2396 while (!list_empty(&wq->maydays)) {
2397 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2398 struct pool_workqueue, mayday_node);
2399 struct worker_pool *pool = pwq->pool;
2400 struct work_struct *work, *n;
2401
2402 __set_current_state(TASK_RUNNING);
2403 list_del_init(&pwq->mayday_node);
2404
2405 spin_unlock_irq(&wq_mayday_lock);
2406
2407
2408 worker_maybe_bind_and_lock(pool);
2409 rescuer->pool = pool;
2410
2411
2412
2413
2414
2415 WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2416 list_for_each_entry_safe(work, n, &pool->worklist, entry)
2417 if (get_work_pwq(work) == pwq)
2418 move_linked_works(work, scheduled, &n);
2419
2420 process_scheduled_works(rescuer);
2421
2422
2423
2424
2425
2426
2427 if (keep_working(pool))
2428 wake_up_worker(pool);
2429
2430 rescuer->pool = NULL;
2431 spin_unlock(&pool->lock);
2432 spin_lock(&wq_mayday_lock);
2433 }
2434
2435 spin_unlock_irq(&wq_mayday_lock);
2436
2437
2438 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2439 schedule();
2440 goto repeat;
2441}
2442
2443struct wq_barrier {
2444 struct work_struct work;
2445 struct completion done;
2446};
2447
2448static void wq_barrier_func(struct work_struct *work)
2449{
2450 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2451 complete(&barr->done);
2452}
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478static void insert_wq_barrier(struct pool_workqueue *pwq,
2479 struct wq_barrier *barr,
2480 struct work_struct *target, struct worker *worker)
2481{
2482 struct list_head *head;
2483 unsigned int linked = 0;
2484
2485
2486
2487
2488
2489
2490
2491 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2492 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2493 init_completion(&barr->done);
2494
2495
2496
2497
2498
2499 if (worker)
2500 head = worker->scheduled.next;
2501 else {
2502 unsigned long *bits = work_data_bits(target);
2503
2504 head = target->entry.next;
2505
2506 linked = *bits & WORK_STRUCT_LINKED;
2507 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2508 }
2509
2510 debug_work_activate(&barr->work);
2511 insert_work(pwq, &barr->work, head,
2512 work_color_to_flags(WORK_NO_COLOR) | linked);
2513}
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2547 int flush_color, int work_color)
2548{
2549 bool wait = false;
2550 struct pool_workqueue *pwq;
2551
2552 if (flush_color >= 0) {
2553 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2554 atomic_set(&wq->nr_pwqs_to_flush, 1);
2555 }
2556
2557 for_each_pwq(pwq, wq) {
2558 struct worker_pool *pool = pwq->pool;
2559
2560 spin_lock_irq(&pool->lock);
2561
2562 if (flush_color >= 0) {
2563 WARN_ON_ONCE(pwq->flush_color != -1);
2564
2565 if (pwq->nr_in_flight[flush_color]) {
2566 pwq->flush_color = flush_color;
2567 atomic_inc(&wq->nr_pwqs_to_flush);
2568 wait = true;
2569 }
2570 }
2571
2572 if (work_color >= 0) {
2573 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2574 pwq->work_color = work_color;
2575 }
2576
2577 spin_unlock_irq(&pool->lock);
2578 }
2579
2580 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2581 complete(&wq->first_flusher->done);
2582
2583 return wait;
2584}
2585
2586
2587
2588
2589
2590
2591
2592
2593void flush_workqueue(struct workqueue_struct *wq)
2594{
2595 struct wq_flusher this_flusher = {
2596 .list = LIST_HEAD_INIT(this_flusher.list),
2597 .flush_color = -1,
2598 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2599 };
2600 int next_color;
2601
2602 lock_map_acquire(&wq->lockdep_map);
2603 lock_map_release(&wq->lockdep_map);
2604
2605 mutex_lock(&wq->mutex);
2606
2607
2608
2609
2610 next_color = work_next_color(wq->work_color);
2611
2612 if (next_color != wq->flush_color) {
2613
2614
2615
2616
2617
2618 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2619 this_flusher.flush_color = wq->work_color;
2620 wq->work_color = next_color;
2621
2622 if (!wq->first_flusher) {
2623
2624 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2625
2626 wq->first_flusher = &this_flusher;
2627
2628 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2629 wq->work_color)) {
2630
2631 wq->flush_color = next_color;
2632 wq->first_flusher = NULL;
2633 goto out_unlock;
2634 }
2635 } else {
2636
2637 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2638 list_add_tail(&this_flusher.list, &wq->flusher_queue);
2639 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2640 }
2641 } else {
2642
2643
2644
2645
2646
2647 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2648 }
2649
2650 mutex_unlock(&wq->mutex);
2651
2652 wait_for_completion(&this_flusher.done);
2653
2654
2655
2656
2657
2658
2659
2660 if (wq->first_flusher != &this_flusher)
2661 return;
2662
2663 mutex_lock(&wq->mutex);
2664
2665
2666 if (wq->first_flusher != &this_flusher)
2667 goto out_unlock;
2668
2669 wq->first_flusher = NULL;
2670
2671 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2672 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2673
2674 while (true) {
2675 struct wq_flusher *next, *tmp;
2676
2677
2678 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2679 if (next->flush_color != wq->flush_color)
2680 break;
2681 list_del_init(&next->list);
2682 complete(&next->done);
2683 }
2684
2685 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2686 wq->flush_color != work_next_color(wq->work_color));
2687
2688
2689 wq->flush_color = work_next_color(wq->flush_color);
2690
2691
2692 if (!list_empty(&wq->flusher_overflow)) {
2693
2694
2695
2696
2697
2698
2699 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2700 tmp->flush_color = wq->work_color;
2701
2702 wq->work_color = work_next_color(wq->work_color);
2703
2704 list_splice_tail_init(&wq->flusher_overflow,
2705 &wq->flusher_queue);
2706 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2707 }
2708
2709 if (list_empty(&wq->flusher_queue)) {
2710 WARN_ON_ONCE(wq->flush_color != wq->work_color);
2711 break;
2712 }
2713
2714
2715
2716
2717
2718 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2719 WARN_ON_ONCE(wq->flush_color != next->flush_color);
2720
2721 list_del_init(&next->list);
2722 wq->first_flusher = next;
2723
2724 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2725 break;
2726
2727
2728
2729
2730
2731 wq->first_flusher = NULL;
2732 }
2733
2734out_unlock:
2735 mutex_unlock(&wq->mutex);
2736}
2737EXPORT_SYMBOL_GPL(flush_workqueue);
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750void drain_workqueue(struct workqueue_struct *wq)
2751{
2752 unsigned int flush_cnt = 0;
2753 struct pool_workqueue *pwq;
2754
2755
2756
2757
2758
2759
2760 mutex_lock(&wq->mutex);
2761 if (!wq->nr_drainers++)
2762 wq->flags |= __WQ_DRAINING;
2763 mutex_unlock(&wq->mutex);
2764reflush:
2765 flush_workqueue(wq);
2766
2767 mutex_lock(&wq->mutex);
2768
2769 for_each_pwq(pwq, wq) {
2770 bool drained;
2771
2772 spin_lock_irq(&pwq->pool->lock);
2773 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2774 spin_unlock_irq(&pwq->pool->lock);
2775
2776 if (drained)
2777 continue;
2778
2779 if (++flush_cnt == 10 ||
2780 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2781 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2782 wq->name, flush_cnt);
2783
2784 mutex_unlock(&wq->mutex);
2785 goto reflush;
2786 }
2787
2788 if (!--wq->nr_drainers)
2789 wq->flags &= ~__WQ_DRAINING;
2790 mutex_unlock(&wq->mutex);
2791}
2792EXPORT_SYMBOL_GPL(drain_workqueue);
2793
2794static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2795{
2796 struct worker *worker = NULL;
2797 struct worker_pool *pool;
2798 struct pool_workqueue *pwq;
2799
2800 might_sleep();
2801
2802 local_irq_disable();
2803 pool = get_work_pool(work);
2804 if (!pool) {
2805 local_irq_enable();
2806 return false;
2807 }
2808
2809 spin_lock(&pool->lock);
2810
2811 pwq = get_work_pwq(work);
2812 if (pwq) {
2813 if (unlikely(pwq->pool != pool))
2814 goto already_gone;
2815 } else {
2816 worker = find_worker_executing_work(pool, work);
2817 if (!worker)
2818 goto already_gone;
2819 pwq = worker->current_pwq;
2820 }
2821
2822 insert_wq_barrier(pwq, barr, work, worker);
2823 spin_unlock_irq(&pool->lock);
2824
2825
2826
2827
2828
2829
2830
2831 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
2832 lock_map_acquire(&pwq->wq->lockdep_map);
2833 else
2834 lock_map_acquire_read(&pwq->wq->lockdep_map);
2835 lock_map_release(&pwq->wq->lockdep_map);
2836
2837 return true;
2838already_gone:
2839 spin_unlock_irq(&pool->lock);
2840 return false;
2841}
2842
2843static bool __flush_work(struct work_struct *work)
2844{
2845 struct wq_barrier barr;
2846
2847 if (start_flush_work(work, &barr)) {
2848 wait_for_completion(&barr.done);
2849 destroy_work_on_stack(&barr.work);
2850 return true;
2851 } else {
2852 return false;
2853 }
2854}
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867bool flush_work(struct work_struct *work)
2868{
2869 lock_map_acquire(&work->lockdep_map);
2870 lock_map_release(&work->lockdep_map);
2871
2872 return __flush_work(work);
2873}
2874EXPORT_SYMBOL_GPL(flush_work);
2875
2876static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2877{
2878 unsigned long flags;
2879 int ret;
2880
2881 do {
2882 ret = try_to_grab_pending(work, is_dwork, &flags);
2883
2884
2885
2886
2887 if (unlikely(ret == -ENOENT))
2888 flush_work(work);
2889 } while (unlikely(ret < 0));
2890
2891
2892 mark_work_canceling(work);
2893 local_irq_restore(flags);
2894
2895 flush_work(work);
2896 clear_work_data(work);
2897 return ret;
2898}
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918bool cancel_work_sync(struct work_struct *work)
2919{
2920 return __cancel_work_timer(work, false);
2921}
2922EXPORT_SYMBOL_GPL(cancel_work_sync);
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936bool flush_delayed_work(struct delayed_work *dwork)
2937{
2938 local_irq_disable();
2939 if (del_timer_sync(&dwork->timer))
2940 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
2941 local_irq_enable();
2942 return flush_work(&dwork->work);
2943}
2944EXPORT_SYMBOL(flush_delayed_work);
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962bool cancel_delayed_work(struct delayed_work *dwork)
2963{
2964 unsigned long flags;
2965 int ret;
2966
2967 do {
2968 ret = try_to_grab_pending(&dwork->work, true, &flags);
2969 } while (unlikely(ret == -EAGAIN));
2970
2971 if (unlikely(ret < 0))
2972 return false;
2973
2974 set_work_pool_and_clear_pending(&dwork->work,
2975 get_work_pool_id(&dwork->work));
2976 local_irq_restore(flags);
2977 return ret;
2978}
2979EXPORT_SYMBOL(cancel_delayed_work);
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990bool cancel_delayed_work_sync(struct delayed_work *dwork)
2991{
2992 return __cancel_work_timer(&dwork->work, true);
2993}
2994EXPORT_SYMBOL(cancel_delayed_work_sync);
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007int schedule_on_each_cpu(work_func_t func)
3008{
3009 int cpu;
3010 struct work_struct __percpu *works;
3011
3012 works = alloc_percpu(struct work_struct);
3013 if (!works)
3014 return -ENOMEM;
3015
3016 get_online_cpus();
3017
3018 for_each_online_cpu(cpu) {
3019 struct work_struct *work = per_cpu_ptr(works, cpu);
3020
3021 INIT_WORK(work, func);
3022 schedule_work_on(cpu, work);
3023 }
3024
3025 for_each_online_cpu(cpu)
3026 flush_work(per_cpu_ptr(works, cpu));
3027
3028 put_online_cpus();
3029 free_percpu(works);
3030 return 0;
3031}
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057void flush_scheduled_work(void)
3058{
3059 flush_workqueue(system_wq);
3060}
3061EXPORT_SYMBOL(flush_scheduled_work);
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3076{
3077 if (!in_interrupt()) {
3078 fn(&ew->work);
3079 return 0;
3080 }
3081
3082 INIT_WORK(&ew->work, fn);
3083 schedule_work(&ew->work);
3084
3085 return 1;
3086}
3087EXPORT_SYMBOL_GPL(execute_in_process_context);
3088
3089#ifdef CONFIG_SYSFS
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104struct wq_device {
3105 struct workqueue_struct *wq;
3106 struct device dev;
3107};
3108
3109static struct workqueue_struct *dev_to_wq(struct device *dev)
3110{
3111 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3112
3113 return wq_dev->wq;
3114}
3115
3116static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
3117 char *buf)
3118{
3119 struct workqueue_struct *wq = dev_to_wq(dev);
3120
3121 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3122}
3123static DEVICE_ATTR_RO(per_cpu);
3124
3125static ssize_t max_active_show(struct device *dev,
3126 struct device_attribute *attr, char *buf)
3127{
3128 struct workqueue_struct *wq = dev_to_wq(dev);
3129
3130 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3131}
3132
3133static ssize_t max_active_store(struct device *dev,
3134 struct device_attribute *attr, const char *buf,
3135 size_t count)
3136{
3137 struct workqueue_struct *wq = dev_to_wq(dev);
3138 int val;
3139
3140 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3141 return -EINVAL;
3142
3143 workqueue_set_max_active(wq, val);
3144 return count;
3145}
3146static DEVICE_ATTR_RW(max_active);
3147
3148static struct attribute *wq_sysfs_attrs[] = {
3149 &dev_attr_per_cpu.attr,
3150 &dev_attr_max_active.attr,
3151 NULL,
3152};
3153ATTRIBUTE_GROUPS(wq_sysfs);
3154
3155static ssize_t wq_pool_ids_show(struct device *dev,
3156 struct device_attribute *attr, char *buf)
3157{
3158 struct workqueue_struct *wq = dev_to_wq(dev);
3159 const char *delim = "";
3160 int node, written = 0;
3161
3162 rcu_read_lock_sched();
3163 for_each_node(node) {
3164 written += scnprintf(buf + written, PAGE_SIZE - written,
3165 "%s%d:%d", delim, node,
3166 unbound_pwq_by_node(wq, node)->pool->id);
3167 delim = " ";
3168 }
3169 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3170 rcu_read_unlock_sched();
3171
3172 return written;
3173}
3174
3175static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3176 char *buf)
3177{
3178 struct workqueue_struct *wq = dev_to_wq(dev);
3179 int written;
3180
3181 mutex_lock(&wq->mutex);
3182 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3183 mutex_unlock(&wq->mutex);
3184
3185 return written;
3186}
3187
3188
3189static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3190{
3191 struct workqueue_attrs *attrs;
3192
3193 attrs = alloc_workqueue_attrs(GFP_KERNEL);
3194 if (!attrs)
3195 return NULL;
3196
3197 mutex_lock(&wq->mutex);
3198 copy_workqueue_attrs(attrs, wq->unbound_attrs);
3199 mutex_unlock(&wq->mutex);
3200 return attrs;
3201}
3202
3203static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3204 const char *buf, size_t count)
3205{
3206 struct workqueue_struct *wq = dev_to_wq(dev);
3207 struct workqueue_attrs *attrs;
3208 int ret;
3209
3210 attrs = wq_sysfs_prep_attrs(wq);
3211 if (!attrs)
3212 return -ENOMEM;
3213
3214 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3215 attrs->nice >= -20 && attrs->nice <= 19)
3216 ret = apply_workqueue_attrs(wq, attrs);
3217 else
3218 ret = -EINVAL;
3219
3220 free_workqueue_attrs(attrs);
3221 return ret ?: count;
3222}
3223
3224static ssize_t wq_cpumask_show(struct device *dev,
3225 struct device_attribute *attr, char *buf)
3226{
3227 struct workqueue_struct *wq = dev_to_wq(dev);
3228 int written;
3229
3230 mutex_lock(&wq->mutex);
3231 written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3232 mutex_unlock(&wq->mutex);
3233
3234 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3235 return written;
3236}
3237
3238static ssize_t wq_cpumask_store(struct device *dev,
3239 struct device_attribute *attr,
3240 const char *buf, size_t count)
3241{
3242 struct workqueue_struct *wq = dev_to_wq(dev);
3243 struct workqueue_attrs *attrs;
3244 int ret;
3245
3246 attrs = wq_sysfs_prep_attrs(wq);
3247 if (!attrs)
3248 return -ENOMEM;
3249
3250 ret = cpumask_parse(buf, attrs->cpumask);
3251 if (!ret)
3252 ret = apply_workqueue_attrs(wq, attrs);
3253
3254 free_workqueue_attrs(attrs);
3255 return ret ?: count;
3256}
3257
3258static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
3259 char *buf)
3260{
3261 struct workqueue_struct *wq = dev_to_wq(dev);
3262 int written;
3263
3264 mutex_lock(&wq->mutex);
3265 written = scnprintf(buf, PAGE_SIZE, "%d\n",
3266 !wq->unbound_attrs->no_numa);
3267 mutex_unlock(&wq->mutex);
3268
3269 return written;
3270}
3271
3272static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
3273 const char *buf, size_t count)
3274{
3275 struct workqueue_struct *wq = dev_to_wq(dev);
3276 struct workqueue_attrs *attrs;
3277 int v, ret;
3278
3279 attrs = wq_sysfs_prep_attrs(wq);
3280 if (!attrs)
3281 return -ENOMEM;
3282
3283 ret = -EINVAL;
3284 if (sscanf(buf, "%d", &v) == 1) {
3285 attrs->no_numa = !v;
3286 ret = apply_workqueue_attrs(wq, attrs);
3287 }
3288
3289 free_workqueue_attrs(attrs);
3290 return ret ?: count;
3291}
3292
3293static struct device_attribute wq_sysfs_unbound_attrs[] = {
3294 __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3295 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3296 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3297 __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3298 __ATTR_NULL,
3299};
3300
3301static struct bus_type wq_subsys = {
3302 .name = "workqueue",
3303 .dev_groups = wq_sysfs_groups,
3304};
3305
3306static int __init wq_sysfs_init(void)
3307{
3308 return subsys_virtual_register(&wq_subsys, NULL);
3309}
3310core_initcall(wq_sysfs_init);
3311
3312static void wq_device_release(struct device *dev)
3313{
3314 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3315
3316 kfree(wq_dev);
3317}
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334int workqueue_sysfs_register(struct workqueue_struct *wq)
3335{
3336 struct wq_device *wq_dev;
3337 int ret;
3338
3339
3340
3341
3342
3343
3344 if (WARN_ON(wq->flags & __WQ_ORDERED))
3345 return -EINVAL;
3346
3347 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3348 if (!wq_dev)
3349 return -ENOMEM;
3350
3351 wq_dev->wq = wq;
3352 wq_dev->dev.bus = &wq_subsys;
3353 wq_dev->dev.init_name = wq->name;
3354 wq_dev->dev.release = wq_device_release;
3355
3356
3357
3358
3359
3360 dev_set_uevent_suppress(&wq_dev->dev, true);
3361
3362 ret = device_register(&wq_dev->dev);
3363 if (ret) {
3364 kfree(wq_dev);
3365 wq->wq_dev = NULL;
3366 return ret;
3367 }
3368
3369 if (wq->flags & WQ_UNBOUND) {
3370 struct device_attribute *attr;
3371
3372 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3373 ret = device_create_file(&wq_dev->dev, attr);
3374 if (ret) {
3375 device_unregister(&wq_dev->dev);
3376 wq->wq_dev = NULL;
3377 return ret;
3378 }
3379 }
3380 }
3381
3382 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3383 return 0;
3384}
3385
3386
3387
3388
3389
3390
3391
3392static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3393{
3394 struct wq_device *wq_dev = wq->wq_dev;
3395
3396 if (!wq->wq_dev)
3397 return;
3398
3399 wq->wq_dev = NULL;
3400 device_unregister(&wq_dev->dev);
3401}
3402#else
3403static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3404#endif
3405
3406
3407
3408
3409
3410
3411
3412void free_workqueue_attrs(struct workqueue_attrs *attrs)
3413{
3414 if (attrs) {
3415 free_cpumask_var(attrs->cpumask);
3416 kfree(attrs);
3417 }
3418}
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3430{
3431 struct workqueue_attrs *attrs;
3432
3433 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3434 if (!attrs)
3435 goto fail;
3436 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3437 goto fail;
3438
3439 cpumask_copy(attrs->cpumask, cpu_possible_mask);
3440 return attrs;
3441fail:
3442 free_workqueue_attrs(attrs);
3443 return NULL;
3444}
3445
3446static void copy_workqueue_attrs(struct workqueue_attrs *to,
3447 const struct workqueue_attrs *from)
3448{
3449 to->nice = from->nice;
3450 cpumask_copy(to->cpumask, from->cpumask);
3451
3452
3453
3454
3455
3456 to->no_numa = from->no_numa;
3457}
3458
3459
3460static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3461{
3462 u32 hash = 0;
3463
3464 hash = jhash_1word(attrs->nice, hash);
3465 hash = jhash(cpumask_bits(attrs->cpumask),
3466 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3467 return hash;
3468}
3469
3470
3471static bool wqattrs_equal(const struct workqueue_attrs *a,
3472 const struct workqueue_attrs *b)
3473{
3474 if (a->nice != b->nice)
3475 return false;
3476 if (!cpumask_equal(a->cpumask, b->cpumask))
3477 return false;
3478 return true;
3479}
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491static int init_worker_pool(struct worker_pool *pool)
3492{
3493 spin_lock_init(&pool->lock);
3494 pool->id = -1;
3495 pool->cpu = -1;
3496 pool->node = NUMA_NO_NODE;
3497 pool->flags |= POOL_DISASSOCIATED;
3498 INIT_LIST_HEAD(&pool->worklist);
3499 INIT_LIST_HEAD(&pool->idle_list);
3500 hash_init(pool->busy_hash);
3501
3502 init_timer_deferrable(&pool->idle_timer);
3503 pool->idle_timer.function = idle_worker_timeout;
3504 pool->idle_timer.data = (unsigned long)pool;
3505
3506 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3507 (unsigned long)pool);
3508
3509 mutex_init(&pool->manager_arb);
3510 mutex_init(&pool->manager_mutex);
3511 idr_init(&pool->worker_idr);
3512
3513 INIT_HLIST_NODE(&pool->hash_node);
3514 pool->refcnt = 1;
3515
3516
3517 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3518 if (!pool->attrs)
3519 return -ENOMEM;
3520 return 0;
3521}
3522
3523static void rcu_free_pool(struct rcu_head *rcu)
3524{
3525 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3526
3527 idr_destroy(&pool->worker_idr);
3528 free_workqueue_attrs(pool->attrs);
3529 kfree(pool);
3530}
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543static void put_unbound_pool(struct worker_pool *pool)
3544{
3545 struct worker *worker;
3546
3547 lockdep_assert_held(&wq_pool_mutex);
3548
3549 if (--pool->refcnt)
3550 return;
3551
3552
3553 if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3554 WARN_ON(!list_empty(&pool->worklist)))
3555 return;
3556
3557
3558 if (pool->id >= 0)
3559 idr_remove(&worker_pool_idr, pool->id);
3560 hash_del(&pool->hash_node);
3561
3562
3563
3564
3565
3566
3567 mutex_lock(&pool->manager_arb);
3568 mutex_lock(&pool->manager_mutex);
3569 spin_lock_irq(&pool->lock);
3570
3571 while ((worker = first_worker(pool)))
3572 destroy_worker(worker);
3573 WARN_ON(pool->nr_workers || pool->nr_idle);
3574
3575 spin_unlock_irq(&pool->lock);
3576 mutex_unlock(&pool->manager_mutex);
3577 mutex_unlock(&pool->manager_arb);
3578
3579
3580 del_timer_sync(&pool->idle_timer);
3581 del_timer_sync(&pool->mayday_timer);
3582
3583
3584 call_rcu_sched(&pool->rcu, rcu_free_pool);
3585}
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3602{
3603 u32 hash = wqattrs_hash(attrs);
3604 struct worker_pool *pool;
3605 int node;
3606
3607 lockdep_assert_held(&wq_pool_mutex);
3608
3609
3610 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3611 if (wqattrs_equal(pool->attrs, attrs)) {
3612 pool->refcnt++;
3613 goto out_unlock;
3614 }
3615 }
3616
3617
3618 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3619 if (!pool || init_worker_pool(pool) < 0)
3620 goto fail;
3621
3622 if (workqueue_freezing)
3623 pool->flags |= POOL_FREEZING;
3624
3625 lockdep_set_subclass(&pool->lock, 1);
3626 copy_workqueue_attrs(pool->attrs, attrs);
3627
3628
3629
3630
3631
3632 pool->attrs->no_numa = false;
3633
3634
3635 if (wq_numa_enabled) {
3636 for_each_node(node) {
3637 if (cpumask_subset(pool->attrs->cpumask,
3638 wq_numa_possible_cpumask[node])) {
3639 pool->node = node;
3640 break;
3641 }
3642 }
3643 }
3644
3645 if (worker_pool_assign_id(pool) < 0)
3646 goto fail;
3647
3648
3649 if (create_and_start_worker(pool) < 0)
3650 goto fail;
3651
3652
3653 hash_add(unbound_pool_hash, &pool->hash_node, hash);
3654out_unlock:
3655 return pool;
3656fail:
3657 if (pool)
3658 put_unbound_pool(pool);
3659 return NULL;
3660}
3661
3662static void rcu_free_pwq(struct rcu_head *rcu)
3663{
3664 kmem_cache_free(pwq_cache,
3665 container_of(rcu, struct pool_workqueue, rcu));
3666}
3667
3668
3669
3670
3671
3672static void pwq_unbound_release_workfn(struct work_struct *work)
3673{
3674 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3675 unbound_release_work);
3676 struct workqueue_struct *wq = pwq->wq;
3677 struct worker_pool *pool = pwq->pool;
3678 bool is_last;
3679
3680 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3681 return;
3682
3683
3684
3685
3686
3687
3688 mutex_lock(&wq->mutex);
3689 list_del_rcu(&pwq->pwqs_node);
3690 is_last = list_empty(&wq->pwqs);
3691 mutex_unlock(&wq->mutex);
3692
3693 mutex_lock(&wq_pool_mutex);
3694 put_unbound_pool(pool);
3695 mutex_unlock(&wq_pool_mutex);
3696
3697 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3698
3699
3700
3701
3702
3703 if (is_last) {
3704 free_workqueue_attrs(wq->unbound_attrs);
3705 kfree(wq);
3706 }
3707}
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3718{
3719 struct workqueue_struct *wq = pwq->wq;
3720 bool freezable = wq->flags & WQ_FREEZABLE;
3721
3722
3723 lockdep_assert_held(&wq->mutex);
3724
3725
3726 if (!freezable && pwq->max_active == wq->saved_max_active)
3727 return;
3728
3729 spin_lock_irq(&pwq->pool->lock);
3730
3731 if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3732 pwq->max_active = wq->saved_max_active;
3733
3734 while (!list_empty(&pwq->delayed_works) &&
3735 pwq->nr_active < pwq->max_active)
3736 pwq_activate_first_delayed(pwq);
3737
3738
3739
3740
3741
3742 wake_up_worker(pwq->pool);
3743 } else {
3744 pwq->max_active = 0;
3745 }
3746
3747 spin_unlock_irq(&pwq->pool->lock);
3748}
3749
3750
3751static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3752 struct worker_pool *pool)
3753{
3754 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3755
3756 memset(pwq, 0, sizeof(*pwq));
3757
3758 pwq->pool = pool;
3759 pwq->wq = wq;
3760 pwq->flush_color = -1;
3761 pwq->refcnt = 1;
3762 INIT_LIST_HEAD(&pwq->delayed_works);
3763 INIT_LIST_HEAD(&pwq->pwqs_node);
3764 INIT_LIST_HEAD(&pwq->mayday_node);
3765 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3766}
3767
3768
3769static void link_pwq(struct pool_workqueue *pwq)
3770{
3771 struct workqueue_struct *wq = pwq->wq;
3772
3773 lockdep_assert_held(&wq->mutex);
3774
3775
3776 if (!list_empty(&pwq->pwqs_node))
3777 return;
3778
3779
3780
3781
3782
3783 pwq->work_color = wq->work_color;
3784
3785
3786 pwq_adjust_max_active(pwq);
3787
3788
3789 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3790}
3791
3792
3793static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3794 const struct workqueue_attrs *attrs)
3795{
3796 struct worker_pool *pool;
3797 struct pool_workqueue *pwq;
3798
3799 lockdep_assert_held(&wq_pool_mutex);
3800
3801 pool = get_unbound_pool(attrs);
3802 if (!pool)
3803 return NULL;
3804
3805 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3806 if (!pwq) {
3807 put_unbound_pool(pool);
3808 return NULL;
3809 }
3810
3811 init_pwq(pwq, wq, pool);
3812 return pwq;
3813}
3814
3815
3816static void free_unbound_pwq(struct pool_workqueue *pwq)
3817{
3818 lockdep_assert_held(&wq_pool_mutex);
3819
3820 if (pwq) {
3821 put_unbound_pool(pwq->pool);
3822 kmem_cache_free(pwq_cache, pwq);
3823 }
3824}
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3849 int cpu_going_down, cpumask_t *cpumask)
3850{
3851 if (!wq_numa_enabled || attrs->no_numa)
3852 goto use_dfl;
3853
3854
3855 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3856 if (cpu_going_down >= 0)
3857 cpumask_clear_cpu(cpu_going_down, cpumask);
3858
3859 if (cpumask_empty(cpumask))
3860 goto use_dfl;
3861
3862
3863 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3864 return !cpumask_equal(cpumask, attrs->cpumask);
3865
3866use_dfl:
3867 cpumask_copy(cpumask, attrs->cpumask);
3868 return false;
3869}
3870
3871
3872static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3873 int node,
3874 struct pool_workqueue *pwq)
3875{
3876 struct pool_workqueue *old_pwq;
3877
3878 lockdep_assert_held(&wq->mutex);
3879
3880
3881 link_pwq(pwq);
3882
3883 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3884 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3885 return old_pwq;
3886}
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904int apply_workqueue_attrs(struct workqueue_struct *wq,
3905 const struct workqueue_attrs *attrs)
3906{
3907 struct workqueue_attrs *new_attrs, *tmp_attrs;
3908 struct pool_workqueue **pwq_tbl, *dfl_pwq;
3909 int node, ret;
3910
3911
3912 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3913 return -EINVAL;
3914
3915
3916 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3917 return -EINVAL;
3918
3919 pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3920 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3921 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3922 if (!pwq_tbl || !new_attrs || !tmp_attrs)
3923 goto enomem;
3924
3925
3926 copy_workqueue_attrs(new_attrs, attrs);
3927 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3928
3929
3930
3931
3932
3933
3934 copy_workqueue_attrs(tmp_attrs, new_attrs);
3935
3936
3937
3938
3939
3940
3941 get_online_cpus();
3942
3943 mutex_lock(&wq_pool_mutex);
3944
3945
3946
3947
3948
3949
3950 dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3951 if (!dfl_pwq)
3952 goto enomem_pwq;
3953
3954 for_each_node(node) {
3955 if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
3956 pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3957 if (!pwq_tbl[node])
3958 goto enomem_pwq;
3959 } else {
3960 dfl_pwq->refcnt++;
3961 pwq_tbl[node] = dfl_pwq;
3962 }
3963 }
3964
3965 mutex_unlock(&wq_pool_mutex);
3966
3967
3968 mutex_lock(&wq->mutex);
3969
3970 copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3971
3972
3973 for_each_node(node)
3974 pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
3975
3976
3977 link_pwq(dfl_pwq);
3978 swap(wq->dfl_pwq, dfl_pwq);
3979
3980 mutex_unlock(&wq->mutex);
3981
3982
3983 for_each_node(node)
3984 put_pwq_unlocked(pwq_tbl[node]);
3985 put_pwq_unlocked(dfl_pwq);
3986
3987 put_online_cpus();
3988 ret = 0;
3989
3990out_free:
3991 free_workqueue_attrs(tmp_attrs);
3992 free_workqueue_attrs(new_attrs);
3993 kfree(pwq_tbl);
3994 return ret;
3995
3996enomem_pwq:
3997 free_unbound_pwq(dfl_pwq);
3998 for_each_node(node)
3999 if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
4000 free_unbound_pwq(pwq_tbl[node]);
4001 mutex_unlock(&wq_pool_mutex);
4002 put_online_cpus();
4003enomem:
4004 ret = -ENOMEM;
4005 goto out_free;
4006}
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4031 bool online)
4032{
4033 int node = cpu_to_node(cpu);
4034 int cpu_off = online ? -1 : cpu;
4035 struct pool_workqueue *old_pwq = NULL, *pwq;
4036 struct workqueue_attrs *target_attrs;
4037 cpumask_t *cpumask;
4038
4039 lockdep_assert_held(&wq_pool_mutex);
4040
4041 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))
4042 return;
4043
4044
4045
4046
4047
4048
4049 target_attrs = wq_update_unbound_numa_attrs_buf;
4050 cpumask = target_attrs->cpumask;
4051
4052 mutex_lock(&wq->mutex);
4053 if (wq->unbound_attrs->no_numa)
4054 goto out_unlock;
4055
4056 copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4057 pwq = unbound_pwq_by_node(wq, node);
4058
4059
4060
4061
4062
4063
4064
4065
4066 if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4067 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4068 goto out_unlock;
4069 } else {
4070 if (pwq == wq->dfl_pwq)
4071 goto out_unlock;
4072 else
4073 goto use_dfl_pwq;
4074 }
4075
4076 mutex_unlock(&wq->mutex);
4077
4078
4079 pwq = alloc_unbound_pwq(wq, target_attrs);
4080 if (!pwq) {
4081 pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4082 wq->name);
4083 goto out_unlock;
4084 }
4085
4086
4087
4088
4089
4090
4091
4092 mutex_lock(&wq->mutex);
4093 old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4094 goto out_unlock;
4095
4096use_dfl_pwq:
4097 spin_lock_irq(&wq->dfl_pwq->pool->lock);
4098 get_pwq(wq->dfl_pwq);
4099 spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4100 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4101out_unlock:
4102 mutex_unlock(&wq->mutex);
4103 put_pwq_unlocked(old_pwq);
4104}
4105
4106static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4107{
4108 bool highpri = wq->flags & WQ_HIGHPRI;
4109 int cpu;
4110
4111 if (!(wq->flags & WQ_UNBOUND)) {
4112 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4113 if (!wq->cpu_pwqs)
4114 return -ENOMEM;
4115
4116 for_each_possible_cpu(cpu) {
4117 struct pool_workqueue *pwq =
4118 per_cpu_ptr(wq->cpu_pwqs, cpu);
4119 struct worker_pool *cpu_pools =
4120 per_cpu(cpu_worker_pools, cpu);
4121
4122 init_pwq(pwq, wq, &cpu_pools[highpri]);
4123
4124 mutex_lock(&wq->mutex);
4125 link_pwq(pwq);
4126 mutex_unlock(&wq->mutex);
4127 }
4128 return 0;
4129 } else {
4130 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4131 }
4132}
4133
4134static int wq_clamp_max_active(int max_active, unsigned int flags,
4135 const char *name)
4136{
4137 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4138
4139 if (max_active < 1 || max_active > lim)
4140 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4141 max_active, name, 1, lim);
4142
4143 return clamp_val(max_active, 1, lim);
4144}
4145
4146struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
4147 unsigned int flags,
4148 int max_active,
4149 struct lock_class_key *key,
4150 const char *lock_name, ...)
4151{
4152 size_t tbl_size = 0;
4153 va_list args;
4154 struct workqueue_struct *wq;
4155 struct pool_workqueue *pwq;
4156
4157
4158 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4159 flags |= WQ_UNBOUND;
4160
4161
4162 if (flags & WQ_UNBOUND)
4163 tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4164
4165 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4166 if (!wq)
4167 return NULL;
4168
4169 if (flags & WQ_UNBOUND) {
4170 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4171 if (!wq->unbound_attrs)
4172 goto err_free_wq;
4173 }
4174
4175 va_start(args, lock_name);
4176 vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4177 va_end(args);
4178
4179 max_active = max_active ?: WQ_DFL_ACTIVE;
4180 max_active = wq_clamp_max_active(max_active, flags, wq->name);
4181
4182
4183 wq->flags = flags;
4184 wq->saved_max_active = max_active;
4185 mutex_init(&wq->mutex);
4186 atomic_set(&wq->nr_pwqs_to_flush, 0);
4187 INIT_LIST_HEAD(&wq->pwqs);
4188 INIT_LIST_HEAD(&wq->flusher_queue);
4189 INIT_LIST_HEAD(&wq->flusher_overflow);
4190 INIT_LIST_HEAD(&wq->maydays);
4191
4192 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4193 INIT_LIST_HEAD(&wq->list);
4194
4195 if (alloc_and_link_pwqs(wq) < 0)
4196 goto err_free_wq;
4197
4198
4199
4200
4201
4202 if (flags & WQ_MEM_RECLAIM) {
4203 struct worker *rescuer;
4204
4205 rescuer = alloc_worker();
4206 if (!rescuer)
4207 goto err_destroy;
4208
4209 rescuer->rescue_wq = wq;
4210 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4211 wq->name);
4212 if (IS_ERR(rescuer->task)) {
4213 kfree(rescuer);
4214 goto err_destroy;
4215 }
4216
4217 wq->rescuer = rescuer;
4218 rescuer->task->flags |= PF_NO_SETAFFINITY;
4219 wake_up_process(rescuer->task);
4220 }
4221
4222 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4223 goto err_destroy;
4224
4225
4226
4227
4228
4229
4230 mutex_lock(&wq_pool_mutex);
4231
4232 mutex_lock(&wq->mutex);
4233 for_each_pwq(pwq, wq)
4234 pwq_adjust_max_active(pwq);
4235 mutex_unlock(&wq->mutex);
4236
4237 list_add(&wq->list, &workqueues);
4238
4239 mutex_unlock(&wq_pool_mutex);
4240
4241 return wq;
4242
4243err_free_wq:
4244 free_workqueue_attrs(wq->unbound_attrs);
4245 kfree(wq);
4246 return NULL;
4247err_destroy:
4248 destroy_workqueue(wq);
4249 return NULL;
4250}
4251EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4252
4253
4254
4255
4256
4257
4258
4259void destroy_workqueue(struct workqueue_struct *wq)
4260{
4261 struct pool_workqueue *pwq;
4262 int node;
4263
4264
4265 drain_workqueue(wq);
4266
4267
4268 mutex_lock(&wq->mutex);
4269 for_each_pwq(pwq, wq) {
4270 int i;
4271
4272 for (i = 0; i < WORK_NR_COLORS; i++) {
4273 if (WARN_ON(pwq->nr_in_flight[i])) {
4274 mutex_unlock(&wq->mutex);
4275 return;
4276 }
4277 }
4278
4279 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
4280 WARN_ON(pwq->nr_active) ||
4281 WARN_ON(!list_empty(&pwq->delayed_works))) {
4282 mutex_unlock(&wq->mutex);
4283 return;
4284 }
4285 }
4286 mutex_unlock(&wq->mutex);
4287
4288
4289
4290
4291
4292 mutex_lock(&wq_pool_mutex);
4293 list_del_init(&wq->list);
4294 mutex_unlock(&wq_pool_mutex);
4295
4296 workqueue_sysfs_unregister(wq);
4297
4298 if (wq->rescuer) {
4299 kthread_stop(wq->rescuer->task);
4300 kfree(wq->rescuer);
4301 wq->rescuer = NULL;
4302 }
4303
4304 if (!(wq->flags & WQ_UNBOUND)) {
4305
4306
4307
4308
4309 free_percpu(wq->cpu_pwqs);
4310 kfree(wq);
4311 } else {
4312
4313
4314
4315
4316
4317 for_each_node(node) {
4318 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4319 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4320 put_pwq_unlocked(pwq);
4321 }
4322
4323
4324
4325
4326
4327 pwq = wq->dfl_pwq;
4328 wq->dfl_pwq = NULL;
4329 put_pwq_unlocked(pwq);
4330 }
4331}
4332EXPORT_SYMBOL_GPL(destroy_workqueue);
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4345{
4346 struct pool_workqueue *pwq;
4347
4348
4349 if (WARN_ON(wq->flags & __WQ_ORDERED))
4350 return;
4351
4352 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4353
4354 mutex_lock(&wq->mutex);
4355
4356 wq->saved_max_active = max_active;
4357
4358 for_each_pwq(pwq, wq)
4359 pwq_adjust_max_active(pwq);
4360
4361 mutex_unlock(&wq->mutex);
4362}
4363EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373bool current_is_workqueue_rescuer(void)
4374{
4375 struct worker *worker = current_wq_worker();
4376
4377 return worker && worker->rescue_wq;
4378}
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4399{
4400 struct pool_workqueue *pwq;
4401 bool ret;
4402
4403 rcu_read_lock_sched();
4404
4405 if (cpu == WORK_CPU_UNBOUND)
4406 cpu = smp_processor_id();
4407
4408 if (!(wq->flags & WQ_UNBOUND))
4409 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4410 else
4411 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4412
4413 ret = !list_empty(&pwq->delayed_works);
4414 rcu_read_unlock_sched();
4415
4416 return ret;
4417}
4418EXPORT_SYMBOL_GPL(workqueue_congested);
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431unsigned int work_busy(struct work_struct *work)
4432{
4433 struct worker_pool *pool;
4434 unsigned long flags;
4435 unsigned int ret = 0;
4436
4437 if (work_pending(work))
4438 ret |= WORK_BUSY_PENDING;
4439
4440 local_irq_save(flags);
4441 pool = get_work_pool(work);
4442 if (pool) {
4443 spin_lock(&pool->lock);
4444 if (find_worker_executing_work(pool, work))
4445 ret |= WORK_BUSY_RUNNING;
4446 spin_unlock(&pool->lock);
4447 }
4448 local_irq_restore(flags);
4449
4450 return ret;
4451}
4452EXPORT_SYMBOL_GPL(work_busy);
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464void set_worker_desc(const char *fmt, ...)
4465{
4466 struct worker *worker = current_wq_worker();
4467 va_list args;
4468
4469 if (worker) {
4470 va_start(args, fmt);
4471 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4472 va_end(args);
4473 worker->desc_valid = true;
4474 }
4475}
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490void print_worker_info(const char *log_lvl, struct task_struct *task)
4491{
4492 work_func_t *fn = NULL;
4493 char name[WQ_NAME_LEN] = { };
4494 char desc[WORKER_DESC_LEN] = { };
4495 struct pool_workqueue *pwq = NULL;
4496 struct workqueue_struct *wq = NULL;
4497 bool desc_valid = false;
4498 struct worker *worker;
4499
4500 if (!(task->flags & PF_WQ_WORKER))
4501 return;
4502
4503
4504
4505
4506
4507 worker = probe_kthread_data(task);
4508
4509
4510
4511
4512
4513 probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4514 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4515 probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4516 probe_kernel_read(name, wq->name, sizeof(name) - 1);
4517
4518
4519 probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4520 if (desc_valid)
4521 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4522
4523 if (fn || name[0] || desc[0]) {
4524 printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4525 if (desc[0])
4526 pr_cont(" (%s)", desc);
4527 pr_cont("\n");
4528 }
4529}
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546static void wq_unbind_fn(struct work_struct *work)
4547{
4548 int cpu = smp_processor_id();
4549 struct worker_pool *pool;
4550 struct worker *worker;
4551 int wi;
4552
4553 for_each_cpu_worker_pool(pool, cpu) {
4554 WARN_ON_ONCE(cpu != smp_processor_id());
4555
4556 mutex_lock(&pool->manager_mutex);
4557 spin_lock_irq(&pool->lock);
4558
4559
4560
4561
4562
4563
4564
4565
4566 for_each_pool_worker(worker, wi, pool)
4567 worker->flags |= WORKER_UNBOUND;
4568
4569 pool->flags |= POOL_DISASSOCIATED;
4570
4571 spin_unlock_irq(&pool->lock);
4572 mutex_unlock(&pool->manager_mutex);
4573
4574
4575
4576
4577
4578
4579
4580 schedule();
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590 atomic_set(&pool->nr_running, 0);
4591
4592
4593
4594
4595
4596
4597 spin_lock_irq(&pool->lock);
4598 wake_up_worker(pool);
4599 spin_unlock_irq(&pool->lock);
4600 }
4601}
4602
4603
4604
4605
4606
4607
4608
4609static void rebind_workers(struct worker_pool *pool)
4610{
4611 struct worker *worker;
4612 int wi;
4613
4614 lockdep_assert_held(&pool->manager_mutex);
4615
4616
4617
4618
4619
4620
4621
4622
4623 for_each_pool_worker(worker, wi, pool)
4624 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4625 pool->attrs->cpumask) < 0);
4626
4627 spin_lock_irq(&pool->lock);
4628
4629 for_each_pool_worker(worker, wi, pool) {
4630 unsigned int worker_flags = worker->flags;
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640 if (worker_flags & WORKER_IDLE)
4641 wake_up_process(worker->task);
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4659 worker_flags |= WORKER_REBOUND;
4660 worker_flags &= ~WORKER_UNBOUND;
4661 ACCESS_ONCE(worker->flags) = worker_flags;
4662 }
4663
4664 spin_unlock_irq(&pool->lock);
4665}
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4678{
4679 static cpumask_t cpumask;
4680 struct worker *worker;
4681 int wi;
4682
4683 lockdep_assert_held(&pool->manager_mutex);
4684
4685
4686 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4687 return;
4688
4689
4690 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4691 if (cpumask_weight(&cpumask) != 1)
4692 return;
4693
4694
4695 for_each_pool_worker(worker, wi, pool)
4696 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4697 pool->attrs->cpumask) < 0);
4698}
4699
4700
4701
4702
4703
4704static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4705 unsigned long action,
4706 void *hcpu)
4707{
4708 int cpu = (unsigned long)hcpu;
4709 struct worker_pool *pool;
4710 struct workqueue_struct *wq;
4711 int pi;
4712
4713 switch (action & ~CPU_TASKS_FROZEN) {
4714 case CPU_UP_PREPARE:
4715 for_each_cpu_worker_pool(pool, cpu) {
4716 if (pool->nr_workers)
4717 continue;
4718 if (create_and_start_worker(pool) < 0)
4719 return NOTIFY_BAD;
4720 }
4721 break;
4722
4723 case CPU_DOWN_FAILED:
4724 case CPU_ONLINE:
4725 mutex_lock(&wq_pool_mutex);
4726
4727 for_each_pool(pool, pi) {
4728 mutex_lock(&pool->manager_mutex);
4729
4730 if (pool->cpu == cpu) {
4731 spin_lock_irq(&pool->lock);
4732 pool->flags &= ~POOL_DISASSOCIATED;
4733 spin_unlock_irq(&pool->lock);
4734
4735 rebind_workers(pool);
4736 } else if (pool->cpu < 0) {
4737 restore_unbound_workers_cpumask(pool, cpu);
4738 }
4739
4740 mutex_unlock(&pool->manager_mutex);
4741 }
4742
4743
4744 list_for_each_entry(wq, &workqueues, list)
4745 wq_update_unbound_numa(wq, cpu, true);
4746
4747 mutex_unlock(&wq_pool_mutex);
4748 break;
4749 }
4750 return NOTIFY_OK;
4751}
4752
4753
4754
4755
4756
4757static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4758 unsigned long action,
4759 void *hcpu)
4760{
4761 int cpu = (unsigned long)hcpu;
4762 struct work_struct unbind_work;
4763 struct workqueue_struct *wq;
4764
4765 switch (action & ~CPU_TASKS_FROZEN) {
4766 case CPU_DOWN_PREPARE:
4767
4768 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4769 queue_work_on(cpu, system_highpri_wq, &unbind_work);
4770
4771
4772 mutex_lock(&wq_pool_mutex);
4773 list_for_each_entry(wq, &workqueues, list)
4774 wq_update_unbound_numa(wq, cpu, false);
4775 mutex_unlock(&wq_pool_mutex);
4776
4777
4778 flush_work(&unbind_work);
4779 break;
4780 }
4781 return NOTIFY_OK;
4782}
4783
4784#ifdef CONFIG_SMP
4785
4786struct work_for_cpu {
4787 struct work_struct work;
4788 long (*fn)(void *);
4789 void *arg;
4790 long ret;
4791};
4792
4793static void work_for_cpu_fn(struct work_struct *work)
4794{
4795 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4796
4797 wfc->ret = wfc->fn(wfc->arg);
4798}
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4812{
4813 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4814
4815 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4816 schedule_work_on(cpu, &wfc.work);
4817
4818
4819
4820
4821
4822
4823 __flush_work(&wfc.work);
4824
4825 return wfc.ret;
4826}
4827EXPORT_SYMBOL_GPL(work_on_cpu);
4828#endif
4829
4830#ifdef CONFIG_FREEZER
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842void freeze_workqueues_begin(void)
4843{
4844 struct worker_pool *pool;
4845 struct workqueue_struct *wq;
4846 struct pool_workqueue *pwq;
4847 int pi;
4848
4849 mutex_lock(&wq_pool_mutex);
4850
4851 WARN_ON_ONCE(workqueue_freezing);
4852 workqueue_freezing = true;
4853
4854
4855 for_each_pool(pool, pi) {
4856 spin_lock_irq(&pool->lock);
4857 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4858 pool->flags |= POOL_FREEZING;
4859 spin_unlock_irq(&pool->lock);
4860 }
4861
4862 list_for_each_entry(wq, &workqueues, list) {
4863 mutex_lock(&wq->mutex);
4864 for_each_pwq(pwq, wq)
4865 pwq_adjust_max_active(pwq);
4866 mutex_unlock(&wq->mutex);
4867 }
4868
4869 mutex_unlock(&wq_pool_mutex);
4870}
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885bool freeze_workqueues_busy(void)
4886{
4887 bool busy = false;
4888 struct workqueue_struct *wq;
4889 struct pool_workqueue *pwq;
4890
4891 mutex_lock(&wq_pool_mutex);
4892
4893 WARN_ON_ONCE(!workqueue_freezing);
4894
4895 list_for_each_entry(wq, &workqueues, list) {
4896 if (!(wq->flags & WQ_FREEZABLE))
4897 continue;
4898
4899
4900
4901
4902 rcu_read_lock_sched();
4903 for_each_pwq(pwq, wq) {
4904 WARN_ON_ONCE(pwq->nr_active < 0);
4905 if (pwq->nr_active) {
4906 busy = true;
4907 rcu_read_unlock_sched();
4908 goto out_unlock;
4909 }
4910 }
4911 rcu_read_unlock_sched();
4912 }
4913out_unlock:
4914 mutex_unlock(&wq_pool_mutex);
4915 return busy;
4916}
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927void thaw_workqueues(void)
4928{
4929 struct workqueue_struct *wq;
4930 struct pool_workqueue *pwq;
4931 struct worker_pool *pool;
4932 int pi;
4933
4934 mutex_lock(&wq_pool_mutex);
4935
4936 if (!workqueue_freezing)
4937 goto out_unlock;
4938
4939
4940 for_each_pool(pool, pi) {
4941 spin_lock_irq(&pool->lock);
4942 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4943 pool->flags &= ~POOL_FREEZING;
4944 spin_unlock_irq(&pool->lock);
4945 }
4946
4947
4948 list_for_each_entry(wq, &workqueues, list) {
4949 mutex_lock(&wq->mutex);
4950 for_each_pwq(pwq, wq)
4951 pwq_adjust_max_active(pwq);
4952 mutex_unlock(&wq->mutex);
4953 }
4954
4955 workqueue_freezing = false;
4956out_unlock:
4957 mutex_unlock(&wq_pool_mutex);
4958}
4959#endif
4960
4961static void __init wq_numa_init(void)
4962{
4963 cpumask_var_t *tbl;
4964 int node, cpu;
4965
4966
4967 for_each_node(node)
4968 wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4969
4970 if (num_possible_nodes() <= 1)
4971 return;
4972
4973 if (wq_disable_numa) {
4974 pr_info("workqueue: NUMA affinity support disabled\n");
4975 return;
4976 }
4977
4978 wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
4979 BUG_ON(!wq_update_unbound_numa_attrs_buf);
4980
4981
4982
4983
4984
4985
4986 tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4987 BUG_ON(!tbl);
4988
4989 for_each_node(node)
4990 BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
4991 node_online(node) ? node : NUMA_NO_NODE));
4992
4993 for_each_possible_cpu(cpu) {
4994 node = cpu_to_node(cpu);
4995 if (WARN_ON(node == NUMA_NO_NODE)) {
4996 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4997
4998 return;
4999 }
5000 cpumask_set_cpu(cpu, tbl[node]);
5001 }
5002
5003 wq_numa_possible_cpumask = tbl;
5004 wq_numa_enabled = true;
5005}
5006
5007static int __init init_workqueues(void)
5008{
5009 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5010 int i, cpu;
5011
5012
5013 BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
5014 WORK_CPU_END * NR_STD_WORKER_POOLS);
5015
5016 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5017
5018 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
5019
5020 cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
5021 hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
5022
5023 wq_numa_init();
5024
5025
5026 for_each_possible_cpu(cpu) {
5027 struct worker_pool *pool;
5028
5029 i = 0;
5030 for_each_cpu_worker_pool(pool, cpu) {
5031 BUG_ON(init_worker_pool(pool));
5032 pool->cpu = cpu;
5033 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5034 pool->attrs->nice = std_nice[i++];
5035 pool->node = cpu_to_node(cpu);
5036
5037
5038 mutex_lock(&wq_pool_mutex);
5039 BUG_ON(worker_pool_assign_id(pool));
5040 mutex_unlock(&wq_pool_mutex);
5041 }
5042 }
5043
5044
5045 for_each_online_cpu(cpu) {
5046 struct worker_pool *pool;
5047
5048 for_each_cpu_worker_pool(pool, cpu) {
5049 pool->flags &= ~POOL_DISASSOCIATED;
5050 BUG_ON(create_and_start_worker(pool) < 0);
5051 }
5052 }
5053
5054
5055 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5056 struct workqueue_attrs *attrs;
5057
5058 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5059 attrs->nice = std_nice[i];
5060 unbound_std_wq_attrs[i] = attrs;
5061 }
5062
5063 system_wq = alloc_workqueue("events", 0, 0);
5064 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5065 system_long_wq = alloc_workqueue("events_long", 0, 0);
5066 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5067 WQ_UNBOUND_MAX_ACTIVE);
5068 system_freezable_wq = alloc_workqueue("events_freezable",
5069 WQ_FREEZABLE, 0);
5070 system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5071 WQ_POWER_EFFICIENT, 0);
5072 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5073 WQ_FREEZABLE | WQ_POWER_EFFICIENT,
5074 0);
5075 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
5076 !system_unbound_wq || !system_freezable_wq ||
5077 !system_power_efficient_wq ||
5078 !system_freezable_power_efficient_wq);
5079 return 0;
5080}
5081early_initcall(init_workqueues);
5082