1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/export.h>
27#include <linux/kernel.h>
28#include <linux/sched.h>
29#include <linux/init.h>
30#include <linux/signal.h>
31#include <linux/completion.h>
32#include <linux/workqueue.h>
33#include <linux/slab.h>
34#include <linux/cpu.h>
35#include <linux/notifier.h>
36#include <linux/kthread.h>
37#include <linux/hardirq.h>
38#include <linux/mempolicy.h>
39#include <linux/freezer.h>
40#include <linux/kallsyms.h>
41#include <linux/debug_locks.h>
42#include <linux/lockdep.h>
43#include <linux/idr.h>
44#include <linux/jhash.h>
45#include <linux/hashtable.h>
46#include <linux/rculist.h>
47#include <linux/nodemask.h>
48#include <linux/moduleparam.h>
49#include <linux/uaccess.h>
50
51#include "workqueue_internal.h"
52
53enum {
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70 POOL_MANAGE_WORKERS = 1 << 0,
71 POOL_DISASSOCIATED = 1 << 2,
72 POOL_FREEZING = 1 << 3,
73
74
75 WORKER_STARTED = 1 << 0,
76 WORKER_DIE = 1 << 1,
77 WORKER_IDLE = 1 << 2,
78 WORKER_PREP = 1 << 3,
79 WORKER_CPU_INTENSIVE = 1 << 6,
80 WORKER_UNBOUND = 1 << 7,
81 WORKER_REBOUND = 1 << 8,
82
83 WORKER_NOT_RUNNING = WORKER_PREP | WORKER_CPU_INTENSIVE |
84 WORKER_UNBOUND | WORKER_REBOUND,
85
86 NR_STD_WORKER_POOLS = 2,
87
88 UNBOUND_POOL_HASH_ORDER = 6,
89 BUSY_WORKER_HASH_ORDER = 6,
90
91 MAX_IDLE_WORKERS_RATIO = 4,
92 IDLE_WORKER_TIMEOUT = 300 * HZ,
93
94 MAYDAY_INITIAL_TIMEOUT = HZ / 100 >= 2 ? HZ / 100 : 2,
95
96
97 MAYDAY_INTERVAL = HZ / 10,
98 CREATE_COOLDOWN = HZ,
99
100
101
102
103
104 RESCUER_NICE_LEVEL = -20,
105 HIGHPRI_NICE_LEVEL = -20,
106
107 WQ_NAME_LEN = 24,
108};
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142struct worker_pool {
143 spinlock_t lock;
144 int cpu;
145 int node;
146 int id;
147 unsigned int flags;
148
149 struct list_head worklist;
150 int nr_workers;
151
152
153 int nr_idle;
154
155 struct list_head idle_list;
156 struct timer_list idle_timer;
157 struct timer_list mayday_timer;
158
159
160 DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
161
162
163
164 struct mutex manager_arb;
165 struct mutex manager_mutex;
166 struct idr worker_idr;
167
168 struct workqueue_attrs *attrs;
169 struct hlist_node hash_node;
170 int refcnt;
171
172
173
174
175
176
177 atomic_t nr_running ____cacheline_aligned_in_smp;
178
179
180
181
182
183 struct rcu_head rcu;
184} ____cacheline_aligned_in_smp;
185
186
187
188
189
190
191
192struct pool_workqueue {
193 struct worker_pool *pool;
194 struct workqueue_struct *wq;
195 int work_color;
196 int flush_color;
197 int refcnt;
198 int nr_in_flight[WORK_NR_COLORS];
199
200 int nr_active;
201 int max_active;
202 struct list_head delayed_works;
203 struct list_head pwqs_node;
204 struct list_head mayday_node;
205
206
207
208
209
210
211
212 struct work_struct unbound_release_work;
213 struct rcu_head rcu;
214} __aligned(1 << WORK_STRUCT_FLAG_BITS);
215
216
217
218
219struct wq_flusher {
220 struct list_head list;
221 int flush_color;
222 struct completion done;
223};
224
225struct wq_device;
226
227
228
229
230
231struct workqueue_struct {
232 struct list_head pwqs;
233 struct list_head list;
234
235 struct mutex mutex;
236 int work_color;
237 int flush_color;
238 atomic_t nr_pwqs_to_flush;
239 struct wq_flusher *first_flusher;
240 struct list_head flusher_queue;
241 struct list_head flusher_overflow;
242
243 struct list_head maydays;
244 struct worker *rescuer;
245
246 int nr_drainers;
247 int saved_max_active;
248
249 struct workqueue_attrs *unbound_attrs;
250 struct pool_workqueue *dfl_pwq;
251
252#ifdef CONFIG_SYSFS
253 struct wq_device *wq_dev;
254#endif
255#ifdef CONFIG_LOCKDEP
256 struct lockdep_map lockdep_map;
257#endif
258 char name[WQ_NAME_LEN];
259
260
261 unsigned int flags ____cacheline_aligned;
262 struct pool_workqueue __percpu *cpu_pwqs;
263 struct pool_workqueue __rcu *numa_pwq_tbl[];
264};
265
266static struct kmem_cache *pwq_cache;
267
268static int wq_numa_tbl_len;
269static cpumask_var_t *wq_numa_possible_cpumask;
270
271
272static bool wq_disable_numa;
273module_param_named(disable_numa, wq_disable_numa, bool, 0444);
274
275
276#ifdef CONFIG_WQ_POWER_EFFICIENT_DEFAULT
277static bool wq_power_efficient = true;
278#else
279static bool wq_power_efficient;
280#endif
281
282module_param_named(power_efficient, wq_power_efficient, bool, 0444);
283
284static bool wq_numa_enabled;
285
286
287static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
288
289static DEFINE_MUTEX(wq_pool_mutex);
290static DEFINE_SPINLOCK(wq_mayday_lock);
291
292static LIST_HEAD(workqueues);
293static bool workqueue_freezing;
294
295
296static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS],
297 cpu_worker_pools);
298
299static DEFINE_IDR(worker_pool_idr);
300
301
302static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
303
304
305static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
306
307struct workqueue_struct *system_wq __read_mostly;
308EXPORT_SYMBOL(system_wq);
309struct workqueue_struct *system_highpri_wq __read_mostly;
310EXPORT_SYMBOL_GPL(system_highpri_wq);
311struct workqueue_struct *system_long_wq __read_mostly;
312EXPORT_SYMBOL_GPL(system_long_wq);
313struct workqueue_struct *system_unbound_wq __read_mostly;
314EXPORT_SYMBOL_GPL(system_unbound_wq);
315struct workqueue_struct *system_freezable_wq __read_mostly;
316EXPORT_SYMBOL_GPL(system_freezable_wq);
317struct workqueue_struct *system_power_efficient_wq __read_mostly;
318EXPORT_SYMBOL_GPL(system_power_efficient_wq);
319struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
320EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
321
322static int worker_thread(void *__worker);
323static void copy_workqueue_attrs(struct workqueue_attrs *to,
324 const struct workqueue_attrs *from);
325
326#define CREATE_TRACE_POINTS
327#include <trace/events/workqueue.h>
328
329#define assert_rcu_or_pool_mutex() \
330 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
331 lockdep_is_held(&wq_pool_mutex), \
332 "sched RCU or wq_pool_mutex should be held")
333
334#define assert_rcu_or_wq_mutex(wq) \
335 rcu_lockdep_assert(rcu_read_lock_sched_held() || \
336 lockdep_is_held(&wq->mutex), \
337 "sched RCU or wq->mutex should be held")
338
339#ifdef CONFIG_LOCKDEP
340#define assert_manager_or_pool_lock(pool) \
341 WARN_ONCE(debug_locks && \
342 !lockdep_is_held(&(pool)->manager_mutex) && \
343 !lockdep_is_held(&(pool)->lock), \
344 "pool->manager_mutex or ->lock should be held")
345#else
346#define assert_manager_or_pool_lock(pool) do { } while (0)
347#endif
348
349#define for_each_cpu_worker_pool(pool, cpu) \
350 for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0]; \
351 (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
352 (pool)++)
353
354
355
356
357
358
359
360
361
362
363
364
365
366#define for_each_pool(pool, pi) \
367 idr_for_each_entry(&worker_pool_idr, pool, pi) \
368 if (({ assert_rcu_or_pool_mutex(); false; })) { } \
369 else
370
371
372
373
374
375
376
377
378
379
380
381
382#define for_each_pool_worker(worker, wi, pool) \
383 idr_for_each_entry(&(pool)->worker_idr, (worker), (wi)) \
384 if (({ assert_manager_or_pool_lock((pool)); false; })) { } \
385 else
386
387
388
389
390
391
392
393
394
395
396
397
398
399#define for_each_pwq(pwq, wq) \
400 list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node) \
401 if (({ assert_rcu_or_wq_mutex(wq); false; })) { } \
402 else
403
404#ifdef CONFIG_DEBUG_OBJECTS_WORK
405
406static struct debug_obj_descr work_debug_descr;
407
408static void *work_debug_hint(void *addr)
409{
410 return ((struct work_struct *) addr)->func;
411}
412
413
414
415
416
417static int work_fixup_init(void *addr, enum debug_obj_state state)
418{
419 struct work_struct *work = addr;
420
421 switch (state) {
422 case ODEBUG_STATE_ACTIVE:
423 cancel_work_sync(work);
424 debug_object_init(work, &work_debug_descr);
425 return 1;
426 default:
427 return 0;
428 }
429}
430
431
432
433
434
435
436static int work_fixup_activate(void *addr, enum debug_obj_state state)
437{
438 struct work_struct *work = addr;
439
440 switch (state) {
441
442 case ODEBUG_STATE_NOTAVAILABLE:
443
444
445
446
447
448 if (test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work))) {
449 debug_object_init(work, &work_debug_descr);
450 debug_object_activate(work, &work_debug_descr);
451 return 0;
452 }
453 WARN_ON_ONCE(1);
454 return 0;
455
456 case ODEBUG_STATE_ACTIVE:
457 WARN_ON(1);
458
459 default:
460 return 0;
461 }
462}
463
464
465
466
467
468static int work_fixup_free(void *addr, enum debug_obj_state state)
469{
470 struct work_struct *work = addr;
471
472 switch (state) {
473 case ODEBUG_STATE_ACTIVE:
474 cancel_work_sync(work);
475 debug_object_free(work, &work_debug_descr);
476 return 1;
477 default:
478 return 0;
479 }
480}
481
482static struct debug_obj_descr work_debug_descr = {
483 .name = "work_struct",
484 .debug_hint = work_debug_hint,
485 .fixup_init = work_fixup_init,
486 .fixup_activate = work_fixup_activate,
487 .fixup_free = work_fixup_free,
488};
489
490static inline void debug_work_activate(struct work_struct *work)
491{
492 debug_object_activate(work, &work_debug_descr);
493}
494
495static inline void debug_work_deactivate(struct work_struct *work)
496{
497 debug_object_deactivate(work, &work_debug_descr);
498}
499
500void __init_work(struct work_struct *work, int onstack)
501{
502 if (onstack)
503 debug_object_init_on_stack(work, &work_debug_descr);
504 else
505 debug_object_init(work, &work_debug_descr);
506}
507EXPORT_SYMBOL_GPL(__init_work);
508
509void destroy_work_on_stack(struct work_struct *work)
510{
511 debug_object_free(work, &work_debug_descr);
512}
513EXPORT_SYMBOL_GPL(destroy_work_on_stack);
514
515#else
516static inline void debug_work_activate(struct work_struct *work) { }
517static inline void debug_work_deactivate(struct work_struct *work) { }
518#endif
519
520
521static int worker_pool_assign_id(struct worker_pool *pool)
522{
523 int ret;
524
525 lockdep_assert_held(&wq_pool_mutex);
526
527 ret = idr_alloc(&worker_pool_idr, pool, 0, 0, GFP_KERNEL);
528 if (ret >= 0) {
529 pool->id = ret;
530 return 0;
531 }
532 return ret;
533}
534
535
536
537
538
539
540
541
542
543
544static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
545 int node)
546{
547 assert_rcu_or_wq_mutex(wq);
548 return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
549}
550
551static unsigned int work_color_to_flags(int color)
552{
553 return color << WORK_STRUCT_COLOR_SHIFT;
554}
555
556static int get_work_color(struct work_struct *work)
557{
558 return (*work_data_bits(work) >> WORK_STRUCT_COLOR_SHIFT) &
559 ((1 << WORK_STRUCT_COLOR_BITS) - 1);
560}
561
562static int work_next_color(int color)
563{
564 return (color + 1) % WORK_NR_COLORS;
565}
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587static inline void set_work_data(struct work_struct *work, unsigned long data,
588 unsigned long flags)
589{
590 WARN_ON_ONCE(!work_pending(work));
591 atomic_long_set(&work->data, data | flags | work_static(work));
592}
593
594static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
595 unsigned long extra_flags)
596{
597 set_work_data(work, (unsigned long)pwq,
598 WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
599}
600
601static void set_work_pool_and_keep_pending(struct work_struct *work,
602 int pool_id)
603{
604 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
605 WORK_STRUCT_PENDING);
606}
607
608static void set_work_pool_and_clear_pending(struct work_struct *work,
609 int pool_id)
610{
611
612
613
614
615
616
617 smp_wmb();
618 set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
619}
620
621static void clear_work_data(struct work_struct *work)
622{
623 smp_wmb();
624 set_work_data(work, WORK_STRUCT_NO_POOL, 0);
625}
626
627static struct pool_workqueue *get_work_pwq(struct work_struct *work)
628{
629 unsigned long data = atomic_long_read(&work->data);
630
631 if (data & WORK_STRUCT_PWQ)
632 return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
633 else
634 return NULL;
635}
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652static struct worker_pool *get_work_pool(struct work_struct *work)
653{
654 unsigned long data = atomic_long_read(&work->data);
655 int pool_id;
656
657 assert_rcu_or_pool_mutex();
658
659 if (data & WORK_STRUCT_PWQ)
660 return ((struct pool_workqueue *)
661 (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
662
663 pool_id = data >> WORK_OFFQ_POOL_SHIFT;
664 if (pool_id == WORK_OFFQ_POOL_NONE)
665 return NULL;
666
667 return idr_find(&worker_pool_idr, pool_id);
668}
669
670
671
672
673
674
675
676
677static int get_work_pool_id(struct work_struct *work)
678{
679 unsigned long data = atomic_long_read(&work->data);
680
681 if (data & WORK_STRUCT_PWQ)
682 return ((struct pool_workqueue *)
683 (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
684
685 return data >> WORK_OFFQ_POOL_SHIFT;
686}
687
688static void mark_work_canceling(struct work_struct *work)
689{
690 unsigned long pool_id = get_work_pool_id(work);
691
692 pool_id <<= WORK_OFFQ_POOL_SHIFT;
693 set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
694}
695
696static bool work_is_canceling(struct work_struct *work)
697{
698 unsigned long data = atomic_long_read(&work->data);
699
700 return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
701}
702
703
704
705
706
707
708
709static bool __need_more_worker(struct worker_pool *pool)
710{
711 return !atomic_read(&pool->nr_running);
712}
713
714
715
716
717
718
719
720
721
722static bool need_more_worker(struct worker_pool *pool)
723{
724 return !list_empty(&pool->worklist) && __need_more_worker(pool);
725}
726
727
728static bool may_start_working(struct worker_pool *pool)
729{
730 return pool->nr_idle;
731}
732
733
734static bool keep_working(struct worker_pool *pool)
735{
736 return !list_empty(&pool->worklist) &&
737 atomic_read(&pool->nr_running) <= 1;
738}
739
740
741static bool need_to_create_worker(struct worker_pool *pool)
742{
743 return need_more_worker(pool) && !may_start_working(pool);
744}
745
746
747static bool need_to_manage_workers(struct worker_pool *pool)
748{
749 return need_to_create_worker(pool) ||
750 (pool->flags & POOL_MANAGE_WORKERS);
751}
752
753
754static bool too_many_workers(struct worker_pool *pool)
755{
756 bool managing = mutex_is_locked(&pool->manager_arb);
757 int nr_idle = pool->nr_idle + managing;
758 int nr_busy = pool->nr_workers - nr_idle;
759
760
761
762
763
764 if (list_empty(&pool->idle_list))
765 return false;
766
767 return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
768}
769
770
771
772
773
774
775static struct worker *first_worker(struct worker_pool *pool)
776{
777 if (unlikely(list_empty(&pool->idle_list)))
778 return NULL;
779
780 return list_first_entry(&pool->idle_list, struct worker, entry);
781}
782
783
784
785
786
787
788
789
790
791
792static void wake_up_worker(struct worker_pool *pool)
793{
794 struct worker *worker = first_worker(pool);
795
796 if (likely(worker))
797 wake_up_process(worker->task);
798}
799
800
801
802
803
804
805
806
807
808
809
810
811void wq_worker_waking_up(struct task_struct *task, int cpu)
812{
813 struct worker *worker = kthread_data(task);
814
815 if (!(worker->flags & WORKER_NOT_RUNNING)) {
816 WARN_ON_ONCE(worker->pool->cpu != cpu);
817 atomic_inc(&worker->pool->nr_running);
818 }
819}
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836struct task_struct *wq_worker_sleeping(struct task_struct *task, int cpu)
837{
838 struct worker *worker = kthread_data(task), *to_wakeup = NULL;
839 struct worker_pool *pool;
840
841
842
843
844
845
846 if (worker->flags & WORKER_NOT_RUNNING)
847 return NULL;
848
849 pool = worker->pool;
850
851
852 if (WARN_ON_ONCE(cpu != raw_smp_processor_id()))
853 return NULL;
854
855
856
857
858
859
860
861
862
863
864
865
866 if (atomic_dec_and_test(&pool->nr_running) &&
867 !list_empty(&pool->worklist))
868 to_wakeup = first_worker(pool);
869 return to_wakeup ? to_wakeup->task : NULL;
870}
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885static inline void worker_set_flags(struct worker *worker, unsigned int flags,
886 bool wakeup)
887{
888 struct worker_pool *pool = worker->pool;
889
890 WARN_ON_ONCE(worker->task != current);
891
892
893
894
895
896
897 if ((flags & WORKER_NOT_RUNNING) &&
898 !(worker->flags & WORKER_NOT_RUNNING)) {
899 if (wakeup) {
900 if (atomic_dec_and_test(&pool->nr_running) &&
901 !list_empty(&pool->worklist))
902 wake_up_worker(pool);
903 } else
904 atomic_dec(&pool->nr_running);
905 }
906
907 worker->flags |= flags;
908}
909
910
911
912
913
914
915
916
917
918
919
920static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
921{
922 struct worker_pool *pool = worker->pool;
923 unsigned int oflags = worker->flags;
924
925 WARN_ON_ONCE(worker->task != current);
926
927 worker->flags &= ~flags;
928
929
930
931
932
933
934 if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
935 if (!(worker->flags & WORKER_NOT_RUNNING))
936 atomic_inc(&pool->nr_running);
937}
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972static struct worker *find_worker_executing_work(struct worker_pool *pool,
973 struct work_struct *work)
974{
975 struct worker *worker;
976
977 hash_for_each_possible(pool->busy_hash, worker, hentry,
978 (unsigned long)work)
979 if (worker->current_work == work &&
980 worker->current_func == work->func)
981 return worker;
982
983 return NULL;
984}
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003static void move_linked_works(struct work_struct *work, struct list_head *head,
1004 struct work_struct **nextp)
1005{
1006 struct work_struct *n;
1007
1008
1009
1010
1011
1012 list_for_each_entry_safe_from(work, n, NULL, entry) {
1013 list_move_tail(&work->entry, head);
1014 if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1015 break;
1016 }
1017
1018
1019
1020
1021
1022
1023 if (nextp)
1024 *nextp = n;
1025}
1026
1027
1028
1029
1030
1031
1032
1033
1034static void get_pwq(struct pool_workqueue *pwq)
1035{
1036 lockdep_assert_held(&pwq->pool->lock);
1037 WARN_ON_ONCE(pwq->refcnt <= 0);
1038 pwq->refcnt++;
1039}
1040
1041
1042
1043
1044
1045
1046
1047
1048static void put_pwq(struct pool_workqueue *pwq)
1049{
1050 lockdep_assert_held(&pwq->pool->lock);
1051 if (likely(--pwq->refcnt))
1052 return;
1053 if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1054 return;
1055
1056
1057
1058
1059
1060
1061
1062
1063 schedule_work(&pwq->unbound_release_work);
1064}
1065
1066
1067
1068
1069
1070
1071
1072static void put_pwq_unlocked(struct pool_workqueue *pwq)
1073{
1074 if (pwq) {
1075
1076
1077
1078
1079 spin_lock_irq(&pwq->pool->lock);
1080 put_pwq(pwq);
1081 spin_unlock_irq(&pwq->pool->lock);
1082 }
1083}
1084
1085static void pwq_activate_delayed_work(struct work_struct *work)
1086{
1087 struct pool_workqueue *pwq = get_work_pwq(work);
1088
1089 trace_workqueue_activate_work(work);
1090 move_linked_works(work, &pwq->pool->worklist, NULL);
1091 __clear_bit(WORK_STRUCT_DELAYED_BIT, work_data_bits(work));
1092 pwq->nr_active++;
1093}
1094
1095static void pwq_activate_first_delayed(struct pool_workqueue *pwq)
1096{
1097 struct work_struct *work = list_first_entry(&pwq->delayed_works,
1098 struct work_struct, entry);
1099
1100 pwq_activate_delayed_work(work);
1101}
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, int color)
1115{
1116
1117 if (color == WORK_NO_COLOR)
1118 goto out_put;
1119
1120 pwq->nr_in_flight[color]--;
1121
1122 pwq->nr_active--;
1123 if (!list_empty(&pwq->delayed_works)) {
1124
1125 if (pwq->nr_active < pwq->max_active)
1126 pwq_activate_first_delayed(pwq);
1127 }
1128
1129
1130 if (likely(pwq->flush_color != color))
1131 goto out_put;
1132
1133
1134 if (pwq->nr_in_flight[color])
1135 goto out_put;
1136
1137
1138 pwq->flush_color = -1;
1139
1140
1141
1142
1143
1144 if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1145 complete(&pwq->wq->first_flusher->done);
1146out_put:
1147 put_pwq(pwq);
1148}
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1176 unsigned long *flags)
1177{
1178 struct worker_pool *pool;
1179 struct pool_workqueue *pwq;
1180
1181 local_irq_save(*flags);
1182
1183
1184 if (is_dwork) {
1185 struct delayed_work *dwork = to_delayed_work(work);
1186
1187
1188
1189
1190
1191
1192 if (likely(del_timer(&dwork->timer)))
1193 return 1;
1194 }
1195
1196
1197 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1198 return 0;
1199
1200
1201
1202
1203
1204 pool = get_work_pool(work);
1205 if (!pool)
1206 goto fail;
1207
1208 spin_lock(&pool->lock);
1209
1210
1211
1212
1213
1214
1215
1216
1217 pwq = get_work_pwq(work);
1218 if (pwq && pwq->pool == pool) {
1219 debug_work_deactivate(work);
1220
1221
1222
1223
1224
1225
1226
1227
1228 if (*work_data_bits(work) & WORK_STRUCT_DELAYED)
1229 pwq_activate_delayed_work(work);
1230
1231 list_del_init(&work->entry);
1232 pwq_dec_nr_in_flight(get_work_pwq(work), get_work_color(work));
1233
1234
1235 set_work_pool_and_keep_pending(work, pool->id);
1236
1237 spin_unlock(&pool->lock);
1238 return 1;
1239 }
1240 spin_unlock(&pool->lock);
1241fail:
1242 local_irq_restore(*flags);
1243 if (work_is_canceling(work))
1244 return -ENOENT;
1245 cpu_relax();
1246 return -EAGAIN;
1247}
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1263 struct list_head *head, unsigned int extra_flags)
1264{
1265 struct worker_pool *pool = pwq->pool;
1266
1267
1268 set_work_pwq(work, pwq, extra_flags);
1269 list_add_tail(&work->entry, head);
1270 get_pwq(pwq);
1271
1272
1273
1274
1275
1276
1277 smp_mb();
1278
1279 if (__need_more_worker(pool))
1280 wake_up_worker(pool);
1281}
1282
1283
1284
1285
1286
1287static bool is_chained_work(struct workqueue_struct *wq)
1288{
1289 struct worker *worker;
1290
1291 worker = current_wq_worker();
1292
1293
1294
1295
1296 return worker && worker->current_pwq->wq == wq;
1297}
1298
1299static void __queue_work(int cpu, struct workqueue_struct *wq,
1300 struct work_struct *work)
1301{
1302 struct pool_workqueue *pwq;
1303 struct worker_pool *last_pool;
1304 struct list_head *worklist;
1305 unsigned int work_flags;
1306 unsigned int req_cpu = cpu;
1307
1308
1309
1310
1311
1312
1313
1314 WARN_ON_ONCE(!irqs_disabled());
1315
1316 debug_work_activate(work);
1317
1318
1319 if (unlikely(wq->flags & __WQ_DRAINING) &&
1320 WARN_ON_ONCE(!is_chained_work(wq)))
1321 return;
1322retry:
1323 if (req_cpu == WORK_CPU_UNBOUND)
1324 cpu = raw_smp_processor_id();
1325
1326
1327 if (!(wq->flags & WQ_UNBOUND))
1328 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1329 else
1330 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1331
1332
1333
1334
1335
1336
1337 last_pool = get_work_pool(work);
1338 if (last_pool && last_pool != pwq->pool) {
1339 struct worker *worker;
1340
1341 spin_lock(&last_pool->lock);
1342
1343 worker = find_worker_executing_work(last_pool, work);
1344
1345 if (worker && worker->current_pwq->wq == wq) {
1346 pwq = worker->current_pwq;
1347 } else {
1348
1349 spin_unlock(&last_pool->lock);
1350 spin_lock(&pwq->pool->lock);
1351 }
1352 } else {
1353 spin_lock(&pwq->pool->lock);
1354 }
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364 if (unlikely(!pwq->refcnt)) {
1365 if (wq->flags & WQ_UNBOUND) {
1366 spin_unlock(&pwq->pool->lock);
1367 cpu_relax();
1368 goto retry;
1369 }
1370
1371 WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1372 wq->name, cpu);
1373 }
1374
1375
1376 trace_workqueue_queue_work(req_cpu, pwq, work);
1377
1378 if (WARN_ON(!list_empty(&work->entry))) {
1379 spin_unlock(&pwq->pool->lock);
1380 return;
1381 }
1382
1383 pwq->nr_in_flight[pwq->work_color]++;
1384 work_flags = work_color_to_flags(pwq->work_color);
1385
1386 if (likely(pwq->nr_active < pwq->max_active)) {
1387 trace_workqueue_activate_work(work);
1388 pwq->nr_active++;
1389 worklist = &pwq->pool->worklist;
1390 } else {
1391 work_flags |= WORK_STRUCT_DELAYED;
1392 worklist = &pwq->delayed_works;
1393 }
1394
1395 insert_work(pwq, work, worklist, work_flags);
1396
1397 spin_unlock(&pwq->pool->lock);
1398}
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411bool queue_work_on(int cpu, struct workqueue_struct *wq,
1412 struct work_struct *work)
1413{
1414 bool ret = false;
1415 unsigned long flags;
1416
1417 local_irq_save(flags);
1418
1419 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1420 __queue_work(cpu, wq, work);
1421 ret = true;
1422 }
1423
1424 local_irq_restore(flags);
1425 return ret;
1426}
1427EXPORT_SYMBOL(queue_work_on);
1428
1429void delayed_work_timer_fn(unsigned long __data)
1430{
1431 struct delayed_work *dwork = (struct delayed_work *)__data;
1432
1433
1434 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1435}
1436EXPORT_SYMBOL(delayed_work_timer_fn);
1437
1438static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1439 struct delayed_work *dwork, unsigned long delay)
1440{
1441 struct timer_list *timer = &dwork->timer;
1442 struct work_struct *work = &dwork->work;
1443
1444 WARN_ON_ONCE(timer->function != delayed_work_timer_fn ||
1445 timer->data != (unsigned long)dwork);
1446 WARN_ON_ONCE(timer_pending(timer));
1447 WARN_ON_ONCE(!list_empty(&work->entry));
1448
1449
1450
1451
1452
1453
1454
1455 if (!delay) {
1456 __queue_work(cpu, wq, &dwork->work);
1457 return;
1458 }
1459
1460 timer_stats_timer_set_start_info(&dwork->timer);
1461
1462 dwork->wq = wq;
1463 dwork->cpu = cpu;
1464 timer->expires = jiffies + delay;
1465
1466 if (unlikely(cpu != WORK_CPU_UNBOUND))
1467 add_timer_on(timer, cpu);
1468 else
1469 add_timer(timer);
1470}
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1484 struct delayed_work *dwork, unsigned long delay)
1485{
1486 struct work_struct *work = &dwork->work;
1487 bool ret = false;
1488 unsigned long flags;
1489
1490
1491 local_irq_save(flags);
1492
1493 if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1494 __queue_delayed_work(cpu, wq, dwork, delay);
1495 ret = true;
1496 }
1497
1498 local_irq_restore(flags);
1499 return ret;
1500}
1501EXPORT_SYMBOL(queue_delayed_work_on);
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1522 struct delayed_work *dwork, unsigned long delay)
1523{
1524 unsigned long flags;
1525 int ret;
1526
1527 do {
1528 ret = try_to_grab_pending(&dwork->work, true, &flags);
1529 } while (unlikely(ret == -EAGAIN));
1530
1531 if (likely(ret >= 0)) {
1532 __queue_delayed_work(cpu, wq, dwork, delay);
1533 local_irq_restore(flags);
1534 }
1535
1536
1537 return ret;
1538}
1539EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551static void worker_enter_idle(struct worker *worker)
1552{
1553 struct worker_pool *pool = worker->pool;
1554
1555 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1556 WARN_ON_ONCE(!list_empty(&worker->entry) &&
1557 (worker->hentry.next || worker->hentry.pprev)))
1558 return;
1559
1560
1561 worker->flags |= WORKER_IDLE;
1562 pool->nr_idle++;
1563 worker->last_active = jiffies;
1564
1565
1566 list_add(&worker->entry, &pool->idle_list);
1567
1568 if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1569 mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1570
1571
1572
1573
1574
1575
1576
1577 WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1578 pool->nr_workers == pool->nr_idle &&
1579 atomic_read(&pool->nr_running));
1580}
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591static void worker_leave_idle(struct worker *worker)
1592{
1593 struct worker_pool *pool = worker->pool;
1594
1595 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1596 return;
1597 worker_clr_flags(worker, WORKER_IDLE);
1598 pool->nr_idle--;
1599 list_del_init(&worker->entry);
1600}
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634static bool worker_maybe_bind_and_lock(struct worker_pool *pool)
1635__acquires(&pool->lock)
1636{
1637 while (true) {
1638
1639
1640
1641
1642
1643
1644 if (!(pool->flags & POOL_DISASSOCIATED))
1645 set_cpus_allowed_ptr(current, pool->attrs->cpumask);
1646
1647 spin_lock_irq(&pool->lock);
1648 if (pool->flags & POOL_DISASSOCIATED)
1649 return false;
1650 if (task_cpu(current) == pool->cpu &&
1651 cpumask_equal(¤t->cpus_allowed, pool->attrs->cpumask))
1652 return true;
1653 spin_unlock_irq(&pool->lock);
1654
1655
1656
1657
1658
1659
1660
1661 cpu_relax();
1662 cond_resched();
1663 }
1664}
1665
1666static struct worker *alloc_worker(void)
1667{
1668 struct worker *worker;
1669
1670 worker = kzalloc(sizeof(*worker), GFP_KERNEL);
1671 if (worker) {
1672 INIT_LIST_HEAD(&worker->entry);
1673 INIT_LIST_HEAD(&worker->scheduled);
1674
1675 worker->flags = WORKER_PREP;
1676 }
1677 return worker;
1678}
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694static struct worker *create_worker(struct worker_pool *pool)
1695{
1696 struct worker *worker = NULL;
1697 int id = -1;
1698 char id_buf[16];
1699
1700 lockdep_assert_held(&pool->manager_mutex);
1701
1702
1703
1704
1705
1706 idr_preload(GFP_KERNEL);
1707 spin_lock_irq(&pool->lock);
1708
1709 id = idr_alloc(&pool->worker_idr, NULL, 0, 0, GFP_NOWAIT);
1710
1711 spin_unlock_irq(&pool->lock);
1712 idr_preload_end();
1713 if (id < 0)
1714 goto fail;
1715
1716 worker = alloc_worker();
1717 if (!worker)
1718 goto fail;
1719
1720 worker->pool = pool;
1721 worker->id = id;
1722
1723 if (pool->cpu >= 0)
1724 snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1725 pool->attrs->nice < 0 ? "H" : "");
1726 else
1727 snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1728
1729 worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1730 "kworker/%s", id_buf);
1731 if (IS_ERR(worker->task))
1732 goto fail;
1733
1734
1735
1736
1737
1738 set_user_nice(worker->task, pool->attrs->nice);
1739 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1740
1741
1742 worker->task->flags |= PF_NO_SETAFFINITY;
1743
1744
1745
1746
1747
1748
1749 if (pool->flags & POOL_DISASSOCIATED)
1750 worker->flags |= WORKER_UNBOUND;
1751
1752
1753 spin_lock_irq(&pool->lock);
1754 idr_replace(&pool->worker_idr, worker, worker->id);
1755 spin_unlock_irq(&pool->lock);
1756
1757 return worker;
1758
1759fail:
1760 if (id >= 0) {
1761 spin_lock_irq(&pool->lock);
1762 idr_remove(&pool->worker_idr, id);
1763 spin_unlock_irq(&pool->lock);
1764 }
1765 kfree(worker);
1766 return NULL;
1767}
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778static void start_worker(struct worker *worker)
1779{
1780 worker->flags |= WORKER_STARTED;
1781 worker->pool->nr_workers++;
1782 worker_enter_idle(worker);
1783 wake_up_process(worker->task);
1784}
1785
1786
1787
1788
1789
1790
1791
1792static int create_and_start_worker(struct worker_pool *pool)
1793{
1794 struct worker *worker;
1795
1796 mutex_lock(&pool->manager_mutex);
1797
1798 worker = create_worker(pool);
1799 if (worker) {
1800 spin_lock_irq(&pool->lock);
1801 start_worker(worker);
1802 spin_unlock_irq(&pool->lock);
1803 }
1804
1805 mutex_unlock(&pool->manager_mutex);
1806
1807 return worker ? 0 : -ENOMEM;
1808}
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819static void destroy_worker(struct worker *worker)
1820{
1821 struct worker_pool *pool = worker->pool;
1822
1823 lockdep_assert_held(&pool->manager_mutex);
1824 lockdep_assert_held(&pool->lock);
1825
1826
1827 if (WARN_ON(worker->current_work) ||
1828 WARN_ON(!list_empty(&worker->scheduled)))
1829 return;
1830
1831 if (worker->flags & WORKER_STARTED)
1832 pool->nr_workers--;
1833 if (worker->flags & WORKER_IDLE)
1834 pool->nr_idle--;
1835
1836 list_del_init(&worker->entry);
1837 worker->flags |= WORKER_DIE;
1838
1839 idr_remove(&pool->worker_idr, worker->id);
1840
1841 spin_unlock_irq(&pool->lock);
1842
1843 kthread_stop(worker->task);
1844 kfree(worker);
1845
1846 spin_lock_irq(&pool->lock);
1847}
1848
1849static void idle_worker_timeout(unsigned long __pool)
1850{
1851 struct worker_pool *pool = (void *)__pool;
1852
1853 spin_lock_irq(&pool->lock);
1854
1855 if (too_many_workers(pool)) {
1856 struct worker *worker;
1857 unsigned long expires;
1858
1859
1860 worker = list_entry(pool->idle_list.prev, struct worker, entry);
1861 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
1862
1863 if (time_before(jiffies, expires))
1864 mod_timer(&pool->idle_timer, expires);
1865 else {
1866
1867 pool->flags |= POOL_MANAGE_WORKERS;
1868 wake_up_worker(pool);
1869 }
1870 }
1871
1872 spin_unlock_irq(&pool->lock);
1873}
1874
1875static void send_mayday(struct work_struct *work)
1876{
1877 struct pool_workqueue *pwq = get_work_pwq(work);
1878 struct workqueue_struct *wq = pwq->wq;
1879
1880 lockdep_assert_held(&wq_mayday_lock);
1881
1882 if (!wq->rescuer)
1883 return;
1884
1885
1886 if (list_empty(&pwq->mayday_node)) {
1887 list_add_tail(&pwq->mayday_node, &wq->maydays);
1888 wake_up_process(wq->rescuer->task);
1889 }
1890}
1891
1892static void pool_mayday_timeout(unsigned long __pool)
1893{
1894 struct worker_pool *pool = (void *)__pool;
1895 struct work_struct *work;
1896
1897 spin_lock_irq(&wq_mayday_lock);
1898 spin_lock(&pool->lock);
1899
1900 if (need_to_create_worker(pool)) {
1901
1902
1903
1904
1905
1906
1907 list_for_each_entry(work, &pool->worklist, entry)
1908 send_mayday(work);
1909 }
1910
1911 spin_unlock(&pool->lock);
1912 spin_unlock_irq(&wq_mayday_lock);
1913
1914 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
1915}
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939static bool maybe_create_worker(struct worker_pool *pool)
1940__releases(&pool->lock)
1941__acquires(&pool->lock)
1942{
1943 if (!need_to_create_worker(pool))
1944 return false;
1945restart:
1946 spin_unlock_irq(&pool->lock);
1947
1948
1949 mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
1950
1951 while (true) {
1952 struct worker *worker;
1953
1954 worker = create_worker(pool);
1955 if (worker) {
1956 del_timer_sync(&pool->mayday_timer);
1957 spin_lock_irq(&pool->lock);
1958 start_worker(worker);
1959 if (WARN_ON_ONCE(need_to_create_worker(pool)))
1960 goto restart;
1961 return true;
1962 }
1963
1964 if (!need_to_create_worker(pool))
1965 break;
1966
1967 __set_current_state(TASK_INTERRUPTIBLE);
1968 schedule_timeout(CREATE_COOLDOWN);
1969
1970 if (!need_to_create_worker(pool))
1971 break;
1972 }
1973
1974 del_timer_sync(&pool->mayday_timer);
1975 spin_lock_irq(&pool->lock);
1976 if (need_to_create_worker(pool))
1977 goto restart;
1978 return true;
1979}
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996static bool maybe_destroy_workers(struct worker_pool *pool)
1997{
1998 bool ret = false;
1999
2000 while (too_many_workers(pool)) {
2001 struct worker *worker;
2002 unsigned long expires;
2003
2004 worker = list_entry(pool->idle_list.prev, struct worker, entry);
2005 expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2006
2007 if (time_before(jiffies, expires)) {
2008 mod_timer(&pool->idle_timer, expires);
2009 break;
2010 }
2011
2012 destroy_worker(worker);
2013 ret = true;
2014 }
2015
2016 return ret;
2017}
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039static bool manage_workers(struct worker *worker)
2040{
2041 struct worker_pool *pool = worker->pool;
2042 bool ret = false;
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065 if (!mutex_trylock(&pool->manager_arb))
2066 return ret;
2067
2068
2069
2070
2071
2072 if (unlikely(!mutex_trylock(&pool->manager_mutex))) {
2073 spin_unlock_irq(&pool->lock);
2074 mutex_lock(&pool->manager_mutex);
2075 spin_lock_irq(&pool->lock);
2076 ret = true;
2077 }
2078
2079 pool->flags &= ~POOL_MANAGE_WORKERS;
2080
2081
2082
2083
2084
2085 ret |= maybe_destroy_workers(pool);
2086 ret |= maybe_create_worker(pool);
2087
2088 mutex_unlock(&pool->manager_mutex);
2089 mutex_unlock(&pool->manager_arb);
2090 return ret;
2091}
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107static void process_one_work(struct worker *worker, struct work_struct *work)
2108__releases(&pool->lock)
2109__acquires(&pool->lock)
2110{
2111 struct pool_workqueue *pwq = get_work_pwq(work);
2112 struct worker_pool *pool = worker->pool;
2113 bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2114 int work_color;
2115 struct worker *collision;
2116#ifdef CONFIG_LOCKDEP
2117
2118
2119
2120
2121
2122
2123
2124 struct lockdep_map lockdep_map;
2125
2126 lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2127#endif
2128
2129
2130
2131
2132
2133 WARN_ON_ONCE(!(worker->flags & WORKER_UNBOUND) &&
2134 !(pool->flags & POOL_DISASSOCIATED) &&
2135 raw_smp_processor_id() != pool->cpu);
2136
2137
2138
2139
2140
2141
2142
2143 collision = find_worker_executing_work(pool, work);
2144 if (unlikely(collision)) {
2145 move_linked_works(work, &collision->scheduled, NULL);
2146 return;
2147 }
2148
2149
2150 debug_work_deactivate(work);
2151 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2152 worker->current_work = work;
2153 worker->current_func = work->func;
2154 worker->current_pwq = pwq;
2155 work_color = get_work_color(work);
2156
2157 list_del_init(&work->entry);
2158
2159
2160
2161
2162
2163 if (unlikely(cpu_intensive))
2164 worker_set_flags(worker, WORKER_CPU_INTENSIVE, true);
2165
2166
2167
2168
2169
2170 if ((worker->flags & WORKER_UNBOUND) && need_more_worker(pool))
2171 wake_up_worker(pool);
2172
2173
2174
2175
2176
2177
2178
2179 set_work_pool_and_clear_pending(work, pool->id);
2180
2181 spin_unlock_irq(&pool->lock);
2182
2183 lock_map_acquire_read(&pwq->wq->lockdep_map);
2184 lock_map_acquire(&lockdep_map);
2185 trace_workqueue_execute_start(work);
2186 worker->current_func(work);
2187
2188
2189
2190
2191 trace_workqueue_execute_end(work);
2192 lock_map_release(&lockdep_map);
2193 lock_map_release(&pwq->wq->lockdep_map);
2194
2195 if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2196 pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2197 " last function: %pf\n",
2198 current->comm, preempt_count(), task_pid_nr(current),
2199 worker->current_func);
2200 debug_show_held_locks(current);
2201 dump_stack();
2202 }
2203
2204
2205
2206
2207
2208
2209
2210
2211 cond_resched();
2212
2213 spin_lock_irq(&pool->lock);
2214
2215
2216 if (unlikely(cpu_intensive))
2217 worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2218
2219
2220 hash_del(&worker->hentry);
2221 worker->current_work = NULL;
2222 worker->current_func = NULL;
2223 worker->current_pwq = NULL;
2224 worker->desc_valid = false;
2225 pwq_dec_nr_in_flight(pwq, work_color);
2226}
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240static void process_scheduled_works(struct worker *worker)
2241{
2242 while (!list_empty(&worker->scheduled)) {
2243 struct work_struct *work = list_first_entry(&worker->scheduled,
2244 struct work_struct, entry);
2245 process_one_work(worker, work);
2246 }
2247}
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259static int worker_thread(void *__worker)
2260{
2261 struct worker *worker = __worker;
2262 struct worker_pool *pool = worker->pool;
2263
2264
2265 worker->task->flags |= PF_WQ_WORKER;
2266woke_up:
2267 spin_lock_irq(&pool->lock);
2268
2269
2270 if (unlikely(worker->flags & WORKER_DIE)) {
2271 spin_unlock_irq(&pool->lock);
2272 WARN_ON_ONCE(!list_empty(&worker->entry));
2273 worker->task->flags &= ~PF_WQ_WORKER;
2274 return 0;
2275 }
2276
2277 worker_leave_idle(worker);
2278recheck:
2279
2280 if (!need_more_worker(pool))
2281 goto sleep;
2282
2283
2284 if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2285 goto recheck;
2286
2287
2288
2289
2290
2291
2292 WARN_ON_ONCE(!list_empty(&worker->scheduled));
2293
2294
2295
2296
2297
2298
2299
2300
2301 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2302
2303 do {
2304 struct work_struct *work =
2305 list_first_entry(&pool->worklist,
2306 struct work_struct, entry);
2307
2308 if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2309
2310 process_one_work(worker, work);
2311 if (unlikely(!list_empty(&worker->scheduled)))
2312 process_scheduled_works(worker);
2313 } else {
2314 move_linked_works(work, &worker->scheduled, NULL);
2315 process_scheduled_works(worker);
2316 }
2317 } while (keep_working(pool));
2318
2319 worker_set_flags(worker, WORKER_PREP, false);
2320sleep:
2321 if (unlikely(need_to_manage_workers(pool)) && manage_workers(worker))
2322 goto recheck;
2323
2324
2325
2326
2327
2328
2329
2330
2331 worker_enter_idle(worker);
2332 __set_current_state(TASK_INTERRUPTIBLE);
2333 spin_unlock_irq(&pool->lock);
2334 schedule();
2335 goto woke_up;
2336}
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357static int rescuer_thread(void *__rescuer)
2358{
2359 struct worker *rescuer = __rescuer;
2360 struct workqueue_struct *wq = rescuer->rescue_wq;
2361 struct list_head *scheduled = &rescuer->scheduled;
2362
2363 set_user_nice(current, RESCUER_NICE_LEVEL);
2364
2365
2366
2367
2368
2369 rescuer->task->flags |= PF_WQ_WORKER;
2370repeat:
2371 set_current_state(TASK_INTERRUPTIBLE);
2372
2373 if (kthread_should_stop()) {
2374 __set_current_state(TASK_RUNNING);
2375 rescuer->task->flags &= ~PF_WQ_WORKER;
2376 return 0;
2377 }
2378
2379
2380 spin_lock_irq(&wq_mayday_lock);
2381
2382 while (!list_empty(&wq->maydays)) {
2383 struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2384 struct pool_workqueue, mayday_node);
2385 struct worker_pool *pool = pwq->pool;
2386 struct work_struct *work, *n;
2387
2388 __set_current_state(TASK_RUNNING);
2389 list_del_init(&pwq->mayday_node);
2390
2391 spin_unlock_irq(&wq_mayday_lock);
2392
2393
2394 worker_maybe_bind_and_lock(pool);
2395 rescuer->pool = pool;
2396
2397
2398
2399
2400
2401 WARN_ON_ONCE(!list_empty(&rescuer->scheduled));
2402 list_for_each_entry_safe(work, n, &pool->worklist, entry)
2403 if (get_work_pwq(work) == pwq)
2404 move_linked_works(work, scheduled, &n);
2405
2406 process_scheduled_works(rescuer);
2407
2408
2409
2410
2411
2412
2413 if (keep_working(pool))
2414 wake_up_worker(pool);
2415
2416 rescuer->pool = NULL;
2417 spin_unlock(&pool->lock);
2418 spin_lock(&wq_mayday_lock);
2419 }
2420
2421 spin_unlock_irq(&wq_mayday_lock);
2422
2423
2424 WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2425 schedule();
2426 goto repeat;
2427}
2428
2429struct wq_barrier {
2430 struct work_struct work;
2431 struct completion done;
2432};
2433
2434static void wq_barrier_func(struct work_struct *work)
2435{
2436 struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2437 complete(&barr->done);
2438}
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464static void insert_wq_barrier(struct pool_workqueue *pwq,
2465 struct wq_barrier *barr,
2466 struct work_struct *target, struct worker *worker)
2467{
2468 struct list_head *head;
2469 unsigned int linked = 0;
2470
2471
2472
2473
2474
2475
2476
2477 INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2478 __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2479 init_completion(&barr->done);
2480
2481
2482
2483
2484
2485 if (worker)
2486 head = worker->scheduled.next;
2487 else {
2488 unsigned long *bits = work_data_bits(target);
2489
2490 head = target->entry.next;
2491
2492 linked = *bits & WORK_STRUCT_LINKED;
2493 __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2494 }
2495
2496 debug_work_activate(&barr->work);
2497 insert_work(pwq, &barr->work, head,
2498 work_color_to_flags(WORK_NO_COLOR) | linked);
2499}
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2533 int flush_color, int work_color)
2534{
2535 bool wait = false;
2536 struct pool_workqueue *pwq;
2537
2538 if (flush_color >= 0) {
2539 WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2540 atomic_set(&wq->nr_pwqs_to_flush, 1);
2541 }
2542
2543 for_each_pwq(pwq, wq) {
2544 struct worker_pool *pool = pwq->pool;
2545
2546 spin_lock_irq(&pool->lock);
2547
2548 if (flush_color >= 0) {
2549 WARN_ON_ONCE(pwq->flush_color != -1);
2550
2551 if (pwq->nr_in_flight[flush_color]) {
2552 pwq->flush_color = flush_color;
2553 atomic_inc(&wq->nr_pwqs_to_flush);
2554 wait = true;
2555 }
2556 }
2557
2558 if (work_color >= 0) {
2559 WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2560 pwq->work_color = work_color;
2561 }
2562
2563 spin_unlock_irq(&pool->lock);
2564 }
2565
2566 if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2567 complete(&wq->first_flusher->done);
2568
2569 return wait;
2570}
2571
2572
2573
2574
2575
2576
2577
2578
2579void flush_workqueue(struct workqueue_struct *wq)
2580{
2581 struct wq_flusher this_flusher = {
2582 .list = LIST_HEAD_INIT(this_flusher.list),
2583 .flush_color = -1,
2584 .done = COMPLETION_INITIALIZER_ONSTACK(this_flusher.done),
2585 };
2586 int next_color;
2587
2588 lock_map_acquire(&wq->lockdep_map);
2589 lock_map_release(&wq->lockdep_map);
2590
2591 mutex_lock(&wq->mutex);
2592
2593
2594
2595
2596 next_color = work_next_color(wq->work_color);
2597
2598 if (next_color != wq->flush_color) {
2599
2600
2601
2602
2603
2604 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2605 this_flusher.flush_color = wq->work_color;
2606 wq->work_color = next_color;
2607
2608 if (!wq->first_flusher) {
2609
2610 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2611
2612 wq->first_flusher = &this_flusher;
2613
2614 if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2615 wq->work_color)) {
2616
2617 wq->flush_color = next_color;
2618 wq->first_flusher = NULL;
2619 goto out_unlock;
2620 }
2621 } else {
2622
2623 WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2624 list_add_tail(&this_flusher.list, &wq->flusher_queue);
2625 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2626 }
2627 } else {
2628
2629
2630
2631
2632
2633 list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2634 }
2635
2636 mutex_unlock(&wq->mutex);
2637
2638 wait_for_completion(&this_flusher.done);
2639
2640
2641
2642
2643
2644
2645
2646 if (wq->first_flusher != &this_flusher)
2647 return;
2648
2649 mutex_lock(&wq->mutex);
2650
2651
2652 if (wq->first_flusher != &this_flusher)
2653 goto out_unlock;
2654
2655 wq->first_flusher = NULL;
2656
2657 WARN_ON_ONCE(!list_empty(&this_flusher.list));
2658 WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2659
2660 while (true) {
2661 struct wq_flusher *next, *tmp;
2662
2663
2664 list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2665 if (next->flush_color != wq->flush_color)
2666 break;
2667 list_del_init(&next->list);
2668 complete(&next->done);
2669 }
2670
2671 WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2672 wq->flush_color != work_next_color(wq->work_color));
2673
2674
2675 wq->flush_color = work_next_color(wq->flush_color);
2676
2677
2678 if (!list_empty(&wq->flusher_overflow)) {
2679
2680
2681
2682
2683
2684
2685 list_for_each_entry(tmp, &wq->flusher_overflow, list)
2686 tmp->flush_color = wq->work_color;
2687
2688 wq->work_color = work_next_color(wq->work_color);
2689
2690 list_splice_tail_init(&wq->flusher_overflow,
2691 &wq->flusher_queue);
2692 flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2693 }
2694
2695 if (list_empty(&wq->flusher_queue)) {
2696 WARN_ON_ONCE(wq->flush_color != wq->work_color);
2697 break;
2698 }
2699
2700
2701
2702
2703
2704 WARN_ON_ONCE(wq->flush_color == wq->work_color);
2705 WARN_ON_ONCE(wq->flush_color != next->flush_color);
2706
2707 list_del_init(&next->list);
2708 wq->first_flusher = next;
2709
2710 if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2711 break;
2712
2713
2714
2715
2716
2717 wq->first_flusher = NULL;
2718 }
2719
2720out_unlock:
2721 mutex_unlock(&wq->mutex);
2722}
2723EXPORT_SYMBOL_GPL(flush_workqueue);
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736void drain_workqueue(struct workqueue_struct *wq)
2737{
2738 unsigned int flush_cnt = 0;
2739 struct pool_workqueue *pwq;
2740
2741
2742
2743
2744
2745
2746 mutex_lock(&wq->mutex);
2747 if (!wq->nr_drainers++)
2748 wq->flags |= __WQ_DRAINING;
2749 mutex_unlock(&wq->mutex);
2750reflush:
2751 flush_workqueue(wq);
2752
2753 mutex_lock(&wq->mutex);
2754
2755 for_each_pwq(pwq, wq) {
2756 bool drained;
2757
2758 spin_lock_irq(&pwq->pool->lock);
2759 drained = !pwq->nr_active && list_empty(&pwq->delayed_works);
2760 spin_unlock_irq(&pwq->pool->lock);
2761
2762 if (drained)
2763 continue;
2764
2765 if (++flush_cnt == 10 ||
2766 (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2767 pr_warn("workqueue %s: drain_workqueue() isn't complete after %u tries\n",
2768 wq->name, flush_cnt);
2769
2770 mutex_unlock(&wq->mutex);
2771 goto reflush;
2772 }
2773
2774 if (!--wq->nr_drainers)
2775 wq->flags &= ~__WQ_DRAINING;
2776 mutex_unlock(&wq->mutex);
2777}
2778EXPORT_SYMBOL_GPL(drain_workqueue);
2779
2780static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr)
2781{
2782 struct worker *worker = NULL;
2783 struct worker_pool *pool;
2784 struct pool_workqueue *pwq;
2785
2786 might_sleep();
2787
2788 local_irq_disable();
2789 pool = get_work_pool(work);
2790 if (!pool) {
2791 local_irq_enable();
2792 return false;
2793 }
2794
2795 spin_lock(&pool->lock);
2796
2797 pwq = get_work_pwq(work);
2798 if (pwq) {
2799 if (unlikely(pwq->pool != pool))
2800 goto already_gone;
2801 } else {
2802 worker = find_worker_executing_work(pool, work);
2803 if (!worker)
2804 goto already_gone;
2805 pwq = worker->current_pwq;
2806 }
2807
2808 insert_wq_barrier(pwq, barr, work, worker);
2809 spin_unlock_irq(&pool->lock);
2810
2811
2812
2813
2814
2815
2816
2817 if (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)
2818 lock_map_acquire(&pwq->wq->lockdep_map);
2819 else
2820 lock_map_acquire_read(&pwq->wq->lockdep_map);
2821 lock_map_release(&pwq->wq->lockdep_map);
2822
2823 return true;
2824already_gone:
2825 spin_unlock_irq(&pool->lock);
2826 return false;
2827}
2828
2829static bool __flush_work(struct work_struct *work)
2830{
2831 struct wq_barrier barr;
2832
2833 if (start_flush_work(work, &barr)) {
2834 wait_for_completion(&barr.done);
2835 destroy_work_on_stack(&barr.work);
2836 return true;
2837 } else {
2838 return false;
2839 }
2840}
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853bool flush_work(struct work_struct *work)
2854{
2855 lock_map_acquire(&work->lockdep_map);
2856 lock_map_release(&work->lockdep_map);
2857
2858 return __flush_work(work);
2859}
2860EXPORT_SYMBOL_GPL(flush_work);
2861
2862static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
2863{
2864 unsigned long flags;
2865 int ret;
2866
2867 do {
2868 ret = try_to_grab_pending(work, is_dwork, &flags);
2869
2870
2871
2872
2873 if (unlikely(ret == -ENOENT))
2874 flush_work(work);
2875 } while (unlikely(ret < 0));
2876
2877
2878 mark_work_canceling(work);
2879 local_irq_restore(flags);
2880
2881 flush_work(work);
2882 clear_work_data(work);
2883 return ret;
2884}
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904bool cancel_work_sync(struct work_struct *work)
2905{
2906 return __cancel_work_timer(work, false);
2907}
2908EXPORT_SYMBOL_GPL(cancel_work_sync);
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922bool flush_delayed_work(struct delayed_work *dwork)
2923{
2924 local_irq_disable();
2925 if (del_timer_sync(&dwork->timer))
2926 __queue_work(dwork->cpu, dwork->wq, &dwork->work);
2927 local_irq_enable();
2928 return flush_work(&dwork->work);
2929}
2930EXPORT_SYMBOL(flush_delayed_work);
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944bool cancel_delayed_work(struct delayed_work *dwork)
2945{
2946 unsigned long flags;
2947 int ret;
2948
2949 do {
2950 ret = try_to_grab_pending(&dwork->work, true, &flags);
2951 } while (unlikely(ret == -EAGAIN));
2952
2953 if (unlikely(ret < 0))
2954 return false;
2955
2956 set_work_pool_and_clear_pending(&dwork->work,
2957 get_work_pool_id(&dwork->work));
2958 local_irq_restore(flags);
2959 return ret;
2960}
2961EXPORT_SYMBOL(cancel_delayed_work);
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972bool cancel_delayed_work_sync(struct delayed_work *dwork)
2973{
2974 return __cancel_work_timer(&dwork->work, true);
2975}
2976EXPORT_SYMBOL(cancel_delayed_work_sync);
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989int schedule_on_each_cpu(work_func_t func)
2990{
2991 int cpu;
2992 struct work_struct __percpu *works;
2993
2994 works = alloc_percpu(struct work_struct);
2995 if (!works)
2996 return -ENOMEM;
2997
2998 get_online_cpus();
2999
3000 for_each_online_cpu(cpu) {
3001 struct work_struct *work = per_cpu_ptr(works, cpu);
3002
3003 INIT_WORK(work, func);
3004 schedule_work_on(cpu, work);
3005 }
3006
3007 for_each_online_cpu(cpu)
3008 flush_work(per_cpu_ptr(works, cpu));
3009
3010 put_online_cpus();
3011 free_percpu(works);
3012 return 0;
3013}
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039void flush_scheduled_work(void)
3040{
3041 flush_workqueue(system_wq);
3042}
3043EXPORT_SYMBOL(flush_scheduled_work);
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3058{
3059 if (!in_interrupt()) {
3060 fn(&ew->work);
3061 return 0;
3062 }
3063
3064 INIT_WORK(&ew->work, fn);
3065 schedule_work(&ew->work);
3066
3067 return 1;
3068}
3069EXPORT_SYMBOL_GPL(execute_in_process_context);
3070
3071#ifdef CONFIG_SYSFS
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086struct wq_device {
3087 struct workqueue_struct *wq;
3088 struct device dev;
3089};
3090
3091static struct workqueue_struct *dev_to_wq(struct device *dev)
3092{
3093 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3094
3095 return wq_dev->wq;
3096}
3097
3098static ssize_t wq_per_cpu_show(struct device *dev,
3099 struct device_attribute *attr, char *buf)
3100{
3101 struct workqueue_struct *wq = dev_to_wq(dev);
3102
3103 return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
3104}
3105
3106static ssize_t wq_max_active_show(struct device *dev,
3107 struct device_attribute *attr, char *buf)
3108{
3109 struct workqueue_struct *wq = dev_to_wq(dev);
3110
3111 return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
3112}
3113
3114static ssize_t wq_max_active_store(struct device *dev,
3115 struct device_attribute *attr,
3116 const char *buf, size_t count)
3117{
3118 struct workqueue_struct *wq = dev_to_wq(dev);
3119 int val;
3120
3121 if (sscanf(buf, "%d", &val) != 1 || val <= 0)
3122 return -EINVAL;
3123
3124 workqueue_set_max_active(wq, val);
3125 return count;
3126}
3127
3128static struct device_attribute wq_sysfs_attrs[] = {
3129 __ATTR(per_cpu, 0444, wq_per_cpu_show, NULL),
3130 __ATTR(max_active, 0644, wq_max_active_show, wq_max_active_store),
3131 __ATTR_NULL,
3132};
3133
3134static ssize_t wq_pool_ids_show(struct device *dev,
3135 struct device_attribute *attr, char *buf)
3136{
3137 struct workqueue_struct *wq = dev_to_wq(dev);
3138 const char *delim = "";
3139 int node, written = 0;
3140
3141 rcu_read_lock_sched();
3142 for_each_node(node) {
3143 written += scnprintf(buf + written, PAGE_SIZE - written,
3144 "%s%d:%d", delim, node,
3145 unbound_pwq_by_node(wq, node)->pool->id);
3146 delim = " ";
3147 }
3148 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3149 rcu_read_unlock_sched();
3150
3151 return written;
3152}
3153
3154static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
3155 char *buf)
3156{
3157 struct workqueue_struct *wq = dev_to_wq(dev);
3158 int written;
3159
3160 mutex_lock(&wq->mutex);
3161 written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
3162 mutex_unlock(&wq->mutex);
3163
3164 return written;
3165}
3166
3167
3168static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
3169{
3170 struct workqueue_attrs *attrs;
3171
3172 attrs = alloc_workqueue_attrs(GFP_KERNEL);
3173 if (!attrs)
3174 return NULL;
3175
3176 mutex_lock(&wq->mutex);
3177 copy_workqueue_attrs(attrs, wq->unbound_attrs);
3178 mutex_unlock(&wq->mutex);
3179 return attrs;
3180}
3181
3182static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
3183 const char *buf, size_t count)
3184{
3185 struct workqueue_struct *wq = dev_to_wq(dev);
3186 struct workqueue_attrs *attrs;
3187 int ret;
3188
3189 attrs = wq_sysfs_prep_attrs(wq);
3190 if (!attrs)
3191 return -ENOMEM;
3192
3193 if (sscanf(buf, "%d", &attrs->nice) == 1 &&
3194 attrs->nice >= -20 && attrs->nice <= 19)
3195 ret = apply_workqueue_attrs(wq, attrs);
3196 else
3197 ret = -EINVAL;
3198
3199 free_workqueue_attrs(attrs);
3200 return ret ?: count;
3201}
3202
3203static ssize_t wq_cpumask_show(struct device *dev,
3204 struct device_attribute *attr, char *buf)
3205{
3206 struct workqueue_struct *wq = dev_to_wq(dev);
3207 int written;
3208
3209 mutex_lock(&wq->mutex);
3210 written = cpumask_scnprintf(buf, PAGE_SIZE, wq->unbound_attrs->cpumask);
3211 mutex_unlock(&wq->mutex);
3212
3213 written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
3214 return written;
3215}
3216
3217static ssize_t wq_cpumask_store(struct device *dev,
3218 struct device_attribute *attr,
3219 const char *buf, size_t count)
3220{
3221 struct workqueue_struct *wq = dev_to_wq(dev);
3222 struct workqueue_attrs *attrs;
3223 int ret;
3224
3225 attrs = wq_sysfs_prep_attrs(wq);
3226 if (!attrs)
3227 return -ENOMEM;
3228
3229 ret = cpumask_parse(buf, attrs->cpumask);
3230 if (!ret)
3231 ret = apply_workqueue_attrs(wq, attrs);
3232
3233 free_workqueue_attrs(attrs);
3234 return ret ?: count;
3235}
3236
3237static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
3238 char *buf)
3239{
3240 struct workqueue_struct *wq = dev_to_wq(dev);
3241 int written;
3242
3243 mutex_lock(&wq->mutex);
3244 written = scnprintf(buf, PAGE_SIZE, "%d\n",
3245 !wq->unbound_attrs->no_numa);
3246 mutex_unlock(&wq->mutex);
3247
3248 return written;
3249}
3250
3251static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
3252 const char *buf, size_t count)
3253{
3254 struct workqueue_struct *wq = dev_to_wq(dev);
3255 struct workqueue_attrs *attrs;
3256 int v, ret;
3257
3258 attrs = wq_sysfs_prep_attrs(wq);
3259 if (!attrs)
3260 return -ENOMEM;
3261
3262 ret = -EINVAL;
3263 if (sscanf(buf, "%d", &v) == 1) {
3264 attrs->no_numa = !v;
3265 ret = apply_workqueue_attrs(wq, attrs);
3266 }
3267
3268 free_workqueue_attrs(attrs);
3269 return ret ?: count;
3270}
3271
3272static struct device_attribute wq_sysfs_unbound_attrs[] = {
3273 __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
3274 __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
3275 __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
3276 __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
3277 __ATTR_NULL,
3278};
3279
3280static struct bus_type wq_subsys = {
3281 .name = "workqueue",
3282 .dev_attrs = wq_sysfs_attrs,
3283};
3284
3285static int __init wq_sysfs_init(void)
3286{
3287 return subsys_virtual_register(&wq_subsys, NULL);
3288}
3289core_initcall(wq_sysfs_init);
3290
3291static void wq_device_release(struct device *dev)
3292{
3293 struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
3294
3295 kfree(wq_dev);
3296}
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313int workqueue_sysfs_register(struct workqueue_struct *wq)
3314{
3315 struct wq_device *wq_dev;
3316 int ret;
3317
3318
3319
3320
3321
3322
3323 if (WARN_ON(wq->flags & __WQ_ORDERED))
3324 return -EINVAL;
3325
3326 wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
3327 if (!wq_dev)
3328 return -ENOMEM;
3329
3330 wq_dev->wq = wq;
3331 wq_dev->dev.bus = &wq_subsys;
3332 wq_dev->dev.init_name = wq->name;
3333 wq_dev->dev.release = wq_device_release;
3334
3335
3336
3337
3338
3339 dev_set_uevent_suppress(&wq_dev->dev, true);
3340
3341 ret = device_register(&wq_dev->dev);
3342 if (ret) {
3343 kfree(wq_dev);
3344 wq->wq_dev = NULL;
3345 return ret;
3346 }
3347
3348 if (wq->flags & WQ_UNBOUND) {
3349 struct device_attribute *attr;
3350
3351 for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
3352 ret = device_create_file(&wq_dev->dev, attr);
3353 if (ret) {
3354 device_unregister(&wq_dev->dev);
3355 wq->wq_dev = NULL;
3356 return ret;
3357 }
3358 }
3359 }
3360
3361 kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
3362 return 0;
3363}
3364
3365
3366
3367
3368
3369
3370
3371static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
3372{
3373 struct wq_device *wq_dev = wq->wq_dev;
3374
3375 if (!wq->wq_dev)
3376 return;
3377
3378 wq->wq_dev = NULL;
3379 device_unregister(&wq_dev->dev);
3380}
3381#else
3382static void workqueue_sysfs_unregister(struct workqueue_struct *wq) { }
3383#endif
3384
3385
3386
3387
3388
3389
3390
3391void free_workqueue_attrs(struct workqueue_attrs *attrs)
3392{
3393 if (attrs) {
3394 free_cpumask_var(attrs->cpumask);
3395 kfree(attrs);
3396 }
3397}
3398
3399
3400
3401
3402
3403
3404
3405
3406struct workqueue_attrs *alloc_workqueue_attrs(gfp_t gfp_mask)
3407{
3408 struct workqueue_attrs *attrs;
3409
3410 attrs = kzalloc(sizeof(*attrs), gfp_mask);
3411 if (!attrs)
3412 goto fail;
3413 if (!alloc_cpumask_var(&attrs->cpumask, gfp_mask))
3414 goto fail;
3415
3416 cpumask_copy(attrs->cpumask, cpu_possible_mask);
3417 return attrs;
3418fail:
3419 free_workqueue_attrs(attrs);
3420 return NULL;
3421}
3422
3423static void copy_workqueue_attrs(struct workqueue_attrs *to,
3424 const struct workqueue_attrs *from)
3425{
3426 to->nice = from->nice;
3427 cpumask_copy(to->cpumask, from->cpumask);
3428
3429
3430
3431
3432
3433 to->no_numa = from->no_numa;
3434}
3435
3436
3437static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3438{
3439 u32 hash = 0;
3440
3441 hash = jhash_1word(attrs->nice, hash);
3442 hash = jhash(cpumask_bits(attrs->cpumask),
3443 BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3444 return hash;
3445}
3446
3447
3448static bool wqattrs_equal(const struct workqueue_attrs *a,
3449 const struct workqueue_attrs *b)
3450{
3451 if (a->nice != b->nice)
3452 return false;
3453 if (!cpumask_equal(a->cpumask, b->cpumask))
3454 return false;
3455 return true;
3456}
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467static int init_worker_pool(struct worker_pool *pool)
3468{
3469 spin_lock_init(&pool->lock);
3470 pool->id = -1;
3471 pool->cpu = -1;
3472 pool->node = NUMA_NO_NODE;
3473 pool->flags |= POOL_DISASSOCIATED;
3474 INIT_LIST_HEAD(&pool->worklist);
3475 INIT_LIST_HEAD(&pool->idle_list);
3476 hash_init(pool->busy_hash);
3477
3478 init_timer_deferrable(&pool->idle_timer);
3479 pool->idle_timer.function = idle_worker_timeout;
3480 pool->idle_timer.data = (unsigned long)pool;
3481
3482 setup_timer(&pool->mayday_timer, pool_mayday_timeout,
3483 (unsigned long)pool);
3484
3485 mutex_init(&pool->manager_arb);
3486 mutex_init(&pool->manager_mutex);
3487 idr_init(&pool->worker_idr);
3488
3489 INIT_HLIST_NODE(&pool->hash_node);
3490 pool->refcnt = 1;
3491
3492
3493 pool->attrs = alloc_workqueue_attrs(GFP_KERNEL);
3494 if (!pool->attrs)
3495 return -ENOMEM;
3496 return 0;
3497}
3498
3499static void rcu_free_pool(struct rcu_head *rcu)
3500{
3501 struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3502
3503 idr_destroy(&pool->worker_idr);
3504 free_workqueue_attrs(pool->attrs);
3505 kfree(pool);
3506}
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519static void put_unbound_pool(struct worker_pool *pool)
3520{
3521 struct worker *worker;
3522
3523 lockdep_assert_held(&wq_pool_mutex);
3524
3525 if (--pool->refcnt)
3526 return;
3527
3528
3529 if (WARN_ON(!(pool->flags & POOL_DISASSOCIATED)) ||
3530 WARN_ON(!list_empty(&pool->worklist)))
3531 return;
3532
3533
3534 if (pool->id >= 0)
3535 idr_remove(&worker_pool_idr, pool->id);
3536 hash_del(&pool->hash_node);
3537
3538
3539
3540
3541
3542
3543 mutex_lock(&pool->manager_arb);
3544 mutex_lock(&pool->manager_mutex);
3545 spin_lock_irq(&pool->lock);
3546
3547 while ((worker = first_worker(pool)))
3548 destroy_worker(worker);
3549 WARN_ON(pool->nr_workers || pool->nr_idle);
3550
3551 spin_unlock_irq(&pool->lock);
3552 mutex_unlock(&pool->manager_mutex);
3553 mutex_unlock(&pool->manager_arb);
3554
3555
3556 del_timer_sync(&pool->idle_timer);
3557 del_timer_sync(&pool->mayday_timer);
3558
3559
3560 call_rcu_sched(&pool->rcu, rcu_free_pool);
3561}
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3575{
3576 u32 hash = wqattrs_hash(attrs);
3577 struct worker_pool *pool;
3578 int node;
3579
3580 lockdep_assert_held(&wq_pool_mutex);
3581
3582
3583 hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3584 if (wqattrs_equal(pool->attrs, attrs)) {
3585 pool->refcnt++;
3586 goto out_unlock;
3587 }
3588 }
3589
3590
3591 pool = kzalloc(sizeof(*pool), GFP_KERNEL);
3592 if (!pool || init_worker_pool(pool) < 0)
3593 goto fail;
3594
3595 if (workqueue_freezing)
3596 pool->flags |= POOL_FREEZING;
3597
3598 lockdep_set_subclass(&pool->lock, 1);
3599 copy_workqueue_attrs(pool->attrs, attrs);
3600
3601
3602
3603
3604
3605 pool->attrs->no_numa = false;
3606
3607
3608 if (wq_numa_enabled) {
3609 for_each_node(node) {
3610 if (cpumask_subset(pool->attrs->cpumask,
3611 wq_numa_possible_cpumask[node])) {
3612 pool->node = node;
3613 break;
3614 }
3615 }
3616 }
3617
3618 if (worker_pool_assign_id(pool) < 0)
3619 goto fail;
3620
3621
3622 if (create_and_start_worker(pool) < 0)
3623 goto fail;
3624
3625
3626 hash_add(unbound_pool_hash, &pool->hash_node, hash);
3627out_unlock:
3628 return pool;
3629fail:
3630 if (pool)
3631 put_unbound_pool(pool);
3632 return NULL;
3633}
3634
3635static void rcu_free_pwq(struct rcu_head *rcu)
3636{
3637 kmem_cache_free(pwq_cache,
3638 container_of(rcu, struct pool_workqueue, rcu));
3639}
3640
3641
3642
3643
3644
3645static void pwq_unbound_release_workfn(struct work_struct *work)
3646{
3647 struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3648 unbound_release_work);
3649 struct workqueue_struct *wq = pwq->wq;
3650 struct worker_pool *pool = pwq->pool;
3651 bool is_last;
3652
3653 if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3654 return;
3655
3656
3657
3658
3659
3660
3661 mutex_lock(&wq->mutex);
3662 list_del_rcu(&pwq->pwqs_node);
3663 is_last = list_empty(&wq->pwqs);
3664 mutex_unlock(&wq->mutex);
3665
3666 mutex_lock(&wq_pool_mutex);
3667 put_unbound_pool(pool);
3668 mutex_unlock(&wq_pool_mutex);
3669
3670 call_rcu_sched(&pwq->rcu, rcu_free_pwq);
3671
3672
3673
3674
3675
3676 if (is_last) {
3677 free_workqueue_attrs(wq->unbound_attrs);
3678 kfree(wq);
3679 }
3680}
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3691{
3692 struct workqueue_struct *wq = pwq->wq;
3693 bool freezable = wq->flags & WQ_FREEZABLE;
3694
3695
3696 lockdep_assert_held(&wq->mutex);
3697
3698
3699 if (!freezable && pwq->max_active == wq->saved_max_active)
3700 return;
3701
3702 spin_lock_irq(&pwq->pool->lock);
3703
3704 if (!freezable || !(pwq->pool->flags & POOL_FREEZING)) {
3705 pwq->max_active = wq->saved_max_active;
3706
3707 while (!list_empty(&pwq->delayed_works) &&
3708 pwq->nr_active < pwq->max_active)
3709 pwq_activate_first_delayed(pwq);
3710
3711
3712
3713
3714
3715 wake_up_worker(pwq->pool);
3716 } else {
3717 pwq->max_active = 0;
3718 }
3719
3720 spin_unlock_irq(&pwq->pool->lock);
3721}
3722
3723
3724static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3725 struct worker_pool *pool)
3726{
3727 BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3728
3729 memset(pwq, 0, sizeof(*pwq));
3730
3731 pwq->pool = pool;
3732 pwq->wq = wq;
3733 pwq->flush_color = -1;
3734 pwq->refcnt = 1;
3735 INIT_LIST_HEAD(&pwq->delayed_works);
3736 INIT_LIST_HEAD(&pwq->pwqs_node);
3737 INIT_LIST_HEAD(&pwq->mayday_node);
3738 INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3739}
3740
3741
3742static void link_pwq(struct pool_workqueue *pwq)
3743{
3744 struct workqueue_struct *wq = pwq->wq;
3745
3746 lockdep_assert_held(&wq->mutex);
3747
3748
3749 if (!list_empty(&pwq->pwqs_node))
3750 return;
3751
3752
3753
3754
3755
3756 pwq->work_color = wq->work_color;
3757
3758
3759 pwq_adjust_max_active(pwq);
3760
3761
3762 list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3763}
3764
3765
3766static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3767 const struct workqueue_attrs *attrs)
3768{
3769 struct worker_pool *pool;
3770 struct pool_workqueue *pwq;
3771
3772 lockdep_assert_held(&wq_pool_mutex);
3773
3774 pool = get_unbound_pool(attrs);
3775 if (!pool)
3776 return NULL;
3777
3778 pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3779 if (!pwq) {
3780 put_unbound_pool(pool);
3781 return NULL;
3782 }
3783
3784 init_pwq(pwq, wq, pool);
3785 return pwq;
3786}
3787
3788
3789static void free_unbound_pwq(struct pool_workqueue *pwq)
3790{
3791 lockdep_assert_held(&wq_pool_mutex);
3792
3793 if (pwq) {
3794 put_unbound_pool(pwq->pool);
3795 kmem_cache_free(pwq_cache, pwq);
3796 }
3797}
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3821 int cpu_going_down, cpumask_t *cpumask)
3822{
3823 if (!wq_numa_enabled || attrs->no_numa)
3824 goto use_dfl;
3825
3826
3827 cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3828 if (cpu_going_down >= 0)
3829 cpumask_clear_cpu(cpu_going_down, cpumask);
3830
3831 if (cpumask_empty(cpumask))
3832 goto use_dfl;
3833
3834
3835 cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3836 return !cpumask_equal(cpumask, attrs->cpumask);
3837
3838use_dfl:
3839 cpumask_copy(cpumask, attrs->cpumask);
3840 return false;
3841}
3842
3843
3844static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3845 int node,
3846 struct pool_workqueue *pwq)
3847{
3848 struct pool_workqueue *old_pwq;
3849
3850 lockdep_assert_held(&wq->mutex);
3851
3852
3853 link_pwq(pwq);
3854
3855 old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3856 rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3857 return old_pwq;
3858}
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875int apply_workqueue_attrs(struct workqueue_struct *wq,
3876 const struct workqueue_attrs *attrs)
3877{
3878 struct workqueue_attrs *new_attrs, *tmp_attrs;
3879 struct pool_workqueue **pwq_tbl, *dfl_pwq;
3880 int node, ret;
3881
3882
3883 if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
3884 return -EINVAL;
3885
3886
3887 if (WARN_ON((wq->flags & __WQ_ORDERED) && !list_empty(&wq->pwqs)))
3888 return -EINVAL;
3889
3890 pwq_tbl = kzalloc(wq_numa_tbl_len * sizeof(pwq_tbl[0]), GFP_KERNEL);
3891 new_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3892 tmp_attrs = alloc_workqueue_attrs(GFP_KERNEL);
3893 if (!pwq_tbl || !new_attrs || !tmp_attrs)
3894 goto enomem;
3895
3896
3897 copy_workqueue_attrs(new_attrs, attrs);
3898 cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
3899
3900
3901
3902
3903
3904
3905 copy_workqueue_attrs(tmp_attrs, new_attrs);
3906
3907
3908
3909
3910
3911
3912 get_online_cpus();
3913
3914 mutex_lock(&wq_pool_mutex);
3915
3916
3917
3918
3919
3920
3921 dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3922 if (!dfl_pwq)
3923 goto enomem_pwq;
3924
3925 for_each_node(node) {
3926 if (wq_calc_node_cpumask(attrs, node, -1, tmp_attrs->cpumask)) {
3927 pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
3928 if (!pwq_tbl[node])
3929 goto enomem_pwq;
3930 } else {
3931 dfl_pwq->refcnt++;
3932 pwq_tbl[node] = dfl_pwq;
3933 }
3934 }
3935
3936 mutex_unlock(&wq_pool_mutex);
3937
3938
3939 mutex_lock(&wq->mutex);
3940
3941 copy_workqueue_attrs(wq->unbound_attrs, new_attrs);
3942
3943
3944 for_each_node(node)
3945 pwq_tbl[node] = numa_pwq_tbl_install(wq, node, pwq_tbl[node]);
3946
3947
3948 link_pwq(dfl_pwq);
3949 swap(wq->dfl_pwq, dfl_pwq);
3950
3951 mutex_unlock(&wq->mutex);
3952
3953
3954 for_each_node(node)
3955 put_pwq_unlocked(pwq_tbl[node]);
3956 put_pwq_unlocked(dfl_pwq);
3957
3958 put_online_cpus();
3959 ret = 0;
3960
3961out_free:
3962 free_workqueue_attrs(tmp_attrs);
3963 free_workqueue_attrs(new_attrs);
3964 kfree(pwq_tbl);
3965 return ret;
3966
3967enomem_pwq:
3968 free_unbound_pwq(dfl_pwq);
3969 for_each_node(node)
3970 if (pwq_tbl && pwq_tbl[node] != dfl_pwq)
3971 free_unbound_pwq(pwq_tbl[node]);
3972 mutex_unlock(&wq_pool_mutex);
3973 put_online_cpus();
3974enomem:
3975 ret = -ENOMEM;
3976 goto out_free;
3977}
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4002 bool online)
4003{
4004 int node = cpu_to_node(cpu);
4005 int cpu_off = online ? -1 : cpu;
4006 struct pool_workqueue *old_pwq = NULL, *pwq;
4007 struct workqueue_attrs *target_attrs;
4008 cpumask_t *cpumask;
4009
4010 lockdep_assert_held(&wq_pool_mutex);
4011
4012 if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND))
4013 return;
4014
4015
4016
4017
4018
4019
4020 target_attrs = wq_update_unbound_numa_attrs_buf;
4021 cpumask = target_attrs->cpumask;
4022
4023 mutex_lock(&wq->mutex);
4024 if (wq->unbound_attrs->no_numa)
4025 goto out_unlock;
4026
4027 copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4028 pwq = unbound_pwq_by_node(wq, node);
4029
4030
4031
4032
4033
4034
4035
4036
4037 if (wq_calc_node_cpumask(wq->unbound_attrs, node, cpu_off, cpumask)) {
4038 if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4039 goto out_unlock;
4040 } else {
4041 if (pwq == wq->dfl_pwq)
4042 goto out_unlock;
4043 else
4044 goto use_dfl_pwq;
4045 }
4046
4047 mutex_unlock(&wq->mutex);
4048
4049
4050 pwq = alloc_unbound_pwq(wq, target_attrs);
4051 if (!pwq) {
4052 pr_warning("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4053 wq->name);
4054 goto out_unlock;
4055 }
4056
4057
4058
4059
4060
4061
4062
4063 mutex_lock(&wq->mutex);
4064 old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4065 goto out_unlock;
4066
4067use_dfl_pwq:
4068 spin_lock_irq(&wq->dfl_pwq->pool->lock);
4069 get_pwq(wq->dfl_pwq);
4070 spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4071 old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4072out_unlock:
4073 mutex_unlock(&wq->mutex);
4074 put_pwq_unlocked(old_pwq);
4075}
4076
4077static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4078{
4079 bool highpri = wq->flags & WQ_HIGHPRI;
4080 int cpu;
4081
4082 if (!(wq->flags & WQ_UNBOUND)) {
4083 wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4084 if (!wq->cpu_pwqs)
4085 return -ENOMEM;
4086
4087 for_each_possible_cpu(cpu) {
4088 struct pool_workqueue *pwq =
4089 per_cpu_ptr(wq->cpu_pwqs, cpu);
4090 struct worker_pool *cpu_pools =
4091 per_cpu(cpu_worker_pools, cpu);
4092
4093 init_pwq(pwq, wq, &cpu_pools[highpri]);
4094
4095 mutex_lock(&wq->mutex);
4096 link_pwq(pwq);
4097 mutex_unlock(&wq->mutex);
4098 }
4099 return 0;
4100 } else {
4101 return apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4102 }
4103}
4104
4105static int wq_clamp_max_active(int max_active, unsigned int flags,
4106 const char *name)
4107{
4108 int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4109
4110 if (max_active < 1 || max_active > lim)
4111 pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4112 max_active, name, 1, lim);
4113
4114 return clamp_val(max_active, 1, lim);
4115}
4116
4117struct workqueue_struct *__alloc_workqueue_key(const char *fmt,
4118 unsigned int flags,
4119 int max_active,
4120 struct lock_class_key *key,
4121 const char *lock_name, ...)
4122{
4123 size_t tbl_size = 0;
4124 va_list args;
4125 struct workqueue_struct *wq;
4126 struct pool_workqueue *pwq;
4127
4128
4129 if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4130 flags |= WQ_UNBOUND;
4131
4132
4133 if (flags & WQ_UNBOUND)
4134 tbl_size = wq_numa_tbl_len * sizeof(wq->numa_pwq_tbl[0]);
4135
4136 wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4137 if (!wq)
4138 return NULL;
4139
4140 if (flags & WQ_UNBOUND) {
4141 wq->unbound_attrs = alloc_workqueue_attrs(GFP_KERNEL);
4142 if (!wq->unbound_attrs)
4143 goto err_free_wq;
4144 }
4145
4146 va_start(args, lock_name);
4147 vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4148 va_end(args);
4149
4150 max_active = max_active ?: WQ_DFL_ACTIVE;
4151 max_active = wq_clamp_max_active(max_active, flags, wq->name);
4152
4153
4154 wq->flags = flags;
4155 wq->saved_max_active = max_active;
4156 mutex_init(&wq->mutex);
4157 atomic_set(&wq->nr_pwqs_to_flush, 0);
4158 INIT_LIST_HEAD(&wq->pwqs);
4159 INIT_LIST_HEAD(&wq->flusher_queue);
4160 INIT_LIST_HEAD(&wq->flusher_overflow);
4161 INIT_LIST_HEAD(&wq->maydays);
4162
4163 lockdep_init_map(&wq->lockdep_map, lock_name, key, 0);
4164 INIT_LIST_HEAD(&wq->list);
4165
4166 if (alloc_and_link_pwqs(wq) < 0)
4167 goto err_free_wq;
4168
4169
4170
4171
4172
4173 if (flags & WQ_MEM_RECLAIM) {
4174 struct worker *rescuer;
4175
4176 rescuer = alloc_worker();
4177 if (!rescuer)
4178 goto err_destroy;
4179
4180 rescuer->rescue_wq = wq;
4181 rescuer->task = kthread_create(rescuer_thread, rescuer, "%s",
4182 wq->name);
4183 if (IS_ERR(rescuer->task)) {
4184 kfree(rescuer);
4185 goto err_destroy;
4186 }
4187
4188 wq->rescuer = rescuer;
4189 rescuer->task->flags |= PF_NO_SETAFFINITY;
4190 wake_up_process(rescuer->task);
4191 }
4192
4193 if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4194 goto err_destroy;
4195
4196
4197
4198
4199
4200
4201 mutex_lock(&wq_pool_mutex);
4202
4203 mutex_lock(&wq->mutex);
4204 for_each_pwq(pwq, wq)
4205 pwq_adjust_max_active(pwq);
4206 mutex_unlock(&wq->mutex);
4207
4208 list_add(&wq->list, &workqueues);
4209
4210 mutex_unlock(&wq_pool_mutex);
4211
4212 return wq;
4213
4214err_free_wq:
4215 free_workqueue_attrs(wq->unbound_attrs);
4216 kfree(wq);
4217 return NULL;
4218err_destroy:
4219 destroy_workqueue(wq);
4220 return NULL;
4221}
4222EXPORT_SYMBOL_GPL(__alloc_workqueue_key);
4223
4224
4225
4226
4227
4228
4229
4230void destroy_workqueue(struct workqueue_struct *wq)
4231{
4232 struct pool_workqueue *pwq;
4233 int node;
4234
4235
4236 drain_workqueue(wq);
4237
4238
4239 mutex_lock(&wq->mutex);
4240 for_each_pwq(pwq, wq) {
4241 int i;
4242
4243 for (i = 0; i < WORK_NR_COLORS; i++) {
4244 if (WARN_ON(pwq->nr_in_flight[i])) {
4245 mutex_unlock(&wq->mutex);
4246 return;
4247 }
4248 }
4249
4250 if (WARN_ON((pwq != wq->dfl_pwq) && (pwq->refcnt > 1)) ||
4251 WARN_ON(pwq->nr_active) ||
4252 WARN_ON(!list_empty(&pwq->delayed_works))) {
4253 mutex_unlock(&wq->mutex);
4254 return;
4255 }
4256 }
4257 mutex_unlock(&wq->mutex);
4258
4259
4260
4261
4262
4263 mutex_lock(&wq_pool_mutex);
4264 list_del_init(&wq->list);
4265 mutex_unlock(&wq_pool_mutex);
4266
4267 workqueue_sysfs_unregister(wq);
4268
4269 if (wq->rescuer) {
4270 kthread_stop(wq->rescuer->task);
4271 kfree(wq->rescuer);
4272 wq->rescuer = NULL;
4273 }
4274
4275 if (!(wq->flags & WQ_UNBOUND)) {
4276
4277
4278
4279
4280 free_percpu(wq->cpu_pwqs);
4281 kfree(wq);
4282 } else {
4283
4284
4285
4286
4287
4288 for_each_node(node) {
4289 pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4290 RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4291 put_pwq_unlocked(pwq);
4292 }
4293
4294
4295
4296
4297
4298 pwq = wq->dfl_pwq;
4299 wq->dfl_pwq = NULL;
4300 put_pwq_unlocked(pwq);
4301 }
4302}
4303EXPORT_SYMBOL_GPL(destroy_workqueue);
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4316{
4317 struct pool_workqueue *pwq;
4318
4319
4320 if (WARN_ON(wq->flags & __WQ_ORDERED))
4321 return;
4322
4323 max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4324
4325 mutex_lock(&wq->mutex);
4326
4327 wq->saved_max_active = max_active;
4328
4329 for_each_pwq(pwq, wq)
4330 pwq_adjust_max_active(pwq);
4331
4332 mutex_unlock(&wq->mutex);
4333}
4334EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4335
4336
4337
4338
4339
4340
4341
4342bool current_is_workqueue_rescuer(void)
4343{
4344 struct worker *worker = current_wq_worker();
4345
4346 return worker && worker->rescue_wq;
4347}
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4368{
4369 struct pool_workqueue *pwq;
4370 bool ret;
4371
4372 rcu_read_lock_sched();
4373
4374 if (cpu == WORK_CPU_UNBOUND)
4375 cpu = smp_processor_id();
4376
4377 if (!(wq->flags & WQ_UNBOUND))
4378 pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4379 else
4380 pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4381
4382 ret = !list_empty(&pwq->delayed_works);
4383 rcu_read_unlock_sched();
4384
4385 return ret;
4386}
4387EXPORT_SYMBOL_GPL(workqueue_congested);
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400unsigned int work_busy(struct work_struct *work)
4401{
4402 struct worker_pool *pool;
4403 unsigned long flags;
4404 unsigned int ret = 0;
4405
4406 if (work_pending(work))
4407 ret |= WORK_BUSY_PENDING;
4408
4409 local_irq_save(flags);
4410 pool = get_work_pool(work);
4411 if (pool) {
4412 spin_lock(&pool->lock);
4413 if (find_worker_executing_work(pool, work))
4414 ret |= WORK_BUSY_RUNNING;
4415 spin_unlock(&pool->lock);
4416 }
4417 local_irq_restore(flags);
4418
4419 return ret;
4420}
4421EXPORT_SYMBOL_GPL(work_busy);
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433void set_worker_desc(const char *fmt, ...)
4434{
4435 struct worker *worker = current_wq_worker();
4436 va_list args;
4437
4438 if (worker) {
4439 va_start(args, fmt);
4440 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4441 va_end(args);
4442 worker->desc_valid = true;
4443 }
4444}
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459void print_worker_info(const char *log_lvl, struct task_struct *task)
4460{
4461 work_func_t *fn = NULL;
4462 char name[WQ_NAME_LEN] = { };
4463 char desc[WORKER_DESC_LEN] = { };
4464 struct pool_workqueue *pwq = NULL;
4465 struct workqueue_struct *wq = NULL;
4466 bool desc_valid = false;
4467 struct worker *worker;
4468
4469 if (!(task->flags & PF_WQ_WORKER))
4470 return;
4471
4472
4473
4474
4475
4476 worker = probe_kthread_data(task);
4477
4478
4479
4480
4481
4482 probe_kernel_read(&fn, &worker->current_func, sizeof(fn));
4483 probe_kernel_read(&pwq, &worker->current_pwq, sizeof(pwq));
4484 probe_kernel_read(&wq, &pwq->wq, sizeof(wq));
4485 probe_kernel_read(name, wq->name, sizeof(name) - 1);
4486
4487
4488 probe_kernel_read(&desc_valid, &worker->desc_valid, sizeof(desc_valid));
4489 if (desc_valid)
4490 probe_kernel_read(desc, worker->desc, sizeof(desc) - 1);
4491
4492 if (fn || name[0] || desc[0]) {
4493 printk("%sWorkqueue: %s %pf", log_lvl, name, fn);
4494 if (desc[0])
4495 pr_cont(" (%s)", desc);
4496 pr_cont("\n");
4497 }
4498}
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515static void wq_unbind_fn(struct work_struct *work)
4516{
4517 int cpu = smp_processor_id();
4518 struct worker_pool *pool;
4519 struct worker *worker;
4520 int wi;
4521
4522 for_each_cpu_worker_pool(pool, cpu) {
4523 WARN_ON_ONCE(cpu != smp_processor_id());
4524
4525 mutex_lock(&pool->manager_mutex);
4526 spin_lock_irq(&pool->lock);
4527
4528
4529
4530
4531
4532
4533
4534
4535 for_each_pool_worker(worker, wi, pool)
4536 worker->flags |= WORKER_UNBOUND;
4537
4538 pool->flags |= POOL_DISASSOCIATED;
4539
4540 spin_unlock_irq(&pool->lock);
4541 mutex_unlock(&pool->manager_mutex);
4542
4543
4544
4545
4546
4547
4548
4549 schedule();
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559 atomic_set(&pool->nr_running, 0);
4560
4561
4562
4563
4564
4565
4566 spin_lock_irq(&pool->lock);
4567 wake_up_worker(pool);
4568 spin_unlock_irq(&pool->lock);
4569 }
4570}
4571
4572
4573
4574
4575
4576
4577
4578static void rebind_workers(struct worker_pool *pool)
4579{
4580 struct worker *worker;
4581 int wi;
4582
4583 lockdep_assert_held(&pool->manager_mutex);
4584
4585
4586
4587
4588
4589
4590
4591
4592 for_each_pool_worker(worker, wi, pool)
4593 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4594 pool->attrs->cpumask) < 0);
4595
4596 spin_lock_irq(&pool->lock);
4597
4598 for_each_pool_worker(worker, wi, pool) {
4599 unsigned int worker_flags = worker->flags;
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609 if (worker_flags & WORKER_IDLE)
4610 wake_up_process(worker->task);
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627 WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
4628 worker_flags |= WORKER_REBOUND;
4629 worker_flags &= ~WORKER_UNBOUND;
4630 ACCESS_ONCE(worker->flags) = worker_flags;
4631 }
4632
4633 spin_unlock_irq(&pool->lock);
4634}
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
4647{
4648 static cpumask_t cpumask;
4649 struct worker *worker;
4650 int wi;
4651
4652 lockdep_assert_held(&pool->manager_mutex);
4653
4654
4655 if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
4656 return;
4657
4658
4659 cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
4660 if (cpumask_weight(&cpumask) != 1)
4661 return;
4662
4663
4664 for_each_pool_worker(worker, wi, pool)
4665 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
4666 pool->attrs->cpumask) < 0);
4667}
4668
4669
4670
4671
4672
4673static int workqueue_cpu_up_callback(struct notifier_block *nfb,
4674 unsigned long action,
4675 void *hcpu)
4676{
4677 int cpu = (unsigned long)hcpu;
4678 struct worker_pool *pool;
4679 struct workqueue_struct *wq;
4680 int pi;
4681
4682 switch (action & ~CPU_TASKS_FROZEN) {
4683 case CPU_UP_PREPARE:
4684 for_each_cpu_worker_pool(pool, cpu) {
4685 if (pool->nr_workers)
4686 continue;
4687 if (create_and_start_worker(pool) < 0)
4688 return NOTIFY_BAD;
4689 }
4690 break;
4691
4692 case CPU_DOWN_FAILED:
4693 case CPU_ONLINE:
4694 mutex_lock(&wq_pool_mutex);
4695
4696 for_each_pool(pool, pi) {
4697 mutex_lock(&pool->manager_mutex);
4698
4699 if (pool->cpu == cpu) {
4700 spin_lock_irq(&pool->lock);
4701 pool->flags &= ~POOL_DISASSOCIATED;
4702 spin_unlock_irq(&pool->lock);
4703
4704 rebind_workers(pool);
4705 } else if (pool->cpu < 0) {
4706 restore_unbound_workers_cpumask(pool, cpu);
4707 }
4708
4709 mutex_unlock(&pool->manager_mutex);
4710 }
4711
4712
4713 list_for_each_entry(wq, &workqueues, list)
4714 wq_update_unbound_numa(wq, cpu, true);
4715
4716 mutex_unlock(&wq_pool_mutex);
4717 break;
4718 }
4719 return NOTIFY_OK;
4720}
4721
4722
4723
4724
4725
4726static int workqueue_cpu_down_callback(struct notifier_block *nfb,
4727 unsigned long action,
4728 void *hcpu)
4729{
4730 int cpu = (unsigned long)hcpu;
4731 struct work_struct unbind_work;
4732 struct workqueue_struct *wq;
4733
4734 switch (action & ~CPU_TASKS_FROZEN) {
4735 case CPU_DOWN_PREPARE:
4736
4737 INIT_WORK_ONSTACK(&unbind_work, wq_unbind_fn);
4738 queue_work_on(cpu, system_highpri_wq, &unbind_work);
4739
4740
4741 mutex_lock(&wq_pool_mutex);
4742 list_for_each_entry(wq, &workqueues, list)
4743 wq_update_unbound_numa(wq, cpu, false);
4744 mutex_unlock(&wq_pool_mutex);
4745
4746
4747 flush_work(&unbind_work);
4748 break;
4749 }
4750 return NOTIFY_OK;
4751}
4752
4753#ifdef CONFIG_SMP
4754
4755struct work_for_cpu {
4756 struct work_struct work;
4757 long (*fn)(void *);
4758 void *arg;
4759 long ret;
4760};
4761
4762static void work_for_cpu_fn(struct work_struct *work)
4763{
4764 struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
4765
4766 wfc->ret = wfc->fn(wfc->arg);
4767}
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
4780{
4781 struct work_for_cpu wfc = { .fn = fn, .arg = arg };
4782
4783 INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
4784 schedule_work_on(cpu, &wfc.work);
4785
4786
4787
4788
4789
4790
4791 __flush_work(&wfc.work);
4792
4793 return wfc.ret;
4794}
4795EXPORT_SYMBOL_GPL(work_on_cpu);
4796#endif
4797
4798#ifdef CONFIG_FREEZER
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810void freeze_workqueues_begin(void)
4811{
4812 struct worker_pool *pool;
4813 struct workqueue_struct *wq;
4814 struct pool_workqueue *pwq;
4815 int pi;
4816
4817 mutex_lock(&wq_pool_mutex);
4818
4819 WARN_ON_ONCE(workqueue_freezing);
4820 workqueue_freezing = true;
4821
4822
4823 for_each_pool(pool, pi) {
4824 spin_lock_irq(&pool->lock);
4825 WARN_ON_ONCE(pool->flags & POOL_FREEZING);
4826 pool->flags |= POOL_FREEZING;
4827 spin_unlock_irq(&pool->lock);
4828 }
4829
4830 list_for_each_entry(wq, &workqueues, list) {
4831 mutex_lock(&wq->mutex);
4832 for_each_pwq(pwq, wq)
4833 pwq_adjust_max_active(pwq);
4834 mutex_unlock(&wq->mutex);
4835 }
4836
4837 mutex_unlock(&wq_pool_mutex);
4838}
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853bool freeze_workqueues_busy(void)
4854{
4855 bool busy = false;
4856 struct workqueue_struct *wq;
4857 struct pool_workqueue *pwq;
4858
4859 mutex_lock(&wq_pool_mutex);
4860
4861 WARN_ON_ONCE(!workqueue_freezing);
4862
4863 list_for_each_entry(wq, &workqueues, list) {
4864 if (!(wq->flags & WQ_FREEZABLE))
4865 continue;
4866
4867
4868
4869
4870 rcu_read_lock_sched();
4871 for_each_pwq(pwq, wq) {
4872 WARN_ON_ONCE(pwq->nr_active < 0);
4873 if (pwq->nr_active) {
4874 busy = true;
4875 rcu_read_unlock_sched();
4876 goto out_unlock;
4877 }
4878 }
4879 rcu_read_unlock_sched();
4880 }
4881out_unlock:
4882 mutex_unlock(&wq_pool_mutex);
4883 return busy;
4884}
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895void thaw_workqueues(void)
4896{
4897 struct workqueue_struct *wq;
4898 struct pool_workqueue *pwq;
4899 struct worker_pool *pool;
4900 int pi;
4901
4902 mutex_lock(&wq_pool_mutex);
4903
4904 if (!workqueue_freezing)
4905 goto out_unlock;
4906
4907
4908 for_each_pool(pool, pi) {
4909 spin_lock_irq(&pool->lock);
4910 WARN_ON_ONCE(!(pool->flags & POOL_FREEZING));
4911 pool->flags &= ~POOL_FREEZING;
4912 spin_unlock_irq(&pool->lock);
4913 }
4914
4915
4916 list_for_each_entry(wq, &workqueues, list) {
4917 mutex_lock(&wq->mutex);
4918 for_each_pwq(pwq, wq)
4919 pwq_adjust_max_active(pwq);
4920 mutex_unlock(&wq->mutex);
4921 }
4922
4923 workqueue_freezing = false;
4924out_unlock:
4925 mutex_unlock(&wq_pool_mutex);
4926}
4927#endif
4928
4929static void __init wq_numa_init(void)
4930{
4931 cpumask_var_t *tbl;
4932 int node, cpu;
4933
4934
4935 for_each_node(node)
4936 wq_numa_tbl_len = max(wq_numa_tbl_len, node + 1);
4937
4938 if (num_possible_nodes() <= 1)
4939 return;
4940
4941 if (wq_disable_numa) {
4942 pr_info("workqueue: NUMA affinity support disabled\n");
4943 return;
4944 }
4945
4946 wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs(GFP_KERNEL);
4947 BUG_ON(!wq_update_unbound_numa_attrs_buf);
4948
4949
4950
4951
4952
4953
4954 tbl = kzalloc(wq_numa_tbl_len * sizeof(tbl[0]), GFP_KERNEL);
4955 BUG_ON(!tbl);
4956
4957 for_each_node(node)
4958 BUG_ON(!alloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
4959 node_online(node) ? node : NUMA_NO_NODE));
4960
4961 for_each_possible_cpu(cpu) {
4962 node = cpu_to_node(cpu);
4963 if (WARN_ON(node == NUMA_NO_NODE)) {
4964 pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
4965
4966 return;
4967 }
4968 cpumask_set_cpu(cpu, tbl[node]);
4969 }
4970
4971 wq_numa_possible_cpumask = tbl;
4972 wq_numa_enabled = true;
4973}
4974
4975static int __init init_workqueues(void)
4976{
4977 int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
4978 int i, cpu;
4979
4980
4981 BUILD_BUG_ON((1LU << (BITS_PER_LONG - WORK_OFFQ_POOL_SHIFT)) <
4982 WORK_CPU_END * NR_STD_WORKER_POOLS);
4983
4984 WARN_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
4985
4986 pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
4987
4988 cpu_notifier(workqueue_cpu_up_callback, CPU_PRI_WORKQUEUE_UP);
4989 hotcpu_notifier(workqueue_cpu_down_callback, CPU_PRI_WORKQUEUE_DOWN);
4990
4991 wq_numa_init();
4992
4993
4994 for_each_possible_cpu(cpu) {
4995 struct worker_pool *pool;
4996
4997 i = 0;
4998 for_each_cpu_worker_pool(pool, cpu) {
4999 BUG_ON(init_worker_pool(pool));
5000 pool->cpu = cpu;
5001 cpumask_copy(pool->attrs->cpumask, cpumask_of(cpu));
5002 pool->attrs->nice = std_nice[i++];
5003 pool->node = cpu_to_node(cpu);
5004
5005
5006 mutex_lock(&wq_pool_mutex);
5007 BUG_ON(worker_pool_assign_id(pool));
5008 mutex_unlock(&wq_pool_mutex);
5009 }
5010 }
5011
5012
5013 for_each_online_cpu(cpu) {
5014 struct worker_pool *pool;
5015
5016 for_each_cpu_worker_pool(pool, cpu) {
5017 pool->flags &= ~POOL_DISASSOCIATED;
5018 BUG_ON(create_and_start_worker(pool) < 0);
5019 }
5020 }
5021
5022
5023 for (i = 0; i < NR_STD_WORKER_POOLS; i++) {
5024 struct workqueue_attrs *attrs;
5025
5026 BUG_ON(!(attrs = alloc_workqueue_attrs(GFP_KERNEL)));
5027 attrs->nice = std_nice[i];
5028 unbound_std_wq_attrs[i] = attrs;
5029 }
5030
5031 system_wq = alloc_workqueue("events", 0, 0);
5032 system_highpri_wq = alloc_workqueue("events_highpri", WQ_HIGHPRI, 0);
5033 system_long_wq = alloc_workqueue("events_long", 0, 0);
5034 system_unbound_wq = alloc_workqueue("events_unbound", WQ_UNBOUND,
5035 WQ_UNBOUND_MAX_ACTIVE);
5036 system_freezable_wq = alloc_workqueue("events_freezable",
5037 WQ_FREEZABLE, 0);
5038 system_power_efficient_wq = alloc_workqueue("events_power_efficient",
5039 WQ_POWER_EFFICIENT, 0);
5040 system_freezable_power_efficient_wq = alloc_workqueue("events_freezable_power_efficient",
5041 WQ_FREEZABLE | WQ_POWER_EFFICIENT,
5042 0);
5043 BUG_ON(!system_wq || !system_highpri_wq || !system_long_wq ||
5044 !system_unbound_wq || !system_freezable_wq ||
5045 !system_power_efficient_wq ||
5046 !system_freezable_power_efficient_wq);
5047 return 0;
5048}
5049early_initcall(init_workqueues);
5050