linux/kernel/workqueue.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * kernel/workqueue.c - generic async execution with shared worker pool
   4 *
   5 * Copyright (C) 2002           Ingo Molnar
   6 *
   7 *   Derived from the taskqueue/keventd code by:
   8 *     David Woodhouse <dwmw2@infradead.org>
   9 *     Andrew Morton
  10 *     Kai Petzke <wpp@marie.physik.tu-berlin.de>
  11 *     Theodore Ts'o <tytso@mit.edu>
  12 *
  13 * Made to use alloc_percpu by Christoph Lameter.
  14 *
  15 * Copyright (C) 2010           SUSE Linux Products GmbH
  16 * Copyright (C) 2010           Tejun Heo <tj@kernel.org>
  17 *
  18 * This is the generic async execution mechanism.  Work items as are
  19 * executed in process context.  The worker pool is shared and
  20 * automatically managed.  There are two worker pools for each CPU (one for
  21 * normal work items and the other for high priority ones) and some extra
  22 * pools for workqueues which are not bound to any specific CPU - the
  23 * number of these backing pools is dynamic.
  24 *
  25 * Please read Documentation/core-api/workqueue.rst for details.
  26 */
  27
  28#include <linux/export.h>
  29#include <linux/kernel.h>
  30#include <linux/sched.h>
  31#include <linux/init.h>
  32#include <linux/signal.h>
  33#include <linux/completion.h>
  34#include <linux/workqueue.h>
  35#include <linux/slab.h>
  36#include <linux/cpu.h>
  37#include <linux/notifier.h>
  38#include <linux/kthread.h>
  39#include <linux/hardirq.h>
  40#include <linux/mempolicy.h>
  41#include <linux/freezer.h>
  42#include <linux/debug_locks.h>
  43#include <linux/lockdep.h>
  44#include <linux/idr.h>
  45#include <linux/jhash.h>
  46#include <linux/hashtable.h>
  47#include <linux/rculist.h>
  48#include <linux/nodemask.h>
  49#include <linux/moduleparam.h>
  50#include <linux/uaccess.h>
  51#include <linux/sched/isolation.h>
  52#include <linux/nmi.h>
  53#include <linux/kvm_para.h>
  54
  55#include "workqueue_internal.h"
  56
  57enum {
  58        /*
  59         * worker_pool flags
  60         *
  61         * A bound pool is either associated or disassociated with its CPU.
  62         * While associated (!DISASSOCIATED), all workers are bound to the
  63         * CPU and none has %WORKER_UNBOUND set and concurrency management
  64         * is in effect.
  65         *
  66         * While DISASSOCIATED, the cpu may be offline and all workers have
  67         * %WORKER_UNBOUND set and concurrency management disabled, and may
  68         * be executing on any CPU.  The pool behaves as an unbound one.
  69         *
  70         * Note that DISASSOCIATED should be flipped only while holding
  71         * wq_pool_attach_mutex to avoid changing binding state while
  72         * worker_attach_to_pool() is in progress.
  73         */
  74        POOL_MANAGER_ACTIVE     = 1 << 0,       /* being managed */
  75        POOL_DISASSOCIATED      = 1 << 2,       /* cpu can't serve workers */
  76
  77        /* worker flags */
  78        WORKER_DIE              = 1 << 1,       /* die die die */
  79        WORKER_IDLE             = 1 << 2,       /* is idle */
  80        WORKER_PREP             = 1 << 3,       /* preparing to run works */
  81        WORKER_CPU_INTENSIVE    = 1 << 6,       /* cpu intensive */
  82        WORKER_UNBOUND          = 1 << 7,       /* worker is unbound */
  83        WORKER_REBOUND          = 1 << 8,       /* worker was rebound */
  84
  85        WORKER_NOT_RUNNING      = WORKER_PREP | WORKER_CPU_INTENSIVE |
  86                                  WORKER_UNBOUND | WORKER_REBOUND,
  87
  88        NR_STD_WORKER_POOLS     = 2,            /* # standard pools per cpu */
  89
  90        UNBOUND_POOL_HASH_ORDER = 6,            /* hashed by pool->attrs */
  91        BUSY_WORKER_HASH_ORDER  = 6,            /* 64 pointers */
  92
  93        MAX_IDLE_WORKERS_RATIO  = 4,            /* 1/4 of busy can be idle */
  94        IDLE_WORKER_TIMEOUT     = 300 * HZ,     /* keep idle ones for 5 mins */
  95
  96        MAYDAY_INITIAL_TIMEOUT  = HZ / 100 >= 2 ? HZ / 100 : 2,
  97                                                /* call for help after 10ms
  98                                                   (min two ticks) */
  99        MAYDAY_INTERVAL         = HZ / 10,      /* and then every 100ms */
 100        CREATE_COOLDOWN         = HZ,           /* time to breath after fail */
 101
 102        /*
 103         * Rescue workers are used only on emergencies and shared by
 104         * all cpus.  Give MIN_NICE.
 105         */
 106        RESCUER_NICE_LEVEL      = MIN_NICE,
 107        HIGHPRI_NICE_LEVEL      = MIN_NICE,
 108
 109        WQ_NAME_LEN             = 24,
 110};
 111
 112/*
 113 * Structure fields follow one of the following exclusion rules.
 114 *
 115 * I: Modifiable by initialization/destruction paths and read-only for
 116 *    everyone else.
 117 *
 118 * P: Preemption protected.  Disabling preemption is enough and should
 119 *    only be modified and accessed from the local cpu.
 120 *
 121 * L: pool->lock protected.  Access with pool->lock held.
 122 *
 123 * X: During normal operation, modification requires pool->lock and should
 124 *    be done only from local cpu.  Either disabling preemption on local
 125 *    cpu or grabbing pool->lock is enough for read access.  If
 126 *    POOL_DISASSOCIATED is set, it's identical to L.
 127 *
 128 * A: wq_pool_attach_mutex protected.
 129 *
 130 * PL: wq_pool_mutex protected.
 131 *
 132 * PR: wq_pool_mutex protected for writes.  RCU protected for reads.
 133 *
 134 * PW: wq_pool_mutex and wq->mutex protected for writes.  Either for reads.
 135 *
 136 * PWR: wq_pool_mutex and wq->mutex protected for writes.  Either or
 137 *      RCU for reads.
 138 *
 139 * WQ: wq->mutex protected.
 140 *
 141 * WR: wq->mutex protected for writes.  RCU protected for reads.
 142 *
 143 * MD: wq_mayday_lock protected.
 144 */
 145
 146/* struct worker is defined in workqueue_internal.h */
 147
 148struct worker_pool {
 149        raw_spinlock_t          lock;           /* the pool lock */
 150        int                     cpu;            /* I: the associated cpu */
 151        int                     node;           /* I: the associated node ID */
 152        int                     id;             /* I: pool ID */
 153        unsigned int            flags;          /* X: flags */
 154
 155        unsigned long           watchdog_ts;    /* L: watchdog timestamp */
 156
 157        struct list_head        worklist;       /* L: list of pending works */
 158
 159        int                     nr_workers;     /* L: total number of workers */
 160        int                     nr_idle;        /* L: currently idle workers */
 161
 162        struct list_head        idle_list;      /* X: list of idle workers */
 163        struct timer_list       idle_timer;     /* L: worker idle timeout */
 164        struct timer_list       mayday_timer;   /* L: SOS timer for workers */
 165
 166        /* a workers is either on busy_hash or idle_list, or the manager */
 167        DECLARE_HASHTABLE(busy_hash, BUSY_WORKER_HASH_ORDER);
 168                                                /* L: hash of busy workers */
 169
 170        struct worker           *manager;       /* L: purely informational */
 171        struct list_head        workers;        /* A: attached workers */
 172        struct completion       *detach_completion; /* all workers detached */
 173
 174        struct ida              worker_ida;     /* worker IDs for task name */
 175
 176        struct workqueue_attrs  *attrs;         /* I: worker attributes */
 177        struct hlist_node       hash_node;      /* PL: unbound_pool_hash node */
 178        int                     refcnt;         /* PL: refcnt for unbound pools */
 179
 180        /*
 181         * The current concurrency level.  As it's likely to be accessed
 182         * from other CPUs during try_to_wake_up(), put it in a separate
 183         * cacheline.
 184         */
 185        atomic_t                nr_running ____cacheline_aligned_in_smp;
 186
 187        /*
 188         * Destruction of pool is RCU protected to allow dereferences
 189         * from get_work_pool().
 190         */
 191        struct rcu_head         rcu;
 192} ____cacheline_aligned_in_smp;
 193
 194/*
 195 * The per-pool workqueue.  While queued, the lower WORK_STRUCT_FLAG_BITS
 196 * of work_struct->data are used for flags and the remaining high bits
 197 * point to the pwq; thus, pwqs need to be aligned at two's power of the
 198 * number of flag bits.
 199 */
 200struct pool_workqueue {
 201        struct worker_pool      *pool;          /* I: the associated pool */
 202        struct workqueue_struct *wq;            /* I: the owning workqueue */
 203        int                     work_color;     /* L: current color */
 204        int                     flush_color;    /* L: flushing color */
 205        int                     refcnt;         /* L: reference count */
 206        int                     nr_in_flight[WORK_NR_COLORS];
 207                                                /* L: nr of in_flight works */
 208
 209        /*
 210         * nr_active management and WORK_STRUCT_INACTIVE:
 211         *
 212         * When pwq->nr_active >= max_active, new work item is queued to
 213         * pwq->inactive_works instead of pool->worklist and marked with
 214         * WORK_STRUCT_INACTIVE.
 215         *
 216         * All work items marked with WORK_STRUCT_INACTIVE do not participate
 217         * in pwq->nr_active and all work items in pwq->inactive_works are
 218         * marked with WORK_STRUCT_INACTIVE.  But not all WORK_STRUCT_INACTIVE
 219         * work items are in pwq->inactive_works.  Some of them are ready to
 220         * run in pool->worklist or worker->scheduled.  Those work itmes are
 221         * only struct wq_barrier which is used for flush_work() and should
 222         * not participate in pwq->nr_active.  For non-barrier work item, it
 223         * is marked with WORK_STRUCT_INACTIVE iff it is in pwq->inactive_works.
 224         */
 225        int                     nr_active;      /* L: nr of active works */
 226        int                     max_active;     /* L: max active works */
 227        struct list_head        inactive_works; /* L: inactive works */
 228        struct list_head        pwqs_node;      /* WR: node on wq->pwqs */
 229        struct list_head        mayday_node;    /* MD: node on wq->maydays */
 230
 231        /*
 232         * Release of unbound pwq is punted to system_wq.  See put_pwq()
 233         * and pwq_unbound_release_workfn() for details.  pool_workqueue
 234         * itself is also RCU protected so that the first pwq can be
 235         * determined without grabbing wq->mutex.
 236         */
 237        struct work_struct      unbound_release_work;
 238        struct rcu_head         rcu;
 239} __aligned(1 << WORK_STRUCT_FLAG_BITS);
 240
 241/*
 242 * Structure used to wait for workqueue flush.
 243 */
 244struct wq_flusher {
 245        struct list_head        list;           /* WQ: list of flushers */
 246        int                     flush_color;    /* WQ: flush color waiting for */
 247        struct completion       done;           /* flush completion */
 248};
 249
 250struct wq_device;
 251
 252/*
 253 * The externally visible workqueue.  It relays the issued work items to
 254 * the appropriate worker_pool through its pool_workqueues.
 255 */
 256struct workqueue_struct {
 257        struct list_head        pwqs;           /* WR: all pwqs of this wq */
 258        struct list_head        list;           /* PR: list of all workqueues */
 259
 260        struct mutex            mutex;          /* protects this wq */
 261        int                     work_color;     /* WQ: current work color */
 262        int                     flush_color;    /* WQ: current flush color */
 263        atomic_t                nr_pwqs_to_flush; /* flush in progress */
 264        struct wq_flusher       *first_flusher; /* WQ: first flusher */
 265        struct list_head        flusher_queue;  /* WQ: flush waiters */
 266        struct list_head        flusher_overflow; /* WQ: flush overflow list */
 267
 268        struct list_head        maydays;        /* MD: pwqs requesting rescue */
 269        struct worker           *rescuer;       /* MD: rescue worker */
 270
 271        int                     nr_drainers;    /* WQ: drain in progress */
 272        int                     saved_max_active; /* WQ: saved pwq max_active */
 273
 274        struct workqueue_attrs  *unbound_attrs; /* PW: only for unbound wqs */
 275        struct pool_workqueue   *dfl_pwq;       /* PW: only for unbound wqs */
 276
 277#ifdef CONFIG_SYSFS
 278        struct wq_device        *wq_dev;        /* I: for sysfs interface */
 279#endif
 280#ifdef CONFIG_LOCKDEP
 281        char                    *lock_name;
 282        struct lock_class_key   key;
 283        struct lockdep_map      lockdep_map;
 284#endif
 285        char                    name[WQ_NAME_LEN]; /* I: workqueue name */
 286
 287        /*
 288         * Destruction of workqueue_struct is RCU protected to allow walking
 289         * the workqueues list without grabbing wq_pool_mutex.
 290         * This is used to dump all workqueues from sysrq.
 291         */
 292        struct rcu_head         rcu;
 293
 294        /* hot fields used during command issue, aligned to cacheline */
 295        unsigned int            flags ____cacheline_aligned; /* WQ: WQ_* flags */
 296        struct pool_workqueue __percpu *cpu_pwqs; /* I: per-cpu pwqs */
 297        struct pool_workqueue __rcu *numa_pwq_tbl[]; /* PWR: unbound pwqs indexed by node */
 298};
 299
 300static struct kmem_cache *pwq_cache;
 301
 302static cpumask_var_t *wq_numa_possible_cpumask;
 303                                        /* possible CPUs of each node */
 304
 305static bool wq_disable_numa;
 306module_param_named(disable_numa, wq_disable_numa, bool, 0444);
 307
 308/* see the comment above the definition of WQ_POWER_EFFICIENT */
 309static bool wq_power_efficient = IS_ENABLED(CONFIG_WQ_POWER_EFFICIENT_DEFAULT);
 310module_param_named(power_efficient, wq_power_efficient, bool, 0444);
 311
 312static bool wq_online;                  /* can kworkers be created yet? */
 313
 314static bool wq_numa_enabled;            /* unbound NUMA affinity enabled */
 315
 316/* buf for wq_update_unbound_numa_attrs(), protected by CPU hotplug exclusion */
 317static struct workqueue_attrs *wq_update_unbound_numa_attrs_buf;
 318
 319static DEFINE_MUTEX(wq_pool_mutex);     /* protects pools and workqueues list */
 320static DEFINE_MUTEX(wq_pool_attach_mutex); /* protects worker attach/detach */
 321static DEFINE_RAW_SPINLOCK(wq_mayday_lock);     /* protects wq->maydays list */
 322/* wait for manager to go away */
 323static struct rcuwait manager_wait = __RCUWAIT_INITIALIZER(manager_wait);
 324
 325static LIST_HEAD(workqueues);           /* PR: list of all workqueues */
 326static bool workqueue_freezing;         /* PL: have wqs started freezing? */
 327
 328/* PL: allowable cpus for unbound wqs and work items */
 329static cpumask_var_t wq_unbound_cpumask;
 330
 331/* CPU where unbound work was last round robin scheduled from this CPU */
 332static DEFINE_PER_CPU(int, wq_rr_cpu_last);
 333
 334/*
 335 * Local execution of unbound work items is no longer guaranteed.  The
 336 * following always forces round-robin CPU selection on unbound work items
 337 * to uncover usages which depend on it.
 338 */
 339#ifdef CONFIG_DEBUG_WQ_FORCE_RR_CPU
 340static bool wq_debug_force_rr_cpu = true;
 341#else
 342static bool wq_debug_force_rr_cpu = false;
 343#endif
 344module_param_named(debug_force_rr_cpu, wq_debug_force_rr_cpu, bool, 0644);
 345
 346/* the per-cpu worker pools */
 347static DEFINE_PER_CPU_SHARED_ALIGNED(struct worker_pool [NR_STD_WORKER_POOLS], cpu_worker_pools);
 348
 349static DEFINE_IDR(worker_pool_idr);     /* PR: idr of all pools */
 350
 351/* PL: hash of all unbound pools keyed by pool->attrs */
 352static DEFINE_HASHTABLE(unbound_pool_hash, UNBOUND_POOL_HASH_ORDER);
 353
 354/* I: attributes used when instantiating standard unbound pools on demand */
 355static struct workqueue_attrs *unbound_std_wq_attrs[NR_STD_WORKER_POOLS];
 356
 357/* I: attributes used when instantiating ordered pools on demand */
 358static struct workqueue_attrs *ordered_wq_attrs[NR_STD_WORKER_POOLS];
 359
 360struct workqueue_struct *system_wq __read_mostly;
 361EXPORT_SYMBOL(system_wq);
 362struct workqueue_struct *system_highpri_wq __read_mostly;
 363EXPORT_SYMBOL_GPL(system_highpri_wq);
 364struct workqueue_struct *system_long_wq __read_mostly;
 365EXPORT_SYMBOL_GPL(system_long_wq);
 366struct workqueue_struct *system_unbound_wq __read_mostly;
 367EXPORT_SYMBOL_GPL(system_unbound_wq);
 368struct workqueue_struct *system_freezable_wq __read_mostly;
 369EXPORT_SYMBOL_GPL(system_freezable_wq);
 370struct workqueue_struct *system_power_efficient_wq __read_mostly;
 371EXPORT_SYMBOL_GPL(system_power_efficient_wq);
 372struct workqueue_struct *system_freezable_power_efficient_wq __read_mostly;
 373EXPORT_SYMBOL_GPL(system_freezable_power_efficient_wq);
 374
 375static int worker_thread(void *__worker);
 376static void workqueue_sysfs_unregister(struct workqueue_struct *wq);
 377static void show_pwq(struct pool_workqueue *pwq);
 378
 379#define CREATE_TRACE_POINTS
 380#include <trace/events/workqueue.h>
 381
 382#define assert_rcu_or_pool_mutex()                                      \
 383        RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&                       \
 384                         !lockdep_is_held(&wq_pool_mutex),              \
 385                         "RCU or wq_pool_mutex should be held")
 386
 387#define assert_rcu_or_wq_mutex_or_pool_mutex(wq)                        \
 388        RCU_LOCKDEP_WARN(!rcu_read_lock_held() &&                       \
 389                         !lockdep_is_held(&wq->mutex) &&                \
 390                         !lockdep_is_held(&wq_pool_mutex),              \
 391                         "RCU, wq->mutex or wq_pool_mutex should be held")
 392
 393#define for_each_cpu_worker_pool(pool, cpu)                             \
 394        for ((pool) = &per_cpu(cpu_worker_pools, cpu)[0];               \
 395             (pool) < &per_cpu(cpu_worker_pools, cpu)[NR_STD_WORKER_POOLS]; \
 396             (pool)++)
 397
 398/**
 399 * for_each_pool - iterate through all worker_pools in the system
 400 * @pool: iteration cursor
 401 * @pi: integer used for iteration
 402 *
 403 * This must be called either with wq_pool_mutex held or RCU read
 404 * locked.  If the pool needs to be used beyond the locking in effect, the
 405 * caller is responsible for guaranteeing that the pool stays online.
 406 *
 407 * The if/else clause exists only for the lockdep assertion and can be
 408 * ignored.
 409 */
 410#define for_each_pool(pool, pi)                                         \
 411        idr_for_each_entry(&worker_pool_idr, pool, pi)                  \
 412                if (({ assert_rcu_or_pool_mutex(); false; })) { }       \
 413                else
 414
 415/**
 416 * for_each_pool_worker - iterate through all workers of a worker_pool
 417 * @worker: iteration cursor
 418 * @pool: worker_pool to iterate workers of
 419 *
 420 * This must be called with wq_pool_attach_mutex.
 421 *
 422 * The if/else clause exists only for the lockdep assertion and can be
 423 * ignored.
 424 */
 425#define for_each_pool_worker(worker, pool)                              \
 426        list_for_each_entry((worker), &(pool)->workers, node)           \
 427                if (({ lockdep_assert_held(&wq_pool_attach_mutex); false; })) { } \
 428                else
 429
 430/**
 431 * for_each_pwq - iterate through all pool_workqueues of the specified workqueue
 432 * @pwq: iteration cursor
 433 * @wq: the target workqueue
 434 *
 435 * This must be called either with wq->mutex held or RCU read locked.
 436 * If the pwq needs to be used beyond the locking in effect, the caller is
 437 * responsible for guaranteeing that the pwq stays online.
 438 *
 439 * The if/else clause exists only for the lockdep assertion and can be
 440 * ignored.
 441 */
 442#define for_each_pwq(pwq, wq)                                           \
 443        list_for_each_entry_rcu((pwq), &(wq)->pwqs, pwqs_node,          \
 444                                 lockdep_is_held(&(wq->mutex)))
 445
 446#ifdef CONFIG_DEBUG_OBJECTS_WORK
 447
 448static const struct debug_obj_descr work_debug_descr;
 449
 450static void *work_debug_hint(void *addr)
 451{
 452        return ((struct work_struct *) addr)->func;
 453}
 454
 455static bool work_is_static_object(void *addr)
 456{
 457        struct work_struct *work = addr;
 458
 459        return test_bit(WORK_STRUCT_STATIC_BIT, work_data_bits(work));
 460}
 461
 462/*
 463 * fixup_init is called when:
 464 * - an active object is initialized
 465 */
 466static bool work_fixup_init(void *addr, enum debug_obj_state state)
 467{
 468        struct work_struct *work = addr;
 469
 470        switch (state) {
 471        case ODEBUG_STATE_ACTIVE:
 472                cancel_work_sync(work);
 473                debug_object_init(work, &work_debug_descr);
 474                return true;
 475        default:
 476                return false;
 477        }
 478}
 479
 480/*
 481 * fixup_free is called when:
 482 * - an active object is freed
 483 */
 484static bool work_fixup_free(void *addr, enum debug_obj_state state)
 485{
 486        struct work_struct *work = addr;
 487
 488        switch (state) {
 489        case ODEBUG_STATE_ACTIVE:
 490                cancel_work_sync(work);
 491                debug_object_free(work, &work_debug_descr);
 492                return true;
 493        default:
 494                return false;
 495        }
 496}
 497
 498static const struct debug_obj_descr work_debug_descr = {
 499        .name           = "work_struct",
 500        .debug_hint     = work_debug_hint,
 501        .is_static_object = work_is_static_object,
 502        .fixup_init     = work_fixup_init,
 503        .fixup_free     = work_fixup_free,
 504};
 505
 506static inline void debug_work_activate(struct work_struct *work)
 507{
 508        debug_object_activate(work, &work_debug_descr);
 509}
 510
 511static inline void debug_work_deactivate(struct work_struct *work)
 512{
 513        debug_object_deactivate(work, &work_debug_descr);
 514}
 515
 516void __init_work(struct work_struct *work, int onstack)
 517{
 518        if (onstack)
 519                debug_object_init_on_stack(work, &work_debug_descr);
 520        else
 521                debug_object_init(work, &work_debug_descr);
 522}
 523EXPORT_SYMBOL_GPL(__init_work);
 524
 525void destroy_work_on_stack(struct work_struct *work)
 526{
 527        debug_object_free(work, &work_debug_descr);
 528}
 529EXPORT_SYMBOL_GPL(destroy_work_on_stack);
 530
 531void destroy_delayed_work_on_stack(struct delayed_work *work)
 532{
 533        destroy_timer_on_stack(&work->timer);
 534        debug_object_free(&work->work, &work_debug_descr);
 535}
 536EXPORT_SYMBOL_GPL(destroy_delayed_work_on_stack);
 537
 538#else
 539static inline void debug_work_activate(struct work_struct *work) { }
 540static inline void debug_work_deactivate(struct work_struct *work) { }
 541#endif
 542
 543/**
 544 * worker_pool_assign_id - allocate ID and assign it to @pool
 545 * @pool: the pool pointer of interest
 546 *
 547 * Returns 0 if ID in [0, WORK_OFFQ_POOL_NONE) is allocated and assigned
 548 * successfully, -errno on failure.
 549 */
 550static int worker_pool_assign_id(struct worker_pool *pool)
 551{
 552        int ret;
 553
 554        lockdep_assert_held(&wq_pool_mutex);
 555
 556        ret = idr_alloc(&worker_pool_idr, pool, 0, WORK_OFFQ_POOL_NONE,
 557                        GFP_KERNEL);
 558        if (ret >= 0) {
 559                pool->id = ret;
 560                return 0;
 561        }
 562        return ret;
 563}
 564
 565/**
 566 * unbound_pwq_by_node - return the unbound pool_workqueue for the given node
 567 * @wq: the target workqueue
 568 * @node: the node ID
 569 *
 570 * This must be called with any of wq_pool_mutex, wq->mutex or RCU
 571 * read locked.
 572 * If the pwq needs to be used beyond the locking in effect, the caller is
 573 * responsible for guaranteeing that the pwq stays online.
 574 *
 575 * Return: The unbound pool_workqueue for @node.
 576 */
 577static struct pool_workqueue *unbound_pwq_by_node(struct workqueue_struct *wq,
 578                                                  int node)
 579{
 580        assert_rcu_or_wq_mutex_or_pool_mutex(wq);
 581
 582        /*
 583         * XXX: @node can be NUMA_NO_NODE if CPU goes offline while a
 584         * delayed item is pending.  The plan is to keep CPU -> NODE
 585         * mapping valid and stable across CPU on/offlines.  Once that
 586         * happens, this workaround can be removed.
 587         */
 588        if (unlikely(node == NUMA_NO_NODE))
 589                return wq->dfl_pwq;
 590
 591        return rcu_dereference_raw(wq->numa_pwq_tbl[node]);
 592}
 593
 594static unsigned int work_color_to_flags(int color)
 595{
 596        return color << WORK_STRUCT_COLOR_SHIFT;
 597}
 598
 599static int get_work_color(unsigned long work_data)
 600{
 601        return (work_data >> WORK_STRUCT_COLOR_SHIFT) &
 602                ((1 << WORK_STRUCT_COLOR_BITS) - 1);
 603}
 604
 605static int work_next_color(int color)
 606{
 607        return (color + 1) % WORK_NR_COLORS;
 608}
 609
 610/*
 611 * While queued, %WORK_STRUCT_PWQ is set and non flag bits of a work's data
 612 * contain the pointer to the queued pwq.  Once execution starts, the flag
 613 * is cleared and the high bits contain OFFQ flags and pool ID.
 614 *
 615 * set_work_pwq(), set_work_pool_and_clear_pending(), mark_work_canceling()
 616 * and clear_work_data() can be used to set the pwq, pool or clear
 617 * work->data.  These functions should only be called while the work is
 618 * owned - ie. while the PENDING bit is set.
 619 *
 620 * get_work_pool() and get_work_pwq() can be used to obtain the pool or pwq
 621 * corresponding to a work.  Pool is available once the work has been
 622 * queued anywhere after initialization until it is sync canceled.  pwq is
 623 * available only while the work item is queued.
 624 *
 625 * %WORK_OFFQ_CANCELING is used to mark a work item which is being
 626 * canceled.  While being canceled, a work item may have its PENDING set
 627 * but stay off timer and worklist for arbitrarily long and nobody should
 628 * try to steal the PENDING bit.
 629 */
 630static inline void set_work_data(struct work_struct *work, unsigned long data,
 631                                 unsigned long flags)
 632{
 633        WARN_ON_ONCE(!work_pending(work));
 634        atomic_long_set(&work->data, data | flags | work_static(work));
 635}
 636
 637static void set_work_pwq(struct work_struct *work, struct pool_workqueue *pwq,
 638                         unsigned long extra_flags)
 639{
 640        set_work_data(work, (unsigned long)pwq,
 641                      WORK_STRUCT_PENDING | WORK_STRUCT_PWQ | extra_flags);
 642}
 643
 644static void set_work_pool_and_keep_pending(struct work_struct *work,
 645                                           int pool_id)
 646{
 647        set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT,
 648                      WORK_STRUCT_PENDING);
 649}
 650
 651static void set_work_pool_and_clear_pending(struct work_struct *work,
 652                                            int pool_id)
 653{
 654        /*
 655         * The following wmb is paired with the implied mb in
 656         * test_and_set_bit(PENDING) and ensures all updates to @work made
 657         * here are visible to and precede any updates by the next PENDING
 658         * owner.
 659         */
 660        smp_wmb();
 661        set_work_data(work, (unsigned long)pool_id << WORK_OFFQ_POOL_SHIFT, 0);
 662        /*
 663         * The following mb guarantees that previous clear of a PENDING bit
 664         * will not be reordered with any speculative LOADS or STORES from
 665         * work->current_func, which is executed afterwards.  This possible
 666         * reordering can lead to a missed execution on attempt to queue
 667         * the same @work.  E.g. consider this case:
 668         *
 669         *   CPU#0                         CPU#1
 670         *   ----------------------------  --------------------------------
 671         *
 672         * 1  STORE event_indicated
 673         * 2  queue_work_on() {
 674         * 3    test_and_set_bit(PENDING)
 675         * 4 }                             set_..._and_clear_pending() {
 676         * 5                                 set_work_data() # clear bit
 677         * 6                                 smp_mb()
 678         * 7                               work->current_func() {
 679         * 8                                  LOAD event_indicated
 680         *                                 }
 681         *
 682         * Without an explicit full barrier speculative LOAD on line 8 can
 683         * be executed before CPU#0 does STORE on line 1.  If that happens,
 684         * CPU#0 observes the PENDING bit is still set and new execution of
 685         * a @work is not queued in a hope, that CPU#1 will eventually
 686         * finish the queued @work.  Meanwhile CPU#1 does not see
 687         * event_indicated is set, because speculative LOAD was executed
 688         * before actual STORE.
 689         */
 690        smp_mb();
 691}
 692
 693static void clear_work_data(struct work_struct *work)
 694{
 695        smp_wmb();      /* see set_work_pool_and_clear_pending() */
 696        set_work_data(work, WORK_STRUCT_NO_POOL, 0);
 697}
 698
 699static struct pool_workqueue *get_work_pwq(struct work_struct *work)
 700{
 701        unsigned long data = atomic_long_read(&work->data);
 702
 703        if (data & WORK_STRUCT_PWQ)
 704                return (void *)(data & WORK_STRUCT_WQ_DATA_MASK);
 705        else
 706                return NULL;
 707}
 708
 709/**
 710 * get_work_pool - return the worker_pool a given work was associated with
 711 * @work: the work item of interest
 712 *
 713 * Pools are created and destroyed under wq_pool_mutex, and allows read
 714 * access under RCU read lock.  As such, this function should be
 715 * called under wq_pool_mutex or inside of a rcu_read_lock() region.
 716 *
 717 * All fields of the returned pool are accessible as long as the above
 718 * mentioned locking is in effect.  If the returned pool needs to be used
 719 * beyond the critical section, the caller is responsible for ensuring the
 720 * returned pool is and stays online.
 721 *
 722 * Return: The worker_pool @work was last associated with.  %NULL if none.
 723 */
 724static struct worker_pool *get_work_pool(struct work_struct *work)
 725{
 726        unsigned long data = atomic_long_read(&work->data);
 727        int pool_id;
 728
 729        assert_rcu_or_pool_mutex();
 730
 731        if (data & WORK_STRUCT_PWQ)
 732                return ((struct pool_workqueue *)
 733                        (data & WORK_STRUCT_WQ_DATA_MASK))->pool;
 734
 735        pool_id = data >> WORK_OFFQ_POOL_SHIFT;
 736        if (pool_id == WORK_OFFQ_POOL_NONE)
 737                return NULL;
 738
 739        return idr_find(&worker_pool_idr, pool_id);
 740}
 741
 742/**
 743 * get_work_pool_id - return the worker pool ID a given work is associated with
 744 * @work: the work item of interest
 745 *
 746 * Return: The worker_pool ID @work was last associated with.
 747 * %WORK_OFFQ_POOL_NONE if none.
 748 */
 749static int get_work_pool_id(struct work_struct *work)
 750{
 751        unsigned long data = atomic_long_read(&work->data);
 752
 753        if (data & WORK_STRUCT_PWQ)
 754                return ((struct pool_workqueue *)
 755                        (data & WORK_STRUCT_WQ_DATA_MASK))->pool->id;
 756
 757        return data >> WORK_OFFQ_POOL_SHIFT;
 758}
 759
 760static void mark_work_canceling(struct work_struct *work)
 761{
 762        unsigned long pool_id = get_work_pool_id(work);
 763
 764        pool_id <<= WORK_OFFQ_POOL_SHIFT;
 765        set_work_data(work, pool_id | WORK_OFFQ_CANCELING, WORK_STRUCT_PENDING);
 766}
 767
 768static bool work_is_canceling(struct work_struct *work)
 769{
 770        unsigned long data = atomic_long_read(&work->data);
 771
 772        return !(data & WORK_STRUCT_PWQ) && (data & WORK_OFFQ_CANCELING);
 773}
 774
 775/*
 776 * Policy functions.  These define the policies on how the global worker
 777 * pools are managed.  Unless noted otherwise, these functions assume that
 778 * they're being called with pool->lock held.
 779 */
 780
 781static bool __need_more_worker(struct worker_pool *pool)
 782{
 783        return !atomic_read(&pool->nr_running);
 784}
 785
 786/*
 787 * Need to wake up a worker?  Called from anything but currently
 788 * running workers.
 789 *
 790 * Note that, because unbound workers never contribute to nr_running, this
 791 * function will always return %true for unbound pools as long as the
 792 * worklist isn't empty.
 793 */
 794static bool need_more_worker(struct worker_pool *pool)
 795{
 796        return !list_empty(&pool->worklist) && __need_more_worker(pool);
 797}
 798
 799/* Can I start working?  Called from busy but !running workers. */
 800static bool may_start_working(struct worker_pool *pool)
 801{
 802        return pool->nr_idle;
 803}
 804
 805/* Do I need to keep working?  Called from currently running workers. */
 806static bool keep_working(struct worker_pool *pool)
 807{
 808        return !list_empty(&pool->worklist) &&
 809                atomic_read(&pool->nr_running) <= 1;
 810}
 811
 812/* Do we need a new worker?  Called from manager. */
 813static bool need_to_create_worker(struct worker_pool *pool)
 814{
 815        return need_more_worker(pool) && !may_start_working(pool);
 816}
 817
 818/* Do we have too many workers and should some go away? */
 819static bool too_many_workers(struct worker_pool *pool)
 820{
 821        bool managing = pool->flags & POOL_MANAGER_ACTIVE;
 822        int nr_idle = pool->nr_idle + managing; /* manager is considered idle */
 823        int nr_busy = pool->nr_workers - nr_idle;
 824
 825        return nr_idle > 2 && (nr_idle - 2) * MAX_IDLE_WORKERS_RATIO >= nr_busy;
 826}
 827
 828/*
 829 * Wake up functions.
 830 */
 831
 832/* Return the first idle worker.  Safe with preemption disabled */
 833static struct worker *first_idle_worker(struct worker_pool *pool)
 834{
 835        if (unlikely(list_empty(&pool->idle_list)))
 836                return NULL;
 837
 838        return list_first_entry(&pool->idle_list, struct worker, entry);
 839}
 840
 841/**
 842 * wake_up_worker - wake up an idle worker
 843 * @pool: worker pool to wake worker from
 844 *
 845 * Wake up the first idle worker of @pool.
 846 *
 847 * CONTEXT:
 848 * raw_spin_lock_irq(pool->lock).
 849 */
 850static void wake_up_worker(struct worker_pool *pool)
 851{
 852        struct worker *worker = first_idle_worker(pool);
 853
 854        if (likely(worker))
 855                wake_up_process(worker->task);
 856}
 857
 858/**
 859 * wq_worker_running - a worker is running again
 860 * @task: task waking up
 861 *
 862 * This function is called when a worker returns from schedule()
 863 */
 864void wq_worker_running(struct task_struct *task)
 865{
 866        struct worker *worker = kthread_data(task);
 867
 868        if (!worker->sleeping)
 869                return;
 870        if (!(worker->flags & WORKER_NOT_RUNNING))
 871                atomic_inc(&worker->pool->nr_running);
 872        worker->sleeping = 0;
 873}
 874
 875/**
 876 * wq_worker_sleeping - a worker is going to sleep
 877 * @task: task going to sleep
 878 *
 879 * This function is called from schedule() when a busy worker is
 880 * going to sleep. Preemption needs to be disabled to protect ->sleeping
 881 * assignment.
 882 */
 883void wq_worker_sleeping(struct task_struct *task)
 884{
 885        struct worker *next, *worker = kthread_data(task);
 886        struct worker_pool *pool;
 887
 888        /*
 889         * Rescuers, which may not have all the fields set up like normal
 890         * workers, also reach here, let's not access anything before
 891         * checking NOT_RUNNING.
 892         */
 893        if (worker->flags & WORKER_NOT_RUNNING)
 894                return;
 895
 896        pool = worker->pool;
 897
 898        /* Return if preempted before wq_worker_running() was reached */
 899        if (worker->sleeping)
 900                return;
 901
 902        worker->sleeping = 1;
 903        raw_spin_lock_irq(&pool->lock);
 904
 905        /*
 906         * The counterpart of the following dec_and_test, implied mb,
 907         * worklist not empty test sequence is in insert_work().
 908         * Please read comment there.
 909         *
 910         * NOT_RUNNING is clear.  This means that we're bound to and
 911         * running on the local cpu w/ rq lock held and preemption
 912         * disabled, which in turn means that none else could be
 913         * manipulating idle_list, so dereferencing idle_list without pool
 914         * lock is safe.
 915         */
 916        if (atomic_dec_and_test(&pool->nr_running) &&
 917            !list_empty(&pool->worklist)) {
 918                next = first_idle_worker(pool);
 919                if (next)
 920                        wake_up_process(next->task);
 921        }
 922        raw_spin_unlock_irq(&pool->lock);
 923}
 924
 925/**
 926 * wq_worker_last_func - retrieve worker's last work function
 927 * @task: Task to retrieve last work function of.
 928 *
 929 * Determine the last function a worker executed. This is called from
 930 * the scheduler to get a worker's last known identity.
 931 *
 932 * CONTEXT:
 933 * raw_spin_lock_irq(rq->lock)
 934 *
 935 * This function is called during schedule() when a kworker is going
 936 * to sleep. It's used by psi to identify aggregation workers during
 937 * dequeuing, to allow periodic aggregation to shut-off when that
 938 * worker is the last task in the system or cgroup to go to sleep.
 939 *
 940 * As this function doesn't involve any workqueue-related locking, it
 941 * only returns stable values when called from inside the scheduler's
 942 * queuing and dequeuing paths, when @task, which must be a kworker,
 943 * is guaranteed to not be processing any works.
 944 *
 945 * Return:
 946 * The last work function %current executed as a worker, NULL if it
 947 * hasn't executed any work yet.
 948 */
 949work_func_t wq_worker_last_func(struct task_struct *task)
 950{
 951        struct worker *worker = kthread_data(task);
 952
 953        return worker->last_func;
 954}
 955
 956/**
 957 * worker_set_flags - set worker flags and adjust nr_running accordingly
 958 * @worker: self
 959 * @flags: flags to set
 960 *
 961 * Set @flags in @worker->flags and adjust nr_running accordingly.
 962 *
 963 * CONTEXT:
 964 * raw_spin_lock_irq(pool->lock)
 965 */
 966static inline void worker_set_flags(struct worker *worker, unsigned int flags)
 967{
 968        struct worker_pool *pool = worker->pool;
 969
 970        WARN_ON_ONCE(worker->task != current);
 971
 972        /* If transitioning into NOT_RUNNING, adjust nr_running. */
 973        if ((flags & WORKER_NOT_RUNNING) &&
 974            !(worker->flags & WORKER_NOT_RUNNING)) {
 975                atomic_dec(&pool->nr_running);
 976        }
 977
 978        worker->flags |= flags;
 979}
 980
 981/**
 982 * worker_clr_flags - clear worker flags and adjust nr_running accordingly
 983 * @worker: self
 984 * @flags: flags to clear
 985 *
 986 * Clear @flags in @worker->flags and adjust nr_running accordingly.
 987 *
 988 * CONTEXT:
 989 * raw_spin_lock_irq(pool->lock)
 990 */
 991static inline void worker_clr_flags(struct worker *worker, unsigned int flags)
 992{
 993        struct worker_pool *pool = worker->pool;
 994        unsigned int oflags = worker->flags;
 995
 996        WARN_ON_ONCE(worker->task != current);
 997
 998        worker->flags &= ~flags;
 999
1000        /*
1001         * If transitioning out of NOT_RUNNING, increment nr_running.  Note
1002         * that the nested NOT_RUNNING is not a noop.  NOT_RUNNING is mask
1003         * of multiple flags, not a single flag.
1004         */
1005        if ((flags & WORKER_NOT_RUNNING) && (oflags & WORKER_NOT_RUNNING))
1006                if (!(worker->flags & WORKER_NOT_RUNNING))
1007                        atomic_inc(&pool->nr_running);
1008}
1009
1010/**
1011 * find_worker_executing_work - find worker which is executing a work
1012 * @pool: pool of interest
1013 * @work: work to find worker for
1014 *
1015 * Find a worker which is executing @work on @pool by searching
1016 * @pool->busy_hash which is keyed by the address of @work.  For a worker
1017 * to match, its current execution should match the address of @work and
1018 * its work function.  This is to avoid unwanted dependency between
1019 * unrelated work executions through a work item being recycled while still
1020 * being executed.
1021 *
1022 * This is a bit tricky.  A work item may be freed once its execution
1023 * starts and nothing prevents the freed area from being recycled for
1024 * another work item.  If the same work item address ends up being reused
1025 * before the original execution finishes, workqueue will identify the
1026 * recycled work item as currently executing and make it wait until the
1027 * current execution finishes, introducing an unwanted dependency.
1028 *
1029 * This function checks the work item address and work function to avoid
1030 * false positives.  Note that this isn't complete as one may construct a
1031 * work function which can introduce dependency onto itself through a
1032 * recycled work item.  Well, if somebody wants to shoot oneself in the
1033 * foot that badly, there's only so much we can do, and if such deadlock
1034 * actually occurs, it should be easy to locate the culprit work function.
1035 *
1036 * CONTEXT:
1037 * raw_spin_lock_irq(pool->lock).
1038 *
1039 * Return:
1040 * Pointer to worker which is executing @work if found, %NULL
1041 * otherwise.
1042 */
1043static struct worker *find_worker_executing_work(struct worker_pool *pool,
1044                                                 struct work_struct *work)
1045{
1046        struct worker *worker;
1047
1048        hash_for_each_possible(pool->busy_hash, worker, hentry,
1049                               (unsigned long)work)
1050                if (worker->current_work == work &&
1051                    worker->current_func == work->func)
1052                        return worker;
1053
1054        return NULL;
1055}
1056
1057/**
1058 * move_linked_works - move linked works to a list
1059 * @work: start of series of works to be scheduled
1060 * @head: target list to append @work to
1061 * @nextp: out parameter for nested worklist walking
1062 *
1063 * Schedule linked works starting from @work to @head.  Work series to
1064 * be scheduled starts at @work and includes any consecutive work with
1065 * WORK_STRUCT_LINKED set in its predecessor.
1066 *
1067 * If @nextp is not NULL, it's updated to point to the next work of
1068 * the last scheduled work.  This allows move_linked_works() to be
1069 * nested inside outer list_for_each_entry_safe().
1070 *
1071 * CONTEXT:
1072 * raw_spin_lock_irq(pool->lock).
1073 */
1074static void move_linked_works(struct work_struct *work, struct list_head *head,
1075                              struct work_struct **nextp)
1076{
1077        struct work_struct *n;
1078
1079        /*
1080         * Linked worklist will always end before the end of the list,
1081         * use NULL for list head.
1082         */
1083        list_for_each_entry_safe_from(work, n, NULL, entry) {
1084                list_move_tail(&work->entry, head);
1085                if (!(*work_data_bits(work) & WORK_STRUCT_LINKED))
1086                        break;
1087        }
1088
1089        /*
1090         * If we're already inside safe list traversal and have moved
1091         * multiple works to the scheduled queue, the next position
1092         * needs to be updated.
1093         */
1094        if (nextp)
1095                *nextp = n;
1096}
1097
1098/**
1099 * get_pwq - get an extra reference on the specified pool_workqueue
1100 * @pwq: pool_workqueue to get
1101 *
1102 * Obtain an extra reference on @pwq.  The caller should guarantee that
1103 * @pwq has positive refcnt and be holding the matching pool->lock.
1104 */
1105static void get_pwq(struct pool_workqueue *pwq)
1106{
1107        lockdep_assert_held(&pwq->pool->lock);
1108        WARN_ON_ONCE(pwq->refcnt <= 0);
1109        pwq->refcnt++;
1110}
1111
1112/**
1113 * put_pwq - put a pool_workqueue reference
1114 * @pwq: pool_workqueue to put
1115 *
1116 * Drop a reference of @pwq.  If its refcnt reaches zero, schedule its
1117 * destruction.  The caller should be holding the matching pool->lock.
1118 */
1119static void put_pwq(struct pool_workqueue *pwq)
1120{
1121        lockdep_assert_held(&pwq->pool->lock);
1122        if (likely(--pwq->refcnt))
1123                return;
1124        if (WARN_ON_ONCE(!(pwq->wq->flags & WQ_UNBOUND)))
1125                return;
1126        /*
1127         * @pwq can't be released under pool->lock, bounce to
1128         * pwq_unbound_release_workfn().  This never recurses on the same
1129         * pool->lock as this path is taken only for unbound workqueues and
1130         * the release work item is scheduled on a per-cpu workqueue.  To
1131         * avoid lockdep warning, unbound pool->locks are given lockdep
1132         * subclass of 1 in get_unbound_pool().
1133         */
1134        schedule_work(&pwq->unbound_release_work);
1135}
1136
1137/**
1138 * put_pwq_unlocked - put_pwq() with surrounding pool lock/unlock
1139 * @pwq: pool_workqueue to put (can be %NULL)
1140 *
1141 * put_pwq() with locking.  This function also allows %NULL @pwq.
1142 */
1143static void put_pwq_unlocked(struct pool_workqueue *pwq)
1144{
1145        if (pwq) {
1146                /*
1147                 * As both pwqs and pools are RCU protected, the
1148                 * following lock operations are safe.
1149                 */
1150                raw_spin_lock_irq(&pwq->pool->lock);
1151                put_pwq(pwq);
1152                raw_spin_unlock_irq(&pwq->pool->lock);
1153        }
1154}
1155
1156static void pwq_activate_inactive_work(struct work_struct *work)
1157{
1158        struct pool_workqueue *pwq = get_work_pwq(work);
1159
1160        trace_workqueue_activate_work(work);
1161        if (list_empty(&pwq->pool->worklist))
1162                pwq->pool->watchdog_ts = jiffies;
1163        move_linked_works(work, &pwq->pool->worklist, NULL);
1164        __clear_bit(WORK_STRUCT_INACTIVE_BIT, work_data_bits(work));
1165        pwq->nr_active++;
1166}
1167
1168static void pwq_activate_first_inactive(struct pool_workqueue *pwq)
1169{
1170        struct work_struct *work = list_first_entry(&pwq->inactive_works,
1171                                                    struct work_struct, entry);
1172
1173        pwq_activate_inactive_work(work);
1174}
1175
1176/**
1177 * pwq_dec_nr_in_flight - decrement pwq's nr_in_flight
1178 * @pwq: pwq of interest
1179 * @work_data: work_data of work which left the queue
1180 *
1181 * A work either has completed or is removed from pending queue,
1182 * decrement nr_in_flight of its pwq and handle workqueue flushing.
1183 *
1184 * CONTEXT:
1185 * raw_spin_lock_irq(pool->lock).
1186 */
1187static void pwq_dec_nr_in_flight(struct pool_workqueue *pwq, unsigned long work_data)
1188{
1189        int color = get_work_color(work_data);
1190
1191        if (!(work_data & WORK_STRUCT_INACTIVE)) {
1192                pwq->nr_active--;
1193                if (!list_empty(&pwq->inactive_works)) {
1194                        /* one down, submit an inactive one */
1195                        if (pwq->nr_active < pwq->max_active)
1196                                pwq_activate_first_inactive(pwq);
1197                }
1198        }
1199
1200        pwq->nr_in_flight[color]--;
1201
1202        /* is flush in progress and are we at the flushing tip? */
1203        if (likely(pwq->flush_color != color))
1204                goto out_put;
1205
1206        /* are there still in-flight works? */
1207        if (pwq->nr_in_flight[color])
1208                goto out_put;
1209
1210        /* this pwq is done, clear flush_color */
1211        pwq->flush_color = -1;
1212
1213        /*
1214         * If this was the last pwq, wake up the first flusher.  It
1215         * will handle the rest.
1216         */
1217        if (atomic_dec_and_test(&pwq->wq->nr_pwqs_to_flush))
1218                complete(&pwq->wq->first_flusher->done);
1219out_put:
1220        put_pwq(pwq);
1221}
1222
1223/**
1224 * try_to_grab_pending - steal work item from worklist and disable irq
1225 * @work: work item to steal
1226 * @is_dwork: @work is a delayed_work
1227 * @flags: place to store irq state
1228 *
1229 * Try to grab PENDING bit of @work.  This function can handle @work in any
1230 * stable state - idle, on timer or on worklist.
1231 *
1232 * Return:
1233 *
1234 *  ========    ================================================================
1235 *  1           if @work was pending and we successfully stole PENDING
1236 *  0           if @work was idle and we claimed PENDING
1237 *  -EAGAIN     if PENDING couldn't be grabbed at the moment, safe to busy-retry
1238 *  -ENOENT     if someone else is canceling @work, this state may persist
1239 *              for arbitrarily long
1240 *  ========    ================================================================
1241 *
1242 * Note:
1243 * On >= 0 return, the caller owns @work's PENDING bit.  To avoid getting
1244 * interrupted while holding PENDING and @work off queue, irq must be
1245 * disabled on entry.  This, combined with delayed_work->timer being
1246 * irqsafe, ensures that we return -EAGAIN for finite short period of time.
1247 *
1248 * On successful return, >= 0, irq is disabled and the caller is
1249 * responsible for releasing it using local_irq_restore(*@flags).
1250 *
1251 * This function is safe to call from any context including IRQ handler.
1252 */
1253static int try_to_grab_pending(struct work_struct *work, bool is_dwork,
1254                               unsigned long *flags)
1255{
1256        struct worker_pool *pool;
1257        struct pool_workqueue *pwq;
1258
1259        local_irq_save(*flags);
1260
1261        /* try to steal the timer if it exists */
1262        if (is_dwork) {
1263                struct delayed_work *dwork = to_delayed_work(work);
1264
1265                /*
1266                 * dwork->timer is irqsafe.  If del_timer() fails, it's
1267                 * guaranteed that the timer is not queued anywhere and not
1268                 * running on the local CPU.
1269                 */
1270                if (likely(del_timer(&dwork->timer)))
1271                        return 1;
1272        }
1273
1274        /* try to claim PENDING the normal way */
1275        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work)))
1276                return 0;
1277
1278        rcu_read_lock();
1279        /*
1280         * The queueing is in progress, or it is already queued. Try to
1281         * steal it from ->worklist without clearing WORK_STRUCT_PENDING.
1282         */
1283        pool = get_work_pool(work);
1284        if (!pool)
1285                goto fail;
1286
1287        raw_spin_lock(&pool->lock);
1288        /*
1289         * work->data is guaranteed to point to pwq only while the work
1290         * item is queued on pwq->wq, and both updating work->data to point
1291         * to pwq on queueing and to pool on dequeueing are done under
1292         * pwq->pool->lock.  This in turn guarantees that, if work->data
1293         * points to pwq which is associated with a locked pool, the work
1294         * item is currently queued on that pool.
1295         */
1296        pwq = get_work_pwq(work);
1297        if (pwq && pwq->pool == pool) {
1298                debug_work_deactivate(work);
1299
1300                /*
1301                 * A cancelable inactive work item must be in the
1302                 * pwq->inactive_works since a queued barrier can't be
1303                 * canceled (see the comments in insert_wq_barrier()).
1304                 *
1305                 * An inactive work item cannot be grabbed directly because
1306                 * it might have linked barrier work items which, if left
1307                 * on the inactive_works list, will confuse pwq->nr_active
1308                 * management later on and cause stall.  Make sure the work
1309                 * item is activated before grabbing.
1310                 */
1311                if (*work_data_bits(work) & WORK_STRUCT_INACTIVE)
1312                        pwq_activate_inactive_work(work);
1313
1314                list_del_init(&work->entry);
1315                pwq_dec_nr_in_flight(pwq, *work_data_bits(work));
1316
1317                /* work->data points to pwq iff queued, point to pool */
1318                set_work_pool_and_keep_pending(work, pool->id);
1319
1320                raw_spin_unlock(&pool->lock);
1321                rcu_read_unlock();
1322                return 1;
1323        }
1324        raw_spin_unlock(&pool->lock);
1325fail:
1326        rcu_read_unlock();
1327        local_irq_restore(*flags);
1328        if (work_is_canceling(work))
1329                return -ENOENT;
1330        cpu_relax();
1331        return -EAGAIN;
1332}
1333
1334/**
1335 * insert_work - insert a work into a pool
1336 * @pwq: pwq @work belongs to
1337 * @work: work to insert
1338 * @head: insertion point
1339 * @extra_flags: extra WORK_STRUCT_* flags to set
1340 *
1341 * Insert @work which belongs to @pwq after @head.  @extra_flags is or'd to
1342 * work_struct flags.
1343 *
1344 * CONTEXT:
1345 * raw_spin_lock_irq(pool->lock).
1346 */
1347static void insert_work(struct pool_workqueue *pwq, struct work_struct *work,
1348                        struct list_head *head, unsigned int extra_flags)
1349{
1350        struct worker_pool *pool = pwq->pool;
1351
1352        /* record the work call stack in order to print it in KASAN reports */
1353        kasan_record_aux_stack(work);
1354
1355        /* we own @work, set data and link */
1356        set_work_pwq(work, pwq, extra_flags);
1357        list_add_tail(&work->entry, head);
1358        get_pwq(pwq);
1359
1360        /*
1361         * Ensure either wq_worker_sleeping() sees the above
1362         * list_add_tail() or we see zero nr_running to avoid workers lying
1363         * around lazily while there are works to be processed.
1364         */
1365        smp_mb();
1366
1367        if (__need_more_worker(pool))
1368                wake_up_worker(pool);
1369}
1370
1371/*
1372 * Test whether @work is being queued from another work executing on the
1373 * same workqueue.
1374 */
1375static bool is_chained_work(struct workqueue_struct *wq)
1376{
1377        struct worker *worker;
1378
1379        worker = current_wq_worker();
1380        /*
1381         * Return %true iff I'm a worker executing a work item on @wq.  If
1382         * I'm @worker, it's safe to dereference it without locking.
1383         */
1384        return worker && worker->current_pwq->wq == wq;
1385}
1386
1387/*
1388 * When queueing an unbound work item to a wq, prefer local CPU if allowed
1389 * by wq_unbound_cpumask.  Otherwise, round robin among the allowed ones to
1390 * avoid perturbing sensitive tasks.
1391 */
1392static int wq_select_unbound_cpu(int cpu)
1393{
1394        static bool printed_dbg_warning;
1395        int new_cpu;
1396
1397        if (likely(!wq_debug_force_rr_cpu)) {
1398                if (cpumask_test_cpu(cpu, wq_unbound_cpumask))
1399                        return cpu;
1400        } else if (!printed_dbg_warning) {
1401                pr_warn("workqueue: round-robin CPU selection forced, expect performance impact\n");
1402                printed_dbg_warning = true;
1403        }
1404
1405        if (cpumask_empty(wq_unbound_cpumask))
1406                return cpu;
1407
1408        new_cpu = __this_cpu_read(wq_rr_cpu_last);
1409        new_cpu = cpumask_next_and(new_cpu, wq_unbound_cpumask, cpu_online_mask);
1410        if (unlikely(new_cpu >= nr_cpu_ids)) {
1411                new_cpu = cpumask_first_and(wq_unbound_cpumask, cpu_online_mask);
1412                if (unlikely(new_cpu >= nr_cpu_ids))
1413                        return cpu;
1414        }
1415        __this_cpu_write(wq_rr_cpu_last, new_cpu);
1416
1417        return new_cpu;
1418}
1419
1420static void __queue_work(int cpu, struct workqueue_struct *wq,
1421                         struct work_struct *work)
1422{
1423        struct pool_workqueue *pwq;
1424        struct worker_pool *last_pool;
1425        struct list_head *worklist;
1426        unsigned int work_flags;
1427        unsigned int req_cpu = cpu;
1428
1429        /*
1430         * While a work item is PENDING && off queue, a task trying to
1431         * steal the PENDING will busy-loop waiting for it to either get
1432         * queued or lose PENDING.  Grabbing PENDING and queueing should
1433         * happen with IRQ disabled.
1434         */
1435        lockdep_assert_irqs_disabled();
1436
1437
1438        /* if draining, only works from the same workqueue are allowed */
1439        if (unlikely(wq->flags & __WQ_DRAINING) &&
1440            WARN_ON_ONCE(!is_chained_work(wq)))
1441                return;
1442        rcu_read_lock();
1443retry:
1444        /* pwq which will be used unless @work is executing elsewhere */
1445        if (wq->flags & WQ_UNBOUND) {
1446                if (req_cpu == WORK_CPU_UNBOUND)
1447                        cpu = wq_select_unbound_cpu(raw_smp_processor_id());
1448                pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
1449        } else {
1450                if (req_cpu == WORK_CPU_UNBOUND)
1451                        cpu = raw_smp_processor_id();
1452                pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
1453        }
1454
1455        /*
1456         * If @work was previously on a different pool, it might still be
1457         * running there, in which case the work needs to be queued on that
1458         * pool to guarantee non-reentrancy.
1459         */
1460        last_pool = get_work_pool(work);
1461        if (last_pool && last_pool != pwq->pool) {
1462                struct worker *worker;
1463
1464                raw_spin_lock(&last_pool->lock);
1465
1466                worker = find_worker_executing_work(last_pool, work);
1467
1468                if (worker && worker->current_pwq->wq == wq) {
1469                        pwq = worker->current_pwq;
1470                } else {
1471                        /* meh... not running there, queue here */
1472                        raw_spin_unlock(&last_pool->lock);
1473                        raw_spin_lock(&pwq->pool->lock);
1474                }
1475        } else {
1476                raw_spin_lock(&pwq->pool->lock);
1477        }
1478
1479        /*
1480         * pwq is determined and locked.  For unbound pools, we could have
1481         * raced with pwq release and it could already be dead.  If its
1482         * refcnt is zero, repeat pwq selection.  Note that pwqs never die
1483         * without another pwq replacing it in the numa_pwq_tbl or while
1484         * work items are executing on it, so the retrying is guaranteed to
1485         * make forward-progress.
1486         */
1487        if (unlikely(!pwq->refcnt)) {
1488                if (wq->flags & WQ_UNBOUND) {
1489                        raw_spin_unlock(&pwq->pool->lock);
1490                        cpu_relax();
1491                        goto retry;
1492                }
1493                /* oops */
1494                WARN_ONCE(true, "workqueue: per-cpu pwq for %s on cpu%d has 0 refcnt",
1495                          wq->name, cpu);
1496        }
1497
1498        /* pwq determined, queue */
1499        trace_workqueue_queue_work(req_cpu, pwq, work);
1500
1501        if (WARN_ON(!list_empty(&work->entry)))
1502                goto out;
1503
1504        pwq->nr_in_flight[pwq->work_color]++;
1505        work_flags = work_color_to_flags(pwq->work_color);
1506
1507        if (likely(pwq->nr_active < pwq->max_active)) {
1508                trace_workqueue_activate_work(work);
1509                pwq->nr_active++;
1510                worklist = &pwq->pool->worklist;
1511                if (list_empty(worklist))
1512                        pwq->pool->watchdog_ts = jiffies;
1513        } else {
1514                work_flags |= WORK_STRUCT_INACTIVE;
1515                worklist = &pwq->inactive_works;
1516        }
1517
1518        debug_work_activate(work);
1519        insert_work(pwq, work, worklist, work_flags);
1520
1521out:
1522        raw_spin_unlock(&pwq->pool->lock);
1523        rcu_read_unlock();
1524}
1525
1526/**
1527 * queue_work_on - queue work on specific cpu
1528 * @cpu: CPU number to execute work on
1529 * @wq: workqueue to use
1530 * @work: work to queue
1531 *
1532 * We queue the work to a specific CPU, the caller must ensure it
1533 * can't go away.
1534 *
1535 * Return: %false if @work was already on a queue, %true otherwise.
1536 */
1537bool queue_work_on(int cpu, struct workqueue_struct *wq,
1538                   struct work_struct *work)
1539{
1540        bool ret = false;
1541        unsigned long flags;
1542
1543        local_irq_save(flags);
1544
1545        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1546                __queue_work(cpu, wq, work);
1547                ret = true;
1548        }
1549
1550        local_irq_restore(flags);
1551        return ret;
1552}
1553EXPORT_SYMBOL(queue_work_on);
1554
1555/**
1556 * workqueue_select_cpu_near - Select a CPU based on NUMA node
1557 * @node: NUMA node ID that we want to select a CPU from
1558 *
1559 * This function will attempt to find a "random" cpu available on a given
1560 * node. If there are no CPUs available on the given node it will return
1561 * WORK_CPU_UNBOUND indicating that we should just schedule to any
1562 * available CPU if we need to schedule this work.
1563 */
1564static int workqueue_select_cpu_near(int node)
1565{
1566        int cpu;
1567
1568        /* No point in doing this if NUMA isn't enabled for workqueues */
1569        if (!wq_numa_enabled)
1570                return WORK_CPU_UNBOUND;
1571
1572        /* Delay binding to CPU if node is not valid or online */
1573        if (node < 0 || node >= MAX_NUMNODES || !node_online(node))
1574                return WORK_CPU_UNBOUND;
1575
1576        /* Use local node/cpu if we are already there */
1577        cpu = raw_smp_processor_id();
1578        if (node == cpu_to_node(cpu))
1579                return cpu;
1580
1581        /* Use "random" otherwise know as "first" online CPU of node */
1582        cpu = cpumask_any_and(cpumask_of_node(node), cpu_online_mask);
1583
1584        /* If CPU is valid return that, otherwise just defer */
1585        return cpu < nr_cpu_ids ? cpu : WORK_CPU_UNBOUND;
1586}
1587
1588/**
1589 * queue_work_node - queue work on a "random" cpu for a given NUMA node
1590 * @node: NUMA node that we are targeting the work for
1591 * @wq: workqueue to use
1592 * @work: work to queue
1593 *
1594 * We queue the work to a "random" CPU within a given NUMA node. The basic
1595 * idea here is to provide a way to somehow associate work with a given
1596 * NUMA node.
1597 *
1598 * This function will only make a best effort attempt at getting this onto
1599 * the right NUMA node. If no node is requested or the requested node is
1600 * offline then we just fall back to standard queue_work behavior.
1601 *
1602 * Currently the "random" CPU ends up being the first available CPU in the
1603 * intersection of cpu_online_mask and the cpumask of the node, unless we
1604 * are running on the node. In that case we just use the current CPU.
1605 *
1606 * Return: %false if @work was already on a queue, %true otherwise.
1607 */
1608bool queue_work_node(int node, struct workqueue_struct *wq,
1609                     struct work_struct *work)
1610{
1611        unsigned long flags;
1612        bool ret = false;
1613
1614        /*
1615         * This current implementation is specific to unbound workqueues.
1616         * Specifically we only return the first available CPU for a given
1617         * node instead of cycling through individual CPUs within the node.
1618         *
1619         * If this is used with a per-cpu workqueue then the logic in
1620         * workqueue_select_cpu_near would need to be updated to allow for
1621         * some round robin type logic.
1622         */
1623        WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND));
1624
1625        local_irq_save(flags);
1626
1627        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1628                int cpu = workqueue_select_cpu_near(node);
1629
1630                __queue_work(cpu, wq, work);
1631                ret = true;
1632        }
1633
1634        local_irq_restore(flags);
1635        return ret;
1636}
1637EXPORT_SYMBOL_GPL(queue_work_node);
1638
1639void delayed_work_timer_fn(struct timer_list *t)
1640{
1641        struct delayed_work *dwork = from_timer(dwork, t, timer);
1642
1643        /* should have been called from irqsafe timer with irq already off */
1644        __queue_work(dwork->cpu, dwork->wq, &dwork->work);
1645}
1646EXPORT_SYMBOL(delayed_work_timer_fn);
1647
1648static void __queue_delayed_work(int cpu, struct workqueue_struct *wq,
1649                                struct delayed_work *dwork, unsigned long delay)
1650{
1651        struct timer_list *timer = &dwork->timer;
1652        struct work_struct *work = &dwork->work;
1653
1654        WARN_ON_ONCE(!wq);
1655        WARN_ON_FUNCTION_MISMATCH(timer->function, delayed_work_timer_fn);
1656        WARN_ON_ONCE(timer_pending(timer));
1657        WARN_ON_ONCE(!list_empty(&work->entry));
1658
1659        /*
1660         * If @delay is 0, queue @dwork->work immediately.  This is for
1661         * both optimization and correctness.  The earliest @timer can
1662         * expire is on the closest next tick and delayed_work users depend
1663         * on that there's no such delay when @delay is 0.
1664         */
1665        if (!delay) {
1666                __queue_work(cpu, wq, &dwork->work);
1667                return;
1668        }
1669
1670        dwork->wq = wq;
1671        dwork->cpu = cpu;
1672        timer->expires = jiffies + delay;
1673
1674        if (unlikely(cpu != WORK_CPU_UNBOUND))
1675                add_timer_on(timer, cpu);
1676        else
1677                add_timer(timer);
1678}
1679
1680/**
1681 * queue_delayed_work_on - queue work on specific CPU after delay
1682 * @cpu: CPU number to execute work on
1683 * @wq: workqueue to use
1684 * @dwork: work to queue
1685 * @delay: number of jiffies to wait before queueing
1686 *
1687 * Return: %false if @work was already on a queue, %true otherwise.  If
1688 * @delay is zero and @dwork is idle, it will be scheduled for immediate
1689 * execution.
1690 */
1691bool queue_delayed_work_on(int cpu, struct workqueue_struct *wq,
1692                           struct delayed_work *dwork, unsigned long delay)
1693{
1694        struct work_struct *work = &dwork->work;
1695        bool ret = false;
1696        unsigned long flags;
1697
1698        /* read the comment in __queue_work() */
1699        local_irq_save(flags);
1700
1701        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1702                __queue_delayed_work(cpu, wq, dwork, delay);
1703                ret = true;
1704        }
1705
1706        local_irq_restore(flags);
1707        return ret;
1708}
1709EXPORT_SYMBOL(queue_delayed_work_on);
1710
1711/**
1712 * mod_delayed_work_on - modify delay of or queue a delayed work on specific CPU
1713 * @cpu: CPU number to execute work on
1714 * @wq: workqueue to use
1715 * @dwork: work to queue
1716 * @delay: number of jiffies to wait before queueing
1717 *
1718 * If @dwork is idle, equivalent to queue_delayed_work_on(); otherwise,
1719 * modify @dwork's timer so that it expires after @delay.  If @delay is
1720 * zero, @work is guaranteed to be scheduled immediately regardless of its
1721 * current state.
1722 *
1723 * Return: %false if @dwork was idle and queued, %true if @dwork was
1724 * pending and its timer was modified.
1725 *
1726 * This function is safe to call from any context including IRQ handler.
1727 * See try_to_grab_pending() for details.
1728 */
1729bool mod_delayed_work_on(int cpu, struct workqueue_struct *wq,
1730                         struct delayed_work *dwork, unsigned long delay)
1731{
1732        unsigned long flags;
1733        int ret;
1734
1735        do {
1736                ret = try_to_grab_pending(&dwork->work, true, &flags);
1737        } while (unlikely(ret == -EAGAIN));
1738
1739        if (likely(ret >= 0)) {
1740                __queue_delayed_work(cpu, wq, dwork, delay);
1741                local_irq_restore(flags);
1742        }
1743
1744        /* -ENOENT from try_to_grab_pending() becomes %true */
1745        return ret;
1746}
1747EXPORT_SYMBOL_GPL(mod_delayed_work_on);
1748
1749static void rcu_work_rcufn(struct rcu_head *rcu)
1750{
1751        struct rcu_work *rwork = container_of(rcu, struct rcu_work, rcu);
1752
1753        /* read the comment in __queue_work() */
1754        local_irq_disable();
1755        __queue_work(WORK_CPU_UNBOUND, rwork->wq, &rwork->work);
1756        local_irq_enable();
1757}
1758
1759/**
1760 * queue_rcu_work - queue work after a RCU grace period
1761 * @wq: workqueue to use
1762 * @rwork: work to queue
1763 *
1764 * Return: %false if @rwork was already pending, %true otherwise.  Note
1765 * that a full RCU grace period is guaranteed only after a %true return.
1766 * While @rwork is guaranteed to be executed after a %false return, the
1767 * execution may happen before a full RCU grace period has passed.
1768 */
1769bool queue_rcu_work(struct workqueue_struct *wq, struct rcu_work *rwork)
1770{
1771        struct work_struct *work = &rwork->work;
1772
1773        if (!test_and_set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(work))) {
1774                rwork->wq = wq;
1775                call_rcu(&rwork->rcu, rcu_work_rcufn);
1776                return true;
1777        }
1778
1779        return false;
1780}
1781EXPORT_SYMBOL(queue_rcu_work);
1782
1783/**
1784 * worker_enter_idle - enter idle state
1785 * @worker: worker which is entering idle state
1786 *
1787 * @worker is entering idle state.  Update stats and idle timer if
1788 * necessary.
1789 *
1790 * LOCKING:
1791 * raw_spin_lock_irq(pool->lock).
1792 */
1793static void worker_enter_idle(struct worker *worker)
1794{
1795        struct worker_pool *pool = worker->pool;
1796
1797        if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) ||
1798            WARN_ON_ONCE(!list_empty(&worker->entry) &&
1799                         (worker->hentry.next || worker->hentry.pprev)))
1800                return;
1801
1802        /* can't use worker_set_flags(), also called from create_worker() */
1803        worker->flags |= WORKER_IDLE;
1804        pool->nr_idle++;
1805        worker->last_active = jiffies;
1806
1807        /* idle_list is LIFO */
1808        list_add(&worker->entry, &pool->idle_list);
1809
1810        if (too_many_workers(pool) && !timer_pending(&pool->idle_timer))
1811                mod_timer(&pool->idle_timer, jiffies + IDLE_WORKER_TIMEOUT);
1812
1813        /*
1814         * Sanity check nr_running.  Because unbind_workers() releases
1815         * pool->lock between setting %WORKER_UNBOUND and zapping
1816         * nr_running, the warning may trigger spuriously.  Check iff
1817         * unbind is not in progress.
1818         */
1819        WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
1820                     pool->nr_workers == pool->nr_idle &&
1821                     atomic_read(&pool->nr_running));
1822}
1823
1824/**
1825 * worker_leave_idle - leave idle state
1826 * @worker: worker which is leaving idle state
1827 *
1828 * @worker is leaving idle state.  Update stats.
1829 *
1830 * LOCKING:
1831 * raw_spin_lock_irq(pool->lock).
1832 */
1833static void worker_leave_idle(struct worker *worker)
1834{
1835        struct worker_pool *pool = worker->pool;
1836
1837        if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE)))
1838                return;
1839        worker_clr_flags(worker, WORKER_IDLE);
1840        pool->nr_idle--;
1841        list_del_init(&worker->entry);
1842}
1843
1844static struct worker *alloc_worker(int node)
1845{
1846        struct worker *worker;
1847
1848        worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node);
1849        if (worker) {
1850                INIT_LIST_HEAD(&worker->entry);
1851                INIT_LIST_HEAD(&worker->scheduled);
1852                INIT_LIST_HEAD(&worker->node);
1853                /* on creation a worker is in !idle && prep state */
1854                worker->flags = WORKER_PREP;
1855        }
1856        return worker;
1857}
1858
1859/**
1860 * worker_attach_to_pool() - attach a worker to a pool
1861 * @worker: worker to be attached
1862 * @pool: the target pool
1863 *
1864 * Attach @worker to @pool.  Once attached, the %WORKER_UNBOUND flag and
1865 * cpu-binding of @worker are kept coordinated with the pool across
1866 * cpu-[un]hotplugs.
1867 */
1868static void worker_attach_to_pool(struct worker *worker,
1869                                   struct worker_pool *pool)
1870{
1871        mutex_lock(&wq_pool_attach_mutex);
1872
1873        /*
1874         * The wq_pool_attach_mutex ensures %POOL_DISASSOCIATED remains
1875         * stable across this function.  See the comments above the flag
1876         * definition for details.
1877         */
1878        if (pool->flags & POOL_DISASSOCIATED)
1879                worker->flags |= WORKER_UNBOUND;
1880        else
1881                kthread_set_per_cpu(worker->task, pool->cpu);
1882
1883        if (worker->rescue_wq)
1884                set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask);
1885
1886        list_add_tail(&worker->node, &pool->workers);
1887        worker->pool = pool;
1888
1889        mutex_unlock(&wq_pool_attach_mutex);
1890}
1891
1892/**
1893 * worker_detach_from_pool() - detach a worker from its pool
1894 * @worker: worker which is attached to its pool
1895 *
1896 * Undo the attaching which had been done in worker_attach_to_pool().  The
1897 * caller worker shouldn't access to the pool after detached except it has
1898 * other reference to the pool.
1899 */
1900static void worker_detach_from_pool(struct worker *worker)
1901{
1902        struct worker_pool *pool = worker->pool;
1903        struct completion *detach_completion = NULL;
1904
1905        mutex_lock(&wq_pool_attach_mutex);
1906
1907        kthread_set_per_cpu(worker->task, -1);
1908        list_del(&worker->node);
1909        worker->pool = NULL;
1910
1911        if (list_empty(&pool->workers))
1912                detach_completion = pool->detach_completion;
1913        mutex_unlock(&wq_pool_attach_mutex);
1914
1915        /* clear leftover flags without pool->lock after it is detached */
1916        worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND);
1917
1918        if (detach_completion)
1919                complete(detach_completion);
1920}
1921
1922/**
1923 * create_worker - create a new workqueue worker
1924 * @pool: pool the new worker will belong to
1925 *
1926 * Create and start a new worker which is attached to @pool.
1927 *
1928 * CONTEXT:
1929 * Might sleep.  Does GFP_KERNEL allocations.
1930 *
1931 * Return:
1932 * Pointer to the newly created worker.
1933 */
1934static struct worker *create_worker(struct worker_pool *pool)
1935{
1936        struct worker *worker;
1937        int id;
1938        char id_buf[16];
1939
1940        /* ID is needed to determine kthread name */
1941        id = ida_alloc(&pool->worker_ida, GFP_KERNEL);
1942        if (id < 0)
1943                return NULL;
1944
1945        worker = alloc_worker(pool->node);
1946        if (!worker)
1947                goto fail;
1948
1949        worker->id = id;
1950
1951        if (pool->cpu >= 0)
1952                snprintf(id_buf, sizeof(id_buf), "%d:%d%s", pool->cpu, id,
1953                         pool->attrs->nice < 0  ? "H" : "");
1954        else
1955                snprintf(id_buf, sizeof(id_buf), "u%d:%d", pool->id, id);
1956
1957        worker->task = kthread_create_on_node(worker_thread, worker, pool->node,
1958                                              "kworker/%s", id_buf);
1959        if (IS_ERR(worker->task))
1960                goto fail;
1961
1962        set_user_nice(worker->task, pool->attrs->nice);
1963        kthread_bind_mask(worker->task, pool->attrs->cpumask);
1964
1965        /* successful, attach the worker to the pool */
1966        worker_attach_to_pool(worker, pool);
1967
1968        /* start the newly created worker */
1969        raw_spin_lock_irq(&pool->lock);
1970        worker->pool->nr_workers++;
1971        worker_enter_idle(worker);
1972        wake_up_process(worker->task);
1973        raw_spin_unlock_irq(&pool->lock);
1974
1975        return worker;
1976
1977fail:
1978        ida_free(&pool->worker_ida, id);
1979        kfree(worker);
1980        return NULL;
1981}
1982
1983/**
1984 * destroy_worker - destroy a workqueue worker
1985 * @worker: worker to be destroyed
1986 *
1987 * Destroy @worker and adjust @pool stats accordingly.  The worker should
1988 * be idle.
1989 *
1990 * CONTEXT:
1991 * raw_spin_lock_irq(pool->lock).
1992 */
1993static void destroy_worker(struct worker *worker)
1994{
1995        struct worker_pool *pool = worker->pool;
1996
1997        lockdep_assert_held(&pool->lock);
1998
1999        /* sanity check frenzy */
2000        if (WARN_ON(worker->current_work) ||
2001            WARN_ON(!list_empty(&worker->scheduled)) ||
2002            WARN_ON(!(worker->flags & WORKER_IDLE)))
2003                return;
2004
2005        pool->nr_workers--;
2006        pool->nr_idle--;
2007
2008        list_del_init(&worker->entry);
2009        worker->flags |= WORKER_DIE;
2010        wake_up_process(worker->task);
2011}
2012
2013static void idle_worker_timeout(struct timer_list *t)
2014{
2015        struct worker_pool *pool = from_timer(pool, t, idle_timer);
2016
2017        raw_spin_lock_irq(&pool->lock);
2018
2019        while (too_many_workers(pool)) {
2020                struct worker *worker;
2021                unsigned long expires;
2022
2023                /* idle_list is kept in LIFO order, check the last one */
2024                worker = list_entry(pool->idle_list.prev, struct worker, entry);
2025                expires = worker->last_active + IDLE_WORKER_TIMEOUT;
2026
2027                if (time_before(jiffies, expires)) {
2028                        mod_timer(&pool->idle_timer, expires);
2029                        break;
2030                }
2031
2032                destroy_worker(worker);
2033        }
2034
2035        raw_spin_unlock_irq(&pool->lock);
2036}
2037
2038static void send_mayday(struct work_struct *work)
2039{
2040        struct pool_workqueue *pwq = get_work_pwq(work);
2041        struct workqueue_struct *wq = pwq->wq;
2042
2043        lockdep_assert_held(&wq_mayday_lock);
2044
2045        if (!wq->rescuer)
2046                return;
2047
2048        /* mayday mayday mayday */
2049        if (list_empty(&pwq->mayday_node)) {
2050                /*
2051                 * If @pwq is for an unbound wq, its base ref may be put at
2052                 * any time due to an attribute change.  Pin @pwq until the
2053                 * rescuer is done with it.
2054                 */
2055                get_pwq(pwq);
2056                list_add_tail(&pwq->mayday_node, &wq->maydays);
2057                wake_up_process(wq->rescuer->task);
2058        }
2059}
2060
2061static void pool_mayday_timeout(struct timer_list *t)
2062{
2063        struct worker_pool *pool = from_timer(pool, t, mayday_timer);
2064        struct work_struct *work;
2065
2066        raw_spin_lock_irq(&pool->lock);
2067        raw_spin_lock(&wq_mayday_lock);         /* for wq->maydays */
2068
2069        if (need_to_create_worker(pool)) {
2070                /*
2071                 * We've been trying to create a new worker but
2072                 * haven't been successful.  We might be hitting an
2073                 * allocation deadlock.  Send distress signals to
2074                 * rescuers.
2075                 */
2076                list_for_each_entry(work, &pool->worklist, entry)
2077                        send_mayday(work);
2078        }
2079
2080        raw_spin_unlock(&wq_mayday_lock);
2081        raw_spin_unlock_irq(&pool->lock);
2082
2083        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INTERVAL);
2084}
2085
2086/**
2087 * maybe_create_worker - create a new worker if necessary
2088 * @pool: pool to create a new worker for
2089 *
2090 * Create a new worker for @pool if necessary.  @pool is guaranteed to
2091 * have at least one idle worker on return from this function.  If
2092 * creating a new worker takes longer than MAYDAY_INTERVAL, mayday is
2093 * sent to all rescuers with works scheduled on @pool to resolve
2094 * possible allocation deadlock.
2095 *
2096 * On return, need_to_create_worker() is guaranteed to be %false and
2097 * may_start_working() %true.
2098 *
2099 * LOCKING:
2100 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2101 * multiple times.  Does GFP_KERNEL allocations.  Called only from
2102 * manager.
2103 */
2104static void maybe_create_worker(struct worker_pool *pool)
2105__releases(&pool->lock)
2106__acquires(&pool->lock)
2107{
2108restart:
2109        raw_spin_unlock_irq(&pool->lock);
2110
2111        /* if we don't make progress in MAYDAY_INITIAL_TIMEOUT, call for help */
2112        mod_timer(&pool->mayday_timer, jiffies + MAYDAY_INITIAL_TIMEOUT);
2113
2114        while (true) {
2115                if (create_worker(pool) || !need_to_create_worker(pool))
2116                        break;
2117
2118                schedule_timeout_interruptible(CREATE_COOLDOWN);
2119
2120                if (!need_to_create_worker(pool))
2121                        break;
2122        }
2123
2124        del_timer_sync(&pool->mayday_timer);
2125        raw_spin_lock_irq(&pool->lock);
2126        /*
2127         * This is necessary even after a new worker was just successfully
2128         * created as @pool->lock was dropped and the new worker might have
2129         * already become busy.
2130         */
2131        if (need_to_create_worker(pool))
2132                goto restart;
2133}
2134
2135/**
2136 * manage_workers - manage worker pool
2137 * @worker: self
2138 *
2139 * Assume the manager role and manage the worker pool @worker belongs
2140 * to.  At any given time, there can be only zero or one manager per
2141 * pool.  The exclusion is handled automatically by this function.
2142 *
2143 * The caller can safely start processing works on false return.  On
2144 * true return, it's guaranteed that need_to_create_worker() is false
2145 * and may_start_working() is true.
2146 *
2147 * CONTEXT:
2148 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2149 * multiple times.  Does GFP_KERNEL allocations.
2150 *
2151 * Return:
2152 * %false if the pool doesn't need management and the caller can safely
2153 * start processing works, %true if management function was performed and
2154 * the conditions that the caller verified before calling the function may
2155 * no longer be true.
2156 */
2157static bool manage_workers(struct worker *worker)
2158{
2159        struct worker_pool *pool = worker->pool;
2160
2161        if (pool->flags & POOL_MANAGER_ACTIVE)
2162                return false;
2163
2164        pool->flags |= POOL_MANAGER_ACTIVE;
2165        pool->manager = worker;
2166
2167        maybe_create_worker(pool);
2168
2169        pool->manager = NULL;
2170        pool->flags &= ~POOL_MANAGER_ACTIVE;
2171        rcuwait_wake_up(&manager_wait);
2172        return true;
2173}
2174
2175/**
2176 * process_one_work - process single work
2177 * @worker: self
2178 * @work: work to process
2179 *
2180 * Process @work.  This function contains all the logics necessary to
2181 * process a single work including synchronization against and
2182 * interaction with other workers on the same cpu, queueing and
2183 * flushing.  As long as context requirement is met, any worker can
2184 * call this function to process a work.
2185 *
2186 * CONTEXT:
2187 * raw_spin_lock_irq(pool->lock) which is released and regrabbed.
2188 */
2189static void process_one_work(struct worker *worker, struct work_struct *work)
2190__releases(&pool->lock)
2191__acquires(&pool->lock)
2192{
2193        struct pool_workqueue *pwq = get_work_pwq(work);
2194        struct worker_pool *pool = worker->pool;
2195        bool cpu_intensive = pwq->wq->flags & WQ_CPU_INTENSIVE;
2196        unsigned long work_data;
2197        struct worker *collision;
2198#ifdef CONFIG_LOCKDEP
2199        /*
2200         * It is permissible to free the struct work_struct from
2201         * inside the function that is called from it, this we need to
2202         * take into account for lockdep too.  To avoid bogus "held
2203         * lock freed" warnings as well as problems when looking into
2204         * work->lockdep_map, make a copy and use that here.
2205         */
2206        struct lockdep_map lockdep_map;
2207
2208        lockdep_copy_map(&lockdep_map, &work->lockdep_map);
2209#endif
2210        /* ensure we're on the correct CPU */
2211        WARN_ON_ONCE(!(pool->flags & POOL_DISASSOCIATED) &&
2212                     raw_smp_processor_id() != pool->cpu);
2213
2214        /*
2215         * A single work shouldn't be executed concurrently by
2216         * multiple workers on a single cpu.  Check whether anyone is
2217         * already processing the work.  If so, defer the work to the
2218         * currently executing one.
2219         */
2220        collision = find_worker_executing_work(pool, work);
2221        if (unlikely(collision)) {
2222                move_linked_works(work, &collision->scheduled, NULL);
2223                return;
2224        }
2225
2226        /* claim and dequeue */
2227        debug_work_deactivate(work);
2228        hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work);
2229        worker->current_work = work;
2230        worker->current_func = work->func;
2231        worker->current_pwq = pwq;
2232        work_data = *work_data_bits(work);
2233        worker->current_color = get_work_color(work_data);
2234
2235        /*
2236         * Record wq name for cmdline and debug reporting, may get
2237         * overridden through set_worker_desc().
2238         */
2239        strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN);
2240
2241        list_del_init(&work->entry);
2242
2243        /*
2244         * CPU intensive works don't participate in concurrency management.
2245         * They're the scheduler's responsibility.  This takes @worker out
2246         * of concurrency management and the next code block will chain
2247         * execution of the pending work items.
2248         */
2249        if (unlikely(cpu_intensive))
2250                worker_set_flags(worker, WORKER_CPU_INTENSIVE);
2251
2252        /*
2253         * Wake up another worker if necessary.  The condition is always
2254         * false for normal per-cpu workers since nr_running would always
2255         * be >= 1 at this point.  This is used to chain execution of the
2256         * pending work items for WORKER_NOT_RUNNING workers such as the
2257         * UNBOUND and CPU_INTENSIVE ones.
2258         */
2259        if (need_more_worker(pool))
2260                wake_up_worker(pool);
2261
2262        /*
2263         * Record the last pool and clear PENDING which should be the last
2264         * update to @work.  Also, do this inside @pool->lock so that
2265         * PENDING and queued state changes happen together while IRQ is
2266         * disabled.
2267         */
2268        set_work_pool_and_clear_pending(work, pool->id);
2269
2270        raw_spin_unlock_irq(&pool->lock);
2271
2272        lock_map_acquire(&pwq->wq->lockdep_map);
2273        lock_map_acquire(&lockdep_map);
2274        /*
2275         * Strictly speaking we should mark the invariant state without holding
2276         * any locks, that is, before these two lock_map_acquire()'s.
2277         *
2278         * However, that would result in:
2279         *
2280         *   A(W1)
2281         *   WFC(C)
2282         *              A(W1)
2283         *              C(C)
2284         *
2285         * Which would create W1->C->W1 dependencies, even though there is no
2286         * actual deadlock possible. There are two solutions, using a
2287         * read-recursive acquire on the work(queue) 'locks', but this will then
2288         * hit the lockdep limitation on recursive locks, or simply discard
2289         * these locks.
2290         *
2291         * AFAICT there is no possible deadlock scenario between the
2292         * flush_work() and complete() primitives (except for single-threaded
2293         * workqueues), so hiding them isn't a problem.
2294         */
2295        lockdep_invariant_state(true);
2296        trace_workqueue_execute_start(work);
2297        worker->current_func(work);
2298        /*
2299         * While we must be careful to not use "work" after this, the trace
2300         * point will only record its address.
2301         */
2302        trace_workqueue_execute_end(work, worker->current_func);
2303        lock_map_release(&lockdep_map);
2304        lock_map_release(&pwq->wq->lockdep_map);
2305
2306        if (unlikely(in_atomic() || lockdep_depth(current) > 0)) {
2307                pr_err("BUG: workqueue leaked lock or atomic: %s/0x%08x/%d\n"
2308                       "     last function: %ps\n",
2309                       current->comm, preempt_count(), task_pid_nr(current),
2310                       worker->current_func);
2311                debug_show_held_locks(current);
2312                dump_stack();
2313        }
2314
2315        /*
2316         * The following prevents a kworker from hogging CPU on !PREEMPTION
2317         * kernels, where a requeueing work item waiting for something to
2318         * happen could deadlock with stop_machine as such work item could
2319         * indefinitely requeue itself while all other CPUs are trapped in
2320         * stop_machine. At the same time, report a quiescent RCU state so
2321         * the same condition doesn't freeze RCU.
2322         */
2323        cond_resched();
2324
2325        raw_spin_lock_irq(&pool->lock);
2326
2327        /* clear cpu intensive status */
2328        if (unlikely(cpu_intensive))
2329                worker_clr_flags(worker, WORKER_CPU_INTENSIVE);
2330
2331        /* tag the worker for identification in schedule() */
2332        worker->last_func = worker->current_func;
2333
2334        /* we're done with it, release */
2335        hash_del(&worker->hentry);
2336        worker->current_work = NULL;
2337        worker->current_func = NULL;
2338        worker->current_pwq = NULL;
2339        worker->current_color = INT_MAX;
2340        pwq_dec_nr_in_flight(pwq, work_data);
2341}
2342
2343/**
2344 * process_scheduled_works - process scheduled works
2345 * @worker: self
2346 *
2347 * Process all scheduled works.  Please note that the scheduled list
2348 * may change while processing a work, so this function repeatedly
2349 * fetches a work from the top and executes it.
2350 *
2351 * CONTEXT:
2352 * raw_spin_lock_irq(pool->lock) which may be released and regrabbed
2353 * multiple times.
2354 */
2355static void process_scheduled_works(struct worker *worker)
2356{
2357        while (!list_empty(&worker->scheduled)) {
2358                struct work_struct *work = list_first_entry(&worker->scheduled,
2359                                                struct work_struct, entry);
2360                process_one_work(worker, work);
2361        }
2362}
2363
2364static void set_pf_worker(bool val)
2365{
2366        mutex_lock(&wq_pool_attach_mutex);
2367        if (val)
2368                current->flags |= PF_WQ_WORKER;
2369        else
2370                current->flags &= ~PF_WQ_WORKER;
2371        mutex_unlock(&wq_pool_attach_mutex);
2372}
2373
2374/**
2375 * worker_thread - the worker thread function
2376 * @__worker: self
2377 *
2378 * The worker thread function.  All workers belong to a worker_pool -
2379 * either a per-cpu one or dynamic unbound one.  These workers process all
2380 * work items regardless of their specific target workqueue.  The only
2381 * exception is work items which belong to workqueues with a rescuer which
2382 * will be explained in rescuer_thread().
2383 *
2384 * Return: 0
2385 */
2386static int worker_thread(void *__worker)
2387{
2388        struct worker *worker = __worker;
2389        struct worker_pool *pool = worker->pool;
2390
2391        /* tell the scheduler that this is a workqueue worker */
2392        set_pf_worker(true);
2393woke_up:
2394        raw_spin_lock_irq(&pool->lock);
2395
2396        /* am I supposed to die? */
2397        if (unlikely(worker->flags & WORKER_DIE)) {
2398                raw_spin_unlock_irq(&pool->lock);
2399                WARN_ON_ONCE(!list_empty(&worker->entry));
2400                set_pf_worker(false);
2401
2402                set_task_comm(worker->task, "kworker/dying");
2403                ida_free(&pool->worker_ida, worker->id);
2404                worker_detach_from_pool(worker);
2405                kfree(worker);
2406                return 0;
2407        }
2408
2409        worker_leave_idle(worker);
2410recheck:
2411        /* no more worker necessary? */
2412        if (!need_more_worker(pool))
2413                goto sleep;
2414
2415        /* do we need to manage? */
2416        if (unlikely(!may_start_working(pool)) && manage_workers(worker))
2417                goto recheck;
2418
2419        /*
2420         * ->scheduled list can only be filled while a worker is
2421         * preparing to process a work or actually processing it.
2422         * Make sure nobody diddled with it while I was sleeping.
2423         */
2424        WARN_ON_ONCE(!list_empty(&worker->scheduled));
2425
2426        /*
2427         * Finish PREP stage.  We're guaranteed to have at least one idle
2428         * worker or that someone else has already assumed the manager
2429         * role.  This is where @worker starts participating in concurrency
2430         * management if applicable and concurrency management is restored
2431         * after being rebound.  See rebind_workers() for details.
2432         */
2433        worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND);
2434
2435        do {
2436                struct work_struct *work =
2437                        list_first_entry(&pool->worklist,
2438                                         struct work_struct, entry);
2439
2440                pool->watchdog_ts = jiffies;
2441
2442                if (likely(!(*work_data_bits(work) & WORK_STRUCT_LINKED))) {
2443                        /* optimization path, not strictly necessary */
2444                        process_one_work(worker, work);
2445                        if (unlikely(!list_empty(&worker->scheduled)))
2446                                process_scheduled_works(worker);
2447                } else {
2448                        move_linked_works(work, &worker->scheduled, NULL);
2449                        process_scheduled_works(worker);
2450                }
2451        } while (keep_working(pool));
2452
2453        worker_set_flags(worker, WORKER_PREP);
2454sleep:
2455        /*
2456         * pool->lock is held and there's no work to process and no need to
2457         * manage, sleep.  Workers are woken up only while holding
2458         * pool->lock or from local cpu, so setting the current state
2459         * before releasing pool->lock is enough to prevent losing any
2460         * event.
2461         */
2462        worker_enter_idle(worker);
2463        __set_current_state(TASK_IDLE);
2464        raw_spin_unlock_irq(&pool->lock);
2465        schedule();
2466        goto woke_up;
2467}
2468
2469/**
2470 * rescuer_thread - the rescuer thread function
2471 * @__rescuer: self
2472 *
2473 * Workqueue rescuer thread function.  There's one rescuer for each
2474 * workqueue which has WQ_MEM_RECLAIM set.
2475 *
2476 * Regular work processing on a pool may block trying to create a new
2477 * worker which uses GFP_KERNEL allocation which has slight chance of
2478 * developing into deadlock if some works currently on the same queue
2479 * need to be processed to satisfy the GFP_KERNEL allocation.  This is
2480 * the problem rescuer solves.
2481 *
2482 * When such condition is possible, the pool summons rescuers of all
2483 * workqueues which have works queued on the pool and let them process
2484 * those works so that forward progress can be guaranteed.
2485 *
2486 * This should happen rarely.
2487 *
2488 * Return: 0
2489 */
2490static int rescuer_thread(void *__rescuer)
2491{
2492        struct worker *rescuer = __rescuer;
2493        struct workqueue_struct *wq = rescuer->rescue_wq;
2494        struct list_head *scheduled = &rescuer->scheduled;
2495        bool should_stop;
2496
2497        set_user_nice(current, RESCUER_NICE_LEVEL);
2498
2499        /*
2500         * Mark rescuer as worker too.  As WORKER_PREP is never cleared, it
2501         * doesn't participate in concurrency management.
2502         */
2503        set_pf_worker(true);
2504repeat:
2505        set_current_state(TASK_IDLE);
2506
2507        /*
2508         * By the time the rescuer is requested to stop, the workqueue
2509         * shouldn't have any work pending, but @wq->maydays may still have
2510         * pwq(s) queued.  This can happen by non-rescuer workers consuming
2511         * all the work items before the rescuer got to them.  Go through
2512         * @wq->maydays processing before acting on should_stop so that the
2513         * list is always empty on exit.
2514         */
2515        should_stop = kthread_should_stop();
2516
2517        /* see whether any pwq is asking for help */
2518        raw_spin_lock_irq(&wq_mayday_lock);
2519
2520        while (!list_empty(&wq->maydays)) {
2521                struct pool_workqueue *pwq = list_first_entry(&wq->maydays,
2522                                        struct pool_workqueue, mayday_node);
2523                struct worker_pool *pool = pwq->pool;
2524                struct work_struct *work, *n;
2525                bool first = true;
2526
2527                __set_current_state(TASK_RUNNING);
2528                list_del_init(&pwq->mayday_node);
2529
2530                raw_spin_unlock_irq(&wq_mayday_lock);
2531
2532                worker_attach_to_pool(rescuer, pool);
2533
2534                raw_spin_lock_irq(&pool->lock);
2535
2536                /*
2537                 * Slurp in all works issued via this workqueue and
2538                 * process'em.
2539                 */
2540                WARN_ON_ONCE(!list_empty(scheduled));
2541                list_for_each_entry_safe(work, n, &pool->worklist, entry) {
2542                        if (get_work_pwq(work) == pwq) {
2543                                if (first)
2544                                        pool->watchdog_ts = jiffies;
2545                                move_linked_works(work, scheduled, &n);
2546                        }
2547                        first = false;
2548                }
2549
2550                if (!list_empty(scheduled)) {
2551                        process_scheduled_works(rescuer);
2552
2553                        /*
2554                         * The above execution of rescued work items could
2555                         * have created more to rescue through
2556                         * pwq_activate_first_inactive() or chained
2557                         * queueing.  Let's put @pwq back on mayday list so
2558                         * that such back-to-back work items, which may be
2559                         * being used to relieve memory pressure, don't
2560                         * incur MAYDAY_INTERVAL delay inbetween.
2561                         */
2562                        if (pwq->nr_active && need_to_create_worker(pool)) {
2563                                raw_spin_lock(&wq_mayday_lock);
2564                                /*
2565                                 * Queue iff we aren't racing destruction
2566                                 * and somebody else hasn't queued it already.
2567                                 */
2568                                if (wq->rescuer && list_empty(&pwq->mayday_node)) {
2569                                        get_pwq(pwq);
2570                                        list_add_tail(&pwq->mayday_node, &wq->maydays);
2571                                }
2572                                raw_spin_unlock(&wq_mayday_lock);
2573                        }
2574                }
2575
2576                /*
2577                 * Put the reference grabbed by send_mayday().  @pool won't
2578                 * go away while we're still attached to it.
2579                 */
2580                put_pwq(pwq);
2581
2582                /*
2583                 * Leave this pool.  If need_more_worker() is %true, notify a
2584                 * regular worker; otherwise, we end up with 0 concurrency
2585                 * and stalling the execution.
2586                 */
2587                if (need_more_worker(pool))
2588                        wake_up_worker(pool);
2589
2590                raw_spin_unlock_irq(&pool->lock);
2591
2592                worker_detach_from_pool(rescuer);
2593
2594                raw_spin_lock_irq(&wq_mayday_lock);
2595        }
2596
2597        raw_spin_unlock_irq(&wq_mayday_lock);
2598
2599        if (should_stop) {
2600                __set_current_state(TASK_RUNNING);
2601                set_pf_worker(false);
2602                return 0;
2603        }
2604
2605        /* rescuers should never participate in concurrency management */
2606        WARN_ON_ONCE(!(rescuer->flags & WORKER_NOT_RUNNING));
2607        schedule();
2608        goto repeat;
2609}
2610
2611/**
2612 * check_flush_dependency - check for flush dependency sanity
2613 * @target_wq: workqueue being flushed
2614 * @target_work: work item being flushed (NULL for workqueue flushes)
2615 *
2616 * %current is trying to flush the whole @target_wq or @target_work on it.
2617 * If @target_wq doesn't have %WQ_MEM_RECLAIM, verify that %current is not
2618 * reclaiming memory or running on a workqueue which doesn't have
2619 * %WQ_MEM_RECLAIM as that can break forward-progress guarantee leading to
2620 * a deadlock.
2621 */
2622static void check_flush_dependency(struct workqueue_struct *target_wq,
2623                                   struct work_struct *target_work)
2624{
2625        work_func_t target_func = target_work ? target_work->func : NULL;
2626        struct worker *worker;
2627
2628        if (target_wq->flags & WQ_MEM_RECLAIM)
2629                return;
2630
2631        worker = current_wq_worker();
2632
2633        WARN_ONCE(current->flags & PF_MEMALLOC,
2634                  "workqueue: PF_MEMALLOC task %d(%s) is flushing !WQ_MEM_RECLAIM %s:%ps",
2635                  current->pid, current->comm, target_wq->name, target_func);
2636        WARN_ONCE(worker && ((worker->current_pwq->wq->flags &
2637                              (WQ_MEM_RECLAIM | __WQ_LEGACY)) == WQ_MEM_RECLAIM),
2638                  "workqueue: WQ_MEM_RECLAIM %s:%ps is flushing !WQ_MEM_RECLAIM %s:%ps",
2639                  worker->current_pwq->wq->name, worker->current_func,
2640                  target_wq->name, target_func);
2641}
2642
2643struct wq_barrier {
2644        struct work_struct      work;
2645        struct completion       done;
2646        struct task_struct      *task;  /* purely informational */
2647};
2648
2649static void wq_barrier_func(struct work_struct *work)
2650{
2651        struct wq_barrier *barr = container_of(work, struct wq_barrier, work);
2652        complete(&barr->done);
2653}
2654
2655/**
2656 * insert_wq_barrier - insert a barrier work
2657 * @pwq: pwq to insert barrier into
2658 * @barr: wq_barrier to insert
2659 * @target: target work to attach @barr to
2660 * @worker: worker currently executing @target, NULL if @target is not executing
2661 *
2662 * @barr is linked to @target such that @barr is completed only after
2663 * @target finishes execution.  Please note that the ordering
2664 * guarantee is observed only with respect to @target and on the local
2665 * cpu.
2666 *
2667 * Currently, a queued barrier can't be canceled.  This is because
2668 * try_to_grab_pending() can't determine whether the work to be
2669 * grabbed is at the head of the queue and thus can't clear LINKED
2670 * flag of the previous work while there must be a valid next work
2671 * after a work with LINKED flag set.
2672 *
2673 * Note that when @worker is non-NULL, @target may be modified
2674 * underneath us, so we can't reliably determine pwq from @target.
2675 *
2676 * CONTEXT:
2677 * raw_spin_lock_irq(pool->lock).
2678 */
2679static void insert_wq_barrier(struct pool_workqueue *pwq,
2680                              struct wq_barrier *barr,
2681                              struct work_struct *target, struct worker *worker)
2682{
2683        unsigned int work_flags = 0;
2684        unsigned int work_color;
2685        struct list_head *head;
2686
2687        /*
2688         * debugobject calls are safe here even with pool->lock locked
2689         * as we know for sure that this will not trigger any of the
2690         * checks and call back into the fixup functions where we
2691         * might deadlock.
2692         */
2693        INIT_WORK_ONSTACK(&barr->work, wq_barrier_func);
2694        __set_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&barr->work));
2695
2696        init_completion_map(&barr->done, &target->lockdep_map);
2697
2698        barr->task = current;
2699
2700        /* The barrier work item does not participate in pwq->nr_active. */
2701        work_flags |= WORK_STRUCT_INACTIVE;
2702
2703        /*
2704         * If @target is currently being executed, schedule the
2705         * barrier to the worker; otherwise, put it after @target.
2706         */
2707        if (worker) {
2708                head = worker->scheduled.next;
2709                work_color = worker->current_color;
2710        } else {
2711                unsigned long *bits = work_data_bits(target);
2712
2713                head = target->entry.next;
2714                /* there can already be other linked works, inherit and set */
2715                work_flags |= *bits & WORK_STRUCT_LINKED;
2716                work_color = get_work_color(*bits);
2717                __set_bit(WORK_STRUCT_LINKED_BIT, bits);
2718        }
2719
2720        pwq->nr_in_flight[work_color]++;
2721        work_flags |= work_color_to_flags(work_color);
2722
2723        debug_work_activate(&barr->work);
2724        insert_work(pwq, &barr->work, head, work_flags);
2725}
2726
2727/**
2728 * flush_workqueue_prep_pwqs - prepare pwqs for workqueue flushing
2729 * @wq: workqueue being flushed
2730 * @flush_color: new flush color, < 0 for no-op
2731 * @work_color: new work color, < 0 for no-op
2732 *
2733 * Prepare pwqs for workqueue flushing.
2734 *
2735 * If @flush_color is non-negative, flush_color on all pwqs should be
2736 * -1.  If no pwq has in-flight commands at the specified color, all
2737 * pwq->flush_color's stay at -1 and %false is returned.  If any pwq
2738 * has in flight commands, its pwq->flush_color is set to
2739 * @flush_color, @wq->nr_pwqs_to_flush is updated accordingly, pwq
2740 * wakeup logic is armed and %true is returned.
2741 *
2742 * The caller should have initialized @wq->first_flusher prior to
2743 * calling this function with non-negative @flush_color.  If
2744 * @flush_color is negative, no flush color update is done and %false
2745 * is returned.
2746 *
2747 * If @work_color is non-negative, all pwqs should have the same
2748 * work_color which is previous to @work_color and all will be
2749 * advanced to @work_color.
2750 *
2751 * CONTEXT:
2752 * mutex_lock(wq->mutex).
2753 *
2754 * Return:
2755 * %true if @flush_color >= 0 and there's something to flush.  %false
2756 * otherwise.
2757 */
2758static bool flush_workqueue_prep_pwqs(struct workqueue_struct *wq,
2759                                      int flush_color, int work_color)
2760{
2761        bool wait = false;
2762        struct pool_workqueue *pwq;
2763
2764        if (flush_color >= 0) {
2765                WARN_ON_ONCE(atomic_read(&wq->nr_pwqs_to_flush));
2766                atomic_set(&wq->nr_pwqs_to_flush, 1);
2767        }
2768
2769        for_each_pwq(pwq, wq) {
2770                struct worker_pool *pool = pwq->pool;
2771
2772                raw_spin_lock_irq(&pool->lock);
2773
2774                if (flush_color >= 0) {
2775                        WARN_ON_ONCE(pwq->flush_color != -1);
2776
2777                        if (pwq->nr_in_flight[flush_color]) {
2778                                pwq->flush_color = flush_color;
2779                                atomic_inc(&wq->nr_pwqs_to_flush);
2780                                wait = true;
2781                        }
2782                }
2783
2784                if (work_color >= 0) {
2785                        WARN_ON_ONCE(work_color != work_next_color(pwq->work_color));
2786                        pwq->work_color = work_color;
2787                }
2788
2789                raw_spin_unlock_irq(&pool->lock);
2790        }
2791
2792        if (flush_color >= 0 && atomic_dec_and_test(&wq->nr_pwqs_to_flush))
2793                complete(&wq->first_flusher->done);
2794
2795        return wait;
2796}
2797
2798/**
2799 * flush_workqueue - ensure that any scheduled work has run to completion.
2800 * @wq: workqueue to flush
2801 *
2802 * This function sleeps until all work items which were queued on entry
2803 * have finished execution, but it is not livelocked by new incoming ones.
2804 */
2805void flush_workqueue(struct workqueue_struct *wq)
2806{
2807        struct wq_flusher this_flusher = {
2808                .list = LIST_HEAD_INIT(this_flusher.list),
2809                .flush_color = -1,
2810                .done = COMPLETION_INITIALIZER_ONSTACK_MAP(this_flusher.done, wq->lockdep_map),
2811        };
2812        int next_color;
2813
2814        if (WARN_ON(!wq_online))
2815                return;
2816
2817        lock_map_acquire(&wq->lockdep_map);
2818        lock_map_release(&wq->lockdep_map);
2819
2820        mutex_lock(&wq->mutex);
2821
2822        /*
2823         * Start-to-wait phase
2824         */
2825        next_color = work_next_color(wq->work_color);
2826
2827        if (next_color != wq->flush_color) {
2828                /*
2829                 * Color space is not full.  The current work_color
2830                 * becomes our flush_color and work_color is advanced
2831                 * by one.
2832                 */
2833                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow));
2834                this_flusher.flush_color = wq->work_color;
2835                wq->work_color = next_color;
2836
2837                if (!wq->first_flusher) {
2838                        /* no flush in progress, become the first flusher */
2839                        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2840
2841                        wq->first_flusher = &this_flusher;
2842
2843                        if (!flush_workqueue_prep_pwqs(wq, wq->flush_color,
2844                                                       wq->work_color)) {
2845                                /* nothing to flush, done */
2846                                wq->flush_color = next_color;
2847                                wq->first_flusher = NULL;
2848                                goto out_unlock;
2849                        }
2850                } else {
2851                        /* wait in queue */
2852                        WARN_ON_ONCE(wq->flush_color == this_flusher.flush_color);
2853                        list_add_tail(&this_flusher.list, &wq->flusher_queue);
2854                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2855                }
2856        } else {
2857                /*
2858                 * Oops, color space is full, wait on overflow queue.
2859                 * The next flush completion will assign us
2860                 * flush_color and transfer to flusher_queue.
2861                 */
2862                list_add_tail(&this_flusher.list, &wq->flusher_overflow);
2863        }
2864
2865        check_flush_dependency(wq, NULL);
2866
2867        mutex_unlock(&wq->mutex);
2868
2869        wait_for_completion(&this_flusher.done);
2870
2871        /*
2872         * Wake-up-and-cascade phase
2873         *
2874         * First flushers are responsible for cascading flushes and
2875         * handling overflow.  Non-first flushers can simply return.
2876         */
2877        if (READ_ONCE(wq->first_flusher) != &this_flusher)
2878                return;
2879
2880        mutex_lock(&wq->mutex);
2881
2882        /* we might have raced, check again with mutex held */
2883        if (wq->first_flusher != &this_flusher)
2884                goto out_unlock;
2885
2886        WRITE_ONCE(wq->first_flusher, NULL);
2887
2888        WARN_ON_ONCE(!list_empty(&this_flusher.list));
2889        WARN_ON_ONCE(wq->flush_color != this_flusher.flush_color);
2890
2891        while (true) {
2892                struct wq_flusher *next, *tmp;
2893
2894                /* complete all the flushers sharing the current flush color */
2895                list_for_each_entry_safe(next, tmp, &wq->flusher_queue, list) {
2896                        if (next->flush_color != wq->flush_color)
2897                                break;
2898                        list_del_init(&next->list);
2899                        complete(&next->done);
2900                }
2901
2902                WARN_ON_ONCE(!list_empty(&wq->flusher_overflow) &&
2903                             wq->flush_color != work_next_color(wq->work_color));
2904
2905                /* this flush_color is finished, advance by one */
2906                wq->flush_color = work_next_color(wq->flush_color);
2907
2908                /* one color has been freed, handle overflow queue */
2909                if (!list_empty(&wq->flusher_overflow)) {
2910                        /*
2911                         * Assign the same color to all overflowed
2912                         * flushers, advance work_color and append to
2913                         * flusher_queue.  This is the start-to-wait
2914                         * phase for these overflowed flushers.
2915                         */
2916                        list_for_each_entry(tmp, &wq->flusher_overflow, list)
2917                                tmp->flush_color = wq->work_color;
2918
2919                        wq->work_color = work_next_color(wq->work_color);
2920
2921                        list_splice_tail_init(&wq->flusher_overflow,
2922                                              &wq->flusher_queue);
2923                        flush_workqueue_prep_pwqs(wq, -1, wq->work_color);
2924                }
2925
2926                if (list_empty(&wq->flusher_queue)) {
2927                        WARN_ON_ONCE(wq->flush_color != wq->work_color);
2928                        break;
2929                }
2930
2931                /*
2932                 * Need to flush more colors.  Make the next flusher
2933                 * the new first flusher and arm pwqs.
2934                 */
2935                WARN_ON_ONCE(wq->flush_color == wq->work_color);
2936                WARN_ON_ONCE(wq->flush_color != next->flush_color);
2937
2938                list_del_init(&next->list);
2939                wq->first_flusher = next;
2940
2941                if (flush_workqueue_prep_pwqs(wq, wq->flush_color, -1))
2942                        break;
2943
2944                /*
2945                 * Meh... this color is already done, clear first
2946                 * flusher and repeat cascading.
2947                 */
2948                wq->first_flusher = NULL;
2949        }
2950
2951out_unlock:
2952        mutex_unlock(&wq->mutex);
2953}
2954EXPORT_SYMBOL(flush_workqueue);
2955
2956/**
2957 * drain_workqueue - drain a workqueue
2958 * @wq: workqueue to drain
2959 *
2960 * Wait until the workqueue becomes empty.  While draining is in progress,
2961 * only chain queueing is allowed.  IOW, only currently pending or running
2962 * work items on @wq can queue further work items on it.  @wq is flushed
2963 * repeatedly until it becomes empty.  The number of flushing is determined
2964 * by the depth of chaining and should be relatively short.  Whine if it
2965 * takes too long.
2966 */
2967void drain_workqueue(struct workqueue_struct *wq)
2968{
2969        unsigned int flush_cnt = 0;
2970        struct pool_workqueue *pwq;
2971
2972        /*
2973         * __queue_work() needs to test whether there are drainers, is much
2974         * hotter than drain_workqueue() and already looks at @wq->flags.
2975         * Use __WQ_DRAINING so that queue doesn't have to check nr_drainers.
2976         */
2977        mutex_lock(&wq->mutex);
2978        if (!wq->nr_drainers++)
2979                wq->flags |= __WQ_DRAINING;
2980        mutex_unlock(&wq->mutex);
2981reflush:
2982        flush_workqueue(wq);
2983
2984        mutex_lock(&wq->mutex);
2985
2986        for_each_pwq(pwq, wq) {
2987                bool drained;
2988
2989                raw_spin_lock_irq(&pwq->pool->lock);
2990                drained = !pwq->nr_active && list_empty(&pwq->inactive_works);
2991                raw_spin_unlock_irq(&pwq->pool->lock);
2992
2993                if (drained)
2994                        continue;
2995
2996                if (++flush_cnt == 10 ||
2997                    (flush_cnt % 100 == 0 && flush_cnt <= 1000))
2998                        pr_warn("workqueue %s: %s() isn't complete after %u tries\n",
2999                                wq->name, __func__, flush_cnt);
3000
3001                mutex_unlock(&wq->mutex);
3002                goto reflush;
3003        }
3004
3005        if (!--wq->nr_drainers)
3006                wq->flags &= ~__WQ_DRAINING;
3007        mutex_unlock(&wq->mutex);
3008}
3009EXPORT_SYMBOL_GPL(drain_workqueue);
3010
3011static bool start_flush_work(struct work_struct *work, struct wq_barrier *barr,
3012                             bool from_cancel)
3013{
3014        struct worker *worker = NULL;
3015        struct worker_pool *pool;
3016        struct pool_workqueue *pwq;
3017
3018        might_sleep();
3019
3020        rcu_read_lock();
3021        pool = get_work_pool(work);
3022        if (!pool) {
3023                rcu_read_unlock();
3024                return false;
3025        }
3026
3027        raw_spin_lock_irq(&pool->lock);
3028        /* see the comment in try_to_grab_pending() with the same code */
3029        pwq = get_work_pwq(work);
3030        if (pwq) {
3031                if (unlikely(pwq->pool != pool))
3032                        goto already_gone;
3033        } else {
3034                worker = find_worker_executing_work(pool, work);
3035                if (!worker)
3036                        goto already_gone;
3037                pwq = worker->current_pwq;
3038        }
3039
3040        check_flush_dependency(pwq->wq, work);
3041
3042        insert_wq_barrier(pwq, barr, work, worker);
3043        raw_spin_unlock_irq(&pool->lock);
3044
3045        /*
3046         * Force a lock recursion deadlock when using flush_work() inside a
3047         * single-threaded or rescuer equipped workqueue.
3048         *
3049         * For single threaded workqueues the deadlock happens when the work
3050         * is after the work issuing the flush_work(). For rescuer equipped
3051         * workqueues the deadlock happens when the rescuer stalls, blocking
3052         * forward progress.
3053         */
3054        if (!from_cancel &&
3055            (pwq->wq->saved_max_active == 1 || pwq->wq->rescuer)) {
3056                lock_map_acquire(&pwq->wq->lockdep_map);
3057                lock_map_release(&pwq->wq->lockdep_map);
3058        }
3059        rcu_read_unlock();
3060        return true;
3061already_gone:
3062        raw_spin_unlock_irq(&pool->lock);
3063        rcu_read_unlock();
3064        return false;
3065}
3066
3067static bool __flush_work(struct work_struct *work, bool from_cancel)
3068{
3069        struct wq_barrier barr;
3070
3071        if (WARN_ON(!wq_online))
3072                return false;
3073
3074        if (WARN_ON(!work->func))
3075                return false;
3076
3077        if (!from_cancel) {
3078                lock_map_acquire(&work->lockdep_map);
3079                lock_map_release(&work->lockdep_map);
3080        }
3081
3082        if (start_flush_work(work, &barr, from_cancel)) {
3083                wait_for_completion(&barr.done);
3084                destroy_work_on_stack(&barr.work);
3085                return true;
3086        } else {
3087                return false;
3088        }
3089}
3090
3091/**
3092 * flush_work - wait for a work to finish executing the last queueing instance
3093 * @work: the work to flush
3094 *
3095 * Wait until @work has finished execution.  @work is guaranteed to be idle
3096 * on return if it hasn't been requeued since flush started.
3097 *
3098 * Return:
3099 * %true if flush_work() waited for the work to finish execution,
3100 * %false if it was already idle.
3101 */
3102bool flush_work(struct work_struct *work)
3103{
3104        return __flush_work(work, false);
3105}
3106EXPORT_SYMBOL_GPL(flush_work);
3107
3108struct cwt_wait {
3109        wait_queue_entry_t              wait;
3110        struct work_struct      *work;
3111};
3112
3113static int cwt_wakefn(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
3114{
3115        struct cwt_wait *cwait = container_of(wait, struct cwt_wait, wait);
3116
3117        if (cwait->work != key)
3118                return 0;
3119        return autoremove_wake_function(wait, mode, sync, key);
3120}
3121
3122static bool __cancel_work_timer(struct work_struct *work, bool is_dwork)
3123{
3124        static DECLARE_WAIT_QUEUE_HEAD(cancel_waitq);
3125        unsigned long flags;
3126        int ret;
3127
3128        do {
3129                ret = try_to_grab_pending(work, is_dwork, &flags);
3130                /*
3131                 * If someone else is already canceling, wait for it to
3132                 * finish.  flush_work() doesn't work for PREEMPT_NONE
3133                 * because we may get scheduled between @work's completion
3134                 * and the other canceling task resuming and clearing
3135                 * CANCELING - flush_work() will return false immediately
3136                 * as @work is no longer busy, try_to_grab_pending() will
3137                 * return -ENOENT as @work is still being canceled and the
3138                 * other canceling task won't be able to clear CANCELING as
3139                 * we're hogging the CPU.
3140                 *
3141                 * Let's wait for completion using a waitqueue.  As this
3142                 * may lead to the thundering herd problem, use a custom
3143                 * wake function which matches @work along with exclusive
3144                 * wait and wakeup.
3145                 */
3146                if (unlikely(ret == -ENOENT)) {
3147                        struct cwt_wait cwait;
3148
3149                        init_wait(&cwait.wait);
3150                        cwait.wait.func = cwt_wakefn;
3151                        cwait.work = work;
3152
3153                        prepare_to_wait_exclusive(&cancel_waitq, &cwait.wait,
3154                                                  TASK_UNINTERRUPTIBLE);
3155                        if (work_is_canceling(work))
3156                                schedule();
3157                        finish_wait(&cancel_waitq, &cwait.wait);
3158                }
3159        } while (unlikely(ret < 0));
3160
3161        /* tell other tasks trying to grab @work to back off */
3162        mark_work_canceling(work);
3163        local_irq_restore(flags);
3164
3165        /*
3166         * This allows canceling during early boot.  We know that @work
3167         * isn't executing.
3168         */
3169        if (wq_online)
3170                __flush_work(work, true);
3171
3172        clear_work_data(work);
3173
3174        /*
3175         * Paired with prepare_to_wait() above so that either
3176         * waitqueue_active() is visible here or !work_is_canceling() is
3177         * visible there.
3178         */
3179        smp_mb();
3180        if (waitqueue_active(&cancel_waitq))
3181                __wake_up(&cancel_waitq, TASK_NORMAL, 1, work);
3182
3183        return ret;
3184}
3185
3186/**
3187 * cancel_work_sync - cancel a work and wait for it to finish
3188 * @work: the work to cancel
3189 *
3190 * Cancel @work and wait for its execution to finish.  This function
3191 * can be used even if the work re-queues itself or migrates to
3192 * another workqueue.  On return from this function, @work is
3193 * guaranteed to be not pending or executing on any CPU.
3194 *
3195 * cancel_work_sync(&delayed_work->work) must not be used for
3196 * delayed_work's.  Use cancel_delayed_work_sync() instead.
3197 *
3198 * The caller must ensure that the workqueue on which @work was last
3199 * queued can't be destroyed before this function returns.
3200 *
3201 * Return:
3202 * %true if @work was pending, %false otherwise.
3203 */
3204bool cancel_work_sync(struct work_struct *work)
3205{
3206        return __cancel_work_timer(work, false);
3207}
3208EXPORT_SYMBOL_GPL(cancel_work_sync);
3209
3210/**
3211 * flush_delayed_work - wait for a dwork to finish executing the last queueing
3212 * @dwork: the delayed work to flush
3213 *
3214 * Delayed timer is cancelled and the pending work is queued for
3215 * immediate execution.  Like flush_work(), this function only
3216 * considers the last queueing instance of @dwork.
3217 *
3218 * Return:
3219 * %true if flush_work() waited for the work to finish execution,
3220 * %false if it was already idle.
3221 */
3222bool flush_delayed_work(struct delayed_work *dwork)
3223{
3224        local_irq_disable();
3225        if (del_timer_sync(&dwork->timer))
3226                __queue_work(dwork->cpu, dwork->wq, &dwork->work);
3227        local_irq_enable();
3228        return flush_work(&dwork->work);
3229}
3230EXPORT_SYMBOL(flush_delayed_work);
3231
3232/**
3233 * flush_rcu_work - wait for a rwork to finish executing the last queueing
3234 * @rwork: the rcu work to flush
3235 *
3236 * Return:
3237 * %true if flush_rcu_work() waited for the work to finish execution,
3238 * %false if it was already idle.
3239 */
3240bool flush_rcu_work(struct rcu_work *rwork)
3241{
3242        if (test_bit(WORK_STRUCT_PENDING_BIT, work_data_bits(&rwork->work))) {
3243                rcu_barrier();
3244                flush_work(&rwork->work);
3245                return true;
3246        } else {
3247                return flush_work(&rwork->work);
3248        }
3249}
3250EXPORT_SYMBOL(flush_rcu_work);
3251
3252static bool __cancel_work(struct work_struct *work, bool is_dwork)
3253{
3254        unsigned long flags;
3255        int ret;
3256
3257        do {
3258                ret = try_to_grab_pending(work, is_dwork, &flags);
3259        } while (unlikely(ret == -EAGAIN));
3260
3261        if (unlikely(ret < 0))
3262                return false;
3263
3264        set_work_pool_and_clear_pending(work, get_work_pool_id(work));
3265        local_irq_restore(flags);
3266        return ret;
3267}
3268
3269/**
3270 * cancel_delayed_work - cancel a delayed work
3271 * @dwork: delayed_work to cancel
3272 *
3273 * Kill off a pending delayed_work.
3274 *
3275 * Return: %true if @dwork was pending and canceled; %false if it wasn't
3276 * pending.
3277 *
3278 * Note:
3279 * The work callback function may still be running on return, unless
3280 * it returns %true and the work doesn't re-arm itself.  Explicitly flush or
3281 * use cancel_delayed_work_sync() to wait on it.
3282 *
3283 * This function is safe to call from any context including IRQ handler.
3284 */
3285bool cancel_delayed_work(struct delayed_work *dwork)
3286{
3287        return __cancel_work(&dwork->work, true);
3288}
3289EXPORT_SYMBOL(cancel_delayed_work);
3290
3291/**
3292 * cancel_delayed_work_sync - cancel a delayed work and wait for it to finish
3293 * @dwork: the delayed work cancel
3294 *
3295 * This is cancel_work_sync() for delayed works.
3296 *
3297 * Return:
3298 * %true if @dwork was pending, %false otherwise.
3299 */
3300bool cancel_delayed_work_sync(struct delayed_work *dwork)
3301{
3302        return __cancel_work_timer(&dwork->work, true);
3303}
3304EXPORT_SYMBOL(cancel_delayed_work_sync);
3305
3306/**
3307 * schedule_on_each_cpu - execute a function synchronously on each online CPU
3308 * @func: the function to call
3309 *
3310 * schedule_on_each_cpu() executes @func on each online CPU using the
3311 * system workqueue and blocks until all CPUs have completed.
3312 * schedule_on_each_cpu() is very slow.
3313 *
3314 * Return:
3315 * 0 on success, -errno on failure.
3316 */
3317int schedule_on_each_cpu(work_func_t func)
3318{
3319        int cpu;
3320        struct work_struct __percpu *works;
3321
3322        works = alloc_percpu(struct work_struct);
3323        if (!works)
3324                return -ENOMEM;
3325
3326        cpus_read_lock();
3327
3328        for_each_online_cpu(cpu) {
3329                struct work_struct *work = per_cpu_ptr(works, cpu);
3330
3331                INIT_WORK(work, func);
3332                schedule_work_on(cpu, work);
3333        }
3334
3335        for_each_online_cpu(cpu)
3336                flush_work(per_cpu_ptr(works, cpu));
3337
3338        cpus_read_unlock();
3339        free_percpu(works);
3340        return 0;
3341}
3342
3343/**
3344 * execute_in_process_context - reliably execute the routine with user context
3345 * @fn:         the function to execute
3346 * @ew:         guaranteed storage for the execute work structure (must
3347 *              be available when the work executes)
3348 *
3349 * Executes the function immediately if process context is available,
3350 * otherwise schedules the function for delayed execution.
3351 *
3352 * Return:      0 - function was executed
3353 *              1 - function was scheduled for execution
3354 */
3355int execute_in_process_context(work_func_t fn, struct execute_work *ew)
3356{
3357        if (!in_interrupt()) {
3358                fn(&ew->work);
3359                return 0;
3360        }
3361
3362        INIT_WORK(&ew->work, fn);
3363        schedule_work(&ew->work);
3364
3365        return 1;
3366}
3367EXPORT_SYMBOL_GPL(execute_in_process_context);
3368
3369/**
3370 * free_workqueue_attrs - free a workqueue_attrs
3371 * @attrs: workqueue_attrs to free
3372 *
3373 * Undo alloc_workqueue_attrs().
3374 */
3375void free_workqueue_attrs(struct workqueue_attrs *attrs)
3376{
3377        if (attrs) {
3378                free_cpumask_var(attrs->cpumask);
3379                kfree(attrs);
3380        }
3381}
3382
3383/**
3384 * alloc_workqueue_attrs - allocate a workqueue_attrs
3385 *
3386 * Allocate a new workqueue_attrs, initialize with default settings and
3387 * return it.
3388 *
3389 * Return: The allocated new workqueue_attr on success. %NULL on failure.
3390 */
3391struct workqueue_attrs *alloc_workqueue_attrs(void)
3392{
3393        struct workqueue_attrs *attrs;
3394
3395        attrs = kzalloc(sizeof(*attrs), GFP_KERNEL);
3396        if (!attrs)
3397                goto fail;
3398        if (!alloc_cpumask_var(&attrs->cpumask, GFP_KERNEL))
3399                goto fail;
3400
3401        cpumask_copy(attrs->cpumask, cpu_possible_mask);
3402        return attrs;
3403fail:
3404        free_workqueue_attrs(attrs);
3405        return NULL;
3406}
3407
3408static void copy_workqueue_attrs(struct workqueue_attrs *to,
3409                                 const struct workqueue_attrs *from)
3410{
3411        to->nice = from->nice;
3412        cpumask_copy(to->cpumask, from->cpumask);
3413        /*
3414         * Unlike hash and equality test, this function doesn't ignore
3415         * ->no_numa as it is used for both pool and wq attrs.  Instead,
3416         * get_unbound_pool() explicitly clears ->no_numa after copying.
3417         */
3418        to->no_numa = from->no_numa;
3419}
3420
3421/* hash value of the content of @attr */
3422static u32 wqattrs_hash(const struct workqueue_attrs *attrs)
3423{
3424        u32 hash = 0;
3425
3426        hash = jhash_1word(attrs->nice, hash);
3427        hash = jhash(cpumask_bits(attrs->cpumask),
3428                     BITS_TO_LONGS(nr_cpumask_bits) * sizeof(long), hash);
3429        return hash;
3430}
3431
3432/* content equality test */
3433static bool wqattrs_equal(const struct workqueue_attrs *a,
3434                          const struct workqueue_attrs *b)
3435{
3436        if (a->nice != b->nice)
3437                return false;
3438        if (!cpumask_equal(a->cpumask, b->cpumask))
3439                return false;
3440        return true;
3441}
3442
3443/**
3444 * init_worker_pool - initialize a newly zalloc'd worker_pool
3445 * @pool: worker_pool to initialize
3446 *
3447 * Initialize a newly zalloc'd @pool.  It also allocates @pool->attrs.
3448 *
3449 * Return: 0 on success, -errno on failure.  Even on failure, all fields
3450 * inside @pool proper are initialized and put_unbound_pool() can be called
3451 * on @pool safely to release it.
3452 */
3453static int init_worker_pool(struct worker_pool *pool)
3454{
3455        raw_spin_lock_init(&pool->lock);
3456        pool->id = -1;
3457        pool->cpu = -1;
3458        pool->node = NUMA_NO_NODE;
3459        pool->flags |= POOL_DISASSOCIATED;
3460        pool->watchdog_ts = jiffies;
3461        INIT_LIST_HEAD(&pool->worklist);
3462        INIT_LIST_HEAD(&pool->idle_list);
3463        hash_init(pool->busy_hash);
3464
3465        timer_setup(&pool->idle_timer, idle_worker_timeout, TIMER_DEFERRABLE);
3466
3467        timer_setup(&pool->mayday_timer, pool_mayday_timeout, 0);
3468
3469        INIT_LIST_HEAD(&pool->workers);
3470
3471        ida_init(&pool->worker_ida);
3472        INIT_HLIST_NODE(&pool->hash_node);
3473        pool->refcnt = 1;
3474
3475        /* shouldn't fail above this point */
3476        pool->attrs = alloc_workqueue_attrs();
3477        if (!pool->attrs)
3478                return -ENOMEM;
3479        return 0;
3480}
3481
3482#ifdef CONFIG_LOCKDEP
3483static void wq_init_lockdep(struct workqueue_struct *wq)
3484{
3485        char *lock_name;
3486
3487        lockdep_register_key(&wq->key);
3488        lock_name = kasprintf(GFP_KERNEL, "%s%s", "(wq_completion)", wq->name);
3489        if (!lock_name)
3490                lock_name = wq->name;
3491
3492        wq->lock_name = lock_name;
3493        lockdep_init_map(&wq->lockdep_map, lock_name, &wq->key, 0);
3494}
3495
3496static void wq_unregister_lockdep(struct workqueue_struct *wq)
3497{
3498        lockdep_unregister_key(&wq->key);
3499}
3500
3501static void wq_free_lockdep(struct workqueue_struct *wq)
3502{
3503        if (wq->lock_name != wq->name)
3504                kfree(wq->lock_name);
3505}
3506#else
3507static void wq_init_lockdep(struct workqueue_struct *wq)
3508{
3509}
3510
3511static void wq_unregister_lockdep(struct workqueue_struct *wq)
3512{
3513}
3514
3515static void wq_free_lockdep(struct workqueue_struct *wq)
3516{
3517}
3518#endif
3519
3520static void rcu_free_wq(struct rcu_head *rcu)
3521{
3522        struct workqueue_struct *wq =
3523                container_of(rcu, struct workqueue_struct, rcu);
3524
3525        wq_free_lockdep(wq);
3526
3527        if (!(wq->flags & WQ_UNBOUND))
3528                free_percpu(wq->cpu_pwqs);
3529        else
3530                free_workqueue_attrs(wq->unbound_attrs);
3531
3532        kfree(wq);
3533}
3534
3535static void rcu_free_pool(struct rcu_head *rcu)
3536{
3537        struct worker_pool *pool = container_of(rcu, struct worker_pool, rcu);
3538
3539        ida_destroy(&pool->worker_ida);
3540        free_workqueue_attrs(pool->attrs);
3541        kfree(pool);
3542}
3543
3544/* This returns with the lock held on success (pool manager is inactive). */
3545static bool wq_manager_inactive(struct worker_pool *pool)
3546{
3547        raw_spin_lock_irq(&pool->lock);
3548
3549        if (pool->flags & POOL_MANAGER_ACTIVE) {
3550                raw_spin_unlock_irq(&pool->lock);
3551                return false;
3552        }
3553        return true;
3554}
3555
3556/**
3557 * put_unbound_pool - put a worker_pool
3558 * @pool: worker_pool to put
3559 *
3560 * Put @pool.  If its refcnt reaches zero, it gets destroyed in RCU
3561 * safe manner.  get_unbound_pool() calls this function on its failure path
3562 * and this function should be able to release pools which went through,
3563 * successfully or not, init_worker_pool().
3564 *
3565 * Should be called with wq_pool_mutex held.
3566 */
3567static void put_unbound_pool(struct worker_pool *pool)
3568{
3569        DECLARE_COMPLETION_ONSTACK(detach_completion);
3570        struct worker *worker;
3571
3572        lockdep_assert_held(&wq_pool_mutex);
3573
3574        if (--pool->refcnt)
3575                return;
3576
3577        /* sanity checks */
3578        if (WARN_ON(!(pool->cpu < 0)) ||
3579            WARN_ON(!list_empty(&pool->worklist)))
3580                return;
3581
3582        /* release id and unhash */
3583        if (pool->id >= 0)
3584                idr_remove(&worker_pool_idr, pool->id);
3585        hash_del(&pool->hash_node);
3586
3587        /*
3588         * Become the manager and destroy all workers.  This prevents
3589         * @pool's workers from blocking on attach_mutex.  We're the last
3590         * manager and @pool gets freed with the flag set.
3591         * Because of how wq_manager_inactive() works, we will hold the
3592         * spinlock after a successful wait.
3593         */
3594        rcuwait_wait_event(&manager_wait, wq_manager_inactive(pool),
3595                           TASK_UNINTERRUPTIBLE);
3596        pool->flags |= POOL_MANAGER_ACTIVE;
3597
3598        while ((worker = first_idle_worker(pool)))
3599                destroy_worker(worker);
3600        WARN_ON(pool->nr_workers || pool->nr_idle);
3601        raw_spin_unlock_irq(&pool->lock);
3602
3603        mutex_lock(&wq_pool_attach_mutex);
3604        if (!list_empty(&pool->workers))
3605                pool->detach_completion = &detach_completion;
3606        mutex_unlock(&wq_pool_attach_mutex);
3607
3608        if (pool->detach_completion)
3609                wait_for_completion(pool->detach_completion);
3610
3611        /* shut down the timers */
3612        del_timer_sync(&pool->idle_timer);
3613        del_timer_sync(&pool->mayday_timer);
3614
3615        /* RCU protected to allow dereferences from get_work_pool() */
3616        call_rcu(&pool->rcu, rcu_free_pool);
3617}
3618
3619/**
3620 * get_unbound_pool - get a worker_pool with the specified attributes
3621 * @attrs: the attributes of the worker_pool to get
3622 *
3623 * Obtain a worker_pool which has the same attributes as @attrs, bump the
3624 * reference count and return it.  If there already is a matching
3625 * worker_pool, it will be used; otherwise, this function attempts to
3626 * create a new one.
3627 *
3628 * Should be called with wq_pool_mutex held.
3629 *
3630 * Return: On success, a worker_pool with the same attributes as @attrs.
3631 * On failure, %NULL.
3632 */
3633static struct worker_pool *get_unbound_pool(const struct workqueue_attrs *attrs)
3634{
3635        u32 hash = wqattrs_hash(attrs);
3636        struct worker_pool *pool;
3637        int node;
3638        int target_node = NUMA_NO_NODE;
3639
3640        lockdep_assert_held(&wq_pool_mutex);
3641
3642        /* do we already have a matching pool? */
3643        hash_for_each_possible(unbound_pool_hash, pool, hash_node, hash) {
3644                if (wqattrs_equal(pool->attrs, attrs)) {
3645                        pool->refcnt++;
3646                        return pool;
3647                }
3648        }
3649
3650        /* if cpumask is contained inside a NUMA node, we belong to that node */
3651        if (wq_numa_enabled) {
3652                for_each_node(node) {
3653                        if (cpumask_subset(attrs->cpumask,
3654                                           wq_numa_possible_cpumask[node])) {
3655                                target_node = node;
3656                                break;
3657                        }
3658                }
3659        }
3660
3661        /* nope, create a new one */
3662        pool = kzalloc_node(sizeof(*pool), GFP_KERNEL, target_node);
3663        if (!pool || init_worker_pool(pool) < 0)
3664                goto fail;
3665
3666        lockdep_set_subclass(&pool->lock, 1);   /* see put_pwq() */
3667        copy_workqueue_attrs(pool->attrs, attrs);
3668        pool->node = target_node;
3669
3670        /*
3671         * no_numa isn't a worker_pool attribute, always clear it.  See
3672         * 'struct workqueue_attrs' comments for detail.
3673         */
3674        pool->attrs->no_numa = false;
3675
3676        if (worker_pool_assign_id(pool) < 0)
3677                goto fail;
3678
3679        /* create and start the initial worker */
3680        if (wq_online && !create_worker(pool))
3681                goto fail;
3682
3683        /* install */
3684        hash_add(unbound_pool_hash, &pool->hash_node, hash);
3685
3686        return pool;
3687fail:
3688        if (pool)
3689                put_unbound_pool(pool);
3690        return NULL;
3691}
3692
3693static void rcu_free_pwq(struct rcu_head *rcu)
3694{
3695        kmem_cache_free(pwq_cache,
3696                        container_of(rcu, struct pool_workqueue, rcu));
3697}
3698
3699/*
3700 * Scheduled on system_wq by put_pwq() when an unbound pwq hits zero refcnt
3701 * and needs to be destroyed.
3702 */
3703static void pwq_unbound_release_workfn(struct work_struct *work)
3704{
3705        struct pool_workqueue *pwq = container_of(work, struct pool_workqueue,
3706                                                  unbound_release_work);
3707        struct workqueue_struct *wq = pwq->wq;
3708        struct worker_pool *pool = pwq->pool;
3709        bool is_last = false;
3710
3711        /*
3712         * when @pwq is not linked, it doesn't hold any reference to the
3713         * @wq, and @wq is invalid to access.
3714         */
3715        if (!list_empty(&pwq->pwqs_node)) {
3716                if (WARN_ON_ONCE(!(wq->flags & WQ_UNBOUND)))
3717                        return;
3718
3719                mutex_lock(&wq->mutex);
3720                list_del_rcu(&pwq->pwqs_node);
3721                is_last = list_empty(&wq->pwqs);
3722                mutex_unlock(&wq->mutex);
3723        }
3724
3725        mutex_lock(&wq_pool_mutex);
3726        put_unbound_pool(pool);
3727        mutex_unlock(&wq_pool_mutex);
3728
3729        call_rcu(&pwq->rcu, rcu_free_pwq);
3730
3731        /*
3732         * If we're the last pwq going away, @wq is already dead and no one
3733         * is gonna access it anymore.  Schedule RCU free.
3734         */
3735        if (is_last) {
3736                wq_unregister_lockdep(wq);
3737                call_rcu(&wq->rcu, rcu_free_wq);
3738        }
3739}
3740
3741/**
3742 * pwq_adjust_max_active - update a pwq's max_active to the current setting
3743 * @pwq: target pool_workqueue
3744 *
3745 * If @pwq isn't freezing, set @pwq->max_active to the associated
3746 * workqueue's saved_max_active and activate inactive work items
3747 * accordingly.  If @pwq is freezing, clear @pwq->max_active to zero.
3748 */
3749static void pwq_adjust_max_active(struct pool_workqueue *pwq)
3750{
3751        struct workqueue_struct *wq = pwq->wq;
3752        bool freezable = wq->flags & WQ_FREEZABLE;
3753        unsigned long flags;
3754
3755        /* for @wq->saved_max_active */
3756        lockdep_assert_held(&wq->mutex);
3757
3758        /* fast exit for non-freezable wqs */
3759        if (!freezable && pwq->max_active == wq->saved_max_active)
3760                return;
3761
3762        /* this function can be called during early boot w/ irq disabled */
3763        raw_spin_lock_irqsave(&pwq->pool->lock, flags);
3764
3765        /*
3766         * During [un]freezing, the caller is responsible for ensuring that
3767         * this function is called at least once after @workqueue_freezing
3768         * is updated and visible.
3769         */
3770        if (!freezable || !workqueue_freezing) {
3771                bool kick = false;
3772
3773                pwq->max_active = wq->saved_max_active;
3774
3775                while (!list_empty(&pwq->inactive_works) &&
3776                       pwq->nr_active < pwq->max_active) {
3777                        pwq_activate_first_inactive(pwq);
3778                        kick = true;
3779                }
3780
3781                /*
3782                 * Need to kick a worker after thawed or an unbound wq's
3783                 * max_active is bumped. In realtime scenarios, always kicking a
3784                 * worker will cause interference on the isolated cpu cores, so
3785                 * let's kick iff work items were activated.
3786                 */
3787                if (kick)
3788                        wake_up_worker(pwq->pool);
3789        } else {
3790                pwq->max_active = 0;
3791        }
3792
3793        raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
3794}
3795
3796/* initialize newly allocated @pwq which is associated with @wq and @pool */
3797static void init_pwq(struct pool_workqueue *pwq, struct workqueue_struct *wq,
3798                     struct worker_pool *pool)
3799{
3800        BUG_ON((unsigned long)pwq & WORK_STRUCT_FLAG_MASK);
3801
3802        memset(pwq, 0, sizeof(*pwq));
3803
3804        pwq->pool = pool;
3805        pwq->wq = wq;
3806        pwq->flush_color = -1;
3807        pwq->refcnt = 1;
3808        INIT_LIST_HEAD(&pwq->inactive_works);
3809        INIT_LIST_HEAD(&pwq->pwqs_node);
3810        INIT_LIST_HEAD(&pwq->mayday_node);
3811        INIT_WORK(&pwq->unbound_release_work, pwq_unbound_release_workfn);
3812}
3813
3814/* sync @pwq with the current state of its associated wq and link it */
3815static void link_pwq(struct pool_workqueue *pwq)
3816{
3817        struct workqueue_struct *wq = pwq->wq;
3818
3819        lockdep_assert_held(&wq->mutex);
3820
3821        /* may be called multiple times, ignore if already linked */
3822        if (!list_empty(&pwq->pwqs_node))
3823                return;
3824
3825        /* set the matching work_color */
3826        pwq->work_color = wq->work_color;
3827
3828        /* sync max_active to the current setting */
3829        pwq_adjust_max_active(pwq);
3830
3831        /* link in @pwq */
3832        list_add_rcu(&pwq->pwqs_node, &wq->pwqs);
3833}
3834
3835/* obtain a pool matching @attr and create a pwq associating the pool and @wq */
3836static struct pool_workqueue *alloc_unbound_pwq(struct workqueue_struct *wq,
3837                                        const struct workqueue_attrs *attrs)
3838{
3839        struct worker_pool *pool;
3840        struct pool_workqueue *pwq;
3841
3842        lockdep_assert_held(&wq_pool_mutex);
3843
3844        pool = get_unbound_pool(attrs);
3845        if (!pool)
3846                return NULL;
3847
3848        pwq = kmem_cache_alloc_node(pwq_cache, GFP_KERNEL, pool->node);
3849        if (!pwq) {
3850                put_unbound_pool(pool);
3851                return NULL;
3852        }
3853
3854        init_pwq(pwq, wq, pool);
3855        return pwq;
3856}
3857
3858/**
3859 * wq_calc_node_cpumask - calculate a wq_attrs' cpumask for the specified node
3860 * @attrs: the wq_attrs of the default pwq of the target workqueue
3861 * @node: the target NUMA node
3862 * @cpu_going_down: if >= 0, the CPU to consider as offline
3863 * @cpumask: outarg, the resulting cpumask
3864 *
3865 * Calculate the cpumask a workqueue with @attrs should use on @node.  If
3866 * @cpu_going_down is >= 0, that cpu is considered offline during
3867 * calculation.  The result is stored in @cpumask.
3868 *
3869 * If NUMA affinity is not enabled, @attrs->cpumask is always used.  If
3870 * enabled and @node has online CPUs requested by @attrs, the returned
3871 * cpumask is the intersection of the possible CPUs of @node and
3872 * @attrs->cpumask.
3873 *
3874 * The caller is responsible for ensuring that the cpumask of @node stays
3875 * stable.
3876 *
3877 * Return: %true if the resulting @cpumask is different from @attrs->cpumask,
3878 * %false if equal.
3879 */
3880static bool wq_calc_node_cpumask(const struct workqueue_attrs *attrs, int node,
3881                                 int cpu_going_down, cpumask_t *cpumask)
3882{
3883        if (!wq_numa_enabled || attrs->no_numa)
3884                goto use_dfl;
3885
3886        /* does @node have any online CPUs @attrs wants? */
3887        cpumask_and(cpumask, cpumask_of_node(node), attrs->cpumask);
3888        if (cpu_going_down >= 0)
3889                cpumask_clear_cpu(cpu_going_down, cpumask);
3890
3891        if (cpumask_empty(cpumask))
3892                goto use_dfl;
3893
3894        /* yeap, return possible CPUs in @node that @attrs wants */
3895        cpumask_and(cpumask, attrs->cpumask, wq_numa_possible_cpumask[node]);
3896
3897        if (cpumask_empty(cpumask)) {
3898                pr_warn_once("WARNING: workqueue cpumask: online intersect > "
3899                                "possible intersect\n");
3900                return false;
3901        }
3902
3903        return !cpumask_equal(cpumask, attrs->cpumask);
3904
3905use_dfl:
3906        cpumask_copy(cpumask, attrs->cpumask);
3907        return false;
3908}
3909
3910/* install @pwq into @wq's numa_pwq_tbl[] for @node and return the old pwq */
3911static struct pool_workqueue *numa_pwq_tbl_install(struct workqueue_struct *wq,
3912                                                   int node,
3913                                                   struct pool_workqueue *pwq)
3914{
3915        struct pool_workqueue *old_pwq;
3916
3917        lockdep_assert_held(&wq_pool_mutex);
3918        lockdep_assert_held(&wq->mutex);
3919
3920        /* link_pwq() can handle duplicate calls */
3921        link_pwq(pwq);
3922
3923        old_pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
3924        rcu_assign_pointer(wq->numa_pwq_tbl[node], pwq);
3925        return old_pwq;
3926}
3927
3928/* context to store the prepared attrs & pwqs before applying */
3929struct apply_wqattrs_ctx {
3930        struct workqueue_struct *wq;            /* target workqueue */
3931        struct workqueue_attrs  *attrs;         /* attrs to apply */
3932        struct list_head        list;           /* queued for batching commit */
3933        struct pool_workqueue   *dfl_pwq;
3934        struct pool_workqueue   *pwq_tbl[];
3935};
3936
3937/* free the resources after success or abort */
3938static void apply_wqattrs_cleanup(struct apply_wqattrs_ctx *ctx)
3939{
3940        if (ctx) {
3941                int node;
3942
3943                for_each_node(node)
3944                        put_pwq_unlocked(ctx->pwq_tbl[node]);
3945                put_pwq_unlocked(ctx->dfl_pwq);
3946
3947                free_workqueue_attrs(ctx->attrs);
3948
3949                kfree(ctx);
3950        }
3951}
3952
3953/* allocate the attrs and pwqs for later installation */
3954static struct apply_wqattrs_ctx *
3955apply_wqattrs_prepare(struct workqueue_struct *wq,
3956                      const struct workqueue_attrs *attrs)
3957{
3958        struct apply_wqattrs_ctx *ctx;
3959        struct workqueue_attrs *new_attrs, *tmp_attrs;
3960        int node;
3961
3962        lockdep_assert_held(&wq_pool_mutex);
3963
3964        ctx = kzalloc(struct_size(ctx, pwq_tbl, nr_node_ids), GFP_KERNEL);
3965
3966        new_attrs = alloc_workqueue_attrs();
3967        tmp_attrs = alloc_workqueue_attrs();
3968        if (!ctx || !new_attrs || !tmp_attrs)
3969                goto out_free;
3970
3971        /*
3972         * Calculate the attrs of the default pwq.
3973         * If the user configured cpumask doesn't overlap with the
3974         * wq_unbound_cpumask, we fallback to the wq_unbound_cpumask.
3975         */
3976        copy_workqueue_attrs(new_attrs, attrs);
3977        cpumask_and(new_attrs->cpumask, new_attrs->cpumask, wq_unbound_cpumask);
3978        if (unlikely(cpumask_empty(new_attrs->cpumask)))
3979                cpumask_copy(new_attrs->cpumask, wq_unbound_cpumask);
3980
3981        /*
3982         * We may create multiple pwqs with differing cpumasks.  Make a
3983         * copy of @new_attrs which will be modified and used to obtain
3984         * pools.
3985         */
3986        copy_workqueue_attrs(tmp_attrs, new_attrs);
3987
3988        /*
3989         * If something goes wrong during CPU up/down, we'll fall back to
3990         * the default pwq covering whole @attrs->cpumask.  Always create
3991         * it even if we don't use it immediately.
3992         */
3993        ctx->dfl_pwq = alloc_unbound_pwq(wq, new_attrs);
3994        if (!ctx->dfl_pwq)
3995                goto out_free;
3996
3997        for_each_node(node) {
3998                if (wq_calc_node_cpumask(new_attrs, node, -1, tmp_attrs->cpumask)) {
3999                        ctx->pwq_tbl[node] = alloc_unbound_pwq(wq, tmp_attrs);
4000                        if (!ctx->pwq_tbl[node])
4001                                goto out_free;
4002                } else {
4003                        ctx->dfl_pwq->refcnt++;
4004                        ctx->pwq_tbl[node] = ctx->dfl_pwq;
4005                }
4006        }
4007
4008        /* save the user configured attrs and sanitize it. */
4009        copy_workqueue_attrs(new_attrs, attrs);
4010        cpumask_and(new_attrs->cpumask, new_attrs->cpumask, cpu_possible_mask);
4011        ctx->attrs = new_attrs;
4012
4013        ctx->wq = wq;
4014        free_workqueue_attrs(tmp_attrs);
4015        return ctx;
4016
4017out_free:
4018        free_workqueue_attrs(tmp_attrs);
4019        free_workqueue_attrs(new_attrs);
4020        apply_wqattrs_cleanup(ctx);
4021        return NULL;
4022}
4023
4024/* set attrs and install prepared pwqs, @ctx points to old pwqs on return */
4025static void apply_wqattrs_commit(struct apply_wqattrs_ctx *ctx)
4026{
4027        int node;
4028
4029        /* all pwqs have been created successfully, let's install'em */
4030        mutex_lock(&ctx->wq->mutex);
4031
4032        copy_workqueue_attrs(ctx->wq->unbound_attrs, ctx->attrs);
4033
4034        /* save the previous pwq and install the new one */
4035        for_each_node(node)
4036                ctx->pwq_tbl[node] = numa_pwq_tbl_install(ctx->wq, node,
4037                                                          ctx->pwq_tbl[node]);
4038
4039        /* @dfl_pwq might not have been used, ensure it's linked */
4040        link_pwq(ctx->dfl_pwq);
4041        swap(ctx->wq->dfl_pwq, ctx->dfl_pwq);
4042
4043        mutex_unlock(&ctx->wq->mutex);
4044}
4045
4046static void apply_wqattrs_lock(void)
4047{
4048        /* CPUs should stay stable across pwq creations and installations */
4049        cpus_read_lock();
4050        mutex_lock(&wq_pool_mutex);
4051}
4052
4053static void apply_wqattrs_unlock(void)
4054{
4055        mutex_unlock(&wq_pool_mutex);
4056        cpus_read_unlock();
4057}
4058
4059static int apply_workqueue_attrs_locked(struct workqueue_struct *wq,
4060                                        const struct workqueue_attrs *attrs)
4061{
4062        struct apply_wqattrs_ctx *ctx;
4063
4064        /* only unbound workqueues can change attributes */
4065        if (WARN_ON(!(wq->flags & WQ_UNBOUND)))
4066                return -EINVAL;
4067
4068        /* creating multiple pwqs breaks ordering guarantee */
4069        if (!list_empty(&wq->pwqs)) {
4070                if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4071                        return -EINVAL;
4072
4073                wq->flags &= ~__WQ_ORDERED;
4074        }
4075
4076        ctx = apply_wqattrs_prepare(wq, attrs);
4077        if (!ctx)
4078                return -ENOMEM;
4079
4080        /* the ctx has been prepared successfully, let's commit it */
4081        apply_wqattrs_commit(ctx);
4082        apply_wqattrs_cleanup(ctx);
4083
4084        return 0;
4085}
4086
4087/**
4088 * apply_workqueue_attrs - apply new workqueue_attrs to an unbound workqueue
4089 * @wq: the target workqueue
4090 * @attrs: the workqueue_attrs to apply, allocated with alloc_workqueue_attrs()
4091 *
4092 * Apply @attrs to an unbound workqueue @wq.  Unless disabled, on NUMA
4093 * machines, this function maps a separate pwq to each NUMA node with
4094 * possibles CPUs in @attrs->cpumask so that work items are affine to the
4095 * NUMA node it was issued on.  Older pwqs are released as in-flight work
4096 * items finish.  Note that a work item which repeatedly requeues itself
4097 * back-to-back will stay on its current pwq.
4098 *
4099 * Performs GFP_KERNEL allocations.
4100 *
4101 * Assumes caller has CPU hotplug read exclusion, i.e. cpus_read_lock().
4102 *
4103 * Return: 0 on success and -errno on failure.
4104 */
4105int apply_workqueue_attrs(struct workqueue_struct *wq,
4106                          const struct workqueue_attrs *attrs)
4107{
4108        int ret;
4109
4110        lockdep_assert_cpus_held();
4111
4112        mutex_lock(&wq_pool_mutex);
4113        ret = apply_workqueue_attrs_locked(wq, attrs);
4114        mutex_unlock(&wq_pool_mutex);
4115
4116        return ret;
4117}
4118
4119/**
4120 * wq_update_unbound_numa - update NUMA affinity of a wq for CPU hot[un]plug
4121 * @wq: the target workqueue
4122 * @cpu: the CPU coming up or going down
4123 * @online: whether @cpu is coming up or going down
4124 *
4125 * This function is to be called from %CPU_DOWN_PREPARE, %CPU_ONLINE and
4126 * %CPU_DOWN_FAILED.  @cpu is being hot[un]plugged, update NUMA affinity of
4127 * @wq accordingly.
4128 *
4129 * If NUMA affinity can't be adjusted due to memory allocation failure, it
4130 * falls back to @wq->dfl_pwq which may not be optimal but is always
4131 * correct.
4132 *
4133 * Note that when the last allowed CPU of a NUMA node goes offline for a
4134 * workqueue with a cpumask spanning multiple nodes, the workers which were
4135 * already executing the work items for the workqueue will lose their CPU
4136 * affinity and may execute on any CPU.  This is similar to how per-cpu
4137 * workqueues behave on CPU_DOWN.  If a workqueue user wants strict
4138 * affinity, it's the user's responsibility to flush the work item from
4139 * CPU_DOWN_PREPARE.
4140 */
4141static void wq_update_unbound_numa(struct workqueue_struct *wq, int cpu,
4142                                   bool online)
4143{
4144        int node = cpu_to_node(cpu);
4145        int cpu_off = online ? -1 : cpu;
4146        struct pool_workqueue *old_pwq = NULL, *pwq;
4147        struct workqueue_attrs *target_attrs;
4148        cpumask_t *cpumask;
4149
4150        lockdep_assert_held(&wq_pool_mutex);
4151
4152        if (!wq_numa_enabled || !(wq->flags & WQ_UNBOUND) ||
4153            wq->unbound_attrs->no_numa)
4154                return;
4155
4156        /*
4157         * We don't wanna alloc/free wq_attrs for each wq for each CPU.
4158         * Let's use a preallocated one.  The following buf is protected by
4159         * CPU hotplug exclusion.
4160         */
4161        target_attrs = wq_update_unbound_numa_attrs_buf;
4162        cpumask = target_attrs->cpumask;
4163
4164        copy_workqueue_attrs(target_attrs, wq->unbound_attrs);
4165        pwq = unbound_pwq_by_node(wq, node);
4166
4167        /*
4168         * Let's determine what needs to be done.  If the target cpumask is
4169         * different from the default pwq's, we need to compare it to @pwq's
4170         * and create a new one if they don't match.  If the target cpumask
4171         * equals the default pwq's, the default pwq should be used.
4172         */
4173        if (wq_calc_node_cpumask(wq->dfl_pwq->pool->attrs, node, cpu_off, cpumask)) {
4174                if (cpumask_equal(cpumask, pwq->pool->attrs->cpumask))
4175                        return;
4176        } else {
4177                goto use_dfl_pwq;
4178        }
4179
4180        /* create a new pwq */
4181        pwq = alloc_unbound_pwq(wq, target_attrs);
4182        if (!pwq) {
4183                pr_warn("workqueue: allocation failed while updating NUMA affinity of \"%s\"\n",
4184                        wq->name);
4185                goto use_dfl_pwq;
4186        }
4187
4188        /* Install the new pwq. */
4189        mutex_lock(&wq->mutex);
4190        old_pwq = numa_pwq_tbl_install(wq, node, pwq);
4191        goto out_unlock;
4192
4193use_dfl_pwq:
4194        mutex_lock(&wq->mutex);
4195        raw_spin_lock_irq(&wq->dfl_pwq->pool->lock);
4196        get_pwq(wq->dfl_pwq);
4197        raw_spin_unlock_irq(&wq->dfl_pwq->pool->lock);
4198        old_pwq = numa_pwq_tbl_install(wq, node, wq->dfl_pwq);
4199out_unlock:
4200        mutex_unlock(&wq->mutex);
4201        put_pwq_unlocked(old_pwq);
4202}
4203
4204static int alloc_and_link_pwqs(struct workqueue_struct *wq)
4205{
4206        bool highpri = wq->flags & WQ_HIGHPRI;
4207        int cpu, ret;
4208
4209        if (!(wq->flags & WQ_UNBOUND)) {
4210                wq->cpu_pwqs = alloc_percpu(struct pool_workqueue);
4211                if (!wq->cpu_pwqs)
4212                        return -ENOMEM;
4213
4214                for_each_possible_cpu(cpu) {
4215                        struct pool_workqueue *pwq =
4216                                per_cpu_ptr(wq->cpu_pwqs, cpu);
4217                        struct worker_pool *cpu_pools =
4218                                per_cpu(cpu_worker_pools, cpu);
4219
4220                        init_pwq(pwq, wq, &cpu_pools[highpri]);
4221
4222                        mutex_lock(&wq->mutex);
4223                        link_pwq(pwq);
4224                        mutex_unlock(&wq->mutex);
4225                }
4226                return 0;
4227        }
4228
4229        cpus_read_lock();
4230        if (wq->flags & __WQ_ORDERED) {
4231                ret = apply_workqueue_attrs(wq, ordered_wq_attrs[highpri]);
4232                /* there should only be single pwq for ordering guarantee */
4233                WARN(!ret && (wq->pwqs.next != &wq->dfl_pwq->pwqs_node ||
4234                              wq->pwqs.prev != &wq->dfl_pwq->pwqs_node),
4235                     "ordering guarantee broken for workqueue %s\n", wq->name);
4236        } else {
4237                ret = apply_workqueue_attrs(wq, unbound_std_wq_attrs[highpri]);
4238        }
4239        cpus_read_unlock();
4240
4241        return ret;
4242}
4243
4244static int wq_clamp_max_active(int max_active, unsigned int flags,
4245                               const char *name)
4246{
4247        int lim = flags & WQ_UNBOUND ? WQ_UNBOUND_MAX_ACTIVE : WQ_MAX_ACTIVE;
4248
4249        if (max_active < 1 || max_active > lim)
4250                pr_warn("workqueue: max_active %d requested for %s is out of range, clamping between %d and %d\n",
4251                        max_active, name, 1, lim);
4252
4253        return clamp_val(max_active, 1, lim);
4254}
4255
4256/*
4257 * Workqueues which may be used during memory reclaim should have a rescuer
4258 * to guarantee forward progress.
4259 */
4260static int init_rescuer(struct workqueue_struct *wq)
4261{
4262        struct worker *rescuer;
4263        int ret;
4264
4265        if (!(wq->flags & WQ_MEM_RECLAIM))
4266                return 0;
4267
4268        rescuer = alloc_worker(NUMA_NO_NODE);
4269        if (!rescuer)
4270                return -ENOMEM;
4271
4272        rescuer->rescue_wq = wq;
4273        rescuer->task = kthread_create(rescuer_thread, rescuer, "%s", wq->name);
4274        if (IS_ERR(rescuer->task)) {
4275                ret = PTR_ERR(rescuer->task);
4276                kfree(rescuer);
4277                return ret;
4278        }
4279
4280        wq->rescuer = rescuer;
4281        kthread_bind_mask(rescuer->task, cpu_possible_mask);
4282        wake_up_process(rescuer->task);
4283
4284        return 0;
4285}
4286
4287__printf(1, 4)
4288struct workqueue_struct *alloc_workqueue(const char *fmt,
4289                                         unsigned int flags,
4290                                         int max_active, ...)
4291{
4292        size_t tbl_size = 0;
4293        va_list args;
4294        struct workqueue_struct *wq;
4295        struct pool_workqueue *pwq;
4296
4297        /*
4298         * Unbound && max_active == 1 used to imply ordered, which is no
4299         * longer the case on NUMA machines due to per-node pools.  While
4300         * alloc_ordered_workqueue() is the right way to create an ordered
4301         * workqueue, keep the previous behavior to avoid subtle breakages
4302         * on NUMA.
4303         */
4304        if ((flags & WQ_UNBOUND) && max_active == 1)
4305                flags |= __WQ_ORDERED;
4306
4307        /* see the comment above the definition of WQ_POWER_EFFICIENT */
4308        if ((flags & WQ_POWER_EFFICIENT) && wq_power_efficient)
4309                flags |= WQ_UNBOUND;
4310
4311        /* allocate wq and format name */
4312        if (flags & WQ_UNBOUND)
4313                tbl_size = nr_node_ids * sizeof(wq->numa_pwq_tbl[0]);
4314
4315        wq = kzalloc(sizeof(*wq) + tbl_size, GFP_KERNEL);
4316        if (!wq)
4317                return NULL;
4318
4319        if (flags & WQ_UNBOUND) {
4320                wq->unbound_attrs = alloc_workqueue_attrs();
4321                if (!wq->unbound_attrs)
4322                        goto err_free_wq;
4323        }
4324
4325        va_start(args, max_active);
4326        vsnprintf(wq->name, sizeof(wq->name), fmt, args);
4327        va_end(args);
4328
4329        max_active = max_active ?: WQ_DFL_ACTIVE;
4330        max_active = wq_clamp_max_active(max_active, flags, wq->name);
4331
4332        /* init wq */
4333        wq->flags = flags;
4334        wq->saved_max_active = max_active;
4335        mutex_init(&wq->mutex);
4336        atomic_set(&wq->nr_pwqs_to_flush, 0);
4337        INIT_LIST_HEAD(&wq->pwqs);
4338        INIT_LIST_HEAD(&wq->flusher_queue);
4339        INIT_LIST_HEAD(&wq->flusher_overflow);
4340        INIT_LIST_HEAD(&wq->maydays);
4341
4342        wq_init_lockdep(wq);
4343        INIT_LIST_HEAD(&wq->list);
4344
4345        if (alloc_and_link_pwqs(wq) < 0)
4346                goto err_unreg_lockdep;
4347
4348        if (wq_online && init_rescuer(wq) < 0)
4349                goto err_destroy;
4350
4351        if ((wq->flags & WQ_SYSFS) && workqueue_sysfs_register(wq))
4352                goto err_destroy;
4353
4354        /*
4355         * wq_pool_mutex protects global freeze state and workqueues list.
4356         * Grab it, adjust max_active and add the new @wq to workqueues
4357         * list.
4358         */
4359        mutex_lock(&wq_pool_mutex);
4360
4361        mutex_lock(&wq->mutex);
4362        for_each_pwq(pwq, wq)
4363                pwq_adjust_max_active(pwq);
4364        mutex_unlock(&wq->mutex);
4365
4366        list_add_tail_rcu(&wq->list, &workqueues);
4367
4368        mutex_unlock(&wq_pool_mutex);
4369
4370        return wq;
4371
4372err_unreg_lockdep:
4373        wq_unregister_lockdep(wq);
4374        wq_free_lockdep(wq);
4375err_free_wq:
4376        free_workqueue_attrs(wq->unbound_attrs);
4377        kfree(wq);
4378        return NULL;
4379err_destroy:
4380        destroy_workqueue(wq);
4381        return NULL;
4382}
4383EXPORT_SYMBOL_GPL(alloc_workqueue);
4384
4385static bool pwq_busy(struct pool_workqueue *pwq)
4386{
4387        int i;
4388
4389        for (i = 0; i < WORK_NR_COLORS; i++)
4390                if (pwq->nr_in_flight[i])
4391                        return true;
4392
4393        if ((pwq != pwq->wq->dfl_pwq) && (pwq->refcnt > 1))
4394                return true;
4395        if (pwq->nr_active || !list_empty(&pwq->inactive_works))
4396                return true;
4397
4398        return false;
4399}
4400
4401/**
4402 * destroy_workqueue - safely terminate a workqueue
4403 * @wq: target workqueue
4404 *
4405 * Safely destroy a workqueue. All work currently pending will be done first.
4406 */
4407void destroy_workqueue(struct workqueue_struct *wq)
4408{
4409        struct pool_workqueue *pwq;
4410        int node;
4411
4412        /*
4413         * Remove it from sysfs first so that sanity check failure doesn't
4414         * lead to sysfs name conflicts.
4415         */
4416        workqueue_sysfs_unregister(wq);
4417
4418        /* drain it before proceeding with destruction */
4419        drain_workqueue(wq);
4420
4421        /* kill rescuer, if sanity checks fail, leave it w/o rescuer */
4422        if (wq->rescuer) {
4423                struct worker *rescuer = wq->rescuer;
4424
4425                /* this prevents new queueing */
4426                raw_spin_lock_irq(&wq_mayday_lock);
4427                wq->rescuer = NULL;
4428                raw_spin_unlock_irq(&wq_mayday_lock);
4429
4430                /* rescuer will empty maydays list before exiting */
4431                kthread_stop(rescuer->task);
4432                kfree(rescuer);
4433        }
4434
4435        /*
4436         * Sanity checks - grab all the locks so that we wait for all
4437         * in-flight operations which may do put_pwq().
4438         */
4439        mutex_lock(&wq_pool_mutex);
4440        mutex_lock(&wq->mutex);
4441        for_each_pwq(pwq, wq) {
4442                raw_spin_lock_irq(&pwq->pool->lock);
4443                if (WARN_ON(pwq_busy(pwq))) {
4444                        pr_warn("%s: %s has the following busy pwq\n",
4445                                __func__, wq->name);
4446                        show_pwq(pwq);
4447                        raw_spin_unlock_irq(&pwq->pool->lock);
4448                        mutex_unlock(&wq->mutex);
4449                        mutex_unlock(&wq_pool_mutex);
4450                        show_workqueue_state();
4451                        return;
4452                }
4453                raw_spin_unlock_irq(&pwq->pool->lock);
4454        }
4455        mutex_unlock(&wq->mutex);
4456
4457        /*
4458         * wq list is used to freeze wq, remove from list after
4459         * flushing is complete in case freeze races us.
4460         */
4461        list_del_rcu(&wq->list);
4462        mutex_unlock(&wq_pool_mutex);
4463
4464        if (!(wq->flags & WQ_UNBOUND)) {
4465                wq_unregister_lockdep(wq);
4466                /*
4467                 * The base ref is never dropped on per-cpu pwqs.  Directly
4468                 * schedule RCU free.
4469                 */
4470                call_rcu(&wq->rcu, rcu_free_wq);
4471        } else {
4472                /*
4473                 * We're the sole accessor of @wq at this point.  Directly
4474                 * access numa_pwq_tbl[] and dfl_pwq to put the base refs.
4475                 * @wq will be freed when the last pwq is released.
4476                 */
4477                for_each_node(node) {
4478                        pwq = rcu_access_pointer(wq->numa_pwq_tbl[node]);
4479                        RCU_INIT_POINTER(wq->numa_pwq_tbl[node], NULL);
4480                        put_pwq_unlocked(pwq);
4481                }
4482
4483                /*
4484                 * Put dfl_pwq.  @wq may be freed any time after dfl_pwq is
4485                 * put.  Don't access it afterwards.
4486                 */
4487                pwq = wq->dfl_pwq;
4488                wq->dfl_pwq = NULL;
4489                put_pwq_unlocked(pwq);
4490        }
4491}
4492EXPORT_SYMBOL_GPL(destroy_workqueue);
4493
4494/**
4495 * workqueue_set_max_active - adjust max_active of a workqueue
4496 * @wq: target workqueue
4497 * @max_active: new max_active value.
4498 *
4499 * Set max_active of @wq to @max_active.
4500 *
4501 * CONTEXT:
4502 * Don't call from IRQ context.
4503 */
4504void workqueue_set_max_active(struct workqueue_struct *wq, int max_active)
4505{
4506        struct pool_workqueue *pwq;
4507
4508        /* disallow meddling with max_active for ordered workqueues */
4509        if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
4510                return;
4511
4512        max_active = wq_clamp_max_active(max_active, wq->flags, wq->name);
4513
4514        mutex_lock(&wq->mutex);
4515
4516        wq->flags &= ~__WQ_ORDERED;
4517        wq->saved_max_active = max_active;
4518
4519        for_each_pwq(pwq, wq)
4520                pwq_adjust_max_active(pwq);
4521
4522        mutex_unlock(&wq->mutex);
4523}
4524EXPORT_SYMBOL_GPL(workqueue_set_max_active);
4525
4526/**
4527 * current_work - retrieve %current task's work struct
4528 *
4529 * Determine if %current task is a workqueue worker and what it's working on.
4530 * Useful to find out the context that the %current task is running in.
4531 *
4532 * Return: work struct if %current task is a workqueue worker, %NULL otherwise.
4533 */
4534struct work_struct *current_work(void)
4535{
4536        struct worker *worker = current_wq_worker();
4537
4538        return worker ? worker->current_work : NULL;
4539}
4540EXPORT_SYMBOL(current_work);
4541
4542/**
4543 * current_is_workqueue_rescuer - is %current workqueue rescuer?
4544 *
4545 * Determine whether %current is a workqueue rescuer.  Can be used from
4546 * work functions to determine whether it's being run off the rescuer task.
4547 *
4548 * Return: %true if %current is a workqueue rescuer. %false otherwise.
4549 */
4550bool current_is_workqueue_rescuer(void)
4551{
4552        struct worker *worker = current_wq_worker();
4553
4554        return worker && worker->rescue_wq;
4555}
4556
4557/**
4558 * workqueue_congested - test whether a workqueue is congested
4559 * @cpu: CPU in question
4560 * @wq: target workqueue
4561 *
4562 * Test whether @wq's cpu workqueue for @cpu is congested.  There is
4563 * no synchronization around this function and the test result is
4564 * unreliable and only useful as advisory hints or for debugging.
4565 *
4566 * If @cpu is WORK_CPU_UNBOUND, the test is performed on the local CPU.
4567 * Note that both per-cpu and unbound workqueues may be associated with
4568 * multiple pool_workqueues which have separate congested states.  A
4569 * workqueue being congested on one CPU doesn't mean the workqueue is also
4570 * contested on other CPUs / NUMA nodes.
4571 *
4572 * Return:
4573 * %true if congested, %false otherwise.
4574 */
4575bool workqueue_congested(int cpu, struct workqueue_struct *wq)
4576{
4577        struct pool_workqueue *pwq;
4578        bool ret;
4579
4580        rcu_read_lock();
4581        preempt_disable();
4582
4583        if (cpu == WORK_CPU_UNBOUND)
4584                cpu = smp_processor_id();
4585
4586        if (!(wq->flags & WQ_UNBOUND))
4587                pwq = per_cpu_ptr(wq->cpu_pwqs, cpu);
4588        else
4589                pwq = unbound_pwq_by_node(wq, cpu_to_node(cpu));
4590
4591        ret = !list_empty(&pwq->inactive_works);
4592        preempt_enable();
4593        rcu_read_unlock();
4594
4595        return ret;
4596}
4597EXPORT_SYMBOL_GPL(workqueue_congested);
4598
4599/**
4600 * work_busy - test whether a work is currently pending or running
4601 * @work: the work to be tested
4602 *
4603 * Test whether @work is currently pending or running.  There is no
4604 * synchronization around this function and the test result is
4605 * unreliable and only useful as advisory hints or for debugging.
4606 *
4607 * Return:
4608 * OR'd bitmask of WORK_BUSY_* bits.
4609 */
4610unsigned int work_busy(struct work_struct *work)
4611{
4612        struct worker_pool *pool;
4613        unsigned long flags;
4614        unsigned int ret = 0;
4615
4616        if (work_pending(work))
4617                ret |= WORK_BUSY_PENDING;
4618
4619        rcu_read_lock();
4620        pool = get_work_pool(work);
4621        if (pool) {
4622                raw_spin_lock_irqsave(&pool->lock, flags);
4623                if (find_worker_executing_work(pool, work))
4624                        ret |= WORK_BUSY_RUNNING;
4625                raw_spin_unlock_irqrestore(&pool->lock, flags);
4626        }
4627        rcu_read_unlock();
4628
4629        return ret;
4630}
4631EXPORT_SYMBOL_GPL(work_busy);
4632
4633/**
4634 * set_worker_desc - set description for the current work item
4635 * @fmt: printf-style format string
4636 * @...: arguments for the format string
4637 *
4638 * This function can be called by a running work function to describe what
4639 * the work item is about.  If the worker task gets dumped, this
4640 * information will be printed out together to help debugging.  The
4641 * description can be at most WORKER_DESC_LEN including the trailing '\0'.
4642 */
4643void set_worker_desc(const char *fmt, ...)
4644{
4645        struct worker *worker = current_wq_worker();
4646        va_list args;
4647
4648        if (worker) {
4649                va_start(args, fmt);
4650                vsnprintf(worker->desc, sizeof(worker->desc), fmt, args);
4651                va_end(args);
4652        }
4653}
4654EXPORT_SYMBOL_GPL(set_worker_desc);
4655
4656/**
4657 * print_worker_info - print out worker information and description
4658 * @log_lvl: the log level to use when printing
4659 * @task: target task
4660 *
4661 * If @task is a worker and currently executing a work item, print out the
4662 * name of the workqueue being serviced and worker description set with
4663 * set_worker_desc() by the currently executing work item.
4664 *
4665 * This function can be safely called on any task as long as the
4666 * task_struct itself is accessible.  While safe, this function isn't
4667 * synchronized and may print out mixups or garbages of limited length.
4668 */
4669void print_worker_info(const char *log_lvl, struct task_struct *task)
4670{
4671        work_func_t *fn = NULL;
4672        char name[WQ_NAME_LEN] = { };
4673        char desc[WORKER_DESC_LEN] = { };
4674        struct pool_workqueue *pwq = NULL;
4675        struct workqueue_struct *wq = NULL;
4676        struct worker *worker;
4677
4678        if (!(task->flags & PF_WQ_WORKER))
4679                return;
4680
4681        /*
4682         * This function is called without any synchronization and @task
4683         * could be in any state.  Be careful with dereferences.
4684         */
4685        worker = kthread_probe_data(task);
4686
4687        /*
4688         * Carefully copy the associated workqueue's workfn, name and desc.
4689         * Keep the original last '\0' in case the original is garbage.
4690         */
4691        copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn));
4692        copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq));
4693        copy_from_kernel_nofault(&wq, &pwq->wq, sizeof(wq));
4694        copy_from_kernel_nofault(name, wq->name, sizeof(name) - 1);
4695        copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1);
4696
4697        if (fn || name[0] || desc[0]) {
4698                printk("%sWorkqueue: %s %ps", log_lvl, name, fn);
4699                if (strcmp(name, desc))
4700                        pr_cont(" (%s)", desc);
4701                pr_cont("\n");
4702        }
4703}
4704
4705static void pr_cont_pool_info(struct worker_pool *pool)
4706{
4707        pr_cont(" cpus=%*pbl", nr_cpumask_bits, pool->attrs->cpumask);
4708        if (pool->node != NUMA_NO_NODE)
4709                pr_cont(" node=%d", pool->node);
4710        pr_cont(" flags=0x%x nice=%d", pool->flags, pool->attrs->nice);
4711}
4712
4713static void pr_cont_work(bool comma, struct work_struct *work)
4714{
4715        if (work->func == wq_barrier_func) {
4716                struct wq_barrier *barr;
4717
4718                barr = container_of(work, struct wq_barrier, work);
4719
4720                pr_cont("%s BAR(%d)", comma ? "," : "",
4721                        task_pid_nr(barr->task));
4722        } else {
4723                pr_cont("%s %ps", comma ? "," : "", work->func);
4724        }
4725}
4726
4727static void show_pwq(struct pool_workqueue *pwq)
4728{
4729        struct worker_pool *pool = pwq->pool;
4730        struct work_struct *work;
4731        struct worker *worker;
4732        bool has_in_flight = false, has_pending = false;
4733        int bkt;
4734
4735        pr_info("  pwq %d:", pool->id);
4736        pr_cont_pool_info(pool);
4737
4738        pr_cont(" active=%d/%d refcnt=%d%s\n",
4739                pwq->nr_active, pwq->max_active, pwq->refcnt,
4740                !list_empty(&pwq->mayday_node) ? " MAYDAY" : "");
4741
4742        hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4743                if (worker->current_pwq == pwq) {
4744                        has_in_flight = true;
4745                        break;
4746                }
4747        }
4748        if (has_in_flight) {
4749                bool comma = false;
4750
4751                pr_info("    in-flight:");
4752                hash_for_each(pool->busy_hash, bkt, worker, hentry) {
4753                        if (worker->current_pwq != pwq)
4754                                continue;
4755
4756                        pr_cont("%s %d%s:%ps", comma ? "," : "",
4757                                task_pid_nr(worker->task),
4758                                worker->rescue_wq ? "(RESCUER)" : "",
4759                                worker->current_func);
4760                        list_for_each_entry(work, &worker->scheduled, entry)
4761                                pr_cont_work(false, work);
4762                        comma = true;
4763                }
4764                pr_cont("\n");
4765        }
4766
4767        list_for_each_entry(work, &pool->worklist, entry) {
4768                if (get_work_pwq(work) == pwq) {
4769                        has_pending = true;
4770                        break;
4771                }
4772        }
4773        if (has_pending) {
4774                bool comma = false;
4775
4776                pr_info("    pending:");
4777                list_for_each_entry(work, &pool->worklist, entry) {
4778                        if (get_work_pwq(work) != pwq)
4779                                continue;
4780
4781                        pr_cont_work(comma, work);
4782                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4783                }
4784                pr_cont("\n");
4785        }
4786
4787        if (!list_empty(&pwq->inactive_works)) {
4788                bool comma = false;
4789
4790                pr_info("    inactive:");
4791                list_for_each_entry(work, &pwq->inactive_works, entry) {
4792                        pr_cont_work(comma, work);
4793                        comma = !(*work_data_bits(work) & WORK_STRUCT_LINKED);
4794                }
4795                pr_cont("\n");
4796        }
4797}
4798
4799/**
4800 * show_workqueue_state - dump workqueue state
4801 *
4802 * Called from a sysrq handler or try_to_freeze_tasks() and prints out
4803 * all busy workqueues and pools.
4804 */
4805void show_workqueue_state(void)
4806{
4807        struct workqueue_struct *wq;
4808        struct worker_pool *pool;
4809        unsigned long flags;
4810        int pi;
4811
4812        rcu_read_lock();
4813
4814        pr_info("Showing busy workqueues and worker pools:\n");
4815
4816        list_for_each_entry_rcu(wq, &workqueues, list) {
4817                struct pool_workqueue *pwq;
4818                bool idle = true;
4819
4820                for_each_pwq(pwq, wq) {
4821                        if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
4822                                idle = false;
4823                                break;
4824                        }
4825                }
4826                if (idle)
4827                        continue;
4828
4829                pr_info("workqueue %s: flags=0x%x\n", wq->name, wq->flags);
4830
4831                for_each_pwq(pwq, wq) {
4832                        raw_spin_lock_irqsave(&pwq->pool->lock, flags);
4833                        if (pwq->nr_active || !list_empty(&pwq->inactive_works)) {
4834                                /*
4835                                 * Defer printing to avoid deadlocks in console
4836                                 * drivers that queue work while holding locks
4837                                 * also taken in their write paths.
4838                                 */
4839                                printk_deferred_enter();
4840                                show_pwq(pwq);
4841                                printk_deferred_exit();
4842                        }
4843                        raw_spin_unlock_irqrestore(&pwq->pool->lock, flags);
4844                        /*
4845                         * We could be printing a lot from atomic context, e.g.
4846                         * sysrq-t -> show_workqueue_state(). Avoid triggering
4847                         * hard lockup.
4848                         */
4849                        touch_nmi_watchdog();
4850                }
4851        }
4852
4853        for_each_pool(pool, pi) {
4854                struct worker *worker;
4855                bool first = true;
4856
4857                raw_spin_lock_irqsave(&pool->lock, flags);
4858                if (pool->nr_workers == pool->nr_idle)
4859                        goto next_pool;
4860                /*
4861                 * Defer printing to avoid deadlocks in console drivers that
4862                 * queue work while holding locks also taken in their write
4863                 * paths.
4864                 */
4865                printk_deferred_enter();
4866                pr_info("pool %d:", pool->id);
4867                pr_cont_pool_info(pool);
4868                pr_cont(" hung=%us workers=%d",
4869                        jiffies_to_msecs(jiffies - pool->watchdog_ts) / 1000,
4870                        pool->nr_workers);
4871                if (pool->manager)
4872                        pr_cont(" manager: %d",
4873                                task_pid_nr(pool->manager->task));
4874                list_for_each_entry(worker, &pool->idle_list, entry) {
4875                        pr_cont(" %s%d", first ? "idle: " : "",
4876                                task_pid_nr(worker->task));
4877                        first = false;
4878                }
4879                pr_cont("\n");
4880                printk_deferred_exit();
4881        next_pool:
4882                raw_spin_unlock_irqrestore(&pool->lock, flags);
4883                /*
4884                 * We could be printing a lot from atomic context, e.g.
4885                 * sysrq-t -> show_workqueue_state(). Avoid triggering
4886                 * hard lockup.
4887                 */
4888                touch_nmi_watchdog();
4889        }
4890
4891        rcu_read_unlock();
4892}
4893
4894/* used to show worker information through /proc/PID/{comm,stat,status} */
4895void wq_worker_comm(char *buf, size_t size, struct task_struct *task)
4896{
4897        int off;
4898
4899        /* always show the actual comm */
4900        off = strscpy(buf, task->comm, size);
4901        if (off < 0)
4902                return;
4903
4904        /* stabilize PF_WQ_WORKER and worker pool association */
4905        mutex_lock(&wq_pool_attach_mutex);
4906
4907        if (task->flags & PF_WQ_WORKER) {
4908                struct worker *worker = kthread_data(task);
4909                struct worker_pool *pool = worker->pool;
4910
4911                if (pool) {
4912                        raw_spin_lock_irq(&pool->lock);
4913                        /*
4914                         * ->desc tracks information (wq name or
4915                         * set_worker_desc()) for the latest execution.  If
4916                         * current, prepend '+', otherwise '-'.
4917                         */
4918                        if (worker->desc[0] != '\0') {
4919                                if (worker->current_work)
4920                                        scnprintf(buf + off, size - off, "+%s",
4921                                                  worker->desc);
4922                                else
4923                                        scnprintf(buf + off, size - off, "-%s",
4924                                                  worker->desc);
4925                        }
4926                        raw_spin_unlock_irq(&pool->lock);
4927                }
4928        }
4929
4930        mutex_unlock(&wq_pool_attach_mutex);
4931}
4932
4933#ifdef CONFIG_SMP
4934
4935/*
4936 * CPU hotplug.
4937 *
4938 * There are two challenges in supporting CPU hotplug.  Firstly, there
4939 * are a lot of assumptions on strong associations among work, pwq and
4940 * pool which make migrating pending and scheduled works very
4941 * difficult to implement without impacting hot paths.  Secondly,
4942 * worker pools serve mix of short, long and very long running works making
4943 * blocked draining impractical.
4944 *
4945 * This is solved by allowing the pools to be disassociated from the CPU
4946 * running as an unbound one and allowing it to be reattached later if the
4947 * cpu comes back online.
4948 */
4949
4950static void unbind_workers(int cpu)
4951{
4952        struct worker_pool *pool;
4953        struct worker *worker;
4954
4955        for_each_cpu_worker_pool(pool, cpu) {
4956                mutex_lock(&wq_pool_attach_mutex);
4957                raw_spin_lock_irq(&pool->lock);
4958
4959                /*
4960                 * We've blocked all attach/detach operations. Make all workers
4961                 * unbound and set DISASSOCIATED.  Before this, all workers
4962                 * except for the ones which are still executing works from
4963                 * before the last CPU down must be on the cpu.  After
4964                 * this, they may become diasporas.
4965                 */
4966                for_each_pool_worker(worker, pool)
4967                        worker->flags |= WORKER_UNBOUND;
4968
4969                pool->flags |= POOL_DISASSOCIATED;
4970
4971                raw_spin_unlock_irq(&pool->lock);
4972
4973                for_each_pool_worker(worker, pool) {
4974                        kthread_set_per_cpu(worker->task, -1);
4975                        WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0);
4976                }
4977
4978                mutex_unlock(&wq_pool_attach_mutex);
4979
4980                /*
4981                 * Call schedule() so that we cross rq->lock and thus can
4982                 * guarantee sched callbacks see the %WORKER_UNBOUND flag.
4983                 * This is necessary as scheduler callbacks may be invoked
4984                 * from other cpus.
4985                 */
4986                schedule();
4987
4988                /*
4989                 * Sched callbacks are disabled now.  Zap nr_running.
4990                 * After this, nr_running stays zero and need_more_worker()
4991                 * and keep_working() are always true as long as the
4992                 * worklist is not empty.  This pool now behaves as an
4993                 * unbound (in terms of concurrency management) pool which
4994                 * are served by workers tied to the pool.
4995                 */
4996                atomic_set(&pool->nr_running, 0);
4997
4998                /*
4999                 * With concurrency management just turned off, a busy
5000                 * worker blocking could lead to lengthy stalls.  Kick off
5001                 * unbound chain execution of currently pending work items.
5002                 */
5003                raw_spin_lock_irq(&pool->lock);
5004                wake_up_worker(pool);
5005                raw_spin_unlock_irq(&pool->lock);
5006        }
5007}
5008
5009/**
5010 * rebind_workers - rebind all workers of a pool to the associated CPU
5011 * @pool: pool of interest
5012 *
5013 * @pool->cpu is coming online.  Rebind all workers to the CPU.
5014 */
5015static void rebind_workers(struct worker_pool *pool)
5016{
5017        struct worker *worker;
5018
5019        lockdep_assert_held(&wq_pool_attach_mutex);
5020
5021        /*
5022         * Restore CPU affinity of all workers.  As all idle workers should
5023         * be on the run-queue of the associated CPU before any local
5024         * wake-ups for concurrency management happen, restore CPU affinity
5025         * of all workers first and then clear UNBOUND.  As we're called
5026         * from CPU_ONLINE, the following shouldn't fail.
5027         */
5028        for_each_pool_worker(worker, pool) {
5029                kthread_set_per_cpu(worker->task, pool->cpu);
5030                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task,
5031                                                  pool->attrs->cpumask) < 0);
5032        }
5033
5034        raw_spin_lock_irq(&pool->lock);
5035
5036        pool->flags &= ~POOL_DISASSOCIATED;
5037
5038        for_each_pool_worker(worker, pool) {
5039                unsigned int worker_flags = worker->flags;
5040
5041                /*
5042                 * A bound idle worker should actually be on the runqueue
5043                 * of the associated CPU for local wake-ups targeting it to
5044                 * work.  Kick all idle workers so that they migrate to the
5045                 * associated CPU.  Doing this in the same loop as
5046                 * replacing UNBOUND with REBOUND is safe as no worker will
5047                 * be bound before @pool->lock is released.
5048                 */
5049                if (worker_flags & WORKER_IDLE)
5050                        wake_up_process(worker->task);
5051
5052                /*
5053                 * We want to clear UNBOUND but can't directly call
5054                 * worker_clr_flags() or adjust nr_running.  Atomically
5055                 * replace UNBOUND with another NOT_RUNNING flag REBOUND.
5056                 * @worker will clear REBOUND using worker_clr_flags() when
5057                 * it initiates the next execution cycle thus restoring
5058                 * concurrency management.  Note that when or whether
5059                 * @worker clears REBOUND doesn't affect correctness.
5060                 *
5061                 * WRITE_ONCE() is necessary because @worker->flags may be
5062                 * tested without holding any lock in
5063                 * wq_worker_running().  Without it, NOT_RUNNING test may
5064                 * fail incorrectly leading to premature concurrency
5065                 * management operations.
5066                 */
5067                WARN_ON_ONCE(!(worker_flags & WORKER_UNBOUND));
5068                worker_flags |= WORKER_REBOUND;
5069                worker_flags &= ~WORKER_UNBOUND;
5070                WRITE_ONCE(worker->flags, worker_flags);
5071        }
5072
5073        raw_spin_unlock_irq(&pool->lock);
5074}
5075
5076/**
5077 * restore_unbound_workers_cpumask - restore cpumask of unbound workers
5078 * @pool: unbound pool of interest
5079 * @cpu: the CPU which is coming up
5080 *
5081 * An unbound pool may end up with a cpumask which doesn't have any online
5082 * CPUs.  When a worker of such pool get scheduled, the scheduler resets
5083 * its cpus_allowed.  If @cpu is in @pool's cpumask which didn't have any
5084 * online CPU before, cpus_allowed of all its workers should be restored.
5085 */
5086static void restore_unbound_workers_cpumask(struct worker_pool *pool, int cpu)
5087{
5088        static cpumask_t cpumask;
5089        struct worker *worker;
5090
5091        lockdep_assert_held(&wq_pool_attach_mutex);
5092
5093        /* is @cpu allowed for @pool? */
5094        if (!cpumask_test_cpu(cpu, pool->attrs->cpumask))
5095                return;
5096
5097        cpumask_and(&cpumask, pool->attrs->cpumask, cpu_online_mask);
5098
5099        /* as we're called from CPU_ONLINE, the following shouldn't fail */
5100        for_each_pool_worker(worker, pool)
5101                WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0);
5102}
5103
5104int workqueue_prepare_cpu(unsigned int cpu)
5105{
5106        struct worker_pool *pool;
5107
5108        for_each_cpu_worker_pool(pool, cpu) {
5109                if (pool->nr_workers)
5110                        continue;
5111                if (!create_worker(pool))
5112                        return -ENOMEM;
5113        }
5114        return 0;
5115}
5116
5117int workqueue_online_cpu(unsigned int cpu)
5118{
5119        struct worker_pool *pool;
5120        struct workqueue_struct *wq;
5121        int pi;
5122
5123        mutex_lock(&wq_pool_mutex);
5124
5125        for_each_pool(pool, pi) {
5126                mutex_lock(&wq_pool_attach_mutex);
5127
5128                if (pool->cpu == cpu)
5129                        rebind_workers(pool);
5130                else if (pool->cpu < 0)
5131                        restore_unbound_workers_cpumask(pool, cpu);
5132
5133                mutex_unlock(&wq_pool_attach_mutex);
5134        }
5135
5136        /* update NUMA affinity of unbound workqueues */
5137        list_for_each_entry(wq, &workqueues, list)
5138                wq_update_unbound_numa(wq, cpu, true);
5139
5140        mutex_unlock(&wq_pool_mutex);
5141        return 0;
5142}
5143
5144int workqueue_offline_cpu(unsigned int cpu)
5145{
5146        struct workqueue_struct *wq;
5147
5148        /* unbinding per-cpu workers should happen on the local CPU */
5149        if (WARN_ON(cpu != smp_processor_id()))
5150                return -1;
5151
5152        unbind_workers(cpu);
5153
5154        /* update NUMA affinity of unbound workqueues */
5155        mutex_lock(&wq_pool_mutex);
5156        list_for_each_entry(wq, &workqueues, list)
5157                wq_update_unbound_numa(wq, cpu, false);
5158        mutex_unlock(&wq_pool_mutex);
5159
5160        return 0;
5161}
5162
5163struct work_for_cpu {
5164        struct work_struct work;
5165        long (*fn)(void *);
5166        void *arg;
5167        long ret;
5168};
5169
5170static void work_for_cpu_fn(struct work_struct *work)
5171{
5172        struct work_for_cpu *wfc = container_of(work, struct work_for_cpu, work);
5173
5174        wfc->ret = wfc->fn(wfc->arg);
5175}
5176
5177/**
5178 * work_on_cpu - run a function in thread context on a particular cpu
5179 * @cpu: the cpu to run on
5180 * @fn: the function to run
5181 * @arg: the function arg
5182 *
5183 * It is up to the caller to ensure that the cpu doesn't go offline.
5184 * The caller must not hold any locks which would prevent @fn from completing.
5185 *
5186 * Return: The value @fn returns.
5187 */
5188long work_on_cpu(int cpu, long (*fn)(void *), void *arg)
5189{
5190        struct work_for_cpu wfc = { .fn = fn, .arg = arg };
5191
5192        INIT_WORK_ONSTACK(&wfc.work, work_for_cpu_fn);
5193        schedule_work_on(cpu, &wfc.work);
5194        flush_work(&wfc.work);
5195        destroy_work_on_stack(&wfc.work);
5196        return wfc.ret;
5197}
5198EXPORT_SYMBOL_GPL(work_on_cpu);
5199
5200/**
5201 * work_on_cpu_safe - run a function in thread context on a particular cpu
5202 * @cpu: the cpu to run on
5203 * @fn:  the function to run
5204 * @arg: the function argument
5205 *
5206 * Disables CPU hotplug and calls work_on_cpu(). The caller must not hold
5207 * any locks which would prevent @fn from completing.
5208 *
5209 * Return: The value @fn returns.
5210 */
5211long work_on_cpu_safe(int cpu, long (*fn)(void *), void *arg)
5212{
5213        long ret = -ENODEV;
5214
5215        cpus_read_lock();
5216        if (cpu_online(cpu))
5217                ret = work_on_cpu(cpu, fn, arg);
5218        cpus_read_unlock();
5219        return ret;
5220}
5221EXPORT_SYMBOL_GPL(work_on_cpu_safe);
5222#endif /* CONFIG_SMP */
5223
5224#ifdef CONFIG_FREEZER
5225
5226/**
5227 * freeze_workqueues_begin - begin freezing workqueues
5228 *
5229 * Start freezing workqueues.  After this function returns, all freezable
5230 * workqueues will queue new works to their inactive_works list instead of
5231 * pool->worklist.
5232 *
5233 * CONTEXT:
5234 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5235 */
5236void freeze_workqueues_begin(void)
5237{
5238        struct workqueue_struct *wq;
5239        struct pool_workqueue *pwq;
5240
5241        mutex_lock(&wq_pool_mutex);
5242
5243        WARN_ON_ONCE(workqueue_freezing);
5244        workqueue_freezing = true;
5245
5246        list_for_each_entry(wq, &workqueues, list) {
5247                mutex_lock(&wq->mutex);
5248                for_each_pwq(pwq, wq)
5249                        pwq_adjust_max_active(pwq);
5250                mutex_unlock(&wq->mutex);
5251        }
5252
5253        mutex_unlock(&wq_pool_mutex);
5254}
5255
5256/**
5257 * freeze_workqueues_busy - are freezable workqueues still busy?
5258 *
5259 * Check whether freezing is complete.  This function must be called
5260 * between freeze_workqueues_begin() and thaw_workqueues().
5261 *
5262 * CONTEXT:
5263 * Grabs and releases wq_pool_mutex.
5264 *
5265 * Return:
5266 * %true if some freezable workqueues are still busy.  %false if freezing
5267 * is complete.
5268 */
5269bool freeze_workqueues_busy(void)
5270{
5271        bool busy = false;
5272        struct workqueue_struct *wq;
5273        struct pool_workqueue *pwq;
5274
5275        mutex_lock(&wq_pool_mutex);
5276
5277        WARN_ON_ONCE(!workqueue_freezing);
5278
5279        list_for_each_entry(wq, &workqueues, list) {
5280                if (!(wq->flags & WQ_FREEZABLE))
5281                        continue;
5282                /*
5283                 * nr_active is monotonically decreasing.  It's safe
5284                 * to peek without lock.
5285                 */
5286                rcu_read_lock();
5287                for_each_pwq(pwq, wq) {
5288                        WARN_ON_ONCE(pwq->nr_active < 0);
5289                        if (pwq->nr_active) {
5290                                busy = true;
5291                                rcu_read_unlock();
5292                                goto out_unlock;
5293                        }
5294                }
5295                rcu_read_unlock();
5296        }
5297out_unlock:
5298        mutex_unlock(&wq_pool_mutex);
5299        return busy;
5300}
5301
5302/**
5303 * thaw_workqueues - thaw workqueues
5304 *
5305 * Thaw workqueues.  Normal queueing is restored and all collected
5306 * frozen works are transferred to their respective pool worklists.
5307 *
5308 * CONTEXT:
5309 * Grabs and releases wq_pool_mutex, wq->mutex and pool->lock's.
5310 */
5311void thaw_workqueues(void)
5312{
5313        struct workqueue_struct *wq;
5314        struct pool_workqueue *pwq;
5315
5316        mutex_lock(&wq_pool_mutex);
5317
5318        if (!workqueue_freezing)
5319                goto out_unlock;
5320
5321        workqueue_freezing = false;
5322
5323        /* restore max_active and repopulate worklist */
5324        list_for_each_entry(wq, &workqueues, list) {
5325                mutex_lock(&wq->mutex);
5326                for_each_pwq(pwq, wq)
5327                        pwq_adjust_max_active(pwq);
5328                mutex_unlock(&wq->mutex);
5329        }
5330
5331out_unlock:
5332        mutex_unlock(&wq_pool_mutex);
5333}
5334#endif /* CONFIG_FREEZER */
5335
5336static int workqueue_apply_unbound_cpumask(void)
5337{
5338        LIST_HEAD(ctxs);
5339        int ret = 0;
5340        struct workqueue_struct *wq;
5341        struct apply_wqattrs_ctx *ctx, *n;
5342
5343        lockdep_assert_held(&wq_pool_mutex);
5344
5345        list_for_each_entry(wq, &workqueues, list) {
5346                if (!(wq->flags & WQ_UNBOUND))
5347                        continue;
5348                /* creating multiple pwqs breaks ordering guarantee */
5349                if (wq->flags & __WQ_ORDERED)
5350                        continue;
5351
5352                ctx = apply_wqattrs_prepare(wq, wq->unbound_attrs);
5353                if (!ctx) {
5354                        ret = -ENOMEM;
5355                        break;
5356                }
5357
5358                list_add_tail(&ctx->list, &ctxs);
5359        }
5360
5361        list_for_each_entry_safe(ctx, n, &ctxs, list) {
5362                if (!ret)
5363                        apply_wqattrs_commit(ctx);
5364                apply_wqattrs_cleanup(ctx);
5365        }
5366
5367        return ret;
5368}
5369
5370/**
5371 *  workqueue_set_unbound_cpumask - Set the low-level unbound cpumask
5372 *  @cpumask: the cpumask to set
5373 *
5374 *  The low-level workqueues cpumask is a global cpumask that limits
5375 *  the affinity of all unbound workqueues.  This function check the @cpumask
5376 *  and apply it to all unbound workqueues and updates all pwqs of them.
5377 *
5378 *  Return:     0       - Success
5379 *              -EINVAL - Invalid @cpumask
5380 *              -ENOMEM - Failed to allocate memory for attrs or pwqs.
5381 */
5382int workqueue_set_unbound_cpumask(cpumask_var_t cpumask)
5383{
5384        int ret = -EINVAL;
5385        cpumask_var_t saved_cpumask;
5386
5387        if (!zalloc_cpumask_var(&saved_cpumask, GFP_KERNEL))
5388                return -ENOMEM;
5389
5390        /*
5391         * Not excluding isolated cpus on purpose.
5392         * If the user wishes to include them, we allow that.
5393         */
5394        cpumask_and(cpumask, cpumask, cpu_possible_mask);
5395        if (!cpumask_empty(cpumask)) {
5396                apply_wqattrs_lock();
5397
5398                /* save the old wq_unbound_cpumask. */
5399                cpumask_copy(saved_cpumask, wq_unbound_cpumask);
5400
5401                /* update wq_unbound_cpumask at first and apply it to wqs. */
5402                cpumask_copy(wq_unbound_cpumask, cpumask);
5403                ret = workqueue_apply_unbound_cpumask();
5404
5405                /* restore the wq_unbound_cpumask when failed. */
5406                if (ret < 0)
5407                        cpumask_copy(wq_unbound_cpumask, saved_cpumask);
5408
5409                apply_wqattrs_unlock();
5410        }
5411
5412        free_cpumask_var(saved_cpumask);
5413        return ret;
5414}
5415
5416#ifdef CONFIG_SYSFS
5417/*
5418 * Workqueues with WQ_SYSFS flag set is visible to userland via
5419 * /sys/bus/workqueue/devices/WQ_NAME.  All visible workqueues have the
5420 * following attributes.
5421 *
5422 *  per_cpu     RO bool : whether the workqueue is per-cpu or unbound
5423 *  max_active  RW int  : maximum number of in-flight work items
5424 *
5425 * Unbound workqueues have the following extra attributes.
5426 *
5427 *  pool_ids    RO int  : the associated pool IDs for each node
5428 *  nice        RW int  : nice value of the workers
5429 *  cpumask     RW mask : bitmask of allowed CPUs for the workers
5430 *  numa        RW bool : whether enable NUMA affinity
5431 */
5432struct wq_device {
5433        struct workqueue_struct         *wq;
5434        struct device                   dev;
5435};
5436
5437static struct workqueue_struct *dev_to_wq(struct device *dev)
5438{
5439        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5440
5441        return wq_dev->wq;
5442}
5443
5444static ssize_t per_cpu_show(struct device *dev, struct device_attribute *attr,
5445                            char *buf)
5446{
5447        struct workqueue_struct *wq = dev_to_wq(dev);
5448
5449        return scnprintf(buf, PAGE_SIZE, "%d\n", (bool)!(wq->flags & WQ_UNBOUND));
5450}
5451static DEVICE_ATTR_RO(per_cpu);
5452
5453static ssize_t max_active_show(struct device *dev,
5454                               struct device_attribute *attr, char *buf)
5455{
5456        struct workqueue_struct *wq = dev_to_wq(dev);
5457
5458        return scnprintf(buf, PAGE_SIZE, "%d\n", wq->saved_max_active);
5459}
5460
5461static ssize_t max_active_store(struct device *dev,
5462                                struct device_attribute *attr, const char *buf,
5463                                size_t count)
5464{
5465        struct workqueue_struct *wq = dev_to_wq(dev);
5466        int val;
5467
5468        if (sscanf(buf, "%d", &val) != 1 || val <= 0)
5469                return -EINVAL;
5470
5471        workqueue_set_max_active(wq, val);
5472        return count;
5473}
5474static DEVICE_ATTR_RW(max_active);
5475
5476static struct attribute *wq_sysfs_attrs[] = {
5477        &dev_attr_per_cpu.attr,
5478        &dev_attr_max_active.attr,
5479        NULL,
5480};
5481ATTRIBUTE_GROUPS(wq_sysfs);
5482
5483static ssize_t wq_pool_ids_show(struct device *dev,
5484                                struct device_attribute *attr, char *buf)
5485{
5486        struct workqueue_struct *wq = dev_to_wq(dev);
5487        const char *delim = "";
5488        int node, written = 0;
5489
5490        cpus_read_lock();
5491        rcu_read_lock();
5492        for_each_node(node) {
5493                written += scnprintf(buf + written, PAGE_SIZE - written,
5494                                     "%s%d:%d", delim, node,
5495                                     unbound_pwq_by_node(wq, node)->pool->id);
5496                delim = " ";
5497        }
5498        written += scnprintf(buf + written, PAGE_SIZE - written, "\n");
5499        rcu_read_unlock();
5500        cpus_read_unlock();
5501
5502        return written;
5503}
5504
5505static ssize_t wq_nice_show(struct device *dev, struct device_attribute *attr,
5506                            char *buf)
5507{
5508        struct workqueue_struct *wq = dev_to_wq(dev);
5509        int written;
5510
5511        mutex_lock(&wq->mutex);
5512        written = scnprintf(buf, PAGE_SIZE, "%d\n", wq->unbound_attrs->nice);
5513        mutex_unlock(&wq->mutex);
5514
5515        return written;
5516}
5517
5518/* prepare workqueue_attrs for sysfs store operations */
5519static struct workqueue_attrs *wq_sysfs_prep_attrs(struct workqueue_struct *wq)
5520{
5521        struct workqueue_attrs *attrs;
5522
5523        lockdep_assert_held(&wq_pool_mutex);
5524
5525        attrs = alloc_workqueue_attrs();
5526        if (!attrs)
5527                return NULL;
5528
5529        copy_workqueue_attrs(attrs, wq->unbound_attrs);
5530        return attrs;
5531}
5532
5533static ssize_t wq_nice_store(struct device *dev, struct device_attribute *attr,
5534                             const char *buf, size_t count)
5535{
5536        struct workqueue_struct *wq = dev_to_wq(dev);
5537        struct workqueue_attrs *attrs;
5538        int ret = -ENOMEM;
5539
5540        apply_wqattrs_lock();
5541
5542        attrs = wq_sysfs_prep_attrs(wq);
5543        if (!attrs)
5544                goto out_unlock;
5545
5546        if (sscanf(buf, "%d", &attrs->nice) == 1 &&
5547            attrs->nice >= MIN_NICE && attrs->nice <= MAX_NICE)
5548                ret = apply_workqueue_attrs_locked(wq, attrs);
5549        else
5550                ret = -EINVAL;
5551
5552out_unlock:
5553        apply_wqattrs_unlock();
5554        free_workqueue_attrs(attrs);
5555        return ret ?: count;
5556}
5557
5558static ssize_t wq_cpumask_show(struct device *dev,
5559                               struct device_attribute *attr, char *buf)
5560{
5561        struct workqueue_struct *wq = dev_to_wq(dev);
5562        int written;
5563
5564        mutex_lock(&wq->mutex);
5565        written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5566                            cpumask_pr_args(wq->unbound_attrs->cpumask));
5567        mutex_unlock(&wq->mutex);
5568        return written;
5569}
5570
5571static ssize_t wq_cpumask_store(struct device *dev,
5572                                struct device_attribute *attr,
5573                                const char *buf, size_t count)
5574{
5575        struct workqueue_struct *wq = dev_to_wq(dev);
5576        struct workqueue_attrs *attrs;
5577        int ret = -ENOMEM;
5578
5579        apply_wqattrs_lock();
5580
5581        attrs = wq_sysfs_prep_attrs(wq);
5582        if (!attrs)
5583                goto out_unlock;
5584
5585        ret = cpumask_parse(buf, attrs->cpumask);
5586        if (!ret)
5587                ret = apply_workqueue_attrs_locked(wq, attrs);
5588
5589out_unlock:
5590        apply_wqattrs_unlock();
5591        free_workqueue_attrs(attrs);
5592        return ret ?: count;
5593}
5594
5595static ssize_t wq_numa_show(struct device *dev, struct device_attribute *attr,
5596                            char *buf)
5597{
5598        struct workqueue_struct *wq = dev_to_wq(dev);
5599        int written;
5600
5601        mutex_lock(&wq->mutex);
5602        written = scnprintf(buf, PAGE_SIZE, "%d\n",
5603                            !wq->unbound_attrs->no_numa);
5604        mutex_unlock(&wq->mutex);
5605
5606        return written;
5607}
5608
5609static ssize_t wq_numa_store(struct device *dev, struct device_attribute *attr,
5610                             const char *buf, size_t count)
5611{
5612        struct workqueue_struct *wq = dev_to_wq(dev);
5613        struct workqueue_attrs *attrs;
5614        int v, ret = -ENOMEM;
5615
5616        apply_wqattrs_lock();
5617
5618        attrs = wq_sysfs_prep_attrs(wq);
5619        if (!attrs)
5620                goto out_unlock;
5621
5622        ret = -EINVAL;
5623        if (sscanf(buf, "%d", &v) == 1) {
5624                attrs->no_numa = !v;
5625                ret = apply_workqueue_attrs_locked(wq, attrs);
5626        }
5627
5628out_unlock:
5629        apply_wqattrs_unlock();
5630        free_workqueue_attrs(attrs);
5631        return ret ?: count;
5632}
5633
5634static struct device_attribute wq_sysfs_unbound_attrs[] = {
5635        __ATTR(pool_ids, 0444, wq_pool_ids_show, NULL),
5636        __ATTR(nice, 0644, wq_nice_show, wq_nice_store),
5637        __ATTR(cpumask, 0644, wq_cpumask_show, wq_cpumask_store),
5638        __ATTR(numa, 0644, wq_numa_show, wq_numa_store),
5639        __ATTR_NULL,
5640};
5641
5642static struct bus_type wq_subsys = {
5643        .name                           = "workqueue",
5644        .dev_groups                     = wq_sysfs_groups,
5645};
5646
5647static ssize_t wq_unbound_cpumask_show(struct device *dev,
5648                struct device_attribute *attr, char *buf)
5649{
5650        int written;
5651
5652        mutex_lock(&wq_pool_mutex);
5653        written = scnprintf(buf, PAGE_SIZE, "%*pb\n",
5654                            cpumask_pr_args(wq_unbound_cpumask));
5655        mutex_unlock(&wq_pool_mutex);
5656
5657        return written;
5658}
5659
5660static ssize_t wq_unbound_cpumask_store(struct device *dev,
5661                struct device_attribute *attr, const char *buf, size_t count)
5662{
5663        cpumask_var_t cpumask;
5664        int ret;
5665
5666        if (!zalloc_cpumask_var(&cpumask, GFP_KERNEL))
5667                return -ENOMEM;
5668
5669        ret = cpumask_parse(buf, cpumask);
5670        if (!ret)
5671                ret = workqueue_set_unbound_cpumask(cpumask);
5672
5673        free_cpumask_var(cpumask);
5674        return ret ? ret : count;
5675}
5676
5677static struct device_attribute wq_sysfs_cpumask_attr =
5678        __ATTR(cpumask, 0644, wq_unbound_cpumask_show,
5679               wq_unbound_cpumask_store);
5680
5681static int __init wq_sysfs_init(void)
5682{
5683        int err;
5684
5685        err = subsys_virtual_register(&wq_subsys, NULL);
5686        if (err)
5687                return err;
5688
5689        return device_create_file(wq_subsys.dev_root, &wq_sysfs_cpumask_attr);
5690}
5691core_initcall(wq_sysfs_init);
5692
5693static void wq_device_release(struct device *dev)
5694{
5695        struct wq_device *wq_dev = container_of(dev, struct wq_device, dev);
5696
5697        kfree(wq_dev);
5698}
5699
5700/**
5701 * workqueue_sysfs_register - make a workqueue visible in sysfs
5702 * @wq: the workqueue to register
5703 *
5704 * Expose @wq in sysfs under /sys/bus/workqueue/devices.
5705 * alloc_workqueue*() automatically calls this function if WQ_SYSFS is set
5706 * which is the preferred method.
5707 *
5708 * Workqueue user should use this function directly iff it wants to apply
5709 * workqueue_attrs before making the workqueue visible in sysfs; otherwise,
5710 * apply_workqueue_attrs() may race against userland updating the
5711 * attributes.
5712 *
5713 * Return: 0 on success, -errno on failure.
5714 */
5715int workqueue_sysfs_register(struct workqueue_struct *wq)
5716{
5717        struct wq_device *wq_dev;
5718        int ret;
5719
5720        /*
5721         * Adjusting max_active or creating new pwqs by applying
5722         * attributes breaks ordering guarantee.  Disallow exposing ordered
5723         * workqueues.
5724         */
5725        if (WARN_ON(wq->flags & __WQ_ORDERED_EXPLICIT))
5726                return -EINVAL;
5727
5728        wq->wq_dev = wq_dev = kzalloc(sizeof(*wq_dev), GFP_KERNEL);
5729        if (!wq_dev)
5730                return -ENOMEM;
5731
5732        wq_dev->wq = wq;
5733        wq_dev->dev.bus = &wq_subsys;
5734        wq_dev->dev.release = wq_device_release;
5735        dev_set_name(&wq_dev->dev, "%s", wq->name);
5736
5737        /*
5738         * unbound_attrs are created separately.  Suppress uevent until
5739         * everything is ready.
5740         */
5741        dev_set_uevent_suppress(&wq_dev->dev, true);
5742
5743        ret = device_register(&wq_dev->dev);
5744        if (ret) {
5745                put_device(&wq_dev->dev);
5746                wq->wq_dev = NULL;
5747                return ret;
5748        }
5749
5750        if (wq->flags & WQ_UNBOUND) {
5751                struct device_attribute *attr;
5752
5753                for (attr = wq_sysfs_unbound_attrs; attr->attr.name; attr++) {
5754                        ret = device_create_file(&wq_dev->dev, attr);
5755                        if (ret) {
5756                                device_unregister(&wq_dev->dev);
5757                                wq->wq_dev = NULL;
5758                                return ret;
5759                        }
5760                }
5761        }
5762
5763        dev_set_uevent_suppress(&wq_dev->dev, false);
5764        kobject_uevent(&wq_dev->dev.kobj, KOBJ_ADD);
5765        return 0;
5766}
5767
5768/**
5769 * workqueue_sysfs_unregister - undo workqueue_sysfs_register()
5770 * @wq: the workqueue to unregister
5771 *
5772 * If @wq is registered to sysfs by workqueue_sysfs_register(), unregister.
5773 */
5774static void workqueue_sysfs_unregister(struct workqueue_struct *wq)
5775{
5776        struct wq_device *wq_dev = wq->wq_dev;
5777
5778        if (!wq->wq_dev)
5779                return;
5780
5781        wq->wq_dev = NULL;
5782        device_unregister(&wq_dev->dev);
5783}
5784#else   /* CONFIG_SYSFS */
5785static void workqueue_sysfs_unregister(struct workqueue_struct *wq)     { }
5786#endif  /* CONFIG_SYSFS */
5787
5788/*
5789 * Workqueue watchdog.
5790 *
5791 * Stall may be caused by various bugs - missing WQ_MEM_RECLAIM, illegal
5792 * flush dependency, a concurrency managed work item which stays RUNNING
5793 * indefinitely.  Workqueue stalls can be very difficult to debug as the
5794 * usual warning mechanisms don't trigger and internal workqueue state is
5795 * largely opaque.
5796 *
5797 * Workqueue watchdog monitors all worker pools periodically and dumps
5798 * state if some pools failed to make forward progress for a while where
5799 * forward progress is defined as the first item on ->worklist changing.
5800 *
5801 * This mechanism is controlled through the kernel parameter
5802 * "workqueue.watchdog_thresh" which can be updated at runtime through the
5803 * corresponding sysfs parameter file.
5804 */
5805#ifdef CONFIG_WQ_WATCHDOG
5806
5807static unsigned long wq_watchdog_thresh = 30;
5808static struct timer_list wq_watchdog_timer;
5809
5810static unsigned long wq_watchdog_touched = INITIAL_JIFFIES;
5811static DEFINE_PER_CPU(unsigned long, wq_watchdog_touched_cpu) = INITIAL_JIFFIES;
5812
5813static void wq_watchdog_reset_touched(void)
5814{
5815        int cpu;
5816
5817        wq_watchdog_touched = jiffies;
5818        for_each_possible_cpu(cpu)
5819                per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5820}
5821
5822static void wq_watchdog_timer_fn(struct timer_list *unused)
5823{
5824        unsigned long thresh = READ_ONCE(wq_watchdog_thresh) * HZ;
5825        bool lockup_detected = false;
5826        unsigned long now = jiffies;
5827        struct worker_pool *pool;
5828        int pi;
5829
5830        if (!thresh)
5831                return;
5832
5833        rcu_read_lock();
5834
5835        for_each_pool(pool, pi) {
5836                unsigned long pool_ts, touched, ts;
5837
5838                if (list_empty(&pool->worklist))
5839                        continue;
5840
5841                /*
5842                 * If a virtual machine is stopped by the host it can look to
5843                 * the watchdog like a stall.
5844                 */
5845                kvm_check_and_clear_guest_paused();
5846
5847                /* get the latest of pool and touched timestamps */
5848                if (pool->cpu >= 0)
5849                        touched = READ_ONCE(per_cpu(wq_watchdog_touched_cpu, pool->cpu));
5850                else
5851                        touched = READ_ONCE(wq_watchdog_touched);
5852                pool_ts = READ_ONCE(pool->watchdog_ts);
5853
5854                if (time_after(pool_ts, touched))
5855                        ts = pool_ts;
5856                else
5857                        ts = touched;
5858
5859                /* did we stall? */
5860                if (time_after(now, ts + thresh)) {
5861                        lockup_detected = true;
5862                        pr_emerg("BUG: workqueue lockup - pool");
5863                        pr_cont_pool_info(pool);
5864                        pr_cont(" stuck for %us!\n",
5865                                jiffies_to_msecs(now - pool_ts) / 1000);
5866                }
5867        }
5868
5869        rcu_read_unlock();
5870
5871        if (lockup_detected)
5872                show_workqueue_state();
5873
5874        wq_watchdog_reset_touched();
5875        mod_timer(&wq_watchdog_timer, jiffies + thresh);
5876}
5877
5878notrace void wq_watchdog_touch(int cpu)
5879{
5880        if (cpu >= 0)
5881                per_cpu(wq_watchdog_touched_cpu, cpu) = jiffies;
5882
5883        wq_watchdog_touched = jiffies;
5884}
5885
5886static void wq_watchdog_set_thresh(unsigned long thresh)
5887{
5888        wq_watchdog_thresh = 0;
5889        del_timer_sync(&wq_watchdog_timer);
5890
5891        if (thresh) {
5892                wq_watchdog_thresh = thresh;
5893                wq_watchdog_reset_touched();
5894                mod_timer(&wq_watchdog_timer, jiffies + thresh * HZ);
5895        }
5896}
5897
5898static int wq_watchdog_param_set_thresh(const char *val,
5899                                        const struct kernel_param *kp)
5900{
5901        unsigned long thresh;
5902        int ret;
5903
5904        ret = kstrtoul(val, 0, &thresh);
5905        if (ret)
5906                return ret;
5907
5908        if (system_wq)
5909                wq_watchdog_set_thresh(thresh);
5910        else
5911                wq_watchdog_thresh = thresh;
5912
5913        return 0;
5914}
5915
5916static const struct kernel_param_ops wq_watchdog_thresh_ops = {
5917        .set    = wq_watchdog_param_set_thresh,
5918        .get    = param_get_ulong,
5919};
5920
5921module_param_cb(watchdog_thresh, &wq_watchdog_thresh_ops, &wq_watchdog_thresh,
5922                0644);
5923
5924static void wq_watchdog_init(void)
5925{
5926        timer_setup(&wq_watchdog_timer, wq_watchdog_timer_fn, TIMER_DEFERRABLE);
5927        wq_watchdog_set_thresh(wq_watchdog_thresh);
5928}
5929
5930#else   /* CONFIG_WQ_WATCHDOG */
5931
5932static inline void wq_watchdog_init(void) { }
5933
5934#endif  /* CONFIG_WQ_WATCHDOG */
5935
5936static void __init wq_numa_init(void)
5937{
5938        cpumask_var_t *tbl;
5939        int node, cpu;
5940
5941        if (num_possible_nodes() <= 1)
5942                return;
5943
5944        if (wq_disable_numa) {
5945                pr_info("workqueue: NUMA affinity support disabled\n");
5946                return;
5947        }
5948
5949        for_each_possible_cpu(cpu) {
5950                if (WARN_ON(cpu_to_node(cpu) == NUMA_NO_NODE)) {
5951                        pr_warn("workqueue: NUMA node mapping not available for cpu%d, disabling NUMA support\n", cpu);
5952                        return;
5953                }
5954        }
5955
5956        wq_update_unbound_numa_attrs_buf = alloc_workqueue_attrs();
5957        BUG_ON(!wq_update_unbound_numa_attrs_buf);
5958
5959        /*
5960         * We want masks of possible CPUs of each node which isn't readily
5961         * available.  Build one from cpu_to_node() which should have been
5962         * fully initialized by now.
5963         */
5964        tbl = kcalloc(nr_node_ids, sizeof(tbl[0]), GFP_KERNEL);
5965        BUG_ON(!tbl);
5966
5967        for_each_node(node)
5968                BUG_ON(!zalloc_cpumask_var_node(&tbl[node], GFP_KERNEL,
5969                                node_online(node) ? node : NUMA_NO_NODE));
5970
5971        for_each_possible_cpu(cpu) {
5972                node = cpu_to_node(cpu);
5973                cpumask_set_cpu(cpu, tbl[node]);
5974        }
5975
5976        wq_numa_possible_cpumask = tbl;
5977        wq_numa_enabled = true;
5978}
5979
5980/**
5981 * workqueue_init_early - early init for workqueue subsystem
5982 *
5983 * This is the first half of two-staged workqueue subsystem initialization
5984 * and invoked as soon as the bare basics - memory allocation, cpumasks and
5985 * idr are up.  It sets up all the data structures and system workqueues
5986 * and allows early boot code to create workqueues and queue/cancel work
5987 * items.  Actual work item execution starts only after kthreads can be
5988 * created and scheduled right before early initcalls.
5989 */
5990void __init workqueue_init_early(void)
5991{
5992        int std_nice[NR_STD_WORKER_POOLS] = { 0, HIGHPRI_NICE_LEVEL };
5993        int hk_flags = HK_FLAG_DOMAIN | HK_FLAG_WQ;
5994        int i, cpu;
5995
5996        BUILD_BUG_ON(__alignof__(struct pool_workqueue) < __alignof__(long long));
5997
5998        BUG_ON(!alloc_cpumask_var(&wq_unbound_cpumask, GFP_KERNEL));
5999        cpumask_copy(wq_unbound_cpumask, housekeeping_cpumask(hk_flags));
6000
6001        pwq_cache = KMEM_CACHE(pool_workqueue, SLAB_PANIC);
6002
6003        /* initialize CPU pools */
6004        for_each_possible_cpu(cpu) {
6005                struct worker_pool *pool;