LXR linux/kernel/task

   1#include <linux/spinlock.h>
   2#include <linux/task_work.h>
   3#include <linux/tracehook.h>
   4
   5static struct callback_head work_exited; /* all we need is ->next == NULL */
   6
   7int
   8task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
   9{
  10        struct callback_head *head;
  11
  12        do {
  13                head = ACCESS_ONCE(task->task_works);
  14                if (unlikely(head == &work_exited))
  15                        return -ESRCH;
  16                work->next = head;
  17        } while (cmpxchg(&task->task_works, head, work) != head);
  18
  19        if (notify)
  20                set_notify_resume(task);
  21        return 0;
  22}
  23
  24struct callback_head *
  25task_work_cancel(struct task_struct *task, task_work_func_t func)
  26{
  27        struct callback_head **pprev = &task->task_works;
  28        struct callback_head *work = NULL;
  29        unsigned long flags;
  30        /*
  31         * If cmpxchg() fails we continue without updating pprev.
  32         * Either we raced with task_work_add() which added the
  33         * new entry before this work, we will find it again. Or
  34         * we raced with task_work_run(), *pprev == NULL/exited.
  35         */
  36        raw_spin_lock_irqsave(&task->pi_lock, flags);
  37        while ((work = ACCESS_ONCE(*pprev))) {
  38                read_barrier_depends();
  39                if (work->func != func)
  40                        pprev = &work->next;
  41                else if (cmpxchg(pprev, work, work->next) == work)
  42                        break;
  43        }
  44        raw_spin_unlock_irqrestore(&task->pi_lock, flags);
  45
  46        return work;
  47}
  48
  49void task_work_run(void)
  50{
  51        struct task_struct *task = current;
  52        struct callback_head *work, *head, *next;
  53
  54        for (;;) {
  55                /*
  56                 * work->func() can do task_work_add(), do not set
  57                 * work_exited unless the list is empty.
  58                 */
  59                do {
  60                        work = ACCESS_ONCE(task->task_works);
  61                        head = !work && (task->flags & PF_EXITING) ?
  62                                &work_exited : NULL;
  63                } while (cmpxchg(&task->task_works, work, head) != work);
  64
  65                if (!work)
  66                        break;
  67                /*
  68                 * Synchronize with task_work_cancel(). It can't remove
  69                 * the first entry == work, cmpxchg(task_works) should
  70                 * fail, but it can play with *work and other entries.
  71                 */
  72                raw_spin_unlock_wait(&task->pi_lock);
  73                smp_mb();
  74
  75                /* Reverse the list to run the works in fifo order */
  76                head = NULL;
  77                do {
  78                        next = work->next;
  79                        work->next = head;
  80                        head = work;
  81                        work = next;
  82                } while (work);
  83
  84                work = head;
  85                do {
  86                        next = work->next;
  87                        work->func(work);
  88                        work = next;
  89                        cond_resched();
  90                } while (work);
  91        }
  92}
  93