1#include <linux/spinlock.h>
2#include <linux/task_work.h>
3#include <linux/tracehook.h>
4
5static struct callback_head work_exited;
6
7int
8task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
9{
10 struct callback_head *head;
11
12 do {
13 head = ACCESS_ONCE(task->task_works);
14 if (unlikely(head == &work_exited))
15 return -ESRCH;
16 work->next = head;
17 } while (cmpxchg(&task->task_works, head, work) != head);
18
19 if (notify)
20 set_notify_resume(task);
21 return 0;
22}
23
24struct callback_head *
25task_work_cancel(struct task_struct *task, task_work_func_t func)
26{
27 struct callback_head **pprev = &task->task_works;
28 struct callback_head *work = NULL;
29 unsigned long flags;
30
31
32
33
34
35
36 raw_spin_lock_irqsave(&task->pi_lock, flags);
37 while ((work = ACCESS_ONCE(*pprev))) {
38 read_barrier_depends();
39 if (work->func != func)
40 pprev = &work->next;
41 else if (cmpxchg(pprev, work, work->next) == work)
42 break;
43 }
44 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
45
46 return work;
47}
48
49void task_work_run(void)
50{
51 struct task_struct *task = current;
52 struct callback_head *work, *head, *next;
53
54 for (;;) {
55
56
57
58
59 do {
60 work = ACCESS_ONCE(task->task_works);
61 head = !work && (task->flags & PF_EXITING) ?
62 &work_exited : NULL;
63 } while (cmpxchg(&task->task_works, work, head) != work);
64
65 if (!work)
66 break;
67
68
69
70
71
72 raw_spin_unlock_wait(&task->pi_lock);
73 smp_mb();
74
75
76 head = NULL;
77 do {
78 next = work->next;
79 work->next = head;
80 head = work;
81 work = next;
82 } while (work);
83
84 work = head;
85 do {
86 next = work->next;
87 work->func(work);
88 work = next;
89 cond_resched();
90 } while (work);
91 }
92}
93