1#include <linux/spinlock.h>
2#include <linux/task_work.h>
3#include <linux/tracehook.h>
4
5static struct callback_head work_exited;
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24int
25task_work_add(struct task_struct *task, struct callback_head *work, bool notify)
26{
27 struct callback_head *head;
28
29 do {
30 head = ACCESS_ONCE(task->task_works);
31 if (unlikely(head == &work_exited))
32 return -ESRCH;
33 work->next = head;
34 } while (cmpxchg(&task->task_works, head, work) != head);
35
36 if (notify)
37 set_notify_resume(task);
38 return 0;
39}
40
41
42
43
44
45
46
47
48
49
50
51
52struct callback_head *
53task_work_cancel(struct task_struct *task, task_work_func_t func)
54{
55 struct callback_head **pprev = &task->task_works;
56 struct callback_head *work;
57 unsigned long flags;
58
59
60
61
62
63
64 raw_spin_lock_irqsave(&task->pi_lock, flags);
65 while ((work = ACCESS_ONCE(*pprev))) {
66 smp_read_barrier_depends();
67 if (work->func != func)
68 pprev = &work->next;
69 else if (cmpxchg(pprev, work, work->next) == work)
70 break;
71 }
72 raw_spin_unlock_irqrestore(&task->pi_lock, flags);
73
74 return work;
75}
76
77
78
79
80
81
82
83
84
85void task_work_run(void)
86{
87 struct task_struct *task = current;
88 struct callback_head *work, *head, *next;
89
90 for (;;) {
91
92
93
94
95 do {
96 work = ACCESS_ONCE(task->task_works);
97 head = !work && (task->flags & PF_EXITING) ?
98 &work_exited : NULL;
99 } while (cmpxchg(&task->task_works, work, head) != work);
100
101 if (!work)
102 break;
103
104
105
106
107
108 raw_spin_unlock_wait(&task->pi_lock);
109 smp_mb();
110
111
112 head = NULL;
113 do {
114 next = work->next;
115 work->next = head;
116 head = work;
117 work = next;
118 } while (work);
119
120 work = head;
121 do {
122 next = work->next;
123 work->func(work);
124 work = next;
125 cond_resched();
126 } while (work);
127 }
128}
129