1
2
3
4
5
6#include <linux/rwsem.h>
7#include <linux/sched.h>
8#include <linux/init.h>
9#include <linux/module.h>
10
11
12
13
14void __init_rwsem(struct rw_semaphore *sem, const char *name,
15 struct lock_class_key *key)
16{
17#ifdef CONFIG_DEBUG_LOCK_ALLOC
18
19
20
21 debug_check_no_locks_freed((void *)sem, sizeof(*sem));
22 lockdep_init_map(&sem->dep_map, name, key, 0);
23#endif
24 sem->count = RWSEM_UNLOCKED_VALUE;
25 raw_spin_lock_init(&sem->wait_lock);
26 INIT_LIST_HEAD(&sem->wait_list);
27}
28
29EXPORT_SYMBOL(__init_rwsem);
30
31struct rwsem_waiter {
32 struct list_head list;
33 struct task_struct *task;
34 unsigned int flags;
35#define RWSEM_WAITING_FOR_READ 0x00000001
36#define RWSEM_WAITING_FOR_WRITE 0x00000002
37};
38
39
40
41
42
43#define RWSEM_WAKE_ANY 0
44#define RWSEM_WAKE_NO_ACTIVE 1
45#define RWSEM_WAKE_READ_OWNED 2
46
47
48
49
50
51
52
53
54
55
56
57static struct rw_semaphore *
58__rwsem_do_wake(struct rw_semaphore *sem, int wake_type)
59{
60 struct rwsem_waiter *waiter;
61 struct task_struct *tsk;
62 struct list_head *next;
63 signed long oldcount, woken, loop, adjustment;
64
65 waiter = list_entry(sem->wait_list.next, struct rwsem_waiter, list);
66 if (!(waiter->flags & RWSEM_WAITING_FOR_WRITE))
67 goto readers_only;
68
69 if (wake_type == RWSEM_WAKE_READ_OWNED)
70
71
72
73 goto out;
74
75
76
77
78
79 adjustment = RWSEM_ACTIVE_WRITE_BIAS;
80 if (waiter->list.next == &sem->wait_list)
81 adjustment -= RWSEM_WAITING_BIAS;
82
83 try_again_write:
84 oldcount = rwsem_atomic_update(adjustment, sem) - adjustment;
85 if (oldcount & RWSEM_ACTIVE_MASK)
86
87 goto undo_write;
88
89
90
91
92
93 list_del(&waiter->list);
94 tsk = waiter->task;
95 smp_mb();
96 waiter->task = NULL;
97 wake_up_process(tsk);
98 put_task_struct(tsk);
99 goto out;
100
101 readers_only:
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116 if (wake_type == RWSEM_WAKE_ANY &&
117 rwsem_atomic_update(0, sem) < RWSEM_WAITING_BIAS)
118
119 goto out;
120
121
122
123
124
125 woken = 0;
126 do {
127 woken++;
128
129 if (waiter->list.next == &sem->wait_list)
130 break;
131
132 waiter = list_entry(waiter->list.next,
133 struct rwsem_waiter, list);
134
135 } while (waiter->flags & RWSEM_WAITING_FOR_READ);
136
137 adjustment = woken * RWSEM_ACTIVE_READ_BIAS;
138 if (waiter->flags & RWSEM_WAITING_FOR_READ)
139
140 adjustment -= RWSEM_WAITING_BIAS;
141
142 rwsem_atomic_add(adjustment, sem);
143
144 next = sem->wait_list.next;
145 for (loop = woken; loop > 0; loop--) {
146 waiter = list_entry(next, struct rwsem_waiter, list);
147 next = waiter->list.next;
148 tsk = waiter->task;
149 smp_mb();
150 waiter->task = NULL;
151 wake_up_process(tsk);
152 put_task_struct(tsk);
153 }
154
155 sem->wait_list.next = next;
156 next->prev = &sem->wait_list;
157
158 out:
159 return sem;
160
161
162
163 undo_write:
164 if (rwsem_atomic_update(-adjustment, sem) & RWSEM_ACTIVE_MASK)
165 goto out;
166 goto try_again_write;
167}
168
169
170
171
172static struct rw_semaphore __sched *
173rwsem_down_failed_common(struct rw_semaphore *sem,
174 unsigned int flags, signed long adjustment)
175{
176 struct rwsem_waiter waiter;
177 struct task_struct *tsk = current;
178 signed long count;
179
180 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
181
182
183 raw_spin_lock_irq(&sem->wait_lock);
184 waiter.task = tsk;
185 waiter.flags = flags;
186 get_task_struct(tsk);
187
188 if (list_empty(&sem->wait_list))
189 adjustment += RWSEM_WAITING_BIAS;
190 list_add_tail(&waiter.list, &sem->wait_list);
191
192
193 count = rwsem_atomic_update(adjustment, sem);
194
195
196
197
198
199
200
201 if (count == RWSEM_WAITING_BIAS)
202 sem = __rwsem_do_wake(sem, RWSEM_WAKE_NO_ACTIVE);
203 else if (count > RWSEM_WAITING_BIAS &&
204 adjustment == -RWSEM_ACTIVE_WRITE_BIAS)
205 sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED);
206
207 raw_spin_unlock_irq(&sem->wait_lock);
208
209
210 for (;;) {
211 if (!waiter.task)
212 break;
213 schedule();
214 set_task_state(tsk, TASK_UNINTERRUPTIBLE);
215 }
216
217 tsk->state = TASK_RUNNING;
218
219 return sem;
220}
221
222
223
224
225struct rw_semaphore __sched *rwsem_down_read_failed(struct rw_semaphore *sem)
226{
227 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_READ,
228 -RWSEM_ACTIVE_READ_BIAS);
229}
230
231
232
233
234struct rw_semaphore __sched *rwsem_down_write_failed(struct rw_semaphore *sem)
235{
236 return rwsem_down_failed_common(sem, RWSEM_WAITING_FOR_WRITE,
237 -RWSEM_ACTIVE_WRITE_BIAS);
238}
239
240
241
242
243
244struct rw_semaphore *rwsem_wake(struct rw_semaphore *sem)
245{
246 unsigned long flags;
247
248 raw_spin_lock_irqsave(&sem->wait_lock, flags);
249
250
251 if (!list_empty(&sem->wait_list))
252 sem = __rwsem_do_wake(sem, RWSEM_WAKE_ANY);
253
254 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
255
256 return sem;
257}
258
259
260
261
262
263
264struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem)
265{
266 unsigned long flags;
267
268 raw_spin_lock_irqsave(&sem->wait_lock, flags);
269
270
271 if (!list_empty(&sem->wait_list))
272 sem = __rwsem_do_wake(sem, RWSEM_WAKE_READ_OWNED);
273
274 raw_spin_unlock_irqrestore(&sem->wait_lock, flags);
275
276 return sem;
277}
278
279EXPORT_SYMBOL(rwsem_down_read_failed);
280EXPORT_SYMBOL(rwsem_down_write_failed);
281EXPORT_SYMBOL(rwsem_wake);
282EXPORT_SYMBOL(rwsem_downgrade_wake);
283