1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include "qemu/osdep.h"
30#include "qemu/rcu.h"
31#include "qemu/atomic.h"
32#include "qemu/thread.h"
33#include "qemu/main-loop.h"
34#include "qemu/lockable.h"
35#if defined(CONFIG_MALLOC_TRIM)
36#include <malloc.h>
37#endif
38
39
40
41
42
43#define RCU_GP_LOCKED (1UL << 0)
44#define RCU_GP_CTR (1UL << 1)
45
46unsigned long rcu_gp_ctr = RCU_GP_LOCKED;
47
48QemuEvent rcu_gp_event;
49static QemuMutex rcu_registry_lock;
50static QemuMutex rcu_sync_lock;
51
52
53
54
55
56static inline int rcu_gp_ongoing(unsigned long *ctr)
57{
58 unsigned long v;
59
60 v = atomic_read(ctr);
61 return v && (v != rcu_gp_ctr);
62}
63
64
65
66
67__thread struct rcu_reader_data rcu_reader;
68
69
70typedef QLIST_HEAD(, rcu_reader_data) ThreadList;
71static ThreadList registry = QLIST_HEAD_INITIALIZER(registry);
72
73
74static void wait_for_readers(void)
75{
76 ThreadList qsreaders = QLIST_HEAD_INITIALIZER(qsreaders);
77 struct rcu_reader_data *index, *tmp;
78
79 for (;;) {
80
81
82
83 qemu_event_reset(&rcu_gp_event);
84
85
86
87
88
89
90
91 QLIST_FOREACH(index, ®istry, node) {
92 atomic_set(&index->waiting, true);
93 }
94
95
96
97
98
99 smp_mb_global();
100
101 QLIST_FOREACH_SAFE(index, ®istry, node, tmp) {
102 if (!rcu_gp_ongoing(&index->ctr)) {
103 QLIST_REMOVE(index, node);
104 QLIST_INSERT_HEAD(&qsreaders, index, node);
105
106
107
108
109 atomic_set(&index->waiting, false);
110 }
111 }
112
113 if (QLIST_EMPTY(®istry)) {
114 break;
115 }
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134 qemu_mutex_unlock(&rcu_registry_lock);
135 qemu_event_wait(&rcu_gp_event);
136 qemu_mutex_lock(&rcu_registry_lock);
137 }
138
139
140 QLIST_SWAP(®istry, &qsreaders, node);
141}
142
143void synchronize_rcu(void)
144{
145 QEMU_LOCK_GUARD(&rcu_sync_lock);
146
147
148
149
150 smp_mb_global();
151
152 QEMU_LOCK_GUARD(&rcu_registry_lock);
153 if (!QLIST_EMPTY(®istry)) {
154
155
156
157 if (sizeof(rcu_gp_ctr) < 8) {
158
159
160
161
162
163 atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
164 wait_for_readers();
165 atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
166 } else {
167
168 atomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
169 }
170
171 wait_for_readers();
172 }
173}
174
175
176#define RCU_CALL_MIN_SIZE 30
177
178
179
180
181static struct rcu_head dummy;
182static struct rcu_head *head = &dummy, **tail = &dummy.next;
183static int rcu_call_count;
184static QemuEvent rcu_call_ready_event;
185
186static void enqueue(struct rcu_head *node)
187{
188 struct rcu_head **old_tail;
189
190 node->next = NULL;
191 old_tail = atomic_xchg(&tail, &node->next);
192 atomic_mb_set(old_tail, node);
193}
194
195static struct rcu_head *try_dequeue(void)
196{
197 struct rcu_head *node, *next;
198
199retry:
200
201
202
203
204
205
206 if (head == &dummy && atomic_mb_read(&tail) == &dummy.next) {
207 abort();
208 }
209
210
211
212
213 node = head;
214 next = atomic_mb_read(&head->next);
215 if (!next) {
216 return NULL;
217 }
218
219
220
221
222
223
224 head = next;
225
226
227 if (node == &dummy) {
228 enqueue(node);
229 goto retry;
230 }
231
232 return node;
233}
234
235static void *call_rcu_thread(void *opaque)
236{
237 struct rcu_head *node;
238
239 rcu_register_thread();
240
241 for (;;) {
242 int tries = 0;
243 int n = atomic_read(&rcu_call_count);
244
245
246
247
248
249 while (n == 0 || (n < RCU_CALL_MIN_SIZE && ++tries <= 5)) {
250 g_usleep(10000);
251 if (n == 0) {
252 qemu_event_reset(&rcu_call_ready_event);
253 n = atomic_read(&rcu_call_count);
254 if (n == 0) {
255#if defined(CONFIG_MALLOC_TRIM)
256 malloc_trim(4 * 1024 * 1024);
257#endif
258 qemu_event_wait(&rcu_call_ready_event);
259 }
260 }
261 n = atomic_read(&rcu_call_count);
262 }
263
264 atomic_sub(&rcu_call_count, n);
265 synchronize_rcu();
266 qemu_mutex_lock_iothread();
267 while (n > 0) {
268 node = try_dequeue();
269 while (!node) {
270 qemu_mutex_unlock_iothread();
271 qemu_event_reset(&rcu_call_ready_event);
272 node = try_dequeue();
273 if (!node) {
274 qemu_event_wait(&rcu_call_ready_event);
275 node = try_dequeue();
276 }
277 qemu_mutex_lock_iothread();
278 }
279
280 n--;
281 node->func(node);
282 }
283 qemu_mutex_unlock_iothread();
284 }
285 abort();
286}
287
288void call_rcu1(struct rcu_head *node, void (*func)(struct rcu_head *node))
289{
290 node->func = func;
291 enqueue(node);
292 atomic_inc(&rcu_call_count);
293 qemu_event_set(&rcu_call_ready_event);
294}
295
296void rcu_register_thread(void)
297{
298 assert(rcu_reader.ctr == 0);
299 qemu_mutex_lock(&rcu_registry_lock);
300 QLIST_INSERT_HEAD(®istry, &rcu_reader, node);
301 qemu_mutex_unlock(&rcu_registry_lock);
302}
303
304void rcu_unregister_thread(void)
305{
306 qemu_mutex_lock(&rcu_registry_lock);
307 QLIST_REMOVE(&rcu_reader, node);
308 qemu_mutex_unlock(&rcu_registry_lock);
309}
310
311static void rcu_init_complete(void)
312{
313 QemuThread thread;
314
315 qemu_mutex_init(&rcu_registry_lock);
316 qemu_mutex_init(&rcu_sync_lock);
317 qemu_event_init(&rcu_gp_event, true);
318
319 qemu_event_init(&rcu_call_ready_event, false);
320
321
322
323
324 qemu_thread_create(&thread, "call_rcu", call_rcu_thread,
325 NULL, QEMU_THREAD_DETACHED);
326
327 rcu_register_thread();
328}
329
330static int atfork_depth = 1;
331
332void rcu_enable_atfork(void)
333{
334 atfork_depth++;
335}
336
337void rcu_disable_atfork(void)
338{
339 atfork_depth--;
340}
341
342#ifdef CONFIG_POSIX
343static void rcu_init_lock(void)
344{
345 if (atfork_depth < 1) {
346 return;
347 }
348
349 qemu_mutex_lock(&rcu_sync_lock);
350 qemu_mutex_lock(&rcu_registry_lock);
351}
352
353static void rcu_init_unlock(void)
354{
355 if (atfork_depth < 1) {
356 return;
357 }
358
359 qemu_mutex_unlock(&rcu_registry_lock);
360 qemu_mutex_unlock(&rcu_sync_lock);
361}
362
363static void rcu_init_child(void)
364{
365 if (atfork_depth < 1) {
366 return;
367 }
368
369 memset(®istry, 0, sizeof(registry));
370 rcu_init_complete();
371}
372#endif
373
374static void __attribute__((__constructor__)) rcu_init(void)
375{
376 smp_mb_global_init();
377#ifdef CONFIG_POSIX
378 pthread_atfork(rcu_init_lock, rcu_init_unlock, rcu_init_child);
379#endif
380 rcu_init_complete();
381}
382