1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29#include "qemu/osdep.h"
30#include "qemu/rcu.h"
31#include "qemu/atomic.h"
32#include "qemu/thread.h"
33#include "qemu/main-loop.h"
34#include "qemu/lockable.h"
35#if defined(CONFIG_MALLOC_TRIM)
36#include <malloc.h>
37#endif
38
39
40
41
42
43#define RCU_GP_LOCKED (1UL << 0)
44#define RCU_GP_CTR (1UL << 1)
45
46unsigned long rcu_gp_ctr = RCU_GP_LOCKED;
47
48QemuEvent rcu_gp_event;
49static QemuMutex rcu_registry_lock;
50static QemuMutex rcu_sync_lock;
51
52
53
54
55
56static inline int rcu_gp_ongoing(unsigned long *ctr)
57{
58 unsigned long v;
59
60 v = qatomic_read(ctr);
61 return v && (v != rcu_gp_ctr);
62}
63
64
65
66
67__thread struct rcu_reader_data rcu_reader;
68
69
70typedef QLIST_HEAD(, rcu_reader_data) ThreadList;
71static ThreadList registry = QLIST_HEAD_INITIALIZER(registry);
72
73
74static void wait_for_readers(void)
75{
76 ThreadList qsreaders = QLIST_HEAD_INITIALIZER(qsreaders);
77 struct rcu_reader_data *index, *tmp;
78
79 for (;;) {
80
81
82
83 qemu_event_reset(&rcu_gp_event);
84
85
86
87
88
89
90
91 QLIST_FOREACH(index, ®istry, node) {
92 qatomic_set(&index->waiting, true);
93 }
94
95
96
97
98
99 smp_mb_global();
100
101 QLIST_FOREACH_SAFE(index, ®istry, node, tmp) {
102 if (!rcu_gp_ongoing(&index->ctr)) {
103 QLIST_REMOVE(index, node);
104 QLIST_INSERT_HEAD(&qsreaders, index, node);
105
106
107
108
109 qatomic_set(&index->waiting, false);
110 }
111 }
112
113 if (QLIST_EMPTY(®istry)) {
114 break;
115 }
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134 qemu_mutex_unlock(&rcu_registry_lock);
135 qemu_event_wait(&rcu_gp_event);
136 qemu_mutex_lock(&rcu_registry_lock);
137 }
138
139
140 QLIST_SWAP(®istry, &qsreaders, node);
141}
142
143void synchronize_rcu(void)
144{
145 QEMU_LOCK_GUARD(&rcu_sync_lock);
146
147
148
149
150 smp_mb_global();
151
152 QEMU_LOCK_GUARD(&rcu_registry_lock);
153 if (!QLIST_EMPTY(®istry)) {
154
155
156
157 if (sizeof(rcu_gp_ctr) < 8) {
158
159
160
161
162
163 qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
164 wait_for_readers();
165 qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr ^ RCU_GP_CTR);
166 } else {
167
168 qatomic_mb_set(&rcu_gp_ctr, rcu_gp_ctr + RCU_GP_CTR);
169 }
170
171 wait_for_readers();
172 }
173}
174
175
176#define RCU_CALL_MIN_SIZE 30
177
178
179
180
181static struct rcu_head dummy;
182static struct rcu_head *head = &dummy, **tail = &dummy.next;
183static int rcu_call_count;
184static QemuEvent rcu_call_ready_event;
185
186static void enqueue(struct rcu_head *node)
187{
188 struct rcu_head **old_tail;
189
190 node->next = NULL;
191 old_tail = qatomic_xchg(&tail, &node->next);
192 qatomic_mb_set(old_tail, node);
193}
194
195static struct rcu_head *try_dequeue(void)
196{
197 struct rcu_head *node, *next;
198
199retry:
200
201
202
203
204
205
206 if (head == &dummy && qatomic_mb_read(&tail) == &dummy.next) {
207 abort();
208 }
209
210
211
212
213 node = head;
214 next = qatomic_mb_read(&head->next);
215 if (!next) {
216 return NULL;
217 }
218
219
220
221
222
223
224 head = next;
225
226
227 if (node == &dummy) {
228 enqueue(node);
229 goto retry;
230 }
231
232 return node;
233}
234
235static void *call_rcu_thread(void *opaque)
236{
237 struct rcu_head *node;
238
239 rcu_register_thread();
240
241 for (;;) {
242 int tries = 0;
243 int n = qatomic_read(&rcu_call_count);
244
245
246
247
248
249 while (n == 0 || (n < RCU_CALL_MIN_SIZE && ++tries <= 5)) {
250 g_usleep(10000);
251 if (n == 0) {
252 qemu_event_reset(&rcu_call_ready_event);
253 n = qatomic_read(&rcu_call_count);
254 if (n == 0) {
255#if defined(CONFIG_MALLOC_TRIM)
256 malloc_trim(4 * 1024 * 1024);
257#endif
258 qemu_event_wait(&rcu_call_ready_event);
259 }
260 }
261 n = qatomic_read(&rcu_call_count);
262 }
263
264 qatomic_sub(&rcu_call_count, n);
265 synchronize_rcu();
266 qemu_mutex_lock_iothread();
267 while (n > 0) {
268 node = try_dequeue();
269 while (!node) {
270 qemu_mutex_unlock_iothread();
271 qemu_event_reset(&rcu_call_ready_event);
272 node = try_dequeue();
273 if (!node) {
274 qemu_event_wait(&rcu_call_ready_event);
275 node = try_dequeue();
276 }
277 qemu_mutex_lock_iothread();
278 }
279
280 n--;
281 node->func(node);
282 }
283 qemu_mutex_unlock_iothread();
284 }
285 abort();
286}
287
288void call_rcu1(struct rcu_head *node, void (*func)(struct rcu_head *node))
289{
290 node->func = func;
291 enqueue(node);
292 qatomic_inc(&rcu_call_count);
293 qemu_event_set(&rcu_call_ready_event);
294}
295
296
297struct rcu_drain {
298 struct rcu_head rcu;
299 QemuEvent drain_complete_event;
300};
301
302static void drain_rcu_callback(struct rcu_head *node)
303{
304 struct rcu_drain *event = (struct rcu_drain *)node;
305 qemu_event_set(&event->drain_complete_event);
306}
307
308
309
310
311
312
313
314
315
316
317void drain_call_rcu(void)
318{
319 struct rcu_drain rcu_drain;
320 bool locked = qemu_mutex_iothread_locked();
321
322 memset(&rcu_drain, 0, sizeof(struct rcu_drain));
323 qemu_event_init(&rcu_drain.drain_complete_event, false);
324
325 if (locked) {
326 qemu_mutex_unlock_iothread();
327 }
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342 call_rcu1(&rcu_drain.rcu, drain_rcu_callback);
343 qemu_event_wait(&rcu_drain.drain_complete_event);
344
345 if (locked) {
346 qemu_mutex_lock_iothread();
347 }
348
349}
350
351void rcu_register_thread(void)
352{
353 assert(rcu_reader.ctr == 0);
354 qemu_mutex_lock(&rcu_registry_lock);
355 QLIST_INSERT_HEAD(®istry, &rcu_reader, node);
356 qemu_mutex_unlock(&rcu_registry_lock);
357}
358
359void rcu_unregister_thread(void)
360{
361 qemu_mutex_lock(&rcu_registry_lock);
362 QLIST_REMOVE(&rcu_reader, node);
363 qemu_mutex_unlock(&rcu_registry_lock);
364}
365
366static void rcu_init_complete(void)
367{
368 QemuThread thread;
369
370 qemu_mutex_init(&rcu_registry_lock);
371 qemu_mutex_init(&rcu_sync_lock);
372 qemu_event_init(&rcu_gp_event, true);
373
374 qemu_event_init(&rcu_call_ready_event, false);
375
376
377
378
379 qemu_thread_create(&thread, "call_rcu", call_rcu_thread,
380 NULL, QEMU_THREAD_DETACHED);
381
382 rcu_register_thread();
383}
384
385static int atfork_depth = 1;
386
387void rcu_enable_atfork(void)
388{
389 atfork_depth++;
390}
391
392void rcu_disable_atfork(void)
393{
394 atfork_depth--;
395}
396
397#ifdef CONFIG_POSIX
398static void rcu_init_lock(void)
399{
400 if (atfork_depth < 1) {
401 return;
402 }
403
404 qemu_mutex_lock(&rcu_sync_lock);
405 qemu_mutex_lock(&rcu_registry_lock);
406}
407
408static void rcu_init_unlock(void)
409{
410 if (atfork_depth < 1) {
411 return;
412 }
413
414 qemu_mutex_unlock(&rcu_registry_lock);
415 qemu_mutex_unlock(&rcu_sync_lock);
416}
417
418static void rcu_init_child(void)
419{
420 if (atfork_depth < 1) {
421 return;
422 }
423
424 memset(®istry, 0, sizeof(registry));
425 rcu_init_complete();
426}
427#endif
428
429static void __attribute__((__constructor__)) rcu_init(void)
430{
431 smp_mb_global_init();
432#ifdef CONFIG_POSIX
433 pthread_atfork(rcu_init_lock, rcu_init_unlock, rcu_init_child);
434#endif
435 rcu_init_complete();
436}
437