1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#ifdef _FORTIFY_SOURCE
23#undef _FORTIFY_SOURCE
24#endif
25#include "qemu/osdep.h"
26#include <ucontext.h>
27#include "qemu/coroutine_int.h"
28
29#ifdef CONFIG_VALGRIND_H
30#include <valgrind/valgrind.h>
31#endif
32
33#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer)
34#ifdef CONFIG_ASAN_IFACE_FIBER
35#define CONFIG_ASAN 1
36#include <sanitizer/asan_interface.h>
37#endif
38#endif
39
40#ifdef CONFIG_TSAN
41#include <sanitizer/tsan_interface.h>
42#endif
43
44typedef struct {
45 Coroutine base;
46 void *stack;
47 size_t stack_size;
48#ifdef CONFIG_SAFESTACK
49
50 void *unsafe_stack;
51 size_t unsafe_stack_size;
52#endif
53 sigjmp_buf env;
54
55#ifdef CONFIG_TSAN
56 void *tsan_co_fiber;
57 void *tsan_caller_fiber;
58#endif
59
60#ifdef CONFIG_VALGRIND_H
61 unsigned int valgrind_stack_id;
62#endif
63
64} CoroutineUContext;
65
66
67
68
69static __thread CoroutineUContext leader;
70static __thread Coroutine *current;
71
72
73
74
75
76
77union cc_arg {
78 void *p;
79 int i[2];
80};
81
82
83
84
85
86static inline __attribute__((always_inline))
87void on_new_fiber(CoroutineUContext *co)
88{
89#ifdef CONFIG_TSAN
90 co->tsan_co_fiber = __tsan_create_fiber(0);
91 co->tsan_caller_fiber = __tsan_get_current_fiber();
92#endif
93}
94
95
96static inline __attribute__((always_inline))
97void finish_switch_fiber(void *fake_stack_save)
98{
99#ifdef CONFIG_ASAN
100 const void *bottom_old;
101 size_t size_old;
102
103 __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old);
104
105 if (!leader.stack) {
106 leader.stack = (void *)bottom_old;
107 leader.stack_size = size_old;
108 }
109#endif
110#ifdef CONFIG_TSAN
111 if (fake_stack_save) {
112 __tsan_release(fake_stack_save);
113 __tsan_switch_to_fiber(fake_stack_save, 0);
114 }
115#endif
116}
117
118
119static inline __attribute__((always_inline))
120void start_switch_fiber_asan(CoroutineAction action, void **fake_stack_save,
121 const void *bottom, size_t size)
122{
123#ifdef CONFIG_ASAN
124 __sanitizer_start_switch_fiber(
125 action == COROUTINE_TERMINATE ? NULL : fake_stack_save,
126 bottom, size);
127#endif
128}
129
130
131static inline __attribute__((always_inline))
132void start_switch_fiber_tsan(void **fake_stack_save,
133 CoroutineUContext *co,
134 bool caller)
135{
136#ifdef CONFIG_TSAN
137 void *new_fiber = caller ?
138 co->tsan_caller_fiber :
139 co->tsan_co_fiber;
140 void *curr_fiber = __tsan_get_current_fiber();
141 __tsan_acquire(curr_fiber);
142
143 *fake_stack_save = curr_fiber;
144 __tsan_switch_to_fiber(new_fiber, 0);
145#endif
146}
147
148static void coroutine_trampoline(int i0, int i1)
149{
150 union cc_arg arg;
151 CoroutineUContext *self;
152 Coroutine *co;
153 void *fake_stack_save = NULL;
154
155 finish_switch_fiber(NULL);
156
157 arg.i[0] = i0;
158 arg.i[1] = i1;
159 self = arg.p;
160 co = &self->base;
161
162
163 if (!sigsetjmp(self->env, 0)) {
164 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, leader.stack,
165 leader.stack_size);
166 start_switch_fiber_tsan(&fake_stack_save, self, true);
167 siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
168 }
169
170 finish_switch_fiber(fake_stack_save);
171
172 while (true) {
173 co->entry(co->entry_arg);
174 qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
175 }
176}
177
178Coroutine *qemu_coroutine_new(void)
179{
180 CoroutineUContext *co;
181 ucontext_t old_uc, uc;
182 sigjmp_buf old_env;
183 union cc_arg arg = {0};
184 void *fake_stack_save = NULL;
185
186
187
188
189
190
191
192
193
194 if (getcontext(&uc) == -1) {
195 abort();
196 }
197
198 co = g_malloc0(sizeof(*co));
199 co->stack_size = COROUTINE_STACK_SIZE;
200 co->stack = qemu_alloc_stack(&co->stack_size);
201#ifdef CONFIG_SAFESTACK
202 co->unsafe_stack_size = COROUTINE_STACK_SIZE;
203 co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size);
204#endif
205 co->base.entry_arg = &old_env;
206
207 uc.uc_link = &old_uc;
208 uc.uc_stack.ss_sp = co->stack;
209 uc.uc_stack.ss_size = co->stack_size;
210 uc.uc_stack.ss_flags = 0;
211
212#ifdef CONFIG_VALGRIND_H
213 co->valgrind_stack_id =
214 VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size);
215#endif
216
217 arg.p = co;
218
219 on_new_fiber(co);
220 makecontext(&uc, (void (*)(void))coroutine_trampoline,
221 2, arg.i[0], arg.i[1]);
222
223
224 if (!sigsetjmp(old_env, 0)) {
225 start_switch_fiber_asan(COROUTINE_YIELD, &fake_stack_save, co->stack,
226 co->stack_size);
227 start_switch_fiber_tsan(&fake_stack_save,
228 co, false);
229
230#ifdef CONFIG_SAFESTACK
231
232
233
234
235
236
237
238
239
240
241 void *usp = co->unsafe_stack + co->unsafe_stack_size;
242 __safestack_unsafe_stack_ptr = usp;
243#endif
244
245 swapcontext(&old_uc, &uc);
246 }
247
248 finish_switch_fiber(fake_stack_save);
249
250 return &co->base;
251}
252
253#ifdef CONFIG_VALGRIND_H
254
255#if !defined(__clang__)
256#pragma GCC diagnostic push
257#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
258#endif
259static inline void valgrind_stack_deregister(CoroutineUContext *co)
260{
261 VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
262}
263#if !defined(__clang__)
264#pragma GCC diagnostic pop
265#endif
266#endif
267
268void qemu_coroutine_delete(Coroutine *co_)
269{
270 CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
271
272#ifdef CONFIG_VALGRIND_H
273 valgrind_stack_deregister(co);
274#endif
275
276 qemu_free_stack(co->stack, co->stack_size);
277#ifdef CONFIG_SAFESTACK
278 qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size);
279#endif
280 g_free(co);
281}
282
283
284
285
286
287
288
289
290
291CoroutineAction __attribute__((noinline))
292qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
293 CoroutineAction action)
294{
295 CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
296 CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
297 int ret;
298 void *fake_stack_save = NULL;
299
300 current = to_;
301
302 ret = sigsetjmp(from->env, 0);
303 if (ret == 0) {
304 start_switch_fiber_asan(action, &fake_stack_save, to->stack,
305 to->stack_size);
306 start_switch_fiber_tsan(&fake_stack_save,
307 to, false);
308 siglongjmp(to->env, action);
309 }
310
311 finish_switch_fiber(fake_stack_save);
312
313 return ret;
314}
315
316Coroutine *qemu_coroutine_self(void)
317{
318 if (!current) {
319 current = &leader.base;
320 }
321#ifdef CONFIG_TSAN
322 if (!leader.tsan_co_fiber) {
323 leader.tsan_co_fiber = __tsan_get_current_fiber();
324 }
325#endif
326 return current;
327}
328
329bool qemu_in_coroutine(void)
330{
331 return current && current->caller;
332}
333