1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22#ifdef _FORTIFY_SOURCE
23#undef _FORTIFY_SOURCE
24#endif
25#include "qemu/osdep.h"
26#include <ucontext.h>
27#include "qemu-common.h"
28#include "qemu/coroutine_int.h"
29
30#ifdef CONFIG_VALGRIND_H
31#include <valgrind/valgrind.h>
32#endif
33
34#if defined(__SANITIZE_ADDRESS__) || __has_feature(address_sanitizer)
35#ifdef CONFIG_ASAN_IFACE_FIBER
36#define CONFIG_ASAN 1
37#include <sanitizer/asan_interface.h>
38#endif
39#endif
40
41typedef struct {
42 Coroutine base;
43 void *stack;
44 size_t stack_size;
45 sigjmp_buf env;
46
47#ifdef CONFIG_VALGRIND_H
48 unsigned int valgrind_stack_id;
49#endif
50
51} CoroutineUContext;
52
53
54
55
56static __thread CoroutineUContext leader;
57static __thread Coroutine *current;
58
59
60
61
62
63
64union cc_arg {
65 void *p;
66 int i[2];
67};
68
69static void finish_switch_fiber(void *fake_stack_save)
70{
71#ifdef CONFIG_ASAN
72 const void *bottom_old;
73 size_t size_old;
74
75 __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old);
76
77 if (!leader.stack) {
78 leader.stack = (void *)bottom_old;
79 leader.stack_size = size_old;
80 }
81#endif
82}
83
84static void start_switch_fiber(void **fake_stack_save,
85 const void *bottom, size_t size)
86{
87#ifdef CONFIG_ASAN
88 __sanitizer_start_switch_fiber(fake_stack_save, bottom, size);
89#endif
90}
91
92static void coroutine_trampoline(int i0, int i1)
93{
94 union cc_arg arg;
95 CoroutineUContext *self;
96 Coroutine *co;
97 void *fake_stack_save = NULL;
98
99 finish_switch_fiber(NULL);
100
101 arg.i[0] = i0;
102 arg.i[1] = i1;
103 self = arg.p;
104 co = &self->base;
105
106
107 if (!sigsetjmp(self->env, 0)) {
108 start_switch_fiber(&fake_stack_save,
109 leader.stack, leader.stack_size);
110 siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
111 }
112
113 finish_switch_fiber(fake_stack_save);
114
115 while (true) {
116 co->entry(co->entry_arg);
117 qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
118 }
119}
120
121Coroutine *qemu_coroutine_new(void)
122{
123 CoroutineUContext *co;
124 ucontext_t old_uc, uc;
125 sigjmp_buf old_env;
126 union cc_arg arg = {0};
127 void *fake_stack_save = NULL;
128
129
130
131
132
133
134
135
136
137 if (getcontext(&uc) == -1) {
138 abort();
139 }
140
141 co = g_malloc0(sizeof(*co));
142 co->stack_size = COROUTINE_STACK_SIZE;
143 co->stack = qemu_alloc_stack(&co->stack_size);
144 co->base.entry_arg = &old_env;
145
146 uc.uc_link = &old_uc;
147 uc.uc_stack.ss_sp = co->stack;
148 uc.uc_stack.ss_size = co->stack_size;
149 uc.uc_stack.ss_flags = 0;
150
151#ifdef CONFIG_VALGRIND_H
152 co->valgrind_stack_id =
153 VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size);
154#endif
155
156 arg.p = co;
157
158 makecontext(&uc, (void (*)(void))coroutine_trampoline,
159 2, arg.i[0], arg.i[1]);
160
161
162 if (!sigsetjmp(old_env, 0)) {
163 start_switch_fiber(&fake_stack_save, co->stack, co->stack_size);
164 swapcontext(&old_uc, &uc);
165 }
166
167 finish_switch_fiber(fake_stack_save);
168
169 return &co->base;
170}
171
172#ifdef CONFIG_VALGRIND_H
173#if defined(CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE) && !defined(__clang__)
174
175#pragma GCC diagnostic push
176#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
177#endif
178static inline void valgrind_stack_deregister(CoroutineUContext *co)
179{
180 VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
181}
182#if defined(CONFIG_PRAGMA_DIAGNOSTIC_AVAILABLE) && !defined(__clang__)
183#pragma GCC diagnostic pop
184#endif
185#endif
186
187void qemu_coroutine_delete(Coroutine *co_)
188{
189 CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
190
191#ifdef CONFIG_VALGRIND_H
192 valgrind_stack_deregister(co);
193#endif
194
195 qemu_free_stack(co->stack, co->stack_size);
196 g_free(co);
197}
198
199
200
201
202
203
204
205
206
207CoroutineAction __attribute__((noinline))
208qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
209 CoroutineAction action)
210{
211 CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
212 CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
213 int ret;
214 void *fake_stack_save = NULL;
215
216 current = to_;
217
218 ret = sigsetjmp(from->env, 0);
219 if (ret == 0) {
220 start_switch_fiber(action == COROUTINE_TERMINATE ?
221 NULL : &fake_stack_save, to->stack, to->stack_size);
222 siglongjmp(to->env, action);
223 }
224
225 finish_switch_fiber(fake_stack_save);
226
227 return ret;
228}
229
230Coroutine *qemu_coroutine_self(void)
231{
232 if (!current) {
233 current = &leader.base;
234 }
235 return current;
236}
237
238bool qemu_in_coroutine(void)
239{
240 return current && current->caller;
241}
242