1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include "qemu/osdep.h"
26#include "block/throttle-groups.h"
27#include "qemu/queue.h"
28#include "qemu/thread.h"
29#include "sysemu/qtest.h"
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55typedef struct ThrottleGroup {
56 char *name;
57
58 QemuMutex lock;
59 ThrottleState ts;
60 QLIST_HEAD(, BlockDriverState) head;
61 BlockDriverState *tokens[2];
62 bool any_timer_armed[2];
63
64
65 unsigned refcount;
66 QTAILQ_ENTRY(ThrottleGroup) list;
67} ThrottleGroup;
68
69static QemuMutex throttle_groups_lock;
70static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
71 QTAILQ_HEAD_INITIALIZER(throttle_groups);
72
73
74
75
76
77
78
79
80
81ThrottleState *throttle_group_incref(const char *name)
82{
83 ThrottleGroup *tg = NULL;
84 ThrottleGroup *iter;
85
86 qemu_mutex_lock(&throttle_groups_lock);
87
88
89 QTAILQ_FOREACH(iter, &throttle_groups, list) {
90 if (!strcmp(name, iter->name)) {
91 tg = iter;
92 break;
93 }
94 }
95
96
97 if (!tg) {
98 tg = g_new0(ThrottleGroup, 1);
99 tg->name = g_strdup(name);
100 qemu_mutex_init(&tg->lock);
101 throttle_init(&tg->ts);
102 QLIST_INIT(&tg->head);
103
104 QTAILQ_INSERT_TAIL(&throttle_groups, tg, list);
105 }
106
107 tg->refcount++;
108
109 qemu_mutex_unlock(&throttle_groups_lock);
110
111 return &tg->ts;
112}
113
114
115
116
117
118
119
120
121void throttle_group_unref(ThrottleState *ts)
122{
123 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
124
125 qemu_mutex_lock(&throttle_groups_lock);
126 if (--tg->refcount == 0) {
127 QTAILQ_REMOVE(&throttle_groups, tg, list);
128 qemu_mutex_destroy(&tg->lock);
129 g_free(tg->name);
130 g_free(tg);
131 }
132 qemu_mutex_unlock(&throttle_groups_lock);
133}
134
135
136
137
138
139
140
141
142const char *throttle_group_get_name(BlockDriverState *bs)
143{
144 ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
145 return tg->name;
146}
147
148
149
150
151
152
153
154
155
156static BlockDriverState *throttle_group_next_bs(BlockDriverState *bs)
157{
158 ThrottleState *ts = bs->throttle_state;
159 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
160 BlockDriverState *next = QLIST_NEXT(bs, round_robin);
161
162 if (!next) {
163 return QLIST_FIRST(&tg->head);
164 }
165
166 return next;
167}
168
169
170
171
172
173
174
175
176
177
178
179static BlockDriverState *next_throttle_token(BlockDriverState *bs,
180 bool is_write)
181{
182 ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
183 BlockDriverState *token, *start;
184
185 start = token = tg->tokens[is_write];
186
187
188 token = throttle_group_next_bs(token);
189 while (token != start && !token->pending_reqs[is_write]) {
190 token = throttle_group_next_bs(token);
191 }
192
193
194
195
196
197 if (token == start && !token->pending_reqs[is_write]) {
198 token = bs;
199 }
200
201 return token;
202}
203
204
205
206
207
208
209
210
211
212
213
214static bool throttle_group_schedule_timer(BlockDriverState *bs,
215 bool is_write)
216{
217 ThrottleState *ts = bs->throttle_state;
218 ThrottleTimers *tt = &bs->throttle_timers;
219 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
220 bool must_wait;
221
222
223 if (tg->any_timer_armed[is_write]) {
224 return true;
225 }
226
227 must_wait = throttle_schedule_timer(ts, tt, is_write);
228
229
230 if (must_wait) {
231 tg->tokens[is_write] = bs;
232 tg->any_timer_armed[is_write] = true;
233 }
234
235 return must_wait;
236}
237
238
239
240
241
242
243
244
245static void schedule_next_request(BlockDriverState *bs, bool is_write)
246{
247 ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
248 bool must_wait;
249 BlockDriverState *token;
250
251
252 token = next_throttle_token(bs, is_write);
253 if (!token->pending_reqs[is_write]) {
254 return;
255 }
256
257
258 must_wait = throttle_group_schedule_timer(token, is_write);
259
260
261 if (!must_wait) {
262
263 if (qemu_in_coroutine() &&
264 qemu_co_queue_next(&bs->throttled_reqs[is_write])) {
265 token = bs;
266 } else {
267 ThrottleTimers *tt = &token->throttle_timers;
268 int64_t now = qemu_clock_get_ns(tt->clock_type);
269 timer_mod(tt->timers[is_write], now + 1);
270 tg->any_timer_armed[is_write] = true;
271 }
272 tg->tokens[is_write] = token;
273 }
274}
275
276
277
278
279
280
281
282
283
284void coroutine_fn throttle_group_co_io_limits_intercept(BlockDriverState *bs,
285 unsigned int bytes,
286 bool is_write)
287{
288 bool must_wait;
289 BlockDriverState *token;
290
291 ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
292 qemu_mutex_lock(&tg->lock);
293
294
295 token = next_throttle_token(bs, is_write);
296 must_wait = throttle_group_schedule_timer(token, is_write);
297
298
299 if (must_wait || bs->pending_reqs[is_write]) {
300 bs->pending_reqs[is_write]++;
301 qemu_mutex_unlock(&tg->lock);
302 qemu_co_queue_wait(&bs->throttled_reqs[is_write]);
303 qemu_mutex_lock(&tg->lock);
304 bs->pending_reqs[is_write]--;
305 }
306
307
308 throttle_account(bs->throttle_state, is_write, bytes);
309
310
311 schedule_next_request(bs, is_write);
312
313 qemu_mutex_unlock(&tg->lock);
314}
315
316
317
318
319
320
321
322
323void throttle_group_config(BlockDriverState *bs, ThrottleConfig *cfg)
324{
325 ThrottleTimers *tt = &bs->throttle_timers;
326 ThrottleState *ts = bs->throttle_state;
327 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
328 qemu_mutex_lock(&tg->lock);
329
330 if (timer_pending(tt->timers[0])) {
331 tg->any_timer_armed[0] = false;
332 }
333 if (timer_pending(tt->timers[1])) {
334 tg->any_timer_armed[1] = false;
335 }
336 throttle_config(ts, tt, cfg);
337 qemu_mutex_unlock(&tg->lock);
338}
339
340
341
342
343
344
345
346
347void throttle_group_get_config(BlockDriverState *bs, ThrottleConfig *cfg)
348{
349 ThrottleState *ts = bs->throttle_state;
350 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
351 qemu_mutex_lock(&tg->lock);
352 throttle_get_config(ts, cfg);
353 qemu_mutex_unlock(&tg->lock);
354}
355
356
357
358
359
360
361
362static void timer_cb(BlockDriverState *bs, bool is_write)
363{
364 ThrottleState *ts = bs->throttle_state;
365 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
366 bool empty_queue;
367
368
369 qemu_mutex_lock(&tg->lock);
370 tg->any_timer_armed[is_write] = false;
371 qemu_mutex_unlock(&tg->lock);
372
373
374 empty_queue = !qemu_co_enter_next(&bs->throttled_reqs[is_write]);
375
376
377
378 if (empty_queue) {
379 qemu_mutex_lock(&tg->lock);
380 schedule_next_request(bs, is_write);
381 qemu_mutex_unlock(&tg->lock);
382 }
383}
384
385static void read_timer_cb(void *opaque)
386{
387 timer_cb(opaque, false);
388}
389
390static void write_timer_cb(void *opaque)
391{
392 timer_cb(opaque, true);
393}
394
395
396
397
398
399
400
401
402
403void throttle_group_register_bs(BlockDriverState *bs, const char *groupname)
404{
405 int i;
406 ThrottleState *ts = throttle_group_incref(groupname);
407 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
408 int clock_type = QEMU_CLOCK_REALTIME;
409
410 if (qtest_enabled()) {
411
412 clock_type = QEMU_CLOCK_VIRTUAL;
413 }
414
415 bs->throttle_state = ts;
416
417 qemu_mutex_lock(&tg->lock);
418
419 for (i = 0; i < 2; i++) {
420 if (!tg->tokens[i]) {
421 tg->tokens[i] = bs;
422 }
423 }
424
425 QLIST_INSERT_HEAD(&tg->head, bs, round_robin);
426
427 throttle_timers_init(&bs->throttle_timers,
428 bdrv_get_aio_context(bs),
429 clock_type,
430 read_timer_cb,
431 write_timer_cb,
432 bs);
433
434 qemu_mutex_unlock(&tg->lock);
435}
436
437
438
439
440
441
442
443
444
445
446
447
448void throttle_group_unregister_bs(BlockDriverState *bs)
449{
450 ThrottleGroup *tg = container_of(bs->throttle_state, ThrottleGroup, ts);
451 int i;
452
453 assert(bs->pending_reqs[0] == 0 && bs->pending_reqs[1] == 0);
454 assert(qemu_co_queue_empty(&bs->throttled_reqs[0]));
455 assert(qemu_co_queue_empty(&bs->throttled_reqs[1]));
456
457 qemu_mutex_lock(&tg->lock);
458 for (i = 0; i < 2; i++) {
459 if (tg->tokens[i] == bs) {
460 BlockDriverState *token = throttle_group_next_bs(bs);
461
462 if (token == bs) {
463 token = NULL;
464 }
465 tg->tokens[i] = token;
466 }
467 }
468
469
470 QLIST_REMOVE(bs, round_robin);
471 throttle_timers_destroy(&bs->throttle_timers);
472 qemu_mutex_unlock(&tg->lock);
473
474 throttle_group_unref(&tg->ts);
475 bs->throttle_state = NULL;
476}
477
478static void throttle_groups_init(void)
479{
480 qemu_mutex_init(&throttle_groups_lock);
481}
482
483block_init(throttle_groups_init);
484