1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25#include "qemu/osdep.h"
26#include "sysemu/block-backend.h"
27#include "block/throttle-groups.h"
28#include "qemu/queue.h"
29#include "qemu/thread.h"
30#include "sysemu/qtest.h"
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56typedef struct ThrottleGroup {
57 char *name;
58
59 QemuMutex lock;
60 ThrottleState ts;
61 QLIST_HEAD(, BlockBackendPublic) head;
62 BlockBackend *tokens[2];
63 bool any_timer_armed[2];
64
65
66 unsigned refcount;
67 QTAILQ_ENTRY(ThrottleGroup) list;
68} ThrottleGroup;
69
70static QemuMutex throttle_groups_lock;
71static QTAILQ_HEAD(, ThrottleGroup) throttle_groups =
72 QTAILQ_HEAD_INITIALIZER(throttle_groups);
73
74
75
76
77
78
79
80
81
82ThrottleState *throttle_group_incref(const char *name)
83{
84 ThrottleGroup *tg = NULL;
85 ThrottleGroup *iter;
86
87 qemu_mutex_lock(&throttle_groups_lock);
88
89
90 QTAILQ_FOREACH(iter, &throttle_groups, list) {
91 if (!strcmp(name, iter->name)) {
92 tg = iter;
93 break;
94 }
95 }
96
97
98 if (!tg) {
99 tg = g_new0(ThrottleGroup, 1);
100 tg->name = g_strdup(name);
101 qemu_mutex_init(&tg->lock);
102 throttle_init(&tg->ts);
103 QLIST_INIT(&tg->head);
104
105 QTAILQ_INSERT_TAIL(&throttle_groups, tg, list);
106 }
107
108 tg->refcount++;
109
110 qemu_mutex_unlock(&throttle_groups_lock);
111
112 return &tg->ts;
113}
114
115
116
117
118
119
120
121
122void throttle_group_unref(ThrottleState *ts)
123{
124 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
125
126 qemu_mutex_lock(&throttle_groups_lock);
127 if (--tg->refcount == 0) {
128 QTAILQ_REMOVE(&throttle_groups, tg, list);
129 qemu_mutex_destroy(&tg->lock);
130 g_free(tg->name);
131 g_free(tg);
132 }
133 qemu_mutex_unlock(&throttle_groups_lock);
134}
135
136
137
138
139
140
141
142const char *throttle_group_get_name(BlockBackend *blk)
143{
144 BlockBackendPublic *blkp = blk_get_public(blk);
145 ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
146 return tg->name;
147}
148
149
150
151
152
153
154
155
156
157static BlockBackend *throttle_group_next_blk(BlockBackend *blk)
158{
159 BlockBackendPublic *blkp = blk_get_public(blk);
160 ThrottleState *ts = blkp->throttle_state;
161 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
162 BlockBackendPublic *next = QLIST_NEXT(blkp, round_robin);
163
164 if (!next) {
165 next = QLIST_FIRST(&tg->head);
166 }
167
168 return blk_by_public(next);
169}
170
171
172
173
174
175
176
177
178
179
180static inline bool blk_has_pending_reqs(BlockBackend *blk,
181 bool is_write)
182{
183 const BlockBackendPublic *blkp = blk_get_public(blk);
184 return blkp->pending_reqs[is_write];
185}
186
187
188
189
190
191
192
193
194
195
196
197static BlockBackend *next_throttle_token(BlockBackend *blk, bool is_write)
198{
199 BlockBackendPublic *blkp = blk_get_public(blk);
200 ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
201 BlockBackend *token, *start;
202
203 start = token = tg->tokens[is_write];
204
205
206 token = throttle_group_next_blk(token);
207 while (token != start && !blk_has_pending_reqs(token, is_write)) {
208 token = throttle_group_next_blk(token);
209 }
210
211
212
213
214
215 if (token == start && !blk_has_pending_reqs(token, is_write)) {
216 token = blk;
217 }
218
219
220 assert(token == blk || blk_has_pending_reqs(token, is_write));
221
222 return token;
223}
224
225
226
227
228
229
230
231
232
233
234
235static bool throttle_group_schedule_timer(BlockBackend *blk, bool is_write)
236{
237 BlockBackendPublic *blkp = blk_get_public(blk);
238 ThrottleState *ts = blkp->throttle_state;
239 ThrottleTimers *tt = &blkp->throttle_timers;
240 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
241 bool must_wait;
242
243 if (blkp->io_limits_disabled) {
244 return false;
245 }
246
247
248 if (tg->any_timer_armed[is_write]) {
249 return true;
250 }
251
252 must_wait = throttle_schedule_timer(ts, tt, is_write);
253
254
255 if (must_wait) {
256 tg->tokens[is_write] = blk;
257 tg->any_timer_armed[is_write] = true;
258 }
259
260 return must_wait;
261}
262
263
264
265
266
267
268
269
270static void schedule_next_request(BlockBackend *blk, bool is_write)
271{
272 BlockBackendPublic *blkp = blk_get_public(blk);
273 ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
274 bool must_wait;
275 BlockBackend *token;
276
277
278 token = next_throttle_token(blk, is_write);
279 if (!blk_has_pending_reqs(token, is_write)) {
280 return;
281 }
282
283
284 must_wait = throttle_group_schedule_timer(token, is_write);
285
286
287 if (!must_wait) {
288
289 if (qemu_in_coroutine() &&
290 qemu_co_queue_next(&blkp->throttled_reqs[is_write])) {
291 token = blk;
292 } else {
293 ThrottleTimers *tt = &blk_get_public(token)->throttle_timers;
294 int64_t now = qemu_clock_get_ns(tt->clock_type);
295 timer_mod(tt->timers[is_write], now + 1);
296 tg->any_timer_armed[is_write] = true;
297 }
298 tg->tokens[is_write] = token;
299 }
300}
301
302
303
304
305
306
307
308
309
310void coroutine_fn throttle_group_co_io_limits_intercept(BlockBackend *blk,
311 unsigned int bytes,
312 bool is_write)
313{
314 bool must_wait;
315 BlockBackend *token;
316
317 BlockBackendPublic *blkp = blk_get_public(blk);
318 ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
319 qemu_mutex_lock(&tg->lock);
320
321
322 token = next_throttle_token(blk, is_write);
323 must_wait = throttle_group_schedule_timer(token, is_write);
324
325
326 if (must_wait || blkp->pending_reqs[is_write]) {
327 blkp->pending_reqs[is_write]++;
328 qemu_mutex_unlock(&tg->lock);
329 qemu_co_queue_wait(&blkp->throttled_reqs[is_write], NULL);
330 qemu_mutex_lock(&tg->lock);
331 blkp->pending_reqs[is_write]--;
332 }
333
334
335 throttle_account(blkp->throttle_state, is_write, bytes);
336
337
338 schedule_next_request(blk, is_write);
339
340 qemu_mutex_unlock(&tg->lock);
341}
342
343void throttle_group_restart_blk(BlockBackend *blk)
344{
345 BlockBackendPublic *blkp = blk_get_public(blk);
346 int i;
347
348 for (i = 0; i < 2; i++) {
349 while (qemu_co_enter_next(&blkp->throttled_reqs[i])) {
350 ;
351 }
352 }
353}
354
355
356
357
358
359
360
361
362void throttle_group_config(BlockBackend *blk, ThrottleConfig *cfg)
363{
364 BlockBackendPublic *blkp = blk_get_public(blk);
365 ThrottleTimers *tt = &blkp->throttle_timers;
366 ThrottleState *ts = blkp->throttle_state;
367 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
368 qemu_mutex_lock(&tg->lock);
369
370 if (timer_pending(tt->timers[0])) {
371 tg->any_timer_armed[0] = false;
372 }
373 if (timer_pending(tt->timers[1])) {
374 tg->any_timer_armed[1] = false;
375 }
376 throttle_config(ts, tt, cfg);
377 qemu_mutex_unlock(&tg->lock);
378
379 qemu_co_enter_next(&blkp->throttled_reqs[0]);
380 qemu_co_enter_next(&blkp->throttled_reqs[1]);
381}
382
383
384
385
386
387
388
389
390void throttle_group_get_config(BlockBackend *blk, ThrottleConfig *cfg)
391{
392 BlockBackendPublic *blkp = blk_get_public(blk);
393 ThrottleState *ts = blkp->throttle_state;
394 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
395 qemu_mutex_lock(&tg->lock);
396 throttle_get_config(ts, cfg);
397 qemu_mutex_unlock(&tg->lock);
398}
399
400
401
402
403
404
405
406static void timer_cb(BlockBackend *blk, bool is_write)
407{
408 BlockBackendPublic *blkp = blk_get_public(blk);
409 ThrottleState *ts = blkp->throttle_state;
410 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
411 bool empty_queue;
412
413
414 qemu_mutex_lock(&tg->lock);
415 tg->any_timer_armed[is_write] = false;
416 qemu_mutex_unlock(&tg->lock);
417
418
419 aio_context_acquire(blk_get_aio_context(blk));
420 empty_queue = !qemu_co_enter_next(&blkp->throttled_reqs[is_write]);
421 aio_context_release(blk_get_aio_context(blk));
422
423
424
425 if (empty_queue) {
426 qemu_mutex_lock(&tg->lock);
427 schedule_next_request(blk, is_write);
428 qemu_mutex_unlock(&tg->lock);
429 }
430}
431
432static void read_timer_cb(void *opaque)
433{
434 timer_cb(opaque, false);
435}
436
437static void write_timer_cb(void *opaque)
438{
439 timer_cb(opaque, true);
440}
441
442
443
444
445
446
447
448
449void throttle_group_register_blk(BlockBackend *blk, const char *groupname)
450{
451 int i;
452 BlockBackendPublic *blkp = blk_get_public(blk);
453 ThrottleState *ts = throttle_group_incref(groupname);
454 ThrottleGroup *tg = container_of(ts, ThrottleGroup, ts);
455 int clock_type = QEMU_CLOCK_REALTIME;
456
457 if (qtest_enabled()) {
458
459 clock_type = QEMU_CLOCK_VIRTUAL;
460 }
461
462 blkp->throttle_state = ts;
463
464 qemu_mutex_lock(&tg->lock);
465
466 for (i = 0; i < 2; i++) {
467 if (!tg->tokens[i]) {
468 tg->tokens[i] = blk;
469 }
470 }
471
472 QLIST_INSERT_HEAD(&tg->head, blkp, round_robin);
473
474 throttle_timers_init(&blkp->throttle_timers,
475 blk_get_aio_context(blk),
476 clock_type,
477 read_timer_cb,
478 write_timer_cb,
479 blk);
480
481 qemu_mutex_unlock(&tg->lock);
482}
483
484
485
486
487
488
489
490
491
492
493
494void throttle_group_unregister_blk(BlockBackend *blk)
495{
496 BlockBackendPublic *blkp = blk_get_public(blk);
497 ThrottleGroup *tg = container_of(blkp->throttle_state, ThrottleGroup, ts);
498 int i;
499
500 assert(blkp->pending_reqs[0] == 0 && blkp->pending_reqs[1] == 0);
501 assert(qemu_co_queue_empty(&blkp->throttled_reqs[0]));
502 assert(qemu_co_queue_empty(&blkp->throttled_reqs[1]));
503
504 qemu_mutex_lock(&tg->lock);
505 for (i = 0; i < 2; i++) {
506 if (tg->tokens[i] == blk) {
507 BlockBackend *token = throttle_group_next_blk(blk);
508
509 if (token == blk) {
510 token = NULL;
511 }
512 tg->tokens[i] = token;
513 }
514 }
515
516
517 QLIST_REMOVE(blkp, round_robin);
518 throttle_timers_destroy(&blkp->throttle_timers);
519 qemu_mutex_unlock(&tg->lock);
520
521 throttle_group_unref(&tg->ts);
522 blkp->throttle_state = NULL;
523}
524
525static void throttle_groups_init(void)
526{
527 qemu_mutex_init(&throttle_groups_lock);
528}
529
530block_init(throttle_groups_init);
531