1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "trace.h"
17#include "block/blockjob_int.h"
18#include "block/block_int.h"
19#include "sysemu/block-backend.h"
20#include "qapi/error.h"
21#include "qapi/qmp/qerror.h"
22#include "qemu/ratelimit.h"
23#include "qemu/bitmap.h"
24
25#define SLICE_TIME 100000000ULL
26#define MAX_IN_FLIGHT 16
27#define MAX_IO_BYTES (1 << 20)
28#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
29
30
31
32
33typedef struct MirrorBuffer {
34 QSIMPLEQ_ENTRY(MirrorBuffer) next;
35} MirrorBuffer;
36
37typedef struct MirrorBlockJob {
38 BlockJob common;
39 RateLimit limit;
40 BlockBackend *target;
41 BlockDriverState *mirror_top_bs;
42 BlockDriverState *source;
43 BlockDriverState *base;
44
45
46 char *replaces;
47
48 BlockDriverState *to_replace;
49
50 Error *replace_blocker;
51 bool is_none_mode;
52 BlockMirrorBackingMode backing_mode;
53 BlockdevOnError on_source_error, on_target_error;
54 bool synced;
55 bool should_complete;
56 int64_t granularity;
57 size_t buf_size;
58 int64_t bdev_length;
59 unsigned long *cow_bitmap;
60 BdrvDirtyBitmap *dirty_bitmap;
61 BdrvDirtyBitmapIter *dbi;
62 uint8_t *buf;
63 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
64 int buf_free_count;
65
66 uint64_t last_pause_ns;
67 unsigned long *in_flight_bitmap;
68 int in_flight;
69 int64_t bytes_in_flight;
70 int ret;
71 bool unmap;
72 bool waiting_for_io;
73 int target_cluster_size;
74 int max_iov;
75 bool initial_zeroing_ongoing;
76} MirrorBlockJob;
77
78typedef struct MirrorOp {
79 MirrorBlockJob *s;
80 QEMUIOVector qiov;
81 int64_t offset;
82 uint64_t bytes;
83} MirrorOp;
84
85static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
86 int error)
87{
88 s->synced = false;
89 if (read) {
90 return block_job_error_action(&s->common, s->on_source_error,
91 true, error);
92 } else {
93 return block_job_error_action(&s->common, s->on_target_error,
94 false, error);
95 }
96}
97
98static void mirror_iteration_done(MirrorOp *op, int ret)
99{
100 MirrorBlockJob *s = op->s;
101 struct iovec *iov;
102 int64_t chunk_num;
103 int i, nb_chunks;
104
105 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
106
107 s->in_flight--;
108 s->bytes_in_flight -= op->bytes;
109 iov = op->qiov.iov;
110 for (i = 0; i < op->qiov.niov; i++) {
111 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
112 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
113 s->buf_free_count++;
114 }
115
116 chunk_num = op->offset / s->granularity;
117 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
118 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
119 if (ret >= 0) {
120 if (s->cow_bitmap) {
121 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
122 }
123 if (!s->initial_zeroing_ongoing) {
124 s->common.offset += op->bytes;
125 }
126 }
127 qemu_iovec_destroy(&op->qiov);
128 g_free(op);
129
130 if (s->waiting_for_io) {
131 qemu_coroutine_enter(s->common.co);
132 }
133}
134
135static void mirror_write_complete(void *opaque, int ret)
136{
137 MirrorOp *op = opaque;
138 MirrorBlockJob *s = op->s;
139
140 aio_context_acquire(blk_get_aio_context(s->common.blk));
141 if (ret < 0) {
142 BlockErrorAction action;
143
144 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset >> BDRV_SECTOR_BITS,
145 op->bytes >> BDRV_SECTOR_BITS);
146 action = mirror_error_action(s, false, -ret);
147 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
148 s->ret = ret;
149 }
150 }
151 mirror_iteration_done(op, ret);
152 aio_context_release(blk_get_aio_context(s->common.blk));
153}
154
155static void mirror_read_complete(void *opaque, int ret)
156{
157 MirrorOp *op = opaque;
158 MirrorBlockJob *s = op->s;
159
160 aio_context_acquire(blk_get_aio_context(s->common.blk));
161 if (ret < 0) {
162 BlockErrorAction action;
163
164 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset >> BDRV_SECTOR_BITS,
165 op->bytes >> BDRV_SECTOR_BITS);
166 action = mirror_error_action(s, true, -ret);
167 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
168 s->ret = ret;
169 }
170
171 mirror_iteration_done(op, ret);
172 } else {
173 blk_aio_pwritev(s->target, op->offset, &op->qiov,
174 0, mirror_write_complete, op);
175 }
176 aio_context_release(blk_get_aio_context(s->common.blk));
177}
178
179
180static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
181 int64_t offset,
182 int64_t bytes)
183{
184 return MIN(bytes, s->bdev_length - offset);
185}
186
187
188
189static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
190 uint64_t *bytes)
191{
192 bool need_cow;
193 int ret = 0;
194 int64_t align_offset = *offset;
195 unsigned int align_bytes = *bytes;
196 int max_bytes = s->granularity * s->max_iov;
197
198 assert(*bytes < INT_MAX);
199 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
200 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
201 s->cow_bitmap);
202 if (need_cow) {
203 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
204 &align_offset, &align_bytes);
205 }
206
207 if (align_bytes > max_bytes) {
208 align_bytes = max_bytes;
209 if (need_cow) {
210 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
211 }
212 }
213
214
215 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
216
217 ret = align_offset + align_bytes - (*offset + *bytes);
218 *offset = align_offset;
219 *bytes = align_bytes;
220 assert(ret >= 0);
221 return ret;
222}
223
224static inline void mirror_wait_for_io(MirrorBlockJob *s)
225{
226 assert(!s->waiting_for_io);
227 s->waiting_for_io = true;
228 qemu_coroutine_yield();
229 s->waiting_for_io = false;
230}
231
232
233
234
235
236
237
238
239static uint64_t mirror_do_read(MirrorBlockJob *s, int64_t offset,
240 uint64_t bytes)
241{
242 BlockBackend *source = s->common.blk;
243 int nb_chunks;
244 uint64_t ret;
245 MirrorOp *op;
246 uint64_t max_bytes;
247
248 max_bytes = s->granularity * s->max_iov;
249
250
251 bytes = MIN(s->buf_size, MIN(max_bytes, bytes));
252 assert(bytes);
253 assert(bytes < BDRV_REQUEST_MAX_BYTES);
254 ret = bytes;
255
256 if (s->cow_bitmap) {
257 ret += mirror_cow_align(s, &offset, &bytes);
258 }
259 assert(bytes <= s->buf_size);
260
261
262
263 assert(QEMU_IS_ALIGNED(offset, s->granularity));
264
265 assert(QEMU_IS_ALIGNED(bytes, BDRV_SECTOR_SIZE));
266 nb_chunks = DIV_ROUND_UP(bytes, s->granularity);
267
268 while (s->buf_free_count < nb_chunks) {
269 trace_mirror_yield_in_flight(s, offset, s->in_flight);
270 mirror_wait_for_io(s);
271 }
272
273
274 op = g_new(MirrorOp, 1);
275 op->s = s;
276 op->offset = offset;
277 op->bytes = bytes;
278
279
280
281
282 qemu_iovec_init(&op->qiov, nb_chunks);
283 while (nb_chunks-- > 0) {
284 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
285 size_t remaining = bytes - op->qiov.size;
286
287 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
288 s->buf_free_count--;
289 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
290 }
291
292
293 s->in_flight++;
294 s->bytes_in_flight += bytes;
295 trace_mirror_one_iteration(s, offset, bytes);
296
297 blk_aio_preadv(source, offset, &op->qiov, 0, mirror_read_complete, op);
298 return ret;
299}
300
301static void mirror_do_zero_or_discard(MirrorBlockJob *s,
302 int64_t offset,
303 uint64_t bytes,
304 bool is_discard)
305{
306 MirrorOp *op;
307
308
309
310 op = g_new0(MirrorOp, 1);
311 op->s = s;
312 op->offset = offset;
313 op->bytes = bytes;
314
315 s->in_flight++;
316 s->bytes_in_flight += bytes;
317 if (is_discard) {
318 blk_aio_pdiscard(s->target, offset,
319 op->bytes, mirror_write_complete, op);
320 } else {
321 blk_aio_pwrite_zeroes(s->target, offset,
322 op->bytes, s->unmap ? BDRV_REQ_MAY_UNMAP : 0,
323 mirror_write_complete, op);
324 }
325}
326
327static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
328{
329 BlockDriverState *source = s->source;
330 int64_t offset, first_chunk;
331 uint64_t delay_ns = 0;
332
333 int nb_chunks = 1;
334 int sectors_per_chunk = s->granularity >> BDRV_SECTOR_BITS;
335 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
336 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
337
338 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
339 offset = bdrv_dirty_iter_next(s->dbi) * BDRV_SECTOR_SIZE;
340 if (offset < 0) {
341 bdrv_set_dirty_iter(s->dbi, 0);
342 offset = bdrv_dirty_iter_next(s->dbi) * BDRV_SECTOR_SIZE;
343 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap) *
344 BDRV_SECTOR_SIZE);
345 assert(offset >= 0);
346 }
347 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
348
349 first_chunk = offset / s->granularity;
350 while (test_bit(first_chunk, s->in_flight_bitmap)) {
351 trace_mirror_yield_in_flight(s, offset, s->in_flight);
352 mirror_wait_for_io(s);
353 }
354
355 block_job_pause_point(&s->common);
356
357
358
359 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
360 while (nb_chunks * s->granularity < s->buf_size) {
361 int64_t next_dirty;
362 int64_t next_offset = offset + nb_chunks * s->granularity;
363 int64_t next_chunk = next_offset / s->granularity;
364 if (next_offset >= s->bdev_length ||
365 !bdrv_get_dirty_locked(source, s->dirty_bitmap,
366 next_offset >> BDRV_SECTOR_BITS)) {
367 break;
368 }
369 if (test_bit(next_chunk, s->in_flight_bitmap)) {
370 break;
371 }
372
373 next_dirty = bdrv_dirty_iter_next(s->dbi) * BDRV_SECTOR_SIZE;
374 if (next_dirty > next_offset || next_dirty < 0) {
375
376 bdrv_set_dirty_iter(s->dbi, next_offset >> BDRV_SECTOR_BITS);
377 next_dirty = bdrv_dirty_iter_next(s->dbi) * BDRV_SECTOR_SIZE;
378 }
379 assert(next_dirty == next_offset);
380 nb_chunks++;
381 }
382
383
384
385
386
387 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset >> BDRV_SECTOR_BITS,
388 nb_chunks * sectors_per_chunk);
389 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
390
391 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
392 while (nb_chunks > 0 && offset < s->bdev_length) {
393 int64_t ret;
394 int io_sectors;
395 unsigned int io_bytes;
396 int64_t io_bytes_acct;
397 BlockDriverState *file;
398 enum MirrorMethod {
399 MIRROR_METHOD_COPY,
400 MIRROR_METHOD_ZERO,
401 MIRROR_METHOD_DISCARD
402 } mirror_method = MIRROR_METHOD_COPY;
403
404 assert(!(offset % s->granularity));
405 ret = bdrv_get_block_status_above(source, NULL,
406 offset >> BDRV_SECTOR_BITS,
407 nb_chunks * sectors_per_chunk,
408 &io_sectors, &file);
409 io_bytes = io_sectors * BDRV_SECTOR_SIZE;
410 if (ret < 0) {
411 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
412 } else if (ret & BDRV_BLOCK_DATA) {
413 io_bytes = MIN(io_bytes, max_io_bytes);
414 }
415
416 io_bytes -= io_bytes % s->granularity;
417 if (io_bytes < s->granularity) {
418 io_bytes = s->granularity;
419 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
420 int64_t target_offset;
421 unsigned int target_bytes;
422 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
423 &target_offset, &target_bytes);
424 if (target_offset == offset &&
425 target_bytes == io_bytes) {
426 mirror_method = ret & BDRV_BLOCK_ZERO ?
427 MIRROR_METHOD_ZERO :
428 MIRROR_METHOD_DISCARD;
429 }
430 }
431
432 while (s->in_flight >= MAX_IN_FLIGHT) {
433 trace_mirror_yield_in_flight(s, offset, s->in_flight);
434 mirror_wait_for_io(s);
435 }
436
437 if (s->ret < 0) {
438 return 0;
439 }
440
441 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
442 switch (mirror_method) {
443 case MIRROR_METHOD_COPY:
444 io_bytes = io_bytes_acct = mirror_do_read(s, offset, io_bytes);
445 break;
446 case MIRROR_METHOD_ZERO:
447 case MIRROR_METHOD_DISCARD:
448 mirror_do_zero_or_discard(s, offset, io_bytes,
449 mirror_method == MIRROR_METHOD_DISCARD);
450 if (write_zeroes_ok) {
451 io_bytes_acct = 0;
452 } else {
453 io_bytes_acct = io_bytes;
454 }
455 break;
456 default:
457 abort();
458 }
459 assert(io_bytes);
460 offset += io_bytes;
461 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
462 if (s->common.speed) {
463 delay_ns = ratelimit_calculate_delay(&s->limit, io_bytes_acct);
464 }
465 }
466 return delay_ns;
467}
468
469static void mirror_free_init(MirrorBlockJob *s)
470{
471 int granularity = s->granularity;
472 size_t buf_size = s->buf_size;
473 uint8_t *buf = s->buf;
474
475 assert(s->buf_free_count == 0);
476 QSIMPLEQ_INIT(&s->buf_free);
477 while (buf_size != 0) {
478 MirrorBuffer *cur = (MirrorBuffer *)buf;
479 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
480 s->buf_free_count++;
481 buf_size -= granularity;
482 buf += granularity;
483 }
484}
485
486
487
488
489
490static void mirror_wait_for_all_io(MirrorBlockJob *s)
491{
492 while (s->in_flight > 0) {
493 mirror_wait_for_io(s);
494 }
495}
496
497typedef struct {
498 int ret;
499} MirrorExitData;
500
501static void mirror_exit(BlockJob *job, void *opaque)
502{
503 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
504 MirrorExitData *data = opaque;
505 AioContext *replace_aio_context = NULL;
506 BlockDriverState *src = s->source;
507 BlockDriverState *target_bs = blk_bs(s->target);
508 BlockDriverState *mirror_top_bs = s->mirror_top_bs;
509 Error *local_err = NULL;
510
511 bdrv_release_dirty_bitmap(src, s->dirty_bitmap);
512
513
514
515 bdrv_ref(src);
516 bdrv_ref(mirror_top_bs);
517 bdrv_ref(target_bs);
518
519
520
521
522
523
524
525
526 blk_set_perm(s->target, 0, BLK_PERM_ALL, &error_abort);
527 blk_unref(s->target);
528 s->target = NULL;
529
530
531
532 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
533 &error_abort);
534 if (s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
535 BlockDriverState *backing = s->is_none_mode ? src : s->base;
536 if (backing_bs(target_bs) != backing) {
537 bdrv_set_backing_hd(target_bs, backing, &local_err);
538 if (local_err) {
539 error_report_err(local_err);
540 data->ret = -EPERM;
541 }
542 }
543 }
544
545 if (s->to_replace) {
546 replace_aio_context = bdrv_get_aio_context(s->to_replace);
547 aio_context_acquire(replace_aio_context);
548 }
549
550 if (s->should_complete && data->ret == 0) {
551 BlockDriverState *to_replace = src;
552 if (s->to_replace) {
553 to_replace = s->to_replace;
554 }
555
556 if (bdrv_get_flags(target_bs) != bdrv_get_flags(to_replace)) {
557 bdrv_reopen(target_bs, bdrv_get_flags(to_replace), NULL);
558 }
559
560
561
562 bdrv_drained_begin(target_bs);
563 bdrv_replace_node(to_replace, target_bs, &local_err);
564 bdrv_drained_end(target_bs);
565 if (local_err) {
566 error_report_err(local_err);
567 data->ret = -EPERM;
568 }
569 }
570 if (s->to_replace) {
571 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
572 error_free(s->replace_blocker);
573 bdrv_unref(s->to_replace);
574 }
575 if (replace_aio_context) {
576 aio_context_release(replace_aio_context);
577 }
578 g_free(s->replaces);
579 bdrv_unref(target_bs);
580
581
582
583
584
585 block_job_remove_all_bdrv(job);
586 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
587 &error_abort);
588 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
589
590
591
592
593 blk_remove_bs(job->blk);
594 blk_set_perm(job->blk, 0, BLK_PERM_ALL, &error_abort);
595 blk_insert_bs(job->blk, mirror_top_bs, &error_abort);
596
597 block_job_completed(&s->common, data->ret);
598
599 g_free(data);
600 bdrv_drained_end(src);
601 bdrv_unref(mirror_top_bs);
602 bdrv_unref(src);
603}
604
605static void mirror_throttle(MirrorBlockJob *s)
606{
607 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
608
609 if (now - s->last_pause_ns > SLICE_TIME) {
610 s->last_pause_ns = now;
611 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, 0);
612 } else {
613 block_job_pause_point(&s->common);
614 }
615}
616
617static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
618{
619 int64_t sector_num, end;
620 BlockDriverState *base = s->base;
621 BlockDriverState *bs = s->source;
622 BlockDriverState *target_bs = blk_bs(s->target);
623 int ret, n;
624 int64_t count;
625
626 end = s->bdev_length / BDRV_SECTOR_SIZE;
627
628 if (base == NULL && !bdrv_has_zero_init(target_bs)) {
629 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
630 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, end);
631 return 0;
632 }
633
634 s->initial_zeroing_ongoing = true;
635 for (sector_num = 0; sector_num < end; ) {
636 int nb_sectors = MIN(end - sector_num,
637 QEMU_ALIGN_DOWN(INT_MAX, s->granularity) >> BDRV_SECTOR_BITS);
638
639 mirror_throttle(s);
640
641 if (block_job_is_cancelled(&s->common)) {
642 s->initial_zeroing_ongoing = false;
643 return 0;
644 }
645
646 if (s->in_flight >= MAX_IN_FLIGHT) {
647 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
648 s->in_flight);
649 mirror_wait_for_io(s);
650 continue;
651 }
652
653 mirror_do_zero_or_discard(s, sector_num * BDRV_SECTOR_SIZE,
654 nb_sectors * BDRV_SECTOR_SIZE, false);
655 sector_num += nb_sectors;
656 }
657
658 mirror_wait_for_all_io(s);
659 s->initial_zeroing_ongoing = false;
660 }
661
662
663 for (sector_num = 0; sector_num < end; ) {
664
665 int nb_sectors = MIN(INT_MAX >> BDRV_SECTOR_BITS,
666 end - sector_num);
667
668 mirror_throttle(s);
669
670 if (block_job_is_cancelled(&s->common)) {
671 return 0;
672 }
673
674 ret = bdrv_is_allocated_above(bs, base, sector_num * BDRV_SECTOR_SIZE,
675 nb_sectors * BDRV_SECTOR_SIZE, &count);
676 if (ret < 0) {
677 return ret;
678 }
679
680
681
682 assert(QEMU_IS_ALIGNED(count, BDRV_SECTOR_SIZE));
683 n = count >> BDRV_SECTOR_BITS;
684 assert(n > 0);
685 if (ret == 1) {
686 bdrv_set_dirty_bitmap(s->dirty_bitmap, sector_num, n);
687 }
688 sector_num += n;
689 }
690 return 0;
691}
692
693
694
695
696static int mirror_flush(MirrorBlockJob *s)
697{
698 int ret = blk_flush(s->target);
699 if (ret < 0) {
700 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
701 s->ret = ret;
702 }
703 }
704 return ret;
705}
706
707static void coroutine_fn mirror_run(void *opaque)
708{
709 MirrorBlockJob *s = opaque;
710 MirrorExitData *data;
711 BlockDriverState *bs = s->source;
712 BlockDriverState *target_bs = blk_bs(s->target);
713 bool need_drain = true;
714 int64_t length;
715 BlockDriverInfo bdi;
716 char backing_filename[2];
717
718 int ret = 0;
719
720 if (block_job_is_cancelled(&s->common)) {
721 goto immediate_exit;
722 }
723
724 s->bdev_length = bdrv_getlength(bs);
725 if (s->bdev_length < 0) {
726 ret = s->bdev_length;
727 goto immediate_exit;
728 }
729
730
731
732 if (s->base == blk_bs(s->target)) {
733 int64_t base_length;
734
735 base_length = blk_getlength(s->target);
736 if (base_length < 0) {
737 ret = base_length;
738 goto immediate_exit;
739 }
740
741 if (s->bdev_length > base_length) {
742 ret = blk_truncate(s->target, s->bdev_length, PREALLOC_MODE_OFF,
743 NULL);
744 if (ret < 0) {
745 goto immediate_exit;
746 }
747 }
748 }
749
750 if (s->bdev_length == 0) {
751
752 block_job_event_ready(&s->common);
753 s->synced = true;
754 while (!block_job_is_cancelled(&s->common) && !s->should_complete) {
755 block_job_yield(&s->common);
756 }
757 s->common.cancelled = false;
758 goto immediate_exit;
759 }
760
761 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
762 s->in_flight_bitmap = bitmap_new(length);
763
764
765
766
767
768 bdrv_get_backing_filename(target_bs, backing_filename,
769 sizeof(backing_filename));
770 if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
771 s->target_cluster_size = bdi.cluster_size;
772 } else {
773 s->target_cluster_size = BDRV_SECTOR_SIZE;
774 }
775 if (backing_filename[0] && !target_bs->backing &&
776 s->granularity < s->target_cluster_size) {
777 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
778 s->cow_bitmap = bitmap_new(length);
779 }
780 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
781
782 s->buf = qemu_try_blockalign(bs, s->buf_size);
783 if (s->buf == NULL) {
784 ret = -ENOMEM;
785 goto immediate_exit;
786 }
787
788 mirror_free_init(s);
789
790 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
791 if (!s->is_none_mode) {
792 ret = mirror_dirty_init(s);
793 if (ret < 0 || block_job_is_cancelled(&s->common)) {
794 goto immediate_exit;
795 }
796 }
797
798 assert(!s->dbi);
799 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap, 0);
800 for (;;) {
801 uint64_t delay_ns = 0;
802 int64_t cnt, delta;
803 bool should_complete;
804
805 if (s->ret < 0) {
806 ret = s->ret;
807 goto immediate_exit;
808 }
809
810 block_job_pause_point(&s->common);
811
812 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
813
814
815
816
817 s->common.len = s->common.offset + s->bytes_in_flight +
818 cnt * BDRV_SECTOR_SIZE;
819
820
821
822
823
824
825 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
826 if (delta < SLICE_TIME &&
827 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
828 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
829 (cnt == 0 && s->in_flight > 0)) {
830 trace_mirror_yield(s, cnt * BDRV_SECTOR_SIZE,
831 s->buf_free_count, s->in_flight);
832 mirror_wait_for_io(s);
833 continue;
834 } else if (cnt != 0) {
835 delay_ns = mirror_iteration(s);
836 }
837 }
838
839 should_complete = false;
840 if (s->in_flight == 0 && cnt == 0) {
841 trace_mirror_before_flush(s);
842 if (!s->synced) {
843 if (mirror_flush(s) < 0) {
844
845 continue;
846 }
847
848
849
850
851
852 block_job_event_ready(&s->common);
853 s->synced = true;
854 }
855
856 should_complete = s->should_complete ||
857 block_job_is_cancelled(&s->common);
858 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
859 }
860
861 if (cnt == 0 && should_complete) {
862
863
864
865
866
867
868
869
870
871
872 trace_mirror_before_drain(s, cnt * BDRV_SECTOR_SIZE);
873
874 bdrv_drained_begin(bs);
875 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
876 if (cnt > 0 || mirror_flush(s) < 0) {
877 bdrv_drained_end(bs);
878 continue;
879 }
880
881
882
883
884 assert(QLIST_EMPTY(&bs->tracked_requests));
885 s->common.cancelled = false;
886 need_drain = false;
887 break;
888 }
889
890 ret = 0;
891 trace_mirror_before_sleep(s, cnt * BDRV_SECTOR_SIZE,
892 s->synced, delay_ns);
893 if (!s->synced) {
894 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns);
895 if (block_job_is_cancelled(&s->common)) {
896 break;
897 }
898 } else if (!should_complete) {
899 delay_ns = (s->in_flight == 0 && cnt == 0 ? SLICE_TIME : 0);
900 block_job_sleep_ns(&s->common, QEMU_CLOCK_REALTIME, delay_ns);
901 }
902 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
903 }
904
905immediate_exit:
906 if (s->in_flight > 0) {
907
908
909
910
911 assert(ret < 0 || (!s->synced && block_job_is_cancelled(&s->common)));
912 assert(need_drain);
913 mirror_wait_for_all_io(s);
914 }
915
916 assert(s->in_flight == 0);
917 qemu_vfree(s->buf);
918 g_free(s->cow_bitmap);
919 g_free(s->in_flight_bitmap);
920 bdrv_dirty_iter_free(s->dbi);
921
922 data = g_malloc(sizeof(*data));
923 data->ret = ret;
924
925 if (need_drain) {
926 bdrv_drained_begin(bs);
927 }
928 block_job_defer_to_main_loop(&s->common, mirror_exit, data);
929}
930
931static void mirror_set_speed(BlockJob *job, int64_t speed, Error **errp)
932{
933 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
934
935 if (speed < 0) {
936 error_setg(errp, QERR_INVALID_PARAMETER, "speed");
937 return;
938 }
939 ratelimit_set_speed(&s->limit, speed, SLICE_TIME);
940}
941
942static void mirror_complete(BlockJob *job, Error **errp)
943{
944 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
945 BlockDriverState *target;
946
947 target = blk_bs(s->target);
948
949 if (!s->synced) {
950 error_setg(errp, "The active block job '%s' cannot be completed",
951 job->id);
952 return;
953 }
954
955 if (s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
956 int ret;
957
958 assert(!target->backing);
959 ret = bdrv_open_backing_file(target, NULL, "backing", errp);
960 if (ret < 0) {
961 return;
962 }
963 }
964
965
966 if (s->replaces) {
967 AioContext *replace_aio_context;
968
969 s->to_replace = bdrv_find_node(s->replaces);
970 if (!s->to_replace) {
971 error_setg(errp, "Node name '%s' not found", s->replaces);
972 return;
973 }
974
975 replace_aio_context = bdrv_get_aio_context(s->to_replace);
976 aio_context_acquire(replace_aio_context);
977
978
979
980
981
982 error_setg(&s->replace_blocker,
983 "block device is in use by block-job-complete");
984 bdrv_op_block_all(s->to_replace, s->replace_blocker);
985 bdrv_ref(s->to_replace);
986
987 aio_context_release(replace_aio_context);
988 }
989
990 s->should_complete = true;
991 block_job_enter(&s->common);
992}
993
994static void mirror_pause(BlockJob *job)
995{
996 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
997
998 mirror_wait_for_all_io(s);
999}
1000
1001static void mirror_attached_aio_context(BlockJob *job, AioContext *new_context)
1002{
1003 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1004
1005 blk_set_aio_context(s->target, new_context);
1006}
1007
1008static void mirror_drain(BlockJob *job)
1009{
1010 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1011
1012
1013
1014
1015 if (s->target) {
1016 BlockBackend *target = s->target;
1017 blk_ref(target);
1018 blk_drain(target);
1019 blk_unref(target);
1020 }
1021}
1022
1023static const BlockJobDriver mirror_job_driver = {
1024 .instance_size = sizeof(MirrorBlockJob),
1025 .job_type = BLOCK_JOB_TYPE_MIRROR,
1026 .set_speed = mirror_set_speed,
1027 .start = mirror_run,
1028 .complete = mirror_complete,
1029 .pause = mirror_pause,
1030 .attached_aio_context = mirror_attached_aio_context,
1031 .drain = mirror_drain,
1032};
1033
1034static const BlockJobDriver commit_active_job_driver = {
1035 .instance_size = sizeof(MirrorBlockJob),
1036 .job_type = BLOCK_JOB_TYPE_COMMIT,
1037 .set_speed = mirror_set_speed,
1038 .start = mirror_run,
1039 .complete = mirror_complete,
1040 .pause = mirror_pause,
1041 .attached_aio_context = mirror_attached_aio_context,
1042 .drain = mirror_drain,
1043};
1044
1045static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
1046 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1047{
1048 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1049}
1050
1051static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
1052 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1053{
1054 return bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1055}
1056
1057static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
1058{
1059 return bdrv_co_flush(bs->backing->bs);
1060}
1061
1062static int64_t coroutine_fn bdrv_mirror_top_get_block_status(
1063 BlockDriverState *bs, int64_t sector_num, int nb_sectors, int *pnum,
1064 BlockDriverState **file)
1065{
1066 *pnum = nb_sectors;
1067 *file = bs->backing->bs;
1068 return BDRV_BLOCK_RAW | BDRV_BLOCK_OFFSET_VALID |
1069 (sector_num << BDRV_SECTOR_BITS);
1070}
1071
1072static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1073 int64_t offset, int bytes, BdrvRequestFlags flags)
1074{
1075 return bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1076}
1077
1078static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1079 int64_t offset, int bytes)
1080{
1081 return bdrv_co_pdiscard(bs->backing->bs, offset, bytes);
1082}
1083
1084static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs, QDict *opts)
1085{
1086 bdrv_refresh_filename(bs->backing->bs);
1087 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1088 bs->backing->bs->filename);
1089}
1090
1091static void bdrv_mirror_top_close(BlockDriverState *bs)
1092{
1093}
1094
1095static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1096 const BdrvChildRole *role,
1097 uint64_t perm, uint64_t shared,
1098 uint64_t *nperm, uint64_t *nshared)
1099{
1100
1101 *nperm = 0;
1102 if (perm & BLK_PERM_WRITE) {
1103 *nperm |= BLK_PERM_WRITE;
1104 }
1105
1106 *nshared = BLK_PERM_ALL;
1107}
1108
1109
1110
1111static BlockDriver bdrv_mirror_top = {
1112 .format_name = "mirror_top",
1113 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1114 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1115 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1116 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1117 .bdrv_co_flush = bdrv_mirror_top_flush,
1118 .bdrv_co_get_block_status = bdrv_mirror_top_get_block_status,
1119 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1120 .bdrv_close = bdrv_mirror_top_close,
1121 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1122};
1123
1124static void mirror_start_job(const char *job_id, BlockDriverState *bs,
1125 int creation_flags, BlockDriverState *target,
1126 const char *replaces, int64_t speed,
1127 uint32_t granularity, int64_t buf_size,
1128 BlockMirrorBackingMode backing_mode,
1129 BlockdevOnError on_source_error,
1130 BlockdevOnError on_target_error,
1131 bool unmap,
1132 BlockCompletionFunc *cb,
1133 void *opaque,
1134 const BlockJobDriver *driver,
1135 bool is_none_mode, BlockDriverState *base,
1136 bool auto_complete, const char *filter_node_name,
1137 bool is_mirror,
1138 Error **errp)
1139{
1140 MirrorBlockJob *s;
1141 BlockDriverState *mirror_top_bs;
1142 bool target_graph_mod;
1143 bool target_is_backing;
1144 Error *local_err = NULL;
1145 int ret;
1146
1147 if (granularity == 0) {
1148 granularity = bdrv_get_default_bitmap_granularity(target);
1149 }
1150
1151 assert ((granularity & (granularity - 1)) == 0);
1152
1153 assert(granularity >= BDRV_SECTOR_SIZE);
1154
1155 if (buf_size < 0) {
1156 error_setg(errp, "Invalid parameter 'buf-size'");
1157 return;
1158 }
1159
1160 if (buf_size == 0) {
1161 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1162 }
1163
1164
1165
1166
1167 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1168 BDRV_O_RDWR, errp);
1169 if (mirror_top_bs == NULL) {
1170 return;
1171 }
1172 if (!filter_node_name) {
1173 mirror_top_bs->implicit = true;
1174 }
1175 mirror_top_bs->total_sectors = bs->total_sectors;
1176 bdrv_set_aio_context(mirror_top_bs, bdrv_get_aio_context(bs));
1177
1178
1179
1180 bdrv_ref(mirror_top_bs);
1181 bdrv_drained_begin(bs);
1182 bdrv_append(mirror_top_bs, bs, &local_err);
1183 bdrv_drained_end(bs);
1184
1185 if (local_err) {
1186 bdrv_unref(mirror_top_bs);
1187 error_propagate(errp, local_err);
1188 return;
1189 }
1190
1191
1192 s = block_job_create(job_id, driver, mirror_top_bs,
1193 BLK_PERM_CONSISTENT_READ,
1194 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1195 BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD, speed,
1196 creation_flags, cb, opaque, errp);
1197 if (!s) {
1198 goto fail;
1199 }
1200
1201 bdrv_unref(mirror_top_bs);
1202
1203 s->source = bs;
1204 s->mirror_top_bs = mirror_top_bs;
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214 target_is_backing = bdrv_chain_contains(bs, target);
1215 target_graph_mod = (backing_mode != MIRROR_LEAVE_BACKING_CHAIN);
1216 s->target = blk_new(BLK_PERM_WRITE | BLK_PERM_RESIZE |
1217 (target_graph_mod ? BLK_PERM_GRAPH_MOD : 0),
1218 BLK_PERM_WRITE_UNCHANGED |
1219 (target_is_backing ? BLK_PERM_CONSISTENT_READ |
1220 BLK_PERM_WRITE |
1221 BLK_PERM_GRAPH_MOD : 0));
1222 ret = blk_insert_bs(s->target, target, errp);
1223 if (ret < 0) {
1224 goto fail;
1225 }
1226 if (is_mirror) {
1227
1228
1229
1230
1231
1232
1233 blk_set_force_allow_inactivate(s->target);
1234 }
1235
1236 s->replaces = g_strdup(replaces);
1237 s->on_source_error = on_source_error;
1238 s->on_target_error = on_target_error;
1239 s->is_none_mode = is_none_mode;
1240 s->backing_mode = backing_mode;
1241 s->base = base;
1242 s->granularity = granularity;
1243 s->buf_size = ROUND_UP(buf_size, granularity);
1244 s->unmap = unmap;
1245 if (auto_complete) {
1246 s->should_complete = true;
1247 }
1248
1249 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1250 if (!s->dirty_bitmap) {
1251 goto fail;
1252 }
1253
1254
1255 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1256 &error_abort);
1257
1258
1259
1260 if (target_is_backing) {
1261 BlockDriverState *iter;
1262 for (iter = backing_bs(bs); iter != target; iter = backing_bs(iter)) {
1263
1264
1265
1266
1267 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1268 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE,
1269 errp);
1270 if (ret < 0) {
1271 goto fail;
1272 }
1273 }
1274 }
1275
1276 trace_mirror_start(bs, s, opaque);
1277 block_job_start(&s->common);
1278 return;
1279
1280fail:
1281 if (s) {
1282
1283
1284 bdrv_ref(mirror_top_bs);
1285
1286 g_free(s->replaces);
1287 blk_unref(s->target);
1288 block_job_early_fail(&s->common);
1289 }
1290
1291 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
1292 &error_abort);
1293 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
1294
1295 bdrv_unref(mirror_top_bs);
1296}
1297
1298void mirror_start(const char *job_id, BlockDriverState *bs,
1299 BlockDriverState *target, const char *replaces,
1300 int64_t speed, uint32_t granularity, int64_t buf_size,
1301 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1302 BlockdevOnError on_source_error,
1303 BlockdevOnError on_target_error,
1304 bool unmap, const char *filter_node_name, Error **errp)
1305{
1306 bool is_none_mode;
1307 BlockDriverState *base;
1308
1309 if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
1310 error_setg(errp, "Sync mode 'incremental' not supported");
1311 return;
1312 }
1313 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1314 base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
1315 mirror_start_job(job_id, bs, BLOCK_JOB_DEFAULT, target, replaces,
1316 speed, granularity, buf_size, backing_mode,
1317 on_source_error, on_target_error, unmap, NULL, NULL,
1318 &mirror_job_driver, is_none_mode, base, false,
1319 filter_node_name, true, errp);
1320}
1321
1322void commit_active_start(const char *job_id, BlockDriverState *bs,
1323 BlockDriverState *base, int creation_flags,
1324 int64_t speed, BlockdevOnError on_error,
1325 const char *filter_node_name,
1326 BlockCompletionFunc *cb, void *opaque,
1327 bool auto_complete, Error **errp)
1328{
1329 int orig_base_flags;
1330 Error *local_err = NULL;
1331
1332 orig_base_flags = bdrv_get_flags(base);
1333
1334 if (bdrv_reopen(base, bs->open_flags, errp)) {
1335 return;
1336 }
1337
1338 mirror_start_job(job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1339 MIRROR_LEAVE_BACKING_CHAIN,
1340 on_error, on_error, true, cb, opaque,
1341 &commit_active_job_driver, false, base, auto_complete,
1342 filter_node_name, false, &local_err);
1343 if (local_err) {
1344 error_propagate(errp, local_err);
1345 goto error_restore_flags;
1346 }
1347
1348 return;
1349
1350error_restore_flags:
1351
1352
1353 bdrv_reopen(base, orig_base_flags, NULL);
1354 return;
1355}
1356