1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "qemu/coroutine.h"
17#include "qemu/range.h"
18#include "trace.h"
19#include "block/blockjob_int.h"
20#include "block/block_int.h"
21#include "sysemu/block-backend.h"
22#include "qapi/error.h"
23#include "qapi/qmp/qerror.h"
24#include "qemu/ratelimit.h"
25#include "qemu/bitmap.h"
26
27#define MAX_IN_FLIGHT 16
28#define MAX_IO_BYTES (1 << 20)
29#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
30
31
32
33
34typedef struct MirrorBuffer {
35 QSIMPLEQ_ENTRY(MirrorBuffer) next;
36} MirrorBuffer;
37
38typedef struct MirrorOp MirrorOp;
39
40typedef struct MirrorBlockJob {
41 BlockJob common;
42 BlockBackend *target;
43 BlockDriverState *mirror_top_bs;
44 BlockDriverState *base;
45 BlockDriverState *base_overlay;
46
47
48 char *replaces;
49
50 BlockDriverState *to_replace;
51
52 Error *replace_blocker;
53 bool is_none_mode;
54 BlockMirrorBackingMode backing_mode;
55
56 bool zero_target;
57 MirrorCopyMode copy_mode;
58 BlockdevOnError on_source_error, on_target_error;
59 bool synced;
60
61
62 bool actively_synced;
63 bool should_complete;
64 int64_t granularity;
65 size_t buf_size;
66 int64_t bdev_length;
67 unsigned long *cow_bitmap;
68 BdrvDirtyBitmap *dirty_bitmap;
69 BdrvDirtyBitmapIter *dbi;
70 uint8_t *buf;
71 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
72 int buf_free_count;
73
74 uint64_t last_pause_ns;
75 unsigned long *in_flight_bitmap;
76 int in_flight;
77 int64_t bytes_in_flight;
78 QTAILQ_HEAD(, MirrorOp) ops_in_flight;
79 int ret;
80 bool unmap;
81 int target_cluster_size;
82 int max_iov;
83 bool initial_zeroing_ongoing;
84 int in_active_write_counter;
85 bool prepared;
86 bool in_drain;
87} MirrorBlockJob;
88
89typedef struct MirrorBDSOpaque {
90 MirrorBlockJob *job;
91 bool stop;
92 bool is_commit;
93} MirrorBDSOpaque;
94
95struct MirrorOp {
96 MirrorBlockJob *s;
97 QEMUIOVector qiov;
98 int64_t offset;
99 uint64_t bytes;
100
101
102
103 int64_t *bytes_handled;
104
105 bool is_pseudo_op;
106 bool is_active_write;
107 bool is_in_flight;
108 CoQueue waiting_requests;
109 Coroutine *co;
110
111 QTAILQ_ENTRY(MirrorOp) next;
112};
113
114typedef enum MirrorMethod {
115 MIRROR_METHOD_COPY,
116 MIRROR_METHOD_ZERO,
117 MIRROR_METHOD_DISCARD,
118} MirrorMethod;
119
120static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
121 int error)
122{
123 s->synced = false;
124 s->actively_synced = false;
125 if (read) {
126 return block_job_error_action(&s->common, s->on_source_error,
127 true, error);
128 } else {
129 return block_job_error_action(&s->common, s->on_target_error,
130 false, error);
131 }
132}
133
134static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
135 MirrorBlockJob *s,
136 uint64_t offset,
137 uint64_t bytes)
138{
139 uint64_t self_start_chunk = offset / s->granularity;
140 uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
141 uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
142
143 while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
144 self_start_chunk) < self_end_chunk &&
145 s->ret >= 0)
146 {
147 MirrorOp *op;
148
149 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
150 uint64_t op_start_chunk = op->offset / s->granularity;
151 uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
152 s->granularity) -
153 op_start_chunk;
154
155 if (op == self) {
156 continue;
157 }
158
159 if (ranges_overlap(self_start_chunk, self_nb_chunks,
160 op_start_chunk, op_nb_chunks))
161 {
162 qemu_co_queue_wait(&op->waiting_requests, NULL);
163 break;
164 }
165 }
166 }
167}
168
169static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
170{
171 MirrorBlockJob *s = op->s;
172 struct iovec *iov;
173 int64_t chunk_num;
174 int i, nb_chunks;
175
176 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
177
178 s->in_flight--;
179 s->bytes_in_flight -= op->bytes;
180 iov = op->qiov.iov;
181 for (i = 0; i < op->qiov.niov; i++) {
182 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
183 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
184 s->buf_free_count++;
185 }
186
187 chunk_num = op->offset / s->granularity;
188 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
189
190 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
191 QTAILQ_REMOVE(&s->ops_in_flight, op, next);
192 if (ret >= 0) {
193 if (s->cow_bitmap) {
194 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
195 }
196 if (!s->initial_zeroing_ongoing) {
197 job_progress_update(&s->common.job, op->bytes);
198 }
199 }
200 qemu_iovec_destroy(&op->qiov);
201
202 qemu_co_queue_restart_all(&op->waiting_requests);
203 g_free(op);
204}
205
206static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
207{
208 MirrorBlockJob *s = op->s;
209
210 if (ret < 0) {
211 BlockErrorAction action;
212
213 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
214 action = mirror_error_action(s, false, -ret);
215 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
216 s->ret = ret;
217 }
218 }
219
220 mirror_iteration_done(op, ret);
221}
222
223static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
224{
225 MirrorBlockJob *s = op->s;
226
227 if (ret < 0) {
228 BlockErrorAction action;
229
230 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
231 action = mirror_error_action(s, true, -ret);
232 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
233 s->ret = ret;
234 }
235
236 mirror_iteration_done(op, ret);
237 return;
238 }
239
240 ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
241 mirror_write_complete(op, ret);
242}
243
244
245static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
246 int64_t offset,
247 int64_t bytes)
248{
249 return MIN(bytes, s->bdev_length - offset);
250}
251
252
253
254static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
255 uint64_t *bytes)
256{
257 bool need_cow;
258 int ret = 0;
259 int64_t align_offset = *offset;
260 int64_t align_bytes = *bytes;
261 int max_bytes = s->granularity * s->max_iov;
262
263 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
264 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
265 s->cow_bitmap);
266 if (need_cow) {
267 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
268 &align_offset, &align_bytes);
269 }
270
271 if (align_bytes > max_bytes) {
272 align_bytes = max_bytes;
273 if (need_cow) {
274 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
275 }
276 }
277
278
279 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
280
281 ret = align_offset + align_bytes - (*offset + *bytes);
282 *offset = align_offset;
283 *bytes = align_bytes;
284 assert(ret >= 0);
285 return ret;
286}
287
288static inline void coroutine_fn
289mirror_wait_for_any_operation(MirrorBlockJob *s, bool active)
290{
291 MirrorOp *op;
292
293 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
294
295
296
297
298
299 if (!op->is_pseudo_op && op->is_in_flight &&
300 op->is_active_write == active)
301 {
302 qemu_co_queue_wait(&op->waiting_requests, NULL);
303 return;
304 }
305 }
306 abort();
307}
308
309static inline void coroutine_fn
310mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
311{
312
313 mirror_wait_for_any_operation(s, false);
314}
315
316
317
318
319
320
321
322
323
324static void coroutine_fn mirror_co_read(void *opaque)
325{
326 MirrorOp *op = opaque;
327 MirrorBlockJob *s = op->s;
328 int nb_chunks;
329 uint64_t ret;
330 uint64_t max_bytes;
331
332 max_bytes = s->granularity * s->max_iov;
333
334
335 op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
336 assert(op->bytes);
337 assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
338 *op->bytes_handled = op->bytes;
339
340 if (s->cow_bitmap) {
341 *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
342 }
343
344 assert(*op->bytes_handled <= UINT_MAX);
345 assert(op->bytes <= s->buf_size);
346
347
348
349 assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
350
351 assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
352 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
353
354 while (s->buf_free_count < nb_chunks) {
355 trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
356 mirror_wait_for_free_in_flight_slot(s);
357 }
358
359
360
361
362 qemu_iovec_init(&op->qiov, nb_chunks);
363 while (nb_chunks-- > 0) {
364 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
365 size_t remaining = op->bytes - op->qiov.size;
366
367 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
368 s->buf_free_count--;
369 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
370 }
371
372
373 s->in_flight++;
374 s->bytes_in_flight += op->bytes;
375 op->is_in_flight = true;
376 trace_mirror_one_iteration(s, op->offset, op->bytes);
377
378 ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
379 &op->qiov, 0);
380 mirror_read_complete(op, ret);
381}
382
383static void coroutine_fn mirror_co_zero(void *opaque)
384{
385 MirrorOp *op = opaque;
386 int ret;
387
388 op->s->in_flight++;
389 op->s->bytes_in_flight += op->bytes;
390 *op->bytes_handled = op->bytes;
391 op->is_in_flight = true;
392
393 ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
394 op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
395 mirror_write_complete(op, ret);
396}
397
398static void coroutine_fn mirror_co_discard(void *opaque)
399{
400 MirrorOp *op = opaque;
401 int ret;
402
403 op->s->in_flight++;
404 op->s->bytes_in_flight += op->bytes;
405 *op->bytes_handled = op->bytes;
406 op->is_in_flight = true;
407
408 ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
409 mirror_write_complete(op, ret);
410}
411
412static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
413 unsigned bytes, MirrorMethod mirror_method)
414{
415 MirrorOp *op;
416 Coroutine *co;
417 int64_t bytes_handled = -1;
418
419 op = g_new(MirrorOp, 1);
420 *op = (MirrorOp){
421 .s = s,
422 .offset = offset,
423 .bytes = bytes,
424 .bytes_handled = &bytes_handled,
425 };
426 qemu_co_queue_init(&op->waiting_requests);
427
428 switch (mirror_method) {
429 case MIRROR_METHOD_COPY:
430 co = qemu_coroutine_create(mirror_co_read, op);
431 break;
432 case MIRROR_METHOD_ZERO:
433 co = qemu_coroutine_create(mirror_co_zero, op);
434 break;
435 case MIRROR_METHOD_DISCARD:
436 co = qemu_coroutine_create(mirror_co_discard, op);
437 break;
438 default:
439 abort();
440 }
441 op->co = co;
442
443 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
444 qemu_coroutine_enter(co);
445
446
447
448
449 assert(bytes_handled >= 0);
450
451
452
453
454 assert(bytes_handled <= UINT_MAX);
455 return bytes_handled;
456}
457
458static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
459{
460 BlockDriverState *source = s->mirror_top_bs->backing->bs;
461 MirrorOp *pseudo_op;
462 int64_t offset;
463 uint64_t delay_ns = 0, ret = 0;
464
465 int nb_chunks = 1;
466 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
467 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
468
469 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
470 offset = bdrv_dirty_iter_next(s->dbi);
471 if (offset < 0) {
472 bdrv_set_dirty_iter(s->dbi, 0);
473 offset = bdrv_dirty_iter_next(s->dbi);
474 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
475 assert(offset >= 0);
476 }
477 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
478
479 mirror_wait_on_conflicts(NULL, s, offset, 1);
480
481 job_pause_point(&s->common.job);
482
483
484
485 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
486 while (nb_chunks * s->granularity < s->buf_size) {
487 int64_t next_dirty;
488 int64_t next_offset = offset + nb_chunks * s->granularity;
489 int64_t next_chunk = next_offset / s->granularity;
490 if (next_offset >= s->bdev_length ||
491 !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
492 break;
493 }
494 if (test_bit(next_chunk, s->in_flight_bitmap)) {
495 break;
496 }
497
498 next_dirty = bdrv_dirty_iter_next(s->dbi);
499 if (next_dirty > next_offset || next_dirty < 0) {
500
501 bdrv_set_dirty_iter(s->dbi, next_offset);
502 next_dirty = bdrv_dirty_iter_next(s->dbi);
503 }
504 assert(next_dirty == next_offset);
505 nb_chunks++;
506 }
507
508
509
510
511
512 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
513 nb_chunks * s->granularity);
514 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
515
516
517
518
519
520
521
522 pseudo_op = g_new(MirrorOp, 1);
523 *pseudo_op = (MirrorOp){
524 .offset = offset,
525 .bytes = nb_chunks * s->granularity,
526 .is_pseudo_op = true,
527 };
528 qemu_co_queue_init(&pseudo_op->waiting_requests);
529 QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
530
531 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
532 while (nb_chunks > 0 && offset < s->bdev_length) {
533 int ret;
534 int64_t io_bytes;
535 int64_t io_bytes_acct;
536 MirrorMethod mirror_method = MIRROR_METHOD_COPY;
537
538 assert(!(offset % s->granularity));
539 ret = bdrv_block_status_above(source, NULL, offset,
540 nb_chunks * s->granularity,
541 &io_bytes, NULL, NULL);
542 if (ret < 0) {
543 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
544 } else if (ret & BDRV_BLOCK_DATA) {
545 io_bytes = MIN(io_bytes, max_io_bytes);
546 }
547
548 io_bytes -= io_bytes % s->granularity;
549 if (io_bytes < s->granularity) {
550 io_bytes = s->granularity;
551 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
552 int64_t target_offset;
553 int64_t target_bytes;
554 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
555 &target_offset, &target_bytes);
556 if (target_offset == offset &&
557 target_bytes == io_bytes) {
558 mirror_method = ret & BDRV_BLOCK_ZERO ?
559 MIRROR_METHOD_ZERO :
560 MIRROR_METHOD_DISCARD;
561 }
562 }
563
564 while (s->in_flight >= MAX_IN_FLIGHT) {
565 trace_mirror_yield_in_flight(s, offset, s->in_flight);
566 mirror_wait_for_free_in_flight_slot(s);
567 }
568
569 if (s->ret < 0) {
570 ret = 0;
571 goto fail;
572 }
573
574 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
575 io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
576 if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
577 io_bytes_acct = 0;
578 } else {
579 io_bytes_acct = io_bytes;
580 }
581 assert(io_bytes);
582 offset += io_bytes;
583 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
584 delay_ns = block_job_ratelimit_get_delay(&s->common, io_bytes_acct);
585 }
586
587 ret = delay_ns;
588fail:
589 QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
590 qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
591 g_free(pseudo_op);
592
593 return ret;
594}
595
596static void mirror_free_init(MirrorBlockJob *s)
597{
598 int granularity = s->granularity;
599 size_t buf_size = s->buf_size;
600 uint8_t *buf = s->buf;
601
602 assert(s->buf_free_count == 0);
603 QSIMPLEQ_INIT(&s->buf_free);
604 while (buf_size != 0) {
605 MirrorBuffer *cur = (MirrorBuffer *)buf;
606 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
607 s->buf_free_count++;
608 buf_size -= granularity;
609 buf += granularity;
610 }
611}
612
613
614
615
616
617static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
618{
619 while (s->in_flight > 0) {
620 mirror_wait_for_free_in_flight_slot(s);
621 }
622}
623
624
625
626
627
628
629static int mirror_exit_common(Job *job)
630{
631 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
632 BlockJob *bjob = &s->common;
633 MirrorBDSOpaque *bs_opaque;
634 AioContext *replace_aio_context = NULL;
635 BlockDriverState *src;
636 BlockDriverState *target_bs;
637 BlockDriverState *mirror_top_bs;
638 Error *local_err = NULL;
639 bool abort = job->ret < 0;
640 int ret = 0;
641
642 if (s->prepared) {
643 return 0;
644 }
645 s->prepared = true;
646
647 mirror_top_bs = s->mirror_top_bs;
648 bs_opaque = mirror_top_bs->opaque;
649 src = mirror_top_bs->backing->bs;
650 target_bs = blk_bs(s->target);
651
652 if (bdrv_chain_contains(src, target_bs)) {
653 bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
654 }
655
656 bdrv_release_dirty_bitmap(s->dirty_bitmap);
657
658
659
660 bdrv_ref(src);
661 bdrv_ref(mirror_top_bs);
662 bdrv_ref(target_bs);
663
664
665
666
667
668
669 blk_unref(s->target);
670 s->target = NULL;
671
672
673
674
675
676 bdrv_drained_begin(mirror_top_bs);
677 bs_opaque->stop = true;
678 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
679 &error_abort);
680 if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
681 BlockDriverState *backing = s->is_none_mode ? src : s->base;
682 BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
683
684 if (bdrv_cow_bs(unfiltered_target) != backing) {
685 bdrv_set_backing_hd(unfiltered_target, backing, &local_err);
686 if (local_err) {
687 error_report_err(local_err);
688 local_err = NULL;
689 ret = -EPERM;
690 }
691 }
692 } else if (!abort && s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
693 assert(!bdrv_backing_chain_next(target_bs));
694 ret = bdrv_open_backing_file(bdrv_skip_filters(target_bs), NULL,
695 "backing", &local_err);
696 if (ret < 0) {
697 error_report_err(local_err);
698 local_err = NULL;
699 }
700 }
701
702 if (s->to_replace) {
703 replace_aio_context = bdrv_get_aio_context(s->to_replace);
704 aio_context_acquire(replace_aio_context);
705 }
706
707 if (s->should_complete && !abort) {
708 BlockDriverState *to_replace = s->to_replace ?: src;
709 bool ro = bdrv_is_read_only(to_replace);
710
711 if (ro != bdrv_is_read_only(target_bs)) {
712 bdrv_reopen_set_read_only(target_bs, ro, NULL);
713 }
714
715
716
717 assert(s->in_drain);
718 bdrv_drained_begin(target_bs);
719
720
721
722
723
724 if (bdrv_recurse_can_replace(src, to_replace)) {
725 bdrv_replace_node(to_replace, target_bs, &local_err);
726 } else {
727 error_setg(&local_err, "Can no longer replace '%s' by '%s', "
728 "because it can no longer be guaranteed that doing so "
729 "would not lead to an abrupt change of visible data",
730 to_replace->node_name, target_bs->node_name);
731 }
732 bdrv_drained_end(target_bs);
733 if (local_err) {
734 error_report_err(local_err);
735 ret = -EPERM;
736 }
737 }
738 if (s->to_replace) {
739 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
740 error_free(s->replace_blocker);
741 bdrv_unref(s->to_replace);
742 }
743 if (replace_aio_context) {
744 aio_context_release(replace_aio_context);
745 }
746 g_free(s->replaces);
747 bdrv_unref(target_bs);
748
749
750
751
752
753
754 block_job_remove_all_bdrv(bjob);
755 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
756
757
758
759
760 blk_remove_bs(bjob->blk);
761 blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort);
762 blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort);
763
764 bs_opaque->job = NULL;
765
766 bdrv_drained_end(src);
767 bdrv_drained_end(mirror_top_bs);
768 s->in_drain = false;
769 bdrv_unref(mirror_top_bs);
770 bdrv_unref(src);
771
772 return ret;
773}
774
775static int mirror_prepare(Job *job)
776{
777 return mirror_exit_common(job);
778}
779
780static void mirror_abort(Job *job)
781{
782 int ret = mirror_exit_common(job);
783 assert(ret == 0);
784}
785
786static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
787{
788 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
789
790 if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
791 s->last_pause_ns = now;
792 job_sleep_ns(&s->common.job, 0);
793 } else {
794 job_pause_point(&s->common.job);
795 }
796}
797
798static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
799{
800 int64_t offset;
801 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
802 BlockDriverState *target_bs = blk_bs(s->target);
803 int ret;
804 int64_t count;
805
806 if (s->zero_target) {
807 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
808 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
809 return 0;
810 }
811
812 s->initial_zeroing_ongoing = true;
813 for (offset = 0; offset < s->bdev_length; ) {
814 int bytes = MIN(s->bdev_length - offset,
815 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
816
817 mirror_throttle(s);
818
819 if (job_is_cancelled(&s->common.job)) {
820 s->initial_zeroing_ongoing = false;
821 return 0;
822 }
823
824 if (s->in_flight >= MAX_IN_FLIGHT) {
825 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
826 s->in_flight);
827 mirror_wait_for_free_in_flight_slot(s);
828 continue;
829 }
830
831 mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
832 offset += bytes;
833 }
834
835 mirror_wait_for_all_io(s);
836 s->initial_zeroing_ongoing = false;
837 }
838
839
840 for (offset = 0; offset < s->bdev_length; ) {
841
842 int bytes = MIN(s->bdev_length - offset,
843 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
844
845 mirror_throttle(s);
846
847 if (job_is_cancelled(&s->common.job)) {
848 return 0;
849 }
850
851 ret = bdrv_is_allocated_above(bs, s->base_overlay, true, offset, bytes,
852 &count);
853 if (ret < 0) {
854 return ret;
855 }
856
857 assert(count);
858 if (ret > 0) {
859 bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
860 }
861 offset += count;
862 }
863 return 0;
864}
865
866
867
868
869static int mirror_flush(MirrorBlockJob *s)
870{
871 int ret = blk_flush(s->target);
872 if (ret < 0) {
873 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
874 s->ret = ret;
875 }
876 }
877 return ret;
878}
879
880static int coroutine_fn mirror_run(Job *job, Error **errp)
881{
882 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
883 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
884 BlockDriverState *target_bs = blk_bs(s->target);
885 bool need_drain = true;
886 int64_t length;
887 int64_t target_length;
888 BlockDriverInfo bdi;
889 char backing_filename[2];
890
891 int ret = 0;
892
893 if (job_is_cancelled(&s->common.job)) {
894 goto immediate_exit;
895 }
896
897 s->bdev_length = bdrv_getlength(bs);
898 if (s->bdev_length < 0) {
899 ret = s->bdev_length;
900 goto immediate_exit;
901 }
902
903 target_length = blk_getlength(s->target);
904 if (target_length < 0) {
905 ret = target_length;
906 goto immediate_exit;
907 }
908
909
910
911 if (s->base == blk_bs(s->target)) {
912 if (s->bdev_length > target_length) {
913 ret = blk_truncate(s->target, s->bdev_length, false,
914 PREALLOC_MODE_OFF, 0, NULL);
915 if (ret < 0) {
916 goto immediate_exit;
917 }
918 }
919 } else if (s->bdev_length != target_length) {
920 error_setg(errp, "Source and target image have different sizes");
921 ret = -EINVAL;
922 goto immediate_exit;
923 }
924
925 if (s->bdev_length == 0) {
926
927 job_transition_to_ready(&s->common.job);
928 s->synced = true;
929 s->actively_synced = true;
930 while (!job_is_cancelled(&s->common.job) && !s->should_complete) {
931 job_yield(&s->common.job);
932 }
933 s->common.job.cancelled = false;
934 goto immediate_exit;
935 }
936
937 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
938 s->in_flight_bitmap = bitmap_new(length);
939
940
941
942
943
944 bdrv_get_backing_filename(target_bs, backing_filename,
945 sizeof(backing_filename));
946 if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
947 s->target_cluster_size = bdi.cluster_size;
948 } else {
949 s->target_cluster_size = BDRV_SECTOR_SIZE;
950 }
951 if (backing_filename[0] && !bdrv_backing_chain_next(target_bs) &&
952 s->granularity < s->target_cluster_size) {
953 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
954 s->cow_bitmap = bitmap_new(length);
955 }
956 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
957
958 s->buf = qemu_try_blockalign(bs, s->buf_size);
959 if (s->buf == NULL) {
960 ret = -ENOMEM;
961 goto immediate_exit;
962 }
963
964 mirror_free_init(s);
965
966 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
967 if (!s->is_none_mode) {
968 ret = mirror_dirty_init(s);
969 if (ret < 0 || job_is_cancelled(&s->common.job)) {
970 goto immediate_exit;
971 }
972 }
973
974 assert(!s->dbi);
975 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
976 for (;;) {
977 uint64_t delay_ns = 0;
978 int64_t cnt, delta;
979 bool should_complete;
980
981
982
983 while (s->in_active_write_counter) {
984 mirror_wait_for_any_operation(s, true);
985 }
986
987 if (s->ret < 0) {
988 ret = s->ret;
989 goto immediate_exit;
990 }
991
992 job_pause_point(&s->common.job);
993
994 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
995
996
997
998 job_progress_set_remaining(&s->common.job, s->bytes_in_flight + cnt);
999
1000
1001
1002
1003
1004 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
1005 if (delta < BLOCK_JOB_SLICE_TIME &&
1006 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1007 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
1008 (cnt == 0 && s->in_flight > 0)) {
1009 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
1010 mirror_wait_for_free_in_flight_slot(s);
1011 continue;
1012 } else if (cnt != 0) {
1013 delay_ns = mirror_iteration(s);
1014 }
1015 }
1016
1017 should_complete = false;
1018 if (s->in_flight == 0 && cnt == 0) {
1019 trace_mirror_before_flush(s);
1020 if (!s->synced) {
1021 if (mirror_flush(s) < 0) {
1022
1023 continue;
1024 }
1025
1026
1027
1028
1029
1030 job_transition_to_ready(&s->common.job);
1031 s->synced = true;
1032 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
1033 s->actively_synced = true;
1034 }
1035 }
1036
1037 should_complete = s->should_complete ||
1038 job_is_cancelled(&s->common.job);
1039 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1040 }
1041
1042 if (cnt == 0 && should_complete) {
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 trace_mirror_before_drain(s, cnt);
1054
1055 s->in_drain = true;
1056 bdrv_drained_begin(bs);
1057 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1058 if (cnt > 0 || mirror_flush(s) < 0) {
1059 bdrv_drained_end(bs);
1060 s->in_drain = false;
1061 continue;
1062 }
1063
1064
1065
1066
1067 assert(QLIST_EMPTY(&bs->tracked_requests));
1068 s->common.job.cancelled = false;
1069 need_drain = false;
1070 break;
1071 }
1072
1073 ret = 0;
1074
1075 if (s->synced && !should_complete) {
1076 delay_ns = (s->in_flight == 0 &&
1077 cnt == 0 ? BLOCK_JOB_SLICE_TIME : 0);
1078 }
1079 trace_mirror_before_sleep(s, cnt, s->synced, delay_ns);
1080 job_sleep_ns(&s->common.job, delay_ns);
1081 if (job_is_cancelled(&s->common.job) &&
1082 (!s->synced || s->common.job.force_cancel))
1083 {
1084 break;
1085 }
1086 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1087 }
1088
1089immediate_exit:
1090 if (s->in_flight > 0) {
1091
1092
1093
1094
1095 assert(ret < 0 || ((s->common.job.force_cancel || !s->synced) &&
1096 job_is_cancelled(&s->common.job)));
1097 assert(need_drain);
1098 mirror_wait_for_all_io(s);
1099 }
1100
1101 assert(s->in_flight == 0);
1102 qemu_vfree(s->buf);
1103 g_free(s->cow_bitmap);
1104 g_free(s->in_flight_bitmap);
1105 bdrv_dirty_iter_free(s->dbi);
1106
1107 if (need_drain) {
1108 s->in_drain = true;
1109 bdrv_drained_begin(bs);
1110 }
1111
1112 return ret;
1113}
1114
1115static void mirror_complete(Job *job, Error **errp)
1116{
1117 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1118
1119 if (!s->synced) {
1120 error_setg(errp, "The active block job '%s' cannot be completed",
1121 job->id);
1122 return;
1123 }
1124
1125
1126 if (s->replaces) {
1127 AioContext *replace_aio_context;
1128
1129 s->to_replace = bdrv_find_node(s->replaces);
1130 if (!s->to_replace) {
1131 error_setg(errp, "Node name '%s' not found", s->replaces);
1132 return;
1133 }
1134
1135 replace_aio_context = bdrv_get_aio_context(s->to_replace);
1136 aio_context_acquire(replace_aio_context);
1137
1138
1139
1140
1141
1142 error_setg(&s->replace_blocker,
1143 "block device is in use by block-job-complete");
1144 bdrv_op_block_all(s->to_replace, s->replace_blocker);
1145 bdrv_ref(s->to_replace);
1146
1147 aio_context_release(replace_aio_context);
1148 }
1149
1150 s->should_complete = true;
1151
1152
1153 if (!job->paused) {
1154 job_enter(job);
1155 }
1156}
1157
1158static void coroutine_fn mirror_pause(Job *job)
1159{
1160 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1161
1162 mirror_wait_for_all_io(s);
1163}
1164
1165static bool mirror_drained_poll(BlockJob *job)
1166{
1167 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1168
1169
1170
1171
1172
1173
1174 if (!s->common.job.paused && !s->common.job.cancelled && !s->in_drain) {
1175 return true;
1176 }
1177
1178 return !!s->in_flight;
1179}
1180
1181static void mirror_cancel(Job *job)
1182{
1183 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1184 BlockDriverState *target = blk_bs(s->target);
1185
1186 bdrv_cancel_in_flight(target);
1187}
1188
1189static const BlockJobDriver mirror_job_driver = {
1190 .job_driver = {
1191 .instance_size = sizeof(MirrorBlockJob),
1192 .job_type = JOB_TYPE_MIRROR,
1193 .free = block_job_free,
1194 .user_resume = block_job_user_resume,
1195 .run = mirror_run,
1196 .prepare = mirror_prepare,
1197 .abort = mirror_abort,
1198 .pause = mirror_pause,
1199 .complete = mirror_complete,
1200 .cancel = mirror_cancel,
1201 },
1202 .drained_poll = mirror_drained_poll,
1203};
1204
1205static const BlockJobDriver commit_active_job_driver = {
1206 .job_driver = {
1207 .instance_size = sizeof(MirrorBlockJob),
1208 .job_type = JOB_TYPE_COMMIT,
1209 .free = block_job_free,
1210 .user_resume = block_job_user_resume,
1211 .run = mirror_run,
1212 .prepare = mirror_prepare,
1213 .abort = mirror_abort,
1214 .pause = mirror_pause,
1215 .complete = mirror_complete,
1216 },
1217 .drained_poll = mirror_drained_poll,
1218};
1219
1220static void coroutine_fn
1221do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1222 uint64_t offset, uint64_t bytes,
1223 QEMUIOVector *qiov, int flags)
1224{
1225 int ret;
1226 size_t qiov_offset = 0;
1227 int64_t bitmap_offset, bitmap_end;
1228
1229 if (!QEMU_IS_ALIGNED(offset, job->granularity) &&
1230 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset))
1231 {
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249 qiov_offset = QEMU_ALIGN_UP(offset, job->granularity) - offset;
1250 if (bytes <= qiov_offset) {
1251
1252 return;
1253 }
1254 offset += qiov_offset;
1255 bytes -= qiov_offset;
1256 }
1257
1258 if (!QEMU_IS_ALIGNED(offset + bytes, job->granularity) &&
1259 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset + bytes - 1))
1260 {
1261 uint64_t tail = (offset + bytes) % job->granularity;
1262
1263 if (bytes <= tail) {
1264
1265 return;
1266 }
1267 bytes -= tail;
1268 }
1269
1270
1271
1272
1273
1274 bitmap_offset = QEMU_ALIGN_UP(offset, job->granularity);
1275 bitmap_end = QEMU_ALIGN_DOWN(offset + bytes, job->granularity);
1276 if (bitmap_offset < bitmap_end) {
1277 bdrv_reset_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1278 bitmap_end - bitmap_offset);
1279 }
1280
1281 job_progress_increase_remaining(&job->common.job, bytes);
1282
1283 switch (method) {
1284 case MIRROR_METHOD_COPY:
1285 ret = blk_co_pwritev_part(job->target, offset, bytes,
1286 qiov, qiov_offset, flags);
1287 break;
1288
1289 case MIRROR_METHOD_ZERO:
1290 assert(!qiov);
1291 ret = blk_co_pwrite_zeroes(job->target, offset, bytes, flags);
1292 break;
1293
1294 case MIRROR_METHOD_DISCARD:
1295 assert(!qiov);
1296 ret = blk_co_pdiscard(job->target, offset, bytes);
1297 break;
1298
1299 default:
1300 abort();
1301 }
1302
1303 if (ret >= 0) {
1304 job_progress_update(&job->common.job, bytes);
1305 } else {
1306 BlockErrorAction action;
1307
1308
1309
1310
1311
1312
1313
1314 bitmap_offset = QEMU_ALIGN_DOWN(offset, job->granularity);
1315 bitmap_end = QEMU_ALIGN_UP(offset + bytes, job->granularity);
1316 bdrv_set_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1317 bitmap_end - bitmap_offset);
1318 job->actively_synced = false;
1319
1320 action = mirror_error_action(job, false, -ret);
1321 if (action == BLOCK_ERROR_ACTION_REPORT) {
1322 if (!job->ret) {
1323 job->ret = ret;
1324 }
1325 }
1326 }
1327}
1328
1329static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1330 uint64_t offset,
1331 uint64_t bytes)
1332{
1333 MirrorOp *op;
1334 uint64_t start_chunk = offset / s->granularity;
1335 uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1336
1337 op = g_new(MirrorOp, 1);
1338 *op = (MirrorOp){
1339 .s = s,
1340 .offset = offset,
1341 .bytes = bytes,
1342 .is_active_write = true,
1343 .is_in_flight = true,
1344 };
1345 qemu_co_queue_init(&op->waiting_requests);
1346 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1347
1348 s->in_active_write_counter++;
1349
1350 mirror_wait_on_conflicts(op, s, offset, bytes);
1351
1352 bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1353
1354 return op;
1355}
1356
1357static void coroutine_fn active_write_settle(MirrorOp *op)
1358{
1359 uint64_t start_chunk = op->offset / op->s->granularity;
1360 uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1361 op->s->granularity);
1362
1363 if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1364 BdrvChild *source = op->s->mirror_top_bs->backing;
1365
1366 if (QLIST_FIRST(&source->bs->parents) == source &&
1367 QLIST_NEXT(source, next_parent) == NULL)
1368 {
1369
1370
1371
1372
1373 assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1374 }
1375 }
1376 bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1377 QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1378 qemu_co_queue_restart_all(&op->waiting_requests);
1379 g_free(op);
1380}
1381
1382static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
1383 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1384{
1385 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1386}
1387
1388static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs,
1389 MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1390 int flags)
1391{
1392 MirrorOp *op = NULL;
1393 MirrorBDSOpaque *s = bs->opaque;
1394 int ret = 0;
1395 bool copy_to_target;
1396
1397 copy_to_target = s->job->ret >= 0 &&
1398 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1399
1400 if (copy_to_target) {
1401 op = active_write_prepare(s->job, offset, bytes);
1402 }
1403
1404 switch (method) {
1405 case MIRROR_METHOD_COPY:
1406 ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1407 break;
1408
1409 case MIRROR_METHOD_ZERO:
1410 ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1411 break;
1412
1413 case MIRROR_METHOD_DISCARD:
1414 ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1415 break;
1416
1417 default:
1418 abort();
1419 }
1420
1421 if (ret < 0) {
1422 goto out;
1423 }
1424
1425 if (copy_to_target) {
1426 do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1427 }
1428
1429out:
1430 if (copy_to_target) {
1431 active_write_settle(op);
1432 }
1433 return ret;
1434}
1435
1436static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
1437 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1438{
1439 MirrorBDSOpaque *s = bs->opaque;
1440 QEMUIOVector bounce_qiov;
1441 void *bounce_buf;
1442 int ret = 0;
1443 bool copy_to_target;
1444
1445 copy_to_target = s->job->ret >= 0 &&
1446 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1447
1448 if (copy_to_target) {
1449
1450
1451
1452
1453 bounce_buf = qemu_blockalign(bs, bytes);
1454 iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1455
1456 qemu_iovec_init(&bounce_qiov, 1);
1457 qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1458 qiov = &bounce_qiov;
1459 }
1460
1461 ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1462 flags);
1463
1464 if (copy_to_target) {
1465 qemu_iovec_destroy(&bounce_qiov);
1466 qemu_vfree(bounce_buf);
1467 }
1468
1469 return ret;
1470}
1471
1472static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
1473{
1474 if (bs->backing == NULL) {
1475
1476 return 0;
1477 }
1478 return bdrv_co_flush(bs->backing->bs);
1479}
1480
1481static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1482 int64_t offset, int bytes, BdrvRequestFlags flags)
1483{
1484 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1485 flags);
1486}
1487
1488static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1489 int64_t offset, int bytes)
1490{
1491 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1492 NULL, 0);
1493}
1494
1495static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1496{
1497 if (bs->backing == NULL) {
1498
1499
1500 return;
1501 }
1502 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1503 bs->backing->bs->filename);
1504}
1505
1506static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1507 BdrvChildRole role,
1508 BlockReopenQueue *reopen_queue,
1509 uint64_t perm, uint64_t shared,
1510 uint64_t *nperm, uint64_t *nshared)
1511{
1512 MirrorBDSOpaque *s = bs->opaque;
1513
1514 if (s->stop) {
1515
1516
1517
1518
1519 *nperm = 0;
1520 *nshared = BLK_PERM_ALL;
1521 return;
1522 }
1523
1524 bdrv_default_perms(bs, c, role, reopen_queue,
1525 perm, shared, nperm, nshared);
1526
1527 if (s->is_commit) {
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542 *nperm &= ~BLK_PERM_CONSISTENT_READ;
1543 *nshared |= BLK_PERM_WRITE;
1544 }
1545}
1546
1547
1548
1549static BlockDriver bdrv_mirror_top = {
1550 .format_name = "mirror_top",
1551 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1552 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1553 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1554 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1555 .bdrv_co_flush = bdrv_mirror_top_flush,
1556 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1557 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1558
1559 .is_filter = true,
1560};
1561
1562static BlockJob *mirror_start_job(
1563 const char *job_id, BlockDriverState *bs,
1564 int creation_flags, BlockDriverState *target,
1565 const char *replaces, int64_t speed,
1566 uint32_t granularity, int64_t buf_size,
1567 BlockMirrorBackingMode backing_mode,
1568 bool zero_target,
1569 BlockdevOnError on_source_error,
1570 BlockdevOnError on_target_error,
1571 bool unmap,
1572 BlockCompletionFunc *cb,
1573 void *opaque,
1574 const BlockJobDriver *driver,
1575 bool is_none_mode, BlockDriverState *base,
1576 bool auto_complete, const char *filter_node_name,
1577 bool is_mirror, MirrorCopyMode copy_mode,
1578 Error **errp)
1579{
1580 MirrorBlockJob *s;
1581 MirrorBDSOpaque *bs_opaque;
1582 BlockDriverState *mirror_top_bs;
1583 bool target_is_backing;
1584 uint64_t target_perms, target_shared_perms;
1585 int ret;
1586
1587 if (granularity == 0) {
1588 granularity = bdrv_get_default_bitmap_granularity(target);
1589 }
1590
1591 assert(is_power_of_2(granularity));
1592
1593 if (buf_size < 0) {
1594 error_setg(errp, "Invalid parameter 'buf-size'");
1595 return NULL;
1596 }
1597
1598 if (buf_size == 0) {
1599 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1600 }
1601
1602 if (bdrv_skip_filters(bs) == bdrv_skip_filters(target)) {
1603 error_setg(errp, "Can't mirror node into itself");
1604 return NULL;
1605 }
1606
1607 target_is_backing = bdrv_chain_contains(bs, target);
1608
1609
1610
1611
1612 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1613 BDRV_O_RDWR, errp);
1614 if (mirror_top_bs == NULL) {
1615 return NULL;
1616 }
1617 if (!filter_node_name) {
1618 mirror_top_bs->implicit = true;
1619 }
1620
1621
1622 mirror_top_bs->never_freeze = true;
1623
1624 mirror_top_bs->total_sectors = bs->total_sectors;
1625 mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1626 mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
1627 BDRV_REQ_NO_FALLBACK;
1628 bs_opaque = g_new0(MirrorBDSOpaque, 1);
1629 mirror_top_bs->opaque = bs_opaque;
1630
1631 bs_opaque->is_commit = target_is_backing;
1632
1633
1634
1635 bdrv_ref(mirror_top_bs);
1636 bdrv_drained_begin(bs);
1637 ret = bdrv_append(mirror_top_bs, bs, errp);
1638 bdrv_drained_end(bs);
1639
1640 if (ret < 0) {
1641 bdrv_unref(mirror_top_bs);
1642 return NULL;
1643 }
1644
1645
1646 s = block_job_create(job_id, driver, NULL, mirror_top_bs,
1647 BLK_PERM_CONSISTENT_READ,
1648 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1649 BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD, speed,
1650 creation_flags, cb, opaque, errp);
1651 if (!s) {
1652 goto fail;
1653 }
1654 bs_opaque->job = s;
1655
1656
1657 bdrv_unref(mirror_top_bs);
1658
1659 s->mirror_top_bs = mirror_top_bs;
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670 target_perms = BLK_PERM_WRITE;
1671 target_shared_perms = BLK_PERM_WRITE_UNCHANGED;
1672
1673 if (target_is_backing) {
1674 int64_t bs_size, target_size;
1675 bs_size = bdrv_getlength(bs);
1676 if (bs_size < 0) {
1677 error_setg_errno(errp, -bs_size,
1678 "Could not inquire top image size");
1679 goto fail;
1680 }
1681
1682 target_size = bdrv_getlength(target);
1683 if (target_size < 0) {
1684 error_setg_errno(errp, -target_size,
1685 "Could not inquire base image size");
1686 goto fail;
1687 }
1688
1689 if (target_size < bs_size) {
1690 target_perms |= BLK_PERM_RESIZE;
1691 }
1692
1693 target_shared_perms |= BLK_PERM_CONSISTENT_READ
1694 | BLK_PERM_WRITE
1695 | BLK_PERM_GRAPH_MOD;
1696 } else if (bdrv_chain_contains(bs, bdrv_skip_filters(target))) {
1697
1698
1699
1700
1701 error_setg(errp, "Cannot mirror to a filter on top of a node in the "
1702 "source's backing chain");
1703 goto fail;
1704 }
1705
1706 if (backing_mode != MIRROR_LEAVE_BACKING_CHAIN) {
1707 target_perms |= BLK_PERM_GRAPH_MOD;
1708 }
1709
1710 s->target = blk_new(s->common.job.aio_context,
1711 target_perms, target_shared_perms);
1712 ret = blk_insert_bs(s->target, target, errp);
1713 if (ret < 0) {
1714 goto fail;
1715 }
1716 if (is_mirror) {
1717
1718
1719
1720
1721
1722
1723 blk_set_force_allow_inactivate(s->target);
1724 }
1725 blk_set_allow_aio_context_change(s->target, true);
1726 blk_set_disable_request_queuing(s->target, true);
1727
1728 s->replaces = g_strdup(replaces);
1729 s->on_source_error = on_source_error;
1730 s->on_target_error = on_target_error;
1731 s->is_none_mode = is_none_mode;
1732 s->backing_mode = backing_mode;
1733 s->zero_target = zero_target;
1734 s->copy_mode = copy_mode;
1735 s->base = base;
1736 s->base_overlay = bdrv_find_overlay(bs, base);
1737 s->granularity = granularity;
1738 s->buf_size = ROUND_UP(buf_size, granularity);
1739 s->unmap = unmap;
1740 if (auto_complete) {
1741 s->should_complete = true;
1742 }
1743
1744 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1745 if (!s->dirty_bitmap) {
1746 goto fail;
1747 }
1748 if (s->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1749 bdrv_disable_dirty_bitmap(s->dirty_bitmap);
1750 }
1751
1752 ret = block_job_add_bdrv(&s->common, "source", bs, 0,
1753 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
1754 BLK_PERM_CONSISTENT_READ,
1755 errp);
1756 if (ret < 0) {
1757 goto fail;
1758 }
1759
1760
1761 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1762 &error_abort);
1763
1764
1765
1766 if (target_is_backing) {
1767 BlockDriverState *iter, *filtered_target;
1768 uint64_t iter_shared_perms;
1769
1770
1771
1772
1773
1774 filtered_target = bdrv_cow_bs(bdrv_find_overlay(bs, target));
1775
1776 assert(bdrv_skip_filters(filtered_target) ==
1777 bdrv_skip_filters(target));
1778
1779
1780
1781
1782
1783
1784
1785 iter_shared_perms = BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE;
1786
1787 for (iter = bdrv_filter_or_cow_bs(bs); iter != target;
1788 iter = bdrv_filter_or_cow_bs(iter))
1789 {
1790 if (iter == filtered_target) {
1791
1792
1793
1794
1795 iter_shared_perms |= BLK_PERM_CONSISTENT_READ;
1796 }
1797
1798 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1799 iter_shared_perms, errp);
1800 if (ret < 0) {
1801 goto fail;
1802 }
1803 }
1804
1805 if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1806 goto fail;
1807 }
1808 }
1809
1810 QTAILQ_INIT(&s->ops_in_flight);
1811
1812 trace_mirror_start(bs, s, opaque);
1813 job_start(&s->common.job);
1814
1815 return &s->common;
1816
1817fail:
1818 if (s) {
1819
1820
1821 bdrv_ref(mirror_top_bs);
1822
1823 g_free(s->replaces);
1824 blk_unref(s->target);
1825 bs_opaque->job = NULL;
1826 if (s->dirty_bitmap) {
1827 bdrv_release_dirty_bitmap(s->dirty_bitmap);
1828 }
1829 job_early_fail(&s->common.job);
1830 }
1831
1832 bs_opaque->stop = true;
1833 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
1834 &error_abort);
1835 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
1836
1837 bdrv_unref(mirror_top_bs);
1838
1839 return NULL;
1840}
1841
1842void mirror_start(const char *job_id, BlockDriverState *bs,
1843 BlockDriverState *target, const char *replaces,
1844 int creation_flags, int64_t speed,
1845 uint32_t granularity, int64_t buf_size,
1846 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1847 bool zero_target,
1848 BlockdevOnError on_source_error,
1849 BlockdevOnError on_target_error,
1850 bool unmap, const char *filter_node_name,
1851 MirrorCopyMode copy_mode, Error **errp)
1852{
1853 bool is_none_mode;
1854 BlockDriverState *base;
1855
1856 if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
1857 (mode == MIRROR_SYNC_MODE_BITMAP)) {
1858 error_setg(errp, "Sync mode '%s' not supported",
1859 MirrorSyncMode_str(mode));
1860 return;
1861 }
1862 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1863 base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
1864 mirror_start_job(job_id, bs, creation_flags, target, replaces,
1865 speed, granularity, buf_size, backing_mode, zero_target,
1866 on_source_error, on_target_error, unmap, NULL, NULL,
1867 &mirror_job_driver, is_none_mode, base, false,
1868 filter_node_name, true, copy_mode, errp);
1869}
1870
1871BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
1872 BlockDriverState *base, int creation_flags,
1873 int64_t speed, BlockdevOnError on_error,
1874 const char *filter_node_name,
1875 BlockCompletionFunc *cb, void *opaque,
1876 bool auto_complete, Error **errp)
1877{
1878 bool base_read_only;
1879 BlockJob *job;
1880
1881 base_read_only = bdrv_is_read_only(base);
1882
1883 if (base_read_only) {
1884 if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1885 return NULL;
1886 }
1887 }
1888
1889 job = mirror_start_job(
1890 job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1891 MIRROR_LEAVE_BACKING_CHAIN, false,
1892 on_error, on_error, true, cb, opaque,
1893 &commit_active_job_driver, false, base, auto_complete,
1894 filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1895 errp);
1896 if (!job) {
1897 goto error_restore_flags;
1898 }
1899
1900 return job;
1901
1902error_restore_flags:
1903
1904
1905 if (base_read_only) {
1906 bdrv_reopen_set_read_only(base, true, NULL);
1907 }
1908 return NULL;
1909}
1910