1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "qemu/coroutine.h"
17#include "qemu/range.h"
18#include "trace.h"
19#include "block/blockjob_int.h"
20#include "block/block_int.h"
21#include "sysemu/block-backend.h"
22#include "qapi/error.h"
23#include "qapi/qmp/qerror.h"
24#include "qemu/ratelimit.h"
25#include "qemu/bitmap.h"
26#include "qemu/memalign.h"
27
28#define MAX_IN_FLIGHT 16
29#define MAX_IO_BYTES (1 << 20)
30#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
31
32
33
34
35typedef struct MirrorBuffer {
36 QSIMPLEQ_ENTRY(MirrorBuffer) next;
37} MirrorBuffer;
38
39typedef struct MirrorOp MirrorOp;
40
41typedef struct MirrorBlockJob {
42 BlockJob common;
43 BlockBackend *target;
44 BlockDriverState *mirror_top_bs;
45 BlockDriverState *base;
46 BlockDriverState *base_overlay;
47
48
49 char *replaces;
50
51 BlockDriverState *to_replace;
52
53 Error *replace_blocker;
54 bool is_none_mode;
55 BlockMirrorBackingMode backing_mode;
56
57 bool zero_target;
58 MirrorCopyMode copy_mode;
59 BlockdevOnError on_source_error, on_target_error;
60
61
62 bool actively_synced;
63 bool should_complete;
64 int64_t granularity;
65 size_t buf_size;
66 int64_t bdev_length;
67 unsigned long *cow_bitmap;
68 BdrvDirtyBitmap *dirty_bitmap;
69 BdrvDirtyBitmapIter *dbi;
70 uint8_t *buf;
71 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
72 int buf_free_count;
73
74 uint64_t last_pause_ns;
75 unsigned long *in_flight_bitmap;
76 unsigned in_flight;
77 int64_t bytes_in_flight;
78 QTAILQ_HEAD(, MirrorOp) ops_in_flight;
79 int ret;
80 bool unmap;
81 int target_cluster_size;
82 int max_iov;
83 bool initial_zeroing_ongoing;
84 int in_active_write_counter;
85 int64_t active_write_bytes_in_flight;
86 bool prepared;
87 bool in_drain;
88} MirrorBlockJob;
89
90typedef struct MirrorBDSOpaque {
91 MirrorBlockJob *job;
92 bool stop;
93 bool is_commit;
94} MirrorBDSOpaque;
95
96struct MirrorOp {
97 MirrorBlockJob *s;
98 QEMUIOVector qiov;
99 int64_t offset;
100 uint64_t bytes;
101
102
103
104 int64_t *bytes_handled;
105
106 bool is_pseudo_op;
107 bool is_active_write;
108 bool is_in_flight;
109 CoQueue waiting_requests;
110 Coroutine *co;
111 MirrorOp *waiting_for_op;
112
113 QTAILQ_ENTRY(MirrorOp) next;
114};
115
116typedef enum MirrorMethod {
117 MIRROR_METHOD_COPY,
118 MIRROR_METHOD_ZERO,
119 MIRROR_METHOD_DISCARD,
120} MirrorMethod;
121
122static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
123 int error)
124{
125 s->actively_synced = false;
126 if (read) {
127 return block_job_error_action(&s->common, s->on_source_error,
128 true, error);
129 } else {
130 return block_job_error_action(&s->common, s->on_target_error,
131 false, error);
132 }
133}
134
135static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
136 MirrorBlockJob *s,
137 uint64_t offset,
138 uint64_t bytes)
139{
140 uint64_t self_start_chunk = offset / s->granularity;
141 uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
142 uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
143
144 while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
145 self_start_chunk) < self_end_chunk &&
146 s->ret >= 0)
147 {
148 MirrorOp *op;
149
150 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
151 uint64_t op_start_chunk = op->offset / s->granularity;
152 uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
153 s->granularity) -
154 op_start_chunk;
155
156 if (op == self) {
157 continue;
158 }
159
160 if (ranges_overlap(self_start_chunk, self_nb_chunks,
161 op_start_chunk, op_nb_chunks))
162 {
163 if (self) {
164
165
166
167
168
169 if (op->waiting_for_op) {
170 continue;
171 }
172
173 self->waiting_for_op = op;
174 }
175
176 qemu_co_queue_wait(&op->waiting_requests, NULL);
177
178 if (self) {
179 self->waiting_for_op = NULL;
180 }
181
182 break;
183 }
184 }
185 }
186}
187
188static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
189{
190 MirrorBlockJob *s = op->s;
191 struct iovec *iov;
192 int64_t chunk_num;
193 int i, nb_chunks;
194
195 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
196
197 s->in_flight--;
198 s->bytes_in_flight -= op->bytes;
199 iov = op->qiov.iov;
200 for (i = 0; i < op->qiov.niov; i++) {
201 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
202 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
203 s->buf_free_count++;
204 }
205
206 chunk_num = op->offset / s->granularity;
207 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
208
209 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
210 QTAILQ_REMOVE(&s->ops_in_flight, op, next);
211 if (ret >= 0) {
212 if (s->cow_bitmap) {
213 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
214 }
215 if (!s->initial_zeroing_ongoing) {
216 job_progress_update(&s->common.job, op->bytes);
217 }
218 }
219 qemu_iovec_destroy(&op->qiov);
220
221 qemu_co_queue_restart_all(&op->waiting_requests);
222 g_free(op);
223}
224
225static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
226{
227 MirrorBlockJob *s = op->s;
228
229 if (ret < 0) {
230 BlockErrorAction action;
231
232 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
233 action = mirror_error_action(s, false, -ret);
234 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
235 s->ret = ret;
236 }
237 }
238
239 mirror_iteration_done(op, ret);
240}
241
242static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
243{
244 MirrorBlockJob *s = op->s;
245
246 if (ret < 0) {
247 BlockErrorAction action;
248
249 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
250 action = mirror_error_action(s, true, -ret);
251 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
252 s->ret = ret;
253 }
254
255 mirror_iteration_done(op, ret);
256 return;
257 }
258
259 ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
260 mirror_write_complete(op, ret);
261}
262
263
264static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
265 int64_t offset,
266 int64_t bytes)
267{
268 return MIN(bytes, s->bdev_length - offset);
269}
270
271
272
273static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
274 uint64_t *bytes)
275{
276 bool need_cow;
277 int ret = 0;
278 int64_t align_offset = *offset;
279 int64_t align_bytes = *bytes;
280 int max_bytes = s->granularity * s->max_iov;
281
282 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
283 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
284 s->cow_bitmap);
285 if (need_cow) {
286 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
287 &align_offset, &align_bytes);
288 }
289
290 if (align_bytes > max_bytes) {
291 align_bytes = max_bytes;
292 if (need_cow) {
293 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
294 }
295 }
296
297
298 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
299
300 ret = align_offset + align_bytes - (*offset + *bytes);
301 *offset = align_offset;
302 *bytes = align_bytes;
303 assert(ret >= 0);
304 return ret;
305}
306
307static inline void coroutine_fn
308mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
309{
310 MirrorOp *op;
311
312 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
313
314
315
316
317
318
319
320
321
322 if (!op->is_pseudo_op && op->is_in_flight && !op->is_active_write) {
323 qemu_co_queue_wait(&op->waiting_requests, NULL);
324 return;
325 }
326 }
327 abort();
328}
329
330
331
332
333
334
335
336
337
338static void coroutine_fn mirror_co_read(void *opaque)
339{
340 MirrorOp *op = opaque;
341 MirrorBlockJob *s = op->s;
342 int nb_chunks;
343 uint64_t ret;
344 uint64_t max_bytes;
345
346 max_bytes = s->granularity * s->max_iov;
347
348
349 op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
350 assert(op->bytes);
351 assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
352 *op->bytes_handled = op->bytes;
353
354 if (s->cow_bitmap) {
355 *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
356 }
357
358 assert(*op->bytes_handled <= UINT_MAX);
359 assert(op->bytes <= s->buf_size);
360
361
362
363 assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
364
365 assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
366 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
367
368 while (s->buf_free_count < nb_chunks) {
369 trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
370 mirror_wait_for_free_in_flight_slot(s);
371 }
372
373
374
375
376 qemu_iovec_init(&op->qiov, nb_chunks);
377 while (nb_chunks-- > 0) {
378 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
379 size_t remaining = op->bytes - op->qiov.size;
380
381 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
382 s->buf_free_count--;
383 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
384 }
385
386
387 s->in_flight++;
388 s->bytes_in_flight += op->bytes;
389 op->is_in_flight = true;
390 trace_mirror_one_iteration(s, op->offset, op->bytes);
391
392 ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
393 &op->qiov, 0);
394 mirror_read_complete(op, ret);
395}
396
397static void coroutine_fn mirror_co_zero(void *opaque)
398{
399 MirrorOp *op = opaque;
400 int ret;
401
402 op->s->in_flight++;
403 op->s->bytes_in_flight += op->bytes;
404 *op->bytes_handled = op->bytes;
405 op->is_in_flight = true;
406
407 ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
408 op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
409 mirror_write_complete(op, ret);
410}
411
412static void coroutine_fn mirror_co_discard(void *opaque)
413{
414 MirrorOp *op = opaque;
415 int ret;
416
417 op->s->in_flight++;
418 op->s->bytes_in_flight += op->bytes;
419 *op->bytes_handled = op->bytes;
420 op->is_in_flight = true;
421
422 ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
423 mirror_write_complete(op, ret);
424}
425
426static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
427 unsigned bytes, MirrorMethod mirror_method)
428{
429 MirrorOp *op;
430 Coroutine *co;
431 int64_t bytes_handled = -1;
432
433 op = g_new(MirrorOp, 1);
434 *op = (MirrorOp){
435 .s = s,
436 .offset = offset,
437 .bytes = bytes,
438 .bytes_handled = &bytes_handled,
439 };
440 qemu_co_queue_init(&op->waiting_requests);
441
442 switch (mirror_method) {
443 case MIRROR_METHOD_COPY:
444 co = qemu_coroutine_create(mirror_co_read, op);
445 break;
446 case MIRROR_METHOD_ZERO:
447 co = qemu_coroutine_create(mirror_co_zero, op);
448 break;
449 case MIRROR_METHOD_DISCARD:
450 co = qemu_coroutine_create(mirror_co_discard, op);
451 break;
452 default:
453 abort();
454 }
455 op->co = co;
456
457 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
458 qemu_coroutine_enter(co);
459
460
461
462
463 assert(bytes_handled >= 0);
464
465
466
467
468 assert(bytes_handled <= UINT_MAX);
469 return bytes_handled;
470}
471
472static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
473{
474 BlockDriverState *source = s->mirror_top_bs->backing->bs;
475 MirrorOp *pseudo_op;
476 int64_t offset;
477 uint64_t delay_ns = 0, ret = 0;
478
479 int nb_chunks = 1;
480 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
481 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
482
483 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
484 offset = bdrv_dirty_iter_next(s->dbi);
485 if (offset < 0) {
486 bdrv_set_dirty_iter(s->dbi, 0);
487 offset = bdrv_dirty_iter_next(s->dbi);
488 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
489 assert(offset >= 0);
490 }
491 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
492
493
494
495
496
497
498
499
500 mirror_wait_on_conflicts(NULL, s, offset, 1);
501
502 job_pause_point(&s->common.job);
503
504
505
506 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
507 while (nb_chunks * s->granularity < s->buf_size) {
508 int64_t next_dirty;
509 int64_t next_offset = offset + nb_chunks * s->granularity;
510 int64_t next_chunk = next_offset / s->granularity;
511 if (next_offset >= s->bdev_length ||
512 !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
513 break;
514 }
515 if (test_bit(next_chunk, s->in_flight_bitmap)) {
516 break;
517 }
518
519 next_dirty = bdrv_dirty_iter_next(s->dbi);
520 if (next_dirty > next_offset || next_dirty < 0) {
521
522 bdrv_set_dirty_iter(s->dbi, next_offset);
523 next_dirty = bdrv_dirty_iter_next(s->dbi);
524 }
525 assert(next_dirty == next_offset);
526 nb_chunks++;
527 }
528
529
530
531
532
533 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
534 nb_chunks * s->granularity);
535 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
536
537
538
539
540
541
542
543 pseudo_op = g_new(MirrorOp, 1);
544 *pseudo_op = (MirrorOp){
545 .offset = offset,
546 .bytes = nb_chunks * s->granularity,
547 .is_pseudo_op = true,
548 };
549 qemu_co_queue_init(&pseudo_op->waiting_requests);
550 QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
551
552 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
553 while (nb_chunks > 0 && offset < s->bdev_length) {
554 int ret;
555 int64_t io_bytes;
556 int64_t io_bytes_acct;
557 MirrorMethod mirror_method = MIRROR_METHOD_COPY;
558
559 assert(!(offset % s->granularity));
560 ret = bdrv_block_status_above(source, NULL, offset,
561 nb_chunks * s->granularity,
562 &io_bytes, NULL, NULL);
563 if (ret < 0) {
564 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
565 } else if (ret & BDRV_BLOCK_DATA) {
566 io_bytes = MIN(io_bytes, max_io_bytes);
567 }
568
569 io_bytes -= io_bytes % s->granularity;
570 if (io_bytes < s->granularity) {
571 io_bytes = s->granularity;
572 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
573 int64_t target_offset;
574 int64_t target_bytes;
575 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
576 &target_offset, &target_bytes);
577 if (target_offset == offset &&
578 target_bytes == io_bytes) {
579 mirror_method = ret & BDRV_BLOCK_ZERO ?
580 MIRROR_METHOD_ZERO :
581 MIRROR_METHOD_DISCARD;
582 }
583 }
584
585 while (s->in_flight >= MAX_IN_FLIGHT) {
586 trace_mirror_yield_in_flight(s, offset, s->in_flight);
587 mirror_wait_for_free_in_flight_slot(s);
588 }
589
590 if (s->ret < 0) {
591 ret = 0;
592 goto fail;
593 }
594
595 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
596 io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
597 if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
598 io_bytes_acct = 0;
599 } else {
600 io_bytes_acct = io_bytes;
601 }
602 assert(io_bytes);
603 offset += io_bytes;
604 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
605 delay_ns = block_job_ratelimit_get_delay(&s->common, io_bytes_acct);
606 }
607
608 ret = delay_ns;
609fail:
610 QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
611 qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
612 g_free(pseudo_op);
613
614 return ret;
615}
616
617static void mirror_free_init(MirrorBlockJob *s)
618{
619 int granularity = s->granularity;
620 size_t buf_size = s->buf_size;
621 uint8_t *buf = s->buf;
622
623 assert(s->buf_free_count == 0);
624 QSIMPLEQ_INIT(&s->buf_free);
625 while (buf_size != 0) {
626 MirrorBuffer *cur = (MirrorBuffer *)buf;
627 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
628 s->buf_free_count++;
629 buf_size -= granularity;
630 buf += granularity;
631 }
632}
633
634
635
636
637
638static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
639{
640 while (s->in_flight > 0) {
641 mirror_wait_for_free_in_flight_slot(s);
642 }
643}
644
645
646
647
648
649
650static int mirror_exit_common(Job *job)
651{
652 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
653 BlockJob *bjob = &s->common;
654 MirrorBDSOpaque *bs_opaque;
655 AioContext *replace_aio_context = NULL;
656 BlockDriverState *src;
657 BlockDriverState *target_bs;
658 BlockDriverState *mirror_top_bs;
659 Error *local_err = NULL;
660 bool abort = job->ret < 0;
661 int ret = 0;
662
663 if (s->prepared) {
664 return 0;
665 }
666 s->prepared = true;
667
668 mirror_top_bs = s->mirror_top_bs;
669 bs_opaque = mirror_top_bs->opaque;
670 src = mirror_top_bs->backing->bs;
671 target_bs = blk_bs(s->target);
672
673 if (bdrv_chain_contains(src, target_bs)) {
674 bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
675 }
676
677 bdrv_release_dirty_bitmap(s->dirty_bitmap);
678
679
680
681 bdrv_ref(src);
682 bdrv_ref(mirror_top_bs);
683 bdrv_ref(target_bs);
684
685
686
687
688
689
690 blk_unref(s->target);
691 s->target = NULL;
692
693
694
695
696
697 bdrv_drained_begin(mirror_top_bs);
698 bs_opaque->stop = true;
699 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
700 &error_abort);
701 if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
702 BlockDriverState *backing = s->is_none_mode ? src : s->base;
703 BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
704
705 if (bdrv_cow_bs(unfiltered_target) != backing) {
706 bdrv_set_backing_hd(unfiltered_target, backing, &local_err);
707 if (local_err) {
708 error_report_err(local_err);
709 local_err = NULL;
710 ret = -EPERM;
711 }
712 }
713 } else if (!abort && s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
714 assert(!bdrv_backing_chain_next(target_bs));
715 ret = bdrv_open_backing_file(bdrv_skip_filters(target_bs), NULL,
716 "backing", &local_err);
717 if (ret < 0) {
718 error_report_err(local_err);
719 local_err = NULL;
720 }
721 }
722
723 if (s->to_replace) {
724 replace_aio_context = bdrv_get_aio_context(s->to_replace);
725 aio_context_acquire(replace_aio_context);
726 }
727
728 if (s->should_complete && !abort) {
729 BlockDriverState *to_replace = s->to_replace ?: src;
730 bool ro = bdrv_is_read_only(to_replace);
731
732 if (ro != bdrv_is_read_only(target_bs)) {
733 bdrv_reopen_set_read_only(target_bs, ro, NULL);
734 }
735
736
737
738 assert(s->in_drain);
739 bdrv_drained_begin(target_bs);
740
741
742
743
744
745 if (bdrv_recurse_can_replace(src, to_replace)) {
746 bdrv_replace_node(to_replace, target_bs, &local_err);
747 } else {
748 error_setg(&local_err, "Can no longer replace '%s' by '%s', "
749 "because it can no longer be guaranteed that doing so "
750 "would not lead to an abrupt change of visible data",
751 to_replace->node_name, target_bs->node_name);
752 }
753 bdrv_drained_end(target_bs);
754 if (local_err) {
755 error_report_err(local_err);
756 ret = -EPERM;
757 }
758 }
759 if (s->to_replace) {
760 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
761 error_free(s->replace_blocker);
762 bdrv_unref(s->to_replace);
763 }
764 if (replace_aio_context) {
765 aio_context_release(replace_aio_context);
766 }
767 g_free(s->replaces);
768 bdrv_unref(target_bs);
769
770
771
772
773
774
775 block_job_remove_all_bdrv(bjob);
776 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
777
778 bs_opaque->job = NULL;
779
780 bdrv_drained_end(src);
781 bdrv_drained_end(mirror_top_bs);
782 s->in_drain = false;
783 bdrv_unref(mirror_top_bs);
784 bdrv_unref(src);
785
786 return ret;
787}
788
789static int mirror_prepare(Job *job)
790{
791 return mirror_exit_common(job);
792}
793
794static void mirror_abort(Job *job)
795{
796 int ret = mirror_exit_common(job);
797 assert(ret == 0);
798}
799
800static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
801{
802 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
803
804 if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
805 s->last_pause_ns = now;
806 job_sleep_ns(&s->common.job, 0);
807 } else {
808 job_pause_point(&s->common.job);
809 }
810}
811
812static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
813{
814 int64_t offset;
815 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
816 BlockDriverState *target_bs = blk_bs(s->target);
817 int ret;
818 int64_t count;
819
820 if (s->zero_target) {
821 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
822 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
823 return 0;
824 }
825
826 s->initial_zeroing_ongoing = true;
827 for (offset = 0; offset < s->bdev_length; ) {
828 int bytes = MIN(s->bdev_length - offset,
829 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
830
831 mirror_throttle(s);
832
833 if (job_is_cancelled(&s->common.job)) {
834 s->initial_zeroing_ongoing = false;
835 return 0;
836 }
837
838 if (s->in_flight >= MAX_IN_FLIGHT) {
839 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
840 s->in_flight);
841 mirror_wait_for_free_in_flight_slot(s);
842 continue;
843 }
844
845 mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
846 offset += bytes;
847 }
848
849 mirror_wait_for_all_io(s);
850 s->initial_zeroing_ongoing = false;
851 }
852
853
854 for (offset = 0; offset < s->bdev_length; ) {
855
856 int bytes = MIN(s->bdev_length - offset,
857 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
858
859 mirror_throttle(s);
860
861 if (job_is_cancelled(&s->common.job)) {
862 return 0;
863 }
864
865 ret = bdrv_is_allocated_above(bs, s->base_overlay, true, offset, bytes,
866 &count);
867 if (ret < 0) {
868 return ret;
869 }
870
871 assert(count);
872 if (ret > 0) {
873 bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
874 }
875 offset += count;
876 }
877 return 0;
878}
879
880
881
882
883static int mirror_flush(MirrorBlockJob *s)
884{
885 int ret = blk_flush(s->target);
886 if (ret < 0) {
887 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
888 s->ret = ret;
889 }
890 }
891 return ret;
892}
893
894static int coroutine_fn mirror_run(Job *job, Error **errp)
895{
896 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
897 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
898 BlockDriverState *target_bs = blk_bs(s->target);
899 bool need_drain = true;
900 BlockDeviceIoStatus iostatus;
901 int64_t length;
902 int64_t target_length;
903 BlockDriverInfo bdi;
904 char backing_filename[2];
905
906 int ret = 0;
907
908 if (job_is_cancelled(&s->common.job)) {
909 goto immediate_exit;
910 }
911
912 s->bdev_length = bdrv_getlength(bs);
913 if (s->bdev_length < 0) {
914 ret = s->bdev_length;
915 goto immediate_exit;
916 }
917
918 target_length = blk_getlength(s->target);
919 if (target_length < 0) {
920 ret = target_length;
921 goto immediate_exit;
922 }
923
924
925
926 if (s->base == blk_bs(s->target)) {
927 if (s->bdev_length > target_length) {
928 ret = blk_co_truncate(s->target, s->bdev_length, false,
929 PREALLOC_MODE_OFF, 0, NULL);
930 if (ret < 0) {
931 goto immediate_exit;
932 }
933 }
934 } else if (s->bdev_length != target_length) {
935 error_setg(errp, "Source and target image have different sizes");
936 ret = -EINVAL;
937 goto immediate_exit;
938 }
939
940 if (s->bdev_length == 0) {
941
942 job_transition_to_ready(&s->common.job);
943 s->actively_synced = true;
944 while (!job_cancel_requested(&s->common.job) && !s->should_complete) {
945 job_yield(&s->common.job);
946 }
947 goto immediate_exit;
948 }
949
950 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
951 s->in_flight_bitmap = bitmap_new(length);
952
953
954
955
956
957 bdrv_get_backing_filename(target_bs, backing_filename,
958 sizeof(backing_filename));
959 if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
960 s->target_cluster_size = bdi.cluster_size;
961 } else {
962 s->target_cluster_size = BDRV_SECTOR_SIZE;
963 }
964 if (backing_filename[0] && !bdrv_backing_chain_next(target_bs) &&
965 s->granularity < s->target_cluster_size) {
966 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
967 s->cow_bitmap = bitmap_new(length);
968 }
969 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
970
971 s->buf = qemu_try_blockalign(bs, s->buf_size);
972 if (s->buf == NULL) {
973 ret = -ENOMEM;
974 goto immediate_exit;
975 }
976
977 mirror_free_init(s);
978
979 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
980 if (!s->is_none_mode) {
981 ret = mirror_dirty_init(s);
982 if (ret < 0 || job_is_cancelled(&s->common.job)) {
983 goto immediate_exit;
984 }
985 }
986
987 assert(!s->dbi);
988 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
989 for (;;) {
990 uint64_t delay_ns = 0;
991 int64_t cnt, delta;
992 bool should_complete;
993
994 if (s->ret < 0) {
995 ret = s->ret;
996 goto immediate_exit;
997 }
998
999 job_pause_point(&s->common.job);
1000
1001 if (job_is_cancelled(&s->common.job)) {
1002 ret = 0;
1003 goto immediate_exit;
1004 }
1005
1006 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1007
1008
1009
1010 job_progress_set_remaining(&s->common.job,
1011 s->bytes_in_flight + cnt +
1012 s->active_write_bytes_in_flight);
1013
1014
1015
1016
1017
1018 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
1019 WITH_JOB_LOCK_GUARD() {
1020 iostatus = s->common.iostatus;
1021 }
1022 if (delta < BLOCK_JOB_SLICE_TIME &&
1023 iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1024 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
1025 (cnt == 0 && s->in_flight > 0)) {
1026 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
1027 mirror_wait_for_free_in_flight_slot(s);
1028 continue;
1029 } else if (cnt != 0) {
1030 delay_ns = mirror_iteration(s);
1031 }
1032 }
1033
1034 should_complete = false;
1035 if (s->in_flight == 0 && cnt == 0) {
1036 trace_mirror_before_flush(s);
1037 if (!job_is_ready(&s->common.job)) {
1038 if (mirror_flush(s) < 0) {
1039
1040 continue;
1041 }
1042
1043
1044
1045
1046
1047 job_transition_to_ready(&s->common.job);
1048 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
1049 s->actively_synced = true;
1050 }
1051 }
1052
1053 should_complete = s->should_complete ||
1054 job_cancel_requested(&s->common.job);
1055 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1056 }
1057
1058 if (cnt == 0 && should_complete) {
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069 trace_mirror_before_drain(s, cnt);
1070
1071 s->in_drain = true;
1072 bdrv_drained_begin(bs);
1073
1074
1075 assert(s->in_active_write_counter == 0);
1076
1077 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1078 if (cnt > 0 || mirror_flush(s) < 0) {
1079 bdrv_drained_end(bs);
1080 s->in_drain = false;
1081 continue;
1082 }
1083
1084
1085
1086
1087 assert(QLIST_EMPTY(&bs->tracked_requests));
1088 need_drain = false;
1089 break;
1090 }
1091
1092 if (job_is_ready(&s->common.job) && !should_complete) {
1093 delay_ns = (s->in_flight == 0 &&
1094 cnt == 0 ? BLOCK_JOB_SLICE_TIME : 0);
1095 }
1096 trace_mirror_before_sleep(s, cnt, job_is_ready(&s->common.job),
1097 delay_ns);
1098 job_sleep_ns(&s->common.job, delay_ns);
1099 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1100 }
1101
1102immediate_exit:
1103 if (s->in_flight > 0) {
1104
1105
1106
1107
1108 assert(ret < 0 || job_is_cancelled(&s->common.job));
1109 assert(need_drain);
1110 mirror_wait_for_all_io(s);
1111 }
1112
1113 assert(s->in_flight == 0);
1114 qemu_vfree(s->buf);
1115 g_free(s->cow_bitmap);
1116 g_free(s->in_flight_bitmap);
1117 bdrv_dirty_iter_free(s->dbi);
1118
1119 if (need_drain) {
1120 s->in_drain = true;
1121 bdrv_drained_begin(bs);
1122 }
1123
1124 return ret;
1125}
1126
1127static void mirror_complete(Job *job, Error **errp)
1128{
1129 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1130
1131 if (!job_is_ready(job)) {
1132 error_setg(errp, "The active block job '%s' cannot be completed",
1133 job->id);
1134 return;
1135 }
1136
1137
1138 if (s->replaces) {
1139 AioContext *replace_aio_context;
1140
1141 s->to_replace = bdrv_find_node(s->replaces);
1142 if (!s->to_replace) {
1143 error_setg(errp, "Node name '%s' not found", s->replaces);
1144 return;
1145 }
1146
1147 replace_aio_context = bdrv_get_aio_context(s->to_replace);
1148 aio_context_acquire(replace_aio_context);
1149
1150
1151 error_setg(&s->replace_blocker,
1152 "block device is in use by block-job-complete");
1153 bdrv_op_block_all(s->to_replace, s->replace_blocker);
1154 bdrv_ref(s->to_replace);
1155
1156 aio_context_release(replace_aio_context);
1157 }
1158
1159 s->should_complete = true;
1160
1161
1162 WITH_JOB_LOCK_GUARD() {
1163 if (!job->paused) {
1164 job_enter_cond_locked(job, NULL);
1165 }
1166 }
1167}
1168
1169static void coroutine_fn mirror_pause(Job *job)
1170{
1171 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1172
1173 mirror_wait_for_all_io(s);
1174}
1175
1176static bool mirror_drained_poll(BlockJob *job)
1177{
1178 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1179
1180
1181
1182
1183
1184
1185 WITH_JOB_LOCK_GUARD() {
1186 if (!s->common.job.paused && !job_is_cancelled_locked(&job->job)
1187 && !s->in_drain) {
1188 return true;
1189 }
1190 }
1191
1192 return !!s->in_flight;
1193}
1194
1195static bool mirror_cancel(Job *job, bool force)
1196{
1197 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1198 BlockDriverState *target = blk_bs(s->target);
1199
1200
1201
1202
1203
1204 force = force || !job_is_ready(job);
1205
1206 if (force) {
1207 bdrv_cancel_in_flight(target);
1208 }
1209 return force;
1210}
1211
1212static bool commit_active_cancel(Job *job, bool force)
1213{
1214
1215 return force || !job_is_ready(job);
1216}
1217
1218static const BlockJobDriver mirror_job_driver = {
1219 .job_driver = {
1220 .instance_size = sizeof(MirrorBlockJob),
1221 .job_type = JOB_TYPE_MIRROR,
1222 .free = block_job_free,
1223 .user_resume = block_job_user_resume,
1224 .run = mirror_run,
1225 .prepare = mirror_prepare,
1226 .abort = mirror_abort,
1227 .pause = mirror_pause,
1228 .complete = mirror_complete,
1229 .cancel = mirror_cancel,
1230 },
1231 .drained_poll = mirror_drained_poll,
1232};
1233
1234static const BlockJobDriver commit_active_job_driver = {
1235 .job_driver = {
1236 .instance_size = sizeof(MirrorBlockJob),
1237 .job_type = JOB_TYPE_COMMIT,
1238 .free = block_job_free,
1239 .user_resume = block_job_user_resume,
1240 .run = mirror_run,
1241 .prepare = mirror_prepare,
1242 .abort = mirror_abort,
1243 .pause = mirror_pause,
1244 .complete = mirror_complete,
1245 .cancel = commit_active_cancel,
1246 },
1247 .drained_poll = mirror_drained_poll,
1248};
1249
1250static void coroutine_fn
1251do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1252 uint64_t offset, uint64_t bytes,
1253 QEMUIOVector *qiov, int flags)
1254{
1255 int ret;
1256 size_t qiov_offset = 0;
1257 int64_t bitmap_offset, bitmap_end;
1258
1259 if (!QEMU_IS_ALIGNED(offset, job->granularity) &&
1260 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset))
1261 {
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279 qiov_offset = QEMU_ALIGN_UP(offset, job->granularity) - offset;
1280 if (bytes <= qiov_offset) {
1281
1282 return;
1283 }
1284 offset += qiov_offset;
1285 bytes -= qiov_offset;
1286 }
1287
1288 if (!QEMU_IS_ALIGNED(offset + bytes, job->granularity) &&
1289 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset + bytes - 1))
1290 {
1291 uint64_t tail = (offset + bytes) % job->granularity;
1292
1293 if (bytes <= tail) {
1294
1295 return;
1296 }
1297 bytes -= tail;
1298 }
1299
1300
1301
1302
1303
1304 bitmap_offset = QEMU_ALIGN_UP(offset, job->granularity);
1305 bitmap_end = QEMU_ALIGN_DOWN(offset + bytes, job->granularity);
1306 if (bitmap_offset < bitmap_end) {
1307 bdrv_reset_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1308 bitmap_end - bitmap_offset);
1309 }
1310
1311 job_progress_increase_remaining(&job->common.job, bytes);
1312 job->active_write_bytes_in_flight += bytes;
1313
1314 switch (method) {
1315 case MIRROR_METHOD_COPY:
1316 ret = blk_co_pwritev_part(job->target, offset, bytes,
1317 qiov, qiov_offset, flags);
1318 break;
1319
1320 case MIRROR_METHOD_ZERO:
1321 assert(!qiov);
1322 ret = blk_co_pwrite_zeroes(job->target, offset, bytes, flags);
1323 break;
1324
1325 case MIRROR_METHOD_DISCARD:
1326 assert(!qiov);
1327 ret = blk_co_pdiscard(job->target, offset, bytes);
1328 break;
1329
1330 default:
1331 abort();
1332 }
1333
1334 job->active_write_bytes_in_flight -= bytes;
1335 if (ret >= 0) {
1336 job_progress_update(&job->common.job, bytes);
1337 } else {
1338 BlockErrorAction action;
1339
1340
1341
1342
1343
1344
1345
1346 bitmap_offset = QEMU_ALIGN_DOWN(offset, job->granularity);
1347 bitmap_end = QEMU_ALIGN_UP(offset + bytes, job->granularity);
1348 bdrv_set_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1349 bitmap_end - bitmap_offset);
1350 job->actively_synced = false;
1351
1352 action = mirror_error_action(job, false, -ret);
1353 if (action == BLOCK_ERROR_ACTION_REPORT) {
1354 if (!job->ret) {
1355 job->ret = ret;
1356 }
1357 }
1358 }
1359}
1360
1361static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1362 uint64_t offset,
1363 uint64_t bytes)
1364{
1365 MirrorOp *op;
1366 uint64_t start_chunk = offset / s->granularity;
1367 uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1368
1369 op = g_new(MirrorOp, 1);
1370 *op = (MirrorOp){
1371 .s = s,
1372 .offset = offset,
1373 .bytes = bytes,
1374 .is_active_write = true,
1375 .is_in_flight = true,
1376 .co = qemu_coroutine_self(),
1377 };
1378 qemu_co_queue_init(&op->waiting_requests);
1379 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1380
1381 s->in_active_write_counter++;
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396 mirror_wait_on_conflicts(op, s, offset, bytes);
1397
1398 bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1399
1400 return op;
1401}
1402
1403static void coroutine_fn active_write_settle(MirrorOp *op)
1404{
1405 uint64_t start_chunk = op->offset / op->s->granularity;
1406 uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1407 op->s->granularity);
1408
1409 if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1410 BdrvChild *source = op->s->mirror_top_bs->backing;
1411
1412 if (QLIST_FIRST(&source->bs->parents) == source &&
1413 QLIST_NEXT(source, next_parent) == NULL)
1414 {
1415
1416
1417
1418
1419 assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1420 }
1421 }
1422 bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1423 QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1424 qemu_co_queue_restart_all(&op->waiting_requests);
1425 g_free(op);
1426}
1427
1428static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
1429 int64_t offset, int64_t bytes, QEMUIOVector *qiov, BdrvRequestFlags flags)
1430{
1431 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1432}
1433
1434static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs,
1435 MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1436 int flags)
1437{
1438 MirrorOp *op = NULL;
1439 MirrorBDSOpaque *s = bs->opaque;
1440 int ret = 0;
1441 bool copy_to_target = false;
1442
1443 if (s->job) {
1444 copy_to_target = s->job->ret >= 0 &&
1445 !job_is_cancelled(&s->job->common.job) &&
1446 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1447 }
1448
1449 if (copy_to_target) {
1450 op = active_write_prepare(s->job, offset, bytes);
1451 }
1452
1453 switch (method) {
1454 case MIRROR_METHOD_COPY:
1455 ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1456 break;
1457
1458 case MIRROR_METHOD_ZERO:
1459 ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1460 break;
1461
1462 case MIRROR_METHOD_DISCARD:
1463 ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1464 break;
1465
1466 default:
1467 abort();
1468 }
1469
1470 if (ret < 0) {
1471 goto out;
1472 }
1473
1474 if (copy_to_target) {
1475 do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1476 }
1477
1478out:
1479 if (copy_to_target) {
1480 active_write_settle(op);
1481 }
1482 return ret;
1483}
1484
1485static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
1486 int64_t offset, int64_t bytes, QEMUIOVector *qiov, BdrvRequestFlags flags)
1487{
1488 MirrorBDSOpaque *s = bs->opaque;
1489 QEMUIOVector bounce_qiov;
1490 void *bounce_buf;
1491 int ret = 0;
1492 bool copy_to_target = false;
1493
1494 if (s->job) {
1495 copy_to_target = s->job->ret >= 0 &&
1496 !job_is_cancelled(&s->job->common.job) &&
1497 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1498 }
1499
1500 if (copy_to_target) {
1501
1502
1503
1504
1505 bounce_buf = qemu_blockalign(bs, bytes);
1506 iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1507
1508 qemu_iovec_init(&bounce_qiov, 1);
1509 qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1510 qiov = &bounce_qiov;
1511
1512 flags &= ~BDRV_REQ_REGISTERED_BUF;
1513 }
1514
1515 ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1516 flags);
1517
1518 if (copy_to_target) {
1519 qemu_iovec_destroy(&bounce_qiov);
1520 qemu_vfree(bounce_buf);
1521 }
1522
1523 return ret;
1524}
1525
1526static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
1527{
1528 if (bs->backing == NULL) {
1529
1530 return 0;
1531 }
1532 return bdrv_co_flush(bs->backing->bs);
1533}
1534
1535static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1536 int64_t offset, int64_t bytes, BdrvRequestFlags flags)
1537{
1538 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1539 flags);
1540}
1541
1542static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1543 int64_t offset, int64_t bytes)
1544{
1545 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1546 NULL, 0);
1547}
1548
1549static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1550{
1551 if (bs->backing == NULL) {
1552
1553
1554 return;
1555 }
1556 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1557 bs->backing->bs->filename);
1558}
1559
1560static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1561 BdrvChildRole role,
1562 BlockReopenQueue *reopen_queue,
1563 uint64_t perm, uint64_t shared,
1564 uint64_t *nperm, uint64_t *nshared)
1565{
1566 MirrorBDSOpaque *s = bs->opaque;
1567
1568 if (s->stop) {
1569
1570
1571
1572
1573 *nperm = 0;
1574 *nshared = BLK_PERM_ALL;
1575 return;
1576 }
1577
1578 bdrv_default_perms(bs, c, role, reopen_queue,
1579 perm, shared, nperm, nshared);
1580
1581 if (s->is_commit) {
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596 *nperm &= ~BLK_PERM_CONSISTENT_READ;
1597 *nshared |= BLK_PERM_WRITE;
1598 }
1599}
1600
1601
1602
1603static BlockDriver bdrv_mirror_top = {
1604 .format_name = "mirror_top",
1605 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1606 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1607 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1608 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1609 .bdrv_co_flush = bdrv_mirror_top_flush,
1610 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1611 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1612
1613 .is_filter = true,
1614 .filtered_child_is_backing = true,
1615};
1616
1617static BlockJob *mirror_start_job(
1618 const char *job_id, BlockDriverState *bs,
1619 int creation_flags, BlockDriverState *target,
1620 const char *replaces, int64_t speed,
1621 uint32_t granularity, int64_t buf_size,
1622 BlockMirrorBackingMode backing_mode,
1623 bool zero_target,
1624 BlockdevOnError on_source_error,
1625 BlockdevOnError on_target_error,
1626 bool unmap,
1627 BlockCompletionFunc *cb,
1628 void *opaque,
1629 const BlockJobDriver *driver,
1630 bool is_none_mode, BlockDriverState *base,
1631 bool auto_complete, const char *filter_node_name,
1632 bool is_mirror, MirrorCopyMode copy_mode,
1633 Error **errp)
1634{
1635 MirrorBlockJob *s;
1636 MirrorBDSOpaque *bs_opaque;
1637 BlockDriverState *mirror_top_bs;
1638 bool target_is_backing;
1639 uint64_t target_perms, target_shared_perms;
1640 int ret;
1641
1642 if (granularity == 0) {
1643 granularity = bdrv_get_default_bitmap_granularity(target);
1644 }
1645
1646 assert(is_power_of_2(granularity));
1647
1648 if (buf_size < 0) {
1649 error_setg(errp, "Invalid parameter 'buf-size'");
1650 return NULL;
1651 }
1652
1653 if (buf_size == 0) {
1654 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1655 }
1656
1657 if (bdrv_skip_filters(bs) == bdrv_skip_filters(target)) {
1658 error_setg(errp, "Can't mirror node into itself");
1659 return NULL;
1660 }
1661
1662 target_is_backing = bdrv_chain_contains(bs, target);
1663
1664
1665
1666
1667 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1668 BDRV_O_RDWR, errp);
1669 if (mirror_top_bs == NULL) {
1670 return NULL;
1671 }
1672 if (!filter_node_name) {
1673 mirror_top_bs->implicit = true;
1674 }
1675
1676
1677 mirror_top_bs->never_freeze = true;
1678
1679 mirror_top_bs->total_sectors = bs->total_sectors;
1680 mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1681 mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
1682 BDRV_REQ_NO_FALLBACK;
1683 bs_opaque = g_new0(MirrorBDSOpaque, 1);
1684 mirror_top_bs->opaque = bs_opaque;
1685
1686 bs_opaque->is_commit = target_is_backing;
1687
1688 bdrv_drained_begin(bs);
1689 ret = bdrv_append(mirror_top_bs, bs, errp);
1690 bdrv_drained_end(bs);
1691
1692 if (ret < 0) {
1693 bdrv_unref(mirror_top_bs);
1694 return NULL;
1695 }
1696
1697
1698 s = block_job_create(job_id, driver, NULL, mirror_top_bs,
1699 BLK_PERM_CONSISTENT_READ,
1700 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1701 BLK_PERM_WRITE, speed,
1702 creation_flags, cb, opaque, errp);
1703 if (!s) {
1704 goto fail;
1705 }
1706 bs_opaque->job = s;
1707
1708
1709 bdrv_unref(mirror_top_bs);
1710
1711 s->mirror_top_bs = mirror_top_bs;
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722 target_perms = BLK_PERM_WRITE;
1723 target_shared_perms = BLK_PERM_WRITE_UNCHANGED;
1724
1725 if (target_is_backing) {
1726 int64_t bs_size, target_size;
1727 bs_size = bdrv_getlength(bs);
1728 if (bs_size < 0) {
1729 error_setg_errno(errp, -bs_size,
1730 "Could not inquire top image size");
1731 goto fail;
1732 }
1733
1734 target_size = bdrv_getlength(target);
1735 if (target_size < 0) {
1736 error_setg_errno(errp, -target_size,
1737 "Could not inquire base image size");
1738 goto fail;
1739 }
1740
1741 if (target_size < bs_size) {
1742 target_perms |= BLK_PERM_RESIZE;
1743 }
1744
1745 target_shared_perms |= BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE;
1746 } else if (bdrv_chain_contains(bs, bdrv_skip_filters(target))) {
1747
1748
1749
1750
1751 error_setg(errp, "Cannot mirror to a filter on top of a node in the "
1752 "source's backing chain");
1753 goto fail;
1754 }
1755
1756 s->target = blk_new(s->common.job.aio_context,
1757 target_perms, target_shared_perms);
1758 ret = blk_insert_bs(s->target, target, errp);
1759 if (ret < 0) {
1760 goto fail;
1761 }
1762 if (is_mirror) {
1763
1764
1765
1766
1767
1768
1769 blk_set_force_allow_inactivate(s->target);
1770 }
1771 blk_set_allow_aio_context_change(s->target, true);
1772 blk_set_disable_request_queuing(s->target, true);
1773
1774 s->replaces = g_strdup(replaces);
1775 s->on_source_error = on_source_error;
1776 s->on_target_error = on_target_error;
1777 s->is_none_mode = is_none_mode;
1778 s->backing_mode = backing_mode;
1779 s->zero_target = zero_target;
1780 s->copy_mode = copy_mode;
1781 s->base = base;
1782 s->base_overlay = bdrv_find_overlay(bs, base);
1783 s->granularity = granularity;
1784 s->buf_size = ROUND_UP(buf_size, granularity);
1785 s->unmap = unmap;
1786 if (auto_complete) {
1787 s->should_complete = true;
1788 }
1789
1790 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1791 if (!s->dirty_bitmap) {
1792 goto fail;
1793 }
1794 if (s->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1795 bdrv_disable_dirty_bitmap(s->dirty_bitmap);
1796 }
1797
1798 ret = block_job_add_bdrv(&s->common, "source", bs, 0,
1799 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
1800 BLK_PERM_CONSISTENT_READ,
1801 errp);
1802 if (ret < 0) {
1803 goto fail;
1804 }
1805
1806
1807 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1808 &error_abort);
1809
1810
1811
1812 if (target_is_backing) {
1813 BlockDriverState *iter, *filtered_target;
1814 uint64_t iter_shared_perms;
1815
1816
1817
1818
1819
1820 filtered_target = bdrv_cow_bs(bdrv_find_overlay(bs, target));
1821
1822 assert(bdrv_skip_filters(filtered_target) ==
1823 bdrv_skip_filters(target));
1824
1825
1826
1827
1828
1829
1830
1831 iter_shared_perms = BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE;
1832
1833 for (iter = bdrv_filter_or_cow_bs(bs); iter != target;
1834 iter = bdrv_filter_or_cow_bs(iter))
1835 {
1836 if (iter == filtered_target) {
1837
1838
1839
1840
1841 iter_shared_perms |= BLK_PERM_CONSISTENT_READ;
1842 }
1843
1844 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1845 iter_shared_perms, errp);
1846 if (ret < 0) {
1847 goto fail;
1848 }
1849 }
1850
1851 if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1852 goto fail;
1853 }
1854 }
1855
1856 QTAILQ_INIT(&s->ops_in_flight);
1857
1858 trace_mirror_start(bs, s, opaque);
1859 job_start(&s->common.job);
1860
1861 return &s->common;
1862
1863fail:
1864 if (s) {
1865
1866
1867 bdrv_ref(mirror_top_bs);
1868
1869 g_free(s->replaces);
1870 blk_unref(s->target);
1871 bs_opaque->job = NULL;
1872 if (s->dirty_bitmap) {
1873 bdrv_release_dirty_bitmap(s->dirty_bitmap);
1874 }
1875 job_early_fail(&s->common.job);
1876 }
1877
1878 bs_opaque->stop = true;
1879 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
1880 &error_abort);
1881 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
1882
1883 bdrv_unref(mirror_top_bs);
1884
1885 return NULL;
1886}
1887
1888void mirror_start(const char *job_id, BlockDriverState *bs,
1889 BlockDriverState *target, const char *replaces,
1890 int creation_flags, int64_t speed,
1891 uint32_t granularity, int64_t buf_size,
1892 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1893 bool zero_target,
1894 BlockdevOnError on_source_error,
1895 BlockdevOnError on_target_error,
1896 bool unmap, const char *filter_node_name,
1897 MirrorCopyMode copy_mode, Error **errp)
1898{
1899 bool is_none_mode;
1900 BlockDriverState *base;
1901
1902 GLOBAL_STATE_CODE();
1903
1904 if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
1905 (mode == MIRROR_SYNC_MODE_BITMAP)) {
1906 error_setg(errp, "Sync mode '%s' not supported",
1907 MirrorSyncMode_str(mode));
1908 return;
1909 }
1910 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1911 base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
1912 mirror_start_job(job_id, bs, creation_flags, target, replaces,
1913 speed, granularity, buf_size, backing_mode, zero_target,
1914 on_source_error, on_target_error, unmap, NULL, NULL,
1915 &mirror_job_driver, is_none_mode, base, false,
1916 filter_node_name, true, copy_mode, errp);
1917}
1918
1919BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
1920 BlockDriverState *base, int creation_flags,
1921 int64_t speed, BlockdevOnError on_error,
1922 const char *filter_node_name,
1923 BlockCompletionFunc *cb, void *opaque,
1924 bool auto_complete, Error **errp)
1925{
1926 bool base_read_only;
1927 BlockJob *job;
1928
1929 GLOBAL_STATE_CODE();
1930
1931 base_read_only = bdrv_is_read_only(base);
1932
1933 if (base_read_only) {
1934 if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1935 return NULL;
1936 }
1937 }
1938
1939 job = mirror_start_job(
1940 job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1941 MIRROR_LEAVE_BACKING_CHAIN, false,
1942 on_error, on_error, true, cb, opaque,
1943 &commit_active_job_driver, false, base, auto_complete,
1944 filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1945 errp);
1946 if (!job) {
1947 goto error_restore_flags;
1948 }
1949
1950 return job;
1951
1952error_restore_flags:
1953
1954
1955 if (base_read_only) {
1956 bdrv_reopen_set_read_only(base, true, NULL);
1957 }
1958 return NULL;
1959}
1960