1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "qemu/coroutine.h"
17#include "qemu/range.h"
18#include "trace.h"
19#include "block/blockjob_int.h"
20#include "block/block_int.h"
21#include "sysemu/block-backend.h"
22#include "qapi/error.h"
23#include "qapi/qmp/qerror.h"
24#include "qemu/ratelimit.h"
25#include "qemu/bitmap.h"
26
27#define MAX_IN_FLIGHT 16
28#define MAX_IO_BYTES (1 << 20)
29#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
30
31
32
33
34typedef struct MirrorBuffer {
35 QSIMPLEQ_ENTRY(MirrorBuffer) next;
36} MirrorBuffer;
37
38typedef struct MirrorOp MirrorOp;
39
40typedef struct MirrorBlockJob {
41 BlockJob common;
42 BlockBackend *target;
43 BlockDriverState *mirror_top_bs;
44 BlockDriverState *base;
45
46
47 char *replaces;
48
49 BlockDriverState *to_replace;
50
51 Error *replace_blocker;
52 bool is_none_mode;
53 BlockMirrorBackingMode backing_mode;
54 MirrorCopyMode copy_mode;
55 BlockdevOnError on_source_error, on_target_error;
56 bool synced;
57
58
59 bool actively_synced;
60 bool should_complete;
61 int64_t granularity;
62 size_t buf_size;
63 int64_t bdev_length;
64 unsigned long *cow_bitmap;
65 BdrvDirtyBitmap *dirty_bitmap;
66 BdrvDirtyBitmapIter *dbi;
67 uint8_t *buf;
68 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
69 int buf_free_count;
70
71 uint64_t last_pause_ns;
72 unsigned long *in_flight_bitmap;
73 int in_flight;
74 int64_t bytes_in_flight;
75 QTAILQ_HEAD(, MirrorOp) ops_in_flight;
76 int ret;
77 bool unmap;
78 int target_cluster_size;
79 int max_iov;
80 bool initial_zeroing_ongoing;
81 int in_active_write_counter;
82 bool prepared;
83 bool in_drain;
84} MirrorBlockJob;
85
86typedef struct MirrorBDSOpaque {
87 MirrorBlockJob *job;
88} MirrorBDSOpaque;
89
90struct MirrorOp {
91 MirrorBlockJob *s;
92 QEMUIOVector qiov;
93 int64_t offset;
94 uint64_t bytes;
95
96
97
98 int64_t *bytes_handled;
99
100 bool is_pseudo_op;
101 bool is_active_write;
102 CoQueue waiting_requests;
103
104 QTAILQ_ENTRY(MirrorOp) next;
105};
106
107typedef enum MirrorMethod {
108 MIRROR_METHOD_COPY,
109 MIRROR_METHOD_ZERO,
110 MIRROR_METHOD_DISCARD,
111} MirrorMethod;
112
113static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
114 int error)
115{
116 s->synced = false;
117 s->actively_synced = false;
118 if (read) {
119 return block_job_error_action(&s->common, s->on_source_error,
120 true, error);
121 } else {
122 return block_job_error_action(&s->common, s->on_target_error,
123 false, error);
124 }
125}
126
127static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
128 MirrorBlockJob *s,
129 uint64_t offset,
130 uint64_t bytes)
131{
132 uint64_t self_start_chunk = offset / s->granularity;
133 uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
134 uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
135
136 while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
137 self_start_chunk) < self_end_chunk &&
138 s->ret >= 0)
139 {
140 MirrorOp *op;
141
142 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
143 uint64_t op_start_chunk = op->offset / s->granularity;
144 uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
145 s->granularity) -
146 op_start_chunk;
147
148 if (op == self) {
149 continue;
150 }
151
152 if (ranges_overlap(self_start_chunk, self_nb_chunks,
153 op_start_chunk, op_nb_chunks))
154 {
155 qemu_co_queue_wait(&op->waiting_requests, NULL);
156 break;
157 }
158 }
159 }
160}
161
162static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
163{
164 MirrorBlockJob *s = op->s;
165 struct iovec *iov;
166 int64_t chunk_num;
167 int i, nb_chunks;
168
169 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
170
171 s->in_flight--;
172 s->bytes_in_flight -= op->bytes;
173 iov = op->qiov.iov;
174 for (i = 0; i < op->qiov.niov; i++) {
175 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
176 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
177 s->buf_free_count++;
178 }
179
180 chunk_num = op->offset / s->granularity;
181 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
182
183 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
184 QTAILQ_REMOVE(&s->ops_in_flight, op, next);
185 if (ret >= 0) {
186 if (s->cow_bitmap) {
187 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
188 }
189 if (!s->initial_zeroing_ongoing) {
190 job_progress_update(&s->common.job, op->bytes);
191 }
192 }
193 qemu_iovec_destroy(&op->qiov);
194
195 qemu_co_queue_restart_all(&op->waiting_requests);
196 g_free(op);
197}
198
199static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
200{
201 MirrorBlockJob *s = op->s;
202
203 if (ret < 0) {
204 BlockErrorAction action;
205
206 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
207 action = mirror_error_action(s, false, -ret);
208 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
209 s->ret = ret;
210 }
211 }
212
213 mirror_iteration_done(op, ret);
214}
215
216static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
217{
218 MirrorBlockJob *s = op->s;
219
220 if (ret < 0) {
221 BlockErrorAction action;
222
223 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
224 action = mirror_error_action(s, true, -ret);
225 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
226 s->ret = ret;
227 }
228
229 mirror_iteration_done(op, ret);
230 return;
231 }
232
233 ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
234 mirror_write_complete(op, ret);
235}
236
237
238static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
239 int64_t offset,
240 int64_t bytes)
241{
242 return MIN(bytes, s->bdev_length - offset);
243}
244
245
246
247static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
248 uint64_t *bytes)
249{
250 bool need_cow;
251 int ret = 0;
252 int64_t align_offset = *offset;
253 int64_t align_bytes = *bytes;
254 int max_bytes = s->granularity * s->max_iov;
255
256 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
257 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
258 s->cow_bitmap);
259 if (need_cow) {
260 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
261 &align_offset, &align_bytes);
262 }
263
264 if (align_bytes > max_bytes) {
265 align_bytes = max_bytes;
266 if (need_cow) {
267 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
268 }
269 }
270
271
272 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
273
274 ret = align_offset + align_bytes - (*offset + *bytes);
275 *offset = align_offset;
276 *bytes = align_bytes;
277 assert(ret >= 0);
278 return ret;
279}
280
281static inline void coroutine_fn
282mirror_wait_for_any_operation(MirrorBlockJob *s, bool active)
283{
284 MirrorOp *op;
285
286 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
287
288
289
290
291
292 if (!op->is_pseudo_op && op->is_active_write == active) {
293 qemu_co_queue_wait(&op->waiting_requests, NULL);
294 return;
295 }
296 }
297 abort();
298}
299
300static inline void coroutine_fn
301mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
302{
303
304 mirror_wait_for_any_operation(s, false);
305}
306
307
308
309
310
311
312
313
314
315static void coroutine_fn mirror_co_read(void *opaque)
316{
317 MirrorOp *op = opaque;
318 MirrorBlockJob *s = op->s;
319 int nb_chunks;
320 uint64_t ret;
321 uint64_t max_bytes;
322
323 max_bytes = s->granularity * s->max_iov;
324
325
326 op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
327 assert(op->bytes);
328 assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
329 *op->bytes_handled = op->bytes;
330
331 if (s->cow_bitmap) {
332 *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
333 }
334
335 assert(*op->bytes_handled <= UINT_MAX);
336 assert(op->bytes <= s->buf_size);
337
338
339
340 assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
341
342 assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
343 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
344
345 while (s->buf_free_count < nb_chunks) {
346 trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
347 mirror_wait_for_free_in_flight_slot(s);
348 }
349
350
351
352
353 qemu_iovec_init(&op->qiov, nb_chunks);
354 while (nb_chunks-- > 0) {
355 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
356 size_t remaining = op->bytes - op->qiov.size;
357
358 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
359 s->buf_free_count--;
360 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
361 }
362
363
364 s->in_flight++;
365 s->bytes_in_flight += op->bytes;
366 trace_mirror_one_iteration(s, op->offset, op->bytes);
367
368 ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
369 &op->qiov, 0);
370 mirror_read_complete(op, ret);
371}
372
373static void coroutine_fn mirror_co_zero(void *opaque)
374{
375 MirrorOp *op = opaque;
376 int ret;
377
378 op->s->in_flight++;
379 op->s->bytes_in_flight += op->bytes;
380 *op->bytes_handled = op->bytes;
381
382 ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
383 op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
384 mirror_write_complete(op, ret);
385}
386
387static void coroutine_fn mirror_co_discard(void *opaque)
388{
389 MirrorOp *op = opaque;
390 int ret;
391
392 op->s->in_flight++;
393 op->s->bytes_in_flight += op->bytes;
394 *op->bytes_handled = op->bytes;
395
396 ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
397 mirror_write_complete(op, ret);
398}
399
400static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
401 unsigned bytes, MirrorMethod mirror_method)
402{
403 MirrorOp *op;
404 Coroutine *co;
405 int64_t bytes_handled = -1;
406
407 op = g_new(MirrorOp, 1);
408 *op = (MirrorOp){
409 .s = s,
410 .offset = offset,
411 .bytes = bytes,
412 .bytes_handled = &bytes_handled,
413 };
414 qemu_co_queue_init(&op->waiting_requests);
415
416 switch (mirror_method) {
417 case MIRROR_METHOD_COPY:
418 co = qemu_coroutine_create(mirror_co_read, op);
419 break;
420 case MIRROR_METHOD_ZERO:
421 co = qemu_coroutine_create(mirror_co_zero, op);
422 break;
423 case MIRROR_METHOD_DISCARD:
424 co = qemu_coroutine_create(mirror_co_discard, op);
425 break;
426 default:
427 abort();
428 }
429
430 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
431 qemu_coroutine_enter(co);
432
433
434
435
436 assert(bytes_handled >= 0);
437
438
439
440
441 assert(bytes_handled <= UINT_MAX);
442 return bytes_handled;
443}
444
445static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
446{
447 BlockDriverState *source = s->mirror_top_bs->backing->bs;
448 MirrorOp *pseudo_op;
449 int64_t offset;
450 uint64_t delay_ns = 0, ret = 0;
451
452 int nb_chunks = 1;
453 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
454 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
455
456 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
457 offset = bdrv_dirty_iter_next(s->dbi);
458 if (offset < 0) {
459 bdrv_set_dirty_iter(s->dbi, 0);
460 offset = bdrv_dirty_iter_next(s->dbi);
461 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
462 assert(offset >= 0);
463 }
464 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
465
466 mirror_wait_on_conflicts(NULL, s, offset, 1);
467
468 job_pause_point(&s->common.job);
469
470
471
472 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
473 while (nb_chunks * s->granularity < s->buf_size) {
474 int64_t next_dirty;
475 int64_t next_offset = offset + nb_chunks * s->granularity;
476 int64_t next_chunk = next_offset / s->granularity;
477 if (next_offset >= s->bdev_length ||
478 !bdrv_get_dirty_locked(source, s->dirty_bitmap, next_offset)) {
479 break;
480 }
481 if (test_bit(next_chunk, s->in_flight_bitmap)) {
482 break;
483 }
484
485 next_dirty = bdrv_dirty_iter_next(s->dbi);
486 if (next_dirty > next_offset || next_dirty < 0) {
487
488 bdrv_set_dirty_iter(s->dbi, next_offset);
489 next_dirty = bdrv_dirty_iter_next(s->dbi);
490 }
491 assert(next_dirty == next_offset);
492 nb_chunks++;
493 }
494
495
496
497
498
499 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
500 nb_chunks * s->granularity);
501 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
502
503
504
505
506
507
508
509 pseudo_op = g_new(MirrorOp, 1);
510 *pseudo_op = (MirrorOp){
511 .offset = offset,
512 .bytes = nb_chunks * s->granularity,
513 .is_pseudo_op = true,
514 };
515 qemu_co_queue_init(&pseudo_op->waiting_requests);
516 QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
517
518 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
519 while (nb_chunks > 0 && offset < s->bdev_length) {
520 int ret;
521 int64_t io_bytes;
522 int64_t io_bytes_acct;
523 MirrorMethod mirror_method = MIRROR_METHOD_COPY;
524
525 assert(!(offset % s->granularity));
526 ret = bdrv_block_status_above(source, NULL, offset,
527 nb_chunks * s->granularity,
528 &io_bytes, NULL, NULL);
529 if (ret < 0) {
530 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
531 } else if (ret & BDRV_BLOCK_DATA) {
532 io_bytes = MIN(io_bytes, max_io_bytes);
533 }
534
535 io_bytes -= io_bytes % s->granularity;
536 if (io_bytes < s->granularity) {
537 io_bytes = s->granularity;
538 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
539 int64_t target_offset;
540 int64_t target_bytes;
541 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
542 &target_offset, &target_bytes);
543 if (target_offset == offset &&
544 target_bytes == io_bytes) {
545 mirror_method = ret & BDRV_BLOCK_ZERO ?
546 MIRROR_METHOD_ZERO :
547 MIRROR_METHOD_DISCARD;
548 }
549 }
550
551 while (s->in_flight >= MAX_IN_FLIGHT) {
552 trace_mirror_yield_in_flight(s, offset, s->in_flight);
553 mirror_wait_for_free_in_flight_slot(s);
554 }
555
556 if (s->ret < 0) {
557 ret = 0;
558 goto fail;
559 }
560
561 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
562 io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
563 if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
564 io_bytes_acct = 0;
565 } else {
566 io_bytes_acct = io_bytes;
567 }
568 assert(io_bytes);
569 offset += io_bytes;
570 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
571 delay_ns = block_job_ratelimit_get_delay(&s->common, io_bytes_acct);
572 }
573
574 ret = delay_ns;
575fail:
576 QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
577 qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
578 g_free(pseudo_op);
579
580 return ret;
581}
582
583static void mirror_free_init(MirrorBlockJob *s)
584{
585 int granularity = s->granularity;
586 size_t buf_size = s->buf_size;
587 uint8_t *buf = s->buf;
588
589 assert(s->buf_free_count == 0);
590 QSIMPLEQ_INIT(&s->buf_free);
591 while (buf_size != 0) {
592 MirrorBuffer *cur = (MirrorBuffer *)buf;
593 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
594 s->buf_free_count++;
595 buf_size -= granularity;
596 buf += granularity;
597 }
598}
599
600
601
602
603
604static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
605{
606 while (s->in_flight > 0) {
607 mirror_wait_for_free_in_flight_slot(s);
608 }
609}
610
611
612
613
614
615
616static int mirror_exit_common(Job *job)
617{
618 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
619 BlockJob *bjob = &s->common;
620 MirrorBDSOpaque *bs_opaque = s->mirror_top_bs->opaque;
621 AioContext *replace_aio_context = NULL;
622 BlockDriverState *src = s->mirror_top_bs->backing->bs;
623 BlockDriverState *target_bs = blk_bs(s->target);
624 BlockDriverState *mirror_top_bs = s->mirror_top_bs;
625 Error *local_err = NULL;
626 bool abort = job->ret < 0;
627 int ret = 0;
628
629 if (s->prepared) {
630 return 0;
631 }
632 s->prepared = true;
633
634 if (bdrv_chain_contains(src, target_bs)) {
635 bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
636 }
637
638 bdrv_release_dirty_bitmap(src, s->dirty_bitmap);
639
640
641
642 bdrv_ref(src);
643 bdrv_ref(mirror_top_bs);
644 bdrv_ref(target_bs);
645
646
647
648
649
650
651
652
653 blk_set_perm(s->target, 0, BLK_PERM_ALL, &error_abort);
654 blk_unref(s->target);
655 s->target = NULL;
656
657
658
659 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
660 &error_abort);
661 if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
662 BlockDriverState *backing = s->is_none_mode ? src : s->base;
663 if (backing_bs(target_bs) != backing) {
664 bdrv_set_backing_hd(target_bs, backing, &local_err);
665 if (local_err) {
666 error_report_err(local_err);
667 ret = -EPERM;
668 }
669 }
670 }
671
672 if (s->to_replace) {
673 replace_aio_context = bdrv_get_aio_context(s->to_replace);
674 aio_context_acquire(replace_aio_context);
675 }
676
677 if (s->should_complete && !abort) {
678 BlockDriverState *to_replace = s->to_replace ?: src;
679 bool ro = bdrv_is_read_only(to_replace);
680
681 if (ro != bdrv_is_read_only(target_bs)) {
682 bdrv_reopen_set_read_only(target_bs, ro, NULL);
683 }
684
685
686
687 assert(s->in_drain);
688 bdrv_drained_begin(target_bs);
689 bdrv_replace_node(to_replace, target_bs, &local_err);
690 bdrv_drained_end(target_bs);
691 if (local_err) {
692 error_report_err(local_err);
693 ret = -EPERM;
694 }
695 }
696 if (s->to_replace) {
697 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
698 error_free(s->replace_blocker);
699 bdrv_unref(s->to_replace);
700 }
701 if (replace_aio_context) {
702 aio_context_release(replace_aio_context);
703 }
704 g_free(s->replaces);
705 bdrv_unref(target_bs);
706
707
708
709
710
711 block_job_remove_all_bdrv(bjob);
712 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
713 &error_abort);
714 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
715
716
717
718
719 blk_remove_bs(bjob->blk);
720 blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort);
721 blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort);
722
723 bs_opaque->job = NULL;
724
725 bdrv_drained_end(src);
726 s->in_drain = false;
727 bdrv_unref(mirror_top_bs);
728 bdrv_unref(src);
729
730 return ret;
731}
732
733static int mirror_prepare(Job *job)
734{
735 return mirror_exit_common(job);
736}
737
738static void mirror_abort(Job *job)
739{
740 int ret = mirror_exit_common(job);
741 assert(ret == 0);
742}
743
744static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
745{
746 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
747
748 if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
749 s->last_pause_ns = now;
750 job_sleep_ns(&s->common.job, 0);
751 } else {
752 job_pause_point(&s->common.job);
753 }
754}
755
756static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
757{
758 int64_t offset;
759 BlockDriverState *base = s->base;
760 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
761 BlockDriverState *target_bs = blk_bs(s->target);
762 int ret;
763 int64_t count;
764
765 if (base == NULL && !bdrv_has_zero_init(target_bs)) {
766 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
767 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
768 return 0;
769 }
770
771 s->initial_zeroing_ongoing = true;
772 for (offset = 0; offset < s->bdev_length; ) {
773 int bytes = MIN(s->bdev_length - offset,
774 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
775
776 mirror_throttle(s);
777
778 if (job_is_cancelled(&s->common.job)) {
779 s->initial_zeroing_ongoing = false;
780 return 0;
781 }
782
783 if (s->in_flight >= MAX_IN_FLIGHT) {
784 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
785 s->in_flight);
786 mirror_wait_for_free_in_flight_slot(s);
787 continue;
788 }
789
790 mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
791 offset += bytes;
792 }
793
794 mirror_wait_for_all_io(s);
795 s->initial_zeroing_ongoing = false;
796 }
797
798
799 for (offset = 0; offset < s->bdev_length; ) {
800
801 int bytes = MIN(s->bdev_length - offset,
802 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
803
804 mirror_throttle(s);
805
806 if (job_is_cancelled(&s->common.job)) {
807 return 0;
808 }
809
810 ret = bdrv_is_allocated_above(bs, base, offset, bytes, &count);
811 if (ret < 0) {
812 return ret;
813 }
814
815 assert(count);
816 if (ret == 1) {
817 bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
818 }
819 offset += count;
820 }
821 return 0;
822}
823
824
825
826
827static int mirror_flush(MirrorBlockJob *s)
828{
829 int ret = blk_flush(s->target);
830 if (ret < 0) {
831 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
832 s->ret = ret;
833 }
834 }
835 return ret;
836}
837
838static int coroutine_fn mirror_run(Job *job, Error **errp)
839{
840 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
841 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
842 BlockDriverState *target_bs = blk_bs(s->target);
843 bool need_drain = true;
844 int64_t length;
845 BlockDriverInfo bdi;
846 char backing_filename[2];
847
848 int ret = 0;
849
850 if (job_is_cancelled(&s->common.job)) {
851 goto immediate_exit;
852 }
853
854 s->bdev_length = bdrv_getlength(bs);
855 if (s->bdev_length < 0) {
856 ret = s->bdev_length;
857 goto immediate_exit;
858 }
859
860
861
862 if (s->base == blk_bs(s->target)) {
863 int64_t base_length;
864
865 base_length = blk_getlength(s->target);
866 if (base_length < 0) {
867 ret = base_length;
868 goto immediate_exit;
869 }
870
871 if (s->bdev_length > base_length) {
872 ret = blk_truncate(s->target, s->bdev_length, PREALLOC_MODE_OFF,
873 NULL);
874 if (ret < 0) {
875 goto immediate_exit;
876 }
877 }
878 }
879
880 if (s->bdev_length == 0) {
881
882 job_transition_to_ready(&s->common.job);
883 s->synced = true;
884 s->actively_synced = true;
885 while (!job_is_cancelled(&s->common.job) && !s->should_complete) {
886 job_yield(&s->common.job);
887 }
888 s->common.job.cancelled = false;
889 goto immediate_exit;
890 }
891
892 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
893 s->in_flight_bitmap = bitmap_new(length);
894
895
896
897
898
899 bdrv_get_backing_filename(target_bs, backing_filename,
900 sizeof(backing_filename));
901 if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
902 s->target_cluster_size = bdi.cluster_size;
903 } else {
904 s->target_cluster_size = BDRV_SECTOR_SIZE;
905 }
906 if (backing_filename[0] && !target_bs->backing &&
907 s->granularity < s->target_cluster_size) {
908 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
909 s->cow_bitmap = bitmap_new(length);
910 }
911 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
912
913 s->buf = qemu_try_blockalign(bs, s->buf_size);
914 if (s->buf == NULL) {
915 ret = -ENOMEM;
916 goto immediate_exit;
917 }
918
919 mirror_free_init(s);
920
921 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
922 if (!s->is_none_mode) {
923 ret = mirror_dirty_init(s);
924 if (ret < 0 || job_is_cancelled(&s->common.job)) {
925 goto immediate_exit;
926 }
927 }
928
929 assert(!s->dbi);
930 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
931 for (;;) {
932 uint64_t delay_ns = 0;
933 int64_t cnt, delta;
934 bool should_complete;
935
936
937
938 while (s->in_active_write_counter) {
939 mirror_wait_for_any_operation(s, true);
940 }
941
942 if (s->ret < 0) {
943 ret = s->ret;
944 goto immediate_exit;
945 }
946
947 job_pause_point(&s->common.job);
948
949 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
950
951
952
953 job_progress_set_remaining(&s->common.job, s->bytes_in_flight + cnt);
954
955
956
957
958
959 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
960 if (delta < BLOCK_JOB_SLICE_TIME &&
961 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
962 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
963 (cnt == 0 && s->in_flight > 0)) {
964 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
965 mirror_wait_for_free_in_flight_slot(s);
966 continue;
967 } else if (cnt != 0) {
968 delay_ns = mirror_iteration(s);
969 }
970 }
971
972 should_complete = false;
973 if (s->in_flight == 0 && cnt == 0) {
974 trace_mirror_before_flush(s);
975 if (!s->synced) {
976 if (mirror_flush(s) < 0) {
977
978 continue;
979 }
980
981
982
983
984
985 job_transition_to_ready(&s->common.job);
986 s->synced = true;
987 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
988 s->actively_synced = true;
989 }
990 }
991
992 should_complete = s->should_complete ||
993 job_is_cancelled(&s->common.job);
994 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
995 }
996
997 if (cnt == 0 && should_complete) {
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008 trace_mirror_before_drain(s, cnt);
1009
1010 s->in_drain = true;
1011 bdrv_drained_begin(bs);
1012 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1013 if (cnt > 0 || mirror_flush(s) < 0) {
1014 bdrv_drained_end(bs);
1015 s->in_drain = false;
1016 continue;
1017 }
1018
1019
1020
1021
1022 assert(QLIST_EMPTY(&bs->tracked_requests));
1023 s->common.job.cancelled = false;
1024 need_drain = false;
1025 break;
1026 }
1027
1028 ret = 0;
1029
1030 if (s->synced && !should_complete) {
1031 delay_ns = (s->in_flight == 0 &&
1032 cnt == 0 ? BLOCK_JOB_SLICE_TIME : 0);
1033 }
1034 trace_mirror_before_sleep(s, cnt, s->synced, delay_ns);
1035 job_sleep_ns(&s->common.job, delay_ns);
1036 if (job_is_cancelled(&s->common.job) &&
1037 (!s->synced || s->common.job.force_cancel))
1038 {
1039 break;
1040 }
1041 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1042 }
1043
1044immediate_exit:
1045 if (s->in_flight > 0) {
1046
1047
1048
1049
1050 assert(ret < 0 || ((s->common.job.force_cancel || !s->synced) &&
1051 job_is_cancelled(&s->common.job)));
1052 assert(need_drain);
1053 mirror_wait_for_all_io(s);
1054 }
1055
1056 assert(s->in_flight == 0);
1057 qemu_vfree(s->buf);
1058 g_free(s->cow_bitmap);
1059 g_free(s->in_flight_bitmap);
1060 bdrv_dirty_iter_free(s->dbi);
1061
1062 if (need_drain) {
1063 s->in_drain = true;
1064 bdrv_drained_begin(bs);
1065 }
1066
1067 return ret;
1068}
1069
1070static void mirror_complete(Job *job, Error **errp)
1071{
1072 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1073 BlockDriverState *target;
1074
1075 target = blk_bs(s->target);
1076
1077 if (!s->synced) {
1078 error_setg(errp, "The active block job '%s' cannot be completed",
1079 job->id);
1080 return;
1081 }
1082
1083 if (s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
1084 int ret;
1085
1086 assert(!target->backing);
1087 ret = bdrv_open_backing_file(target, NULL, "backing", errp);
1088 if (ret < 0) {
1089 return;
1090 }
1091 }
1092
1093
1094 if (s->replaces) {
1095 AioContext *replace_aio_context;
1096
1097 s->to_replace = bdrv_find_node(s->replaces);
1098 if (!s->to_replace) {
1099 error_setg(errp, "Node name '%s' not found", s->replaces);
1100 return;
1101 }
1102
1103 replace_aio_context = bdrv_get_aio_context(s->to_replace);
1104 aio_context_acquire(replace_aio_context);
1105
1106
1107
1108
1109
1110 error_setg(&s->replace_blocker,
1111 "block device is in use by block-job-complete");
1112 bdrv_op_block_all(s->to_replace, s->replace_blocker);
1113 bdrv_ref(s->to_replace);
1114
1115 aio_context_release(replace_aio_context);
1116 }
1117
1118 s->should_complete = true;
1119 job_enter(job);
1120}
1121
1122static void coroutine_fn mirror_pause(Job *job)
1123{
1124 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1125
1126 mirror_wait_for_all_io(s);
1127}
1128
1129static bool mirror_drained_poll(BlockJob *job)
1130{
1131 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1132
1133
1134
1135
1136
1137
1138 if (!s->common.job.paused && !s->common.job.cancelled && !s->in_drain) {
1139 return true;
1140 }
1141
1142 return !!s->in_flight;
1143}
1144
1145static void mirror_attached_aio_context(BlockJob *job, AioContext *new_context)
1146{
1147 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1148
1149 blk_set_aio_context(s->target, new_context);
1150}
1151
1152static void mirror_drain(BlockJob *job)
1153{
1154 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1155
1156
1157
1158
1159 if (s->target) {
1160 BlockBackend *target = s->target;
1161 blk_ref(target);
1162 blk_drain(target);
1163 blk_unref(target);
1164 }
1165}
1166
1167static const BlockJobDriver mirror_job_driver = {
1168 .job_driver = {
1169 .instance_size = sizeof(MirrorBlockJob),
1170 .job_type = JOB_TYPE_MIRROR,
1171 .free = block_job_free,
1172 .user_resume = block_job_user_resume,
1173 .drain = block_job_drain,
1174 .run = mirror_run,
1175 .prepare = mirror_prepare,
1176 .abort = mirror_abort,
1177 .pause = mirror_pause,
1178 .complete = mirror_complete,
1179 },
1180 .drained_poll = mirror_drained_poll,
1181 .attached_aio_context = mirror_attached_aio_context,
1182 .drain = mirror_drain,
1183};
1184
1185static const BlockJobDriver commit_active_job_driver = {
1186 .job_driver = {
1187 .instance_size = sizeof(MirrorBlockJob),
1188 .job_type = JOB_TYPE_COMMIT,
1189 .free = block_job_free,
1190 .user_resume = block_job_user_resume,
1191 .drain = block_job_drain,
1192 .run = mirror_run,
1193 .prepare = mirror_prepare,
1194 .abort = mirror_abort,
1195 .pause = mirror_pause,
1196 .complete = mirror_complete,
1197 },
1198 .drained_poll = mirror_drained_poll,
1199 .attached_aio_context = mirror_attached_aio_context,
1200 .drain = mirror_drain,
1201};
1202
1203static void coroutine_fn
1204do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1205 uint64_t offset, uint64_t bytes,
1206 QEMUIOVector *qiov, int flags)
1207{
1208 QEMUIOVector target_qiov;
1209 uint64_t dirty_offset = offset;
1210 uint64_t dirty_bytes;
1211
1212 if (qiov) {
1213 qemu_iovec_init(&target_qiov, qiov->niov);
1214 }
1215
1216 while (true) {
1217 bool valid_area;
1218 int ret;
1219
1220 bdrv_dirty_bitmap_lock(job->dirty_bitmap);
1221 dirty_bytes = MIN(offset + bytes - dirty_offset, INT_MAX);
1222 valid_area = bdrv_dirty_bitmap_next_dirty_area(job->dirty_bitmap,
1223 &dirty_offset,
1224 &dirty_bytes);
1225 if (!valid_area) {
1226 bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1227 break;
1228 }
1229
1230 bdrv_reset_dirty_bitmap_locked(job->dirty_bitmap,
1231 dirty_offset, dirty_bytes);
1232 bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1233
1234 job_progress_increase_remaining(&job->common.job, dirty_bytes);
1235
1236 assert(dirty_offset - offset <= SIZE_MAX);
1237 if (qiov) {
1238 qemu_iovec_reset(&target_qiov);
1239 qemu_iovec_concat(&target_qiov, qiov,
1240 dirty_offset - offset, dirty_bytes);
1241 }
1242
1243 switch (method) {
1244 case MIRROR_METHOD_COPY:
1245 ret = blk_co_pwritev(job->target, dirty_offset, dirty_bytes,
1246 qiov ? &target_qiov : NULL, flags);
1247 break;
1248
1249 case MIRROR_METHOD_ZERO:
1250 assert(!qiov);
1251 ret = blk_co_pwrite_zeroes(job->target, dirty_offset, dirty_bytes,
1252 flags);
1253 break;
1254
1255 case MIRROR_METHOD_DISCARD:
1256 assert(!qiov);
1257 ret = blk_co_pdiscard(job->target, dirty_offset, dirty_bytes);
1258 break;
1259
1260 default:
1261 abort();
1262 }
1263
1264 if (ret >= 0) {
1265 job_progress_update(&job->common.job, dirty_bytes);
1266 } else {
1267 BlockErrorAction action;
1268
1269 bdrv_set_dirty_bitmap(job->dirty_bitmap, dirty_offset, dirty_bytes);
1270 job->actively_synced = false;
1271
1272 action = mirror_error_action(job, false, -ret);
1273 if (action == BLOCK_ERROR_ACTION_REPORT) {
1274 if (!job->ret) {
1275 job->ret = ret;
1276 }
1277 break;
1278 }
1279 }
1280
1281 dirty_offset += dirty_bytes;
1282 }
1283
1284 if (qiov) {
1285 qemu_iovec_destroy(&target_qiov);
1286 }
1287}
1288
1289static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1290 uint64_t offset,
1291 uint64_t bytes)
1292{
1293 MirrorOp *op;
1294 uint64_t start_chunk = offset / s->granularity;
1295 uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1296
1297 op = g_new(MirrorOp, 1);
1298 *op = (MirrorOp){
1299 .s = s,
1300 .offset = offset,
1301 .bytes = bytes,
1302 .is_active_write = true,
1303 };
1304 qemu_co_queue_init(&op->waiting_requests);
1305 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1306
1307 s->in_active_write_counter++;
1308
1309 mirror_wait_on_conflicts(op, s, offset, bytes);
1310
1311 bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1312
1313 return op;
1314}
1315
1316static void coroutine_fn active_write_settle(MirrorOp *op)
1317{
1318 uint64_t start_chunk = op->offset / op->s->granularity;
1319 uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1320 op->s->granularity);
1321
1322 if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1323 BdrvChild *source = op->s->mirror_top_bs->backing;
1324
1325 if (QLIST_FIRST(&source->bs->parents) == source &&
1326 QLIST_NEXT(source, next_parent) == NULL)
1327 {
1328
1329
1330
1331
1332 assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1333 }
1334 }
1335 bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1336 QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1337 qemu_co_queue_restart_all(&op->waiting_requests);
1338 g_free(op);
1339}
1340
1341static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
1342 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1343{
1344 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1345}
1346
1347static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs,
1348 MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1349 int flags)
1350{
1351 MirrorOp *op = NULL;
1352 MirrorBDSOpaque *s = bs->opaque;
1353 int ret = 0;
1354 bool copy_to_target;
1355
1356 copy_to_target = s->job->ret >= 0 &&
1357 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1358
1359 if (copy_to_target) {
1360 op = active_write_prepare(s->job, offset, bytes);
1361 }
1362
1363 switch (method) {
1364 case MIRROR_METHOD_COPY:
1365 ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1366 break;
1367
1368 case MIRROR_METHOD_ZERO:
1369 ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1370 break;
1371
1372 case MIRROR_METHOD_DISCARD:
1373 ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1374 break;
1375
1376 default:
1377 abort();
1378 }
1379
1380 if (ret < 0) {
1381 goto out;
1382 }
1383
1384 if (copy_to_target) {
1385 do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1386 }
1387
1388out:
1389 if (copy_to_target) {
1390 active_write_settle(op);
1391 }
1392 return ret;
1393}
1394
1395static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
1396 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1397{
1398 MirrorBDSOpaque *s = bs->opaque;
1399 QEMUIOVector bounce_qiov;
1400 void *bounce_buf;
1401 int ret = 0;
1402 bool copy_to_target;
1403
1404 copy_to_target = s->job->ret >= 0 &&
1405 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1406
1407 if (copy_to_target) {
1408
1409
1410
1411
1412 bounce_buf = qemu_blockalign(bs, bytes);
1413 iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1414
1415 qemu_iovec_init(&bounce_qiov, 1);
1416 qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1417 qiov = &bounce_qiov;
1418 }
1419
1420 ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1421 flags);
1422
1423 if (copy_to_target) {
1424 qemu_iovec_destroy(&bounce_qiov);
1425 qemu_vfree(bounce_buf);
1426 }
1427
1428 return ret;
1429}
1430
1431static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
1432{
1433 if (bs->backing == NULL) {
1434
1435 return 0;
1436 }
1437 return bdrv_co_flush(bs->backing->bs);
1438}
1439
1440static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1441 int64_t offset, int bytes, BdrvRequestFlags flags)
1442{
1443 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1444 flags);
1445}
1446
1447static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1448 int64_t offset, int bytes)
1449{
1450 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1451 NULL, 0);
1452}
1453
1454static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1455{
1456 if (bs->backing == NULL) {
1457
1458
1459 return;
1460 }
1461 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1462 bs->backing->bs->filename);
1463}
1464
1465static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1466 const BdrvChildRole *role,
1467 BlockReopenQueue *reopen_queue,
1468 uint64_t perm, uint64_t shared,
1469 uint64_t *nperm, uint64_t *nshared)
1470{
1471
1472 *nperm = 0;
1473 if (perm & BLK_PERM_WRITE) {
1474 *nperm |= BLK_PERM_WRITE;
1475 }
1476
1477 *nshared = BLK_PERM_ALL;
1478}
1479
1480static void bdrv_mirror_top_refresh_limits(BlockDriverState *bs, Error **errp)
1481{
1482 MirrorBDSOpaque *s = bs->opaque;
1483
1484 if (s && s->job && s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1485 bs->bl.request_alignment = s->job->granularity;
1486 }
1487}
1488
1489
1490
1491static BlockDriver bdrv_mirror_top = {
1492 .format_name = "mirror_top",
1493 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1494 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1495 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1496 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1497 .bdrv_co_flush = bdrv_mirror_top_flush,
1498 .bdrv_co_block_status = bdrv_co_block_status_from_backing,
1499 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1500 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1501 .bdrv_refresh_limits = bdrv_mirror_top_refresh_limits,
1502};
1503
1504static void mirror_start_job(const char *job_id, BlockDriverState *bs,
1505 int creation_flags, BlockDriverState *target,
1506 const char *replaces, int64_t speed,
1507 uint32_t granularity, int64_t buf_size,
1508 BlockMirrorBackingMode backing_mode,
1509 BlockdevOnError on_source_error,
1510 BlockdevOnError on_target_error,
1511 bool unmap,
1512 BlockCompletionFunc *cb,
1513 void *opaque,
1514 const BlockJobDriver *driver,
1515 bool is_none_mode, BlockDriverState *base,
1516 bool auto_complete, const char *filter_node_name,
1517 bool is_mirror, MirrorCopyMode copy_mode,
1518 Error **errp)
1519{
1520 MirrorBlockJob *s;
1521 MirrorBDSOpaque *bs_opaque;
1522 BlockDriverState *mirror_top_bs;
1523 bool target_graph_mod;
1524 bool target_is_backing;
1525 Error *local_err = NULL;
1526 int ret;
1527
1528 if (granularity == 0) {
1529 granularity = bdrv_get_default_bitmap_granularity(target);
1530 }
1531
1532 assert(is_power_of_2(granularity));
1533
1534 if (buf_size < 0) {
1535 error_setg(errp, "Invalid parameter 'buf-size'");
1536 return;
1537 }
1538
1539 if (buf_size == 0) {
1540 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1541 }
1542
1543 if (bs == target) {
1544 error_setg(errp, "Can't mirror node into itself");
1545 return;
1546 }
1547
1548
1549
1550
1551 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1552 BDRV_O_RDWR, errp);
1553 if (mirror_top_bs == NULL) {
1554 return;
1555 }
1556 if (!filter_node_name) {
1557 mirror_top_bs->implicit = true;
1558 }
1559 mirror_top_bs->total_sectors = bs->total_sectors;
1560 mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1561 mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
1562 BDRV_REQ_NO_FALLBACK;
1563 bs_opaque = g_new0(MirrorBDSOpaque, 1);
1564 mirror_top_bs->opaque = bs_opaque;
1565 bdrv_set_aio_context(mirror_top_bs, bdrv_get_aio_context(bs));
1566
1567
1568
1569 bdrv_ref(mirror_top_bs);
1570 bdrv_drained_begin(bs);
1571 bdrv_append(mirror_top_bs, bs, &local_err);
1572 bdrv_drained_end(bs);
1573
1574 if (local_err) {
1575 bdrv_unref(mirror_top_bs);
1576 error_propagate(errp, local_err);
1577 return;
1578 }
1579
1580
1581 s = block_job_create(job_id, driver, NULL, mirror_top_bs,
1582 BLK_PERM_CONSISTENT_READ,
1583 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1584 BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD, speed,
1585 creation_flags, cb, opaque, errp);
1586 if (!s) {
1587 goto fail;
1588 }
1589 bs_opaque->job = s;
1590
1591
1592 bdrv_unref(mirror_top_bs);
1593
1594 s->mirror_top_bs = mirror_top_bs;
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604 target_is_backing = bdrv_chain_contains(bs, target);
1605 target_graph_mod = (backing_mode != MIRROR_LEAVE_BACKING_CHAIN);
1606 s->target = blk_new(BLK_PERM_WRITE | BLK_PERM_RESIZE |
1607 (target_graph_mod ? BLK_PERM_GRAPH_MOD : 0),
1608 BLK_PERM_WRITE_UNCHANGED |
1609 (target_is_backing ? BLK_PERM_CONSISTENT_READ |
1610 BLK_PERM_WRITE |
1611 BLK_PERM_GRAPH_MOD : 0));
1612 ret = blk_insert_bs(s->target, target, errp);
1613 if (ret < 0) {
1614 goto fail;
1615 }
1616 if (is_mirror) {
1617
1618
1619
1620
1621
1622
1623 blk_set_force_allow_inactivate(s->target);
1624 }
1625
1626 s->replaces = g_strdup(replaces);
1627 s->on_source_error = on_source_error;
1628 s->on_target_error = on_target_error;
1629 s->is_none_mode = is_none_mode;
1630 s->backing_mode = backing_mode;
1631 s->copy_mode = copy_mode;
1632 s->base = base;
1633 s->granularity = granularity;
1634 s->buf_size = ROUND_UP(buf_size, granularity);
1635 s->unmap = unmap;
1636 if (auto_complete) {
1637 s->should_complete = true;
1638 }
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653 bdrv_refresh_limits(mirror_top_bs, &local_err);
1654 if (local_err) {
1655 error_propagate(errp, local_err);
1656 goto fail;
1657 }
1658
1659 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1660 if (!s->dirty_bitmap) {
1661 goto fail;
1662 }
1663
1664 ret = block_job_add_bdrv(&s->common, "source", bs, 0,
1665 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
1666 BLK_PERM_CONSISTENT_READ,
1667 errp);
1668 if (ret < 0) {
1669 goto fail;
1670 }
1671
1672
1673 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1674 &error_abort);
1675
1676
1677
1678 if (target_is_backing) {
1679 BlockDriverState *iter;
1680 for (iter = backing_bs(bs); iter != target; iter = backing_bs(iter)) {
1681
1682
1683
1684
1685 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1686 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE,
1687 errp);
1688 if (ret < 0) {
1689 goto fail;
1690 }
1691 }
1692
1693 if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1694 goto fail;
1695 }
1696 }
1697
1698 QTAILQ_INIT(&s->ops_in_flight);
1699
1700 trace_mirror_start(bs, s, opaque);
1701 job_start(&s->common.job);
1702 return;
1703
1704fail:
1705 if (s) {
1706
1707
1708 bdrv_ref(mirror_top_bs);
1709
1710 g_free(s->replaces);
1711 blk_unref(s->target);
1712 bs_opaque->job = NULL;
1713 if (s->dirty_bitmap) {
1714 bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
1715 }
1716 job_early_fail(&s->common.job);
1717 }
1718
1719 bdrv_child_try_set_perm(mirror_top_bs->backing, 0, BLK_PERM_ALL,
1720 &error_abort);
1721 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
1722
1723 bdrv_unref(mirror_top_bs);
1724}
1725
1726void mirror_start(const char *job_id, BlockDriverState *bs,
1727 BlockDriverState *target, const char *replaces,
1728 int creation_flags, int64_t speed,
1729 uint32_t granularity, int64_t buf_size,
1730 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1731 BlockdevOnError on_source_error,
1732 BlockdevOnError on_target_error,
1733 bool unmap, const char *filter_node_name,
1734 MirrorCopyMode copy_mode, Error **errp)
1735{
1736 bool is_none_mode;
1737 BlockDriverState *base;
1738
1739 if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
1740 error_setg(errp, "Sync mode 'incremental' not supported");
1741 return;
1742 }
1743 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1744 base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
1745 mirror_start_job(job_id, bs, creation_flags, target, replaces,
1746 speed, granularity, buf_size, backing_mode,
1747 on_source_error, on_target_error, unmap, NULL, NULL,
1748 &mirror_job_driver, is_none_mode, base, false,
1749 filter_node_name, true, copy_mode, errp);
1750}
1751
1752void commit_active_start(const char *job_id, BlockDriverState *bs,
1753 BlockDriverState *base, int creation_flags,
1754 int64_t speed, BlockdevOnError on_error,
1755 const char *filter_node_name,
1756 BlockCompletionFunc *cb, void *opaque,
1757 bool auto_complete, Error **errp)
1758{
1759 bool base_read_only;
1760 Error *local_err = NULL;
1761
1762 base_read_only = bdrv_is_read_only(base);
1763
1764 if (base_read_only) {
1765 if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1766 return;
1767 }
1768 }
1769
1770 mirror_start_job(job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1771 MIRROR_LEAVE_BACKING_CHAIN,
1772 on_error, on_error, true, cb, opaque,
1773 &commit_active_job_driver, false, base, auto_complete,
1774 filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1775 &local_err);
1776 if (local_err) {
1777 error_propagate(errp, local_err);
1778 goto error_restore_flags;
1779 }
1780
1781 return;
1782
1783error_restore_flags:
1784
1785
1786 if (base_read_only) {
1787 bdrv_reopen_set_read_only(base, true, NULL);
1788 }
1789 return;
1790}
1791