1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "qemu/coroutine.h"
17#include "qemu/range.h"
18#include "trace.h"
19#include "block/blockjob_int.h"
20#include "block/block_int.h"
21#include "sysemu/block-backend.h"
22#include "qapi/error.h"
23#include "qapi/qmp/qerror.h"
24#include "qemu/ratelimit.h"
25#include "qemu/bitmap.h"
26
27#define MAX_IN_FLIGHT 16
28#define MAX_IO_BYTES (1 << 20)
29#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
30
31
32
33
34typedef struct MirrorBuffer {
35 QSIMPLEQ_ENTRY(MirrorBuffer) next;
36} MirrorBuffer;
37
38typedef struct MirrorOp MirrorOp;
39
40typedef struct MirrorBlockJob {
41 BlockJob common;
42 BlockBackend *target;
43 BlockDriverState *mirror_top_bs;
44 BlockDriverState *base;
45
46
47 char *replaces;
48
49 BlockDriverState *to_replace;
50
51 Error *replace_blocker;
52 bool is_none_mode;
53 BlockMirrorBackingMode backing_mode;
54 MirrorCopyMode copy_mode;
55 BlockdevOnError on_source_error, on_target_error;
56 bool synced;
57
58
59 bool actively_synced;
60 bool should_complete;
61 int64_t granularity;
62 size_t buf_size;
63 int64_t bdev_length;
64 unsigned long *cow_bitmap;
65 BdrvDirtyBitmap *dirty_bitmap;
66 BdrvDirtyBitmapIter *dbi;
67 uint8_t *buf;
68 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
69 int buf_free_count;
70
71 uint64_t last_pause_ns;
72 unsigned long *in_flight_bitmap;
73 int in_flight;
74 int64_t bytes_in_flight;
75 QTAILQ_HEAD(, MirrorOp) ops_in_flight;
76 int ret;
77 bool unmap;
78 int target_cluster_size;
79 int max_iov;
80 bool initial_zeroing_ongoing;
81 int in_active_write_counter;
82 bool prepared;
83 bool in_drain;
84} MirrorBlockJob;
85
86typedef struct MirrorBDSOpaque {
87 MirrorBlockJob *job;
88 bool stop;
89} MirrorBDSOpaque;
90
91struct MirrorOp {
92 MirrorBlockJob *s;
93 QEMUIOVector qiov;
94 int64_t offset;
95 uint64_t bytes;
96
97
98
99 int64_t *bytes_handled;
100
101 bool is_pseudo_op;
102 bool is_active_write;
103 CoQueue waiting_requests;
104
105 QTAILQ_ENTRY(MirrorOp) next;
106};
107
108typedef enum MirrorMethod {
109 MIRROR_METHOD_COPY,
110 MIRROR_METHOD_ZERO,
111 MIRROR_METHOD_DISCARD,
112} MirrorMethod;
113
114static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
115 int error)
116{
117 s->synced = false;
118 s->actively_synced = false;
119 if (read) {
120 return block_job_error_action(&s->common, s->on_source_error,
121 true, error);
122 } else {
123 return block_job_error_action(&s->common, s->on_target_error,
124 false, error);
125 }
126}
127
128static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
129 MirrorBlockJob *s,
130 uint64_t offset,
131 uint64_t bytes)
132{
133 uint64_t self_start_chunk = offset / s->granularity;
134 uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
135 uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
136
137 while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
138 self_start_chunk) < self_end_chunk &&
139 s->ret >= 0)
140 {
141 MirrorOp *op;
142
143 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
144 uint64_t op_start_chunk = op->offset / s->granularity;
145 uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
146 s->granularity) -
147 op_start_chunk;
148
149 if (op == self) {
150 continue;
151 }
152
153 if (ranges_overlap(self_start_chunk, self_nb_chunks,
154 op_start_chunk, op_nb_chunks))
155 {
156 qemu_co_queue_wait(&op->waiting_requests, NULL);
157 break;
158 }
159 }
160 }
161}
162
163static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
164{
165 MirrorBlockJob *s = op->s;
166 struct iovec *iov;
167 int64_t chunk_num;
168 int i, nb_chunks;
169
170 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
171
172 s->in_flight--;
173 s->bytes_in_flight -= op->bytes;
174 iov = op->qiov.iov;
175 for (i = 0; i < op->qiov.niov; i++) {
176 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
177 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
178 s->buf_free_count++;
179 }
180
181 chunk_num = op->offset / s->granularity;
182 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
183
184 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
185 QTAILQ_REMOVE(&s->ops_in_flight, op, next);
186 if (ret >= 0) {
187 if (s->cow_bitmap) {
188 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
189 }
190 if (!s->initial_zeroing_ongoing) {
191 job_progress_update(&s->common.job, op->bytes);
192 }
193 }
194 qemu_iovec_destroy(&op->qiov);
195
196 qemu_co_queue_restart_all(&op->waiting_requests);
197 g_free(op);
198}
199
200static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
201{
202 MirrorBlockJob *s = op->s;
203
204 if (ret < 0) {
205 BlockErrorAction action;
206
207 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
208 action = mirror_error_action(s, false, -ret);
209 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
210 s->ret = ret;
211 }
212 }
213
214 mirror_iteration_done(op, ret);
215}
216
217static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
218{
219 MirrorBlockJob *s = op->s;
220
221 if (ret < 0) {
222 BlockErrorAction action;
223
224 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
225 action = mirror_error_action(s, true, -ret);
226 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
227 s->ret = ret;
228 }
229
230 mirror_iteration_done(op, ret);
231 return;
232 }
233
234 ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
235 mirror_write_complete(op, ret);
236}
237
238
239static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
240 int64_t offset,
241 int64_t bytes)
242{
243 return MIN(bytes, s->bdev_length - offset);
244}
245
246
247
248static int mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
249 uint64_t *bytes)
250{
251 bool need_cow;
252 int ret = 0;
253 int64_t align_offset = *offset;
254 int64_t align_bytes = *bytes;
255 int max_bytes = s->granularity * s->max_iov;
256
257 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
258 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
259 s->cow_bitmap);
260 if (need_cow) {
261 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
262 &align_offset, &align_bytes);
263 }
264
265 if (align_bytes > max_bytes) {
266 align_bytes = max_bytes;
267 if (need_cow) {
268 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
269 }
270 }
271
272
273 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
274
275 ret = align_offset + align_bytes - (*offset + *bytes);
276 *offset = align_offset;
277 *bytes = align_bytes;
278 assert(ret >= 0);
279 return ret;
280}
281
282static inline void coroutine_fn
283mirror_wait_for_any_operation(MirrorBlockJob *s, bool active)
284{
285 MirrorOp *op;
286
287 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
288
289
290
291
292
293 if (!op->is_pseudo_op && op->is_active_write == active) {
294 qemu_co_queue_wait(&op->waiting_requests, NULL);
295 return;
296 }
297 }
298 abort();
299}
300
301static inline void coroutine_fn
302mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
303{
304
305 mirror_wait_for_any_operation(s, false);
306}
307
308
309
310
311
312
313
314
315
316static void coroutine_fn mirror_co_read(void *opaque)
317{
318 MirrorOp *op = opaque;
319 MirrorBlockJob *s = op->s;
320 int nb_chunks;
321 uint64_t ret;
322 uint64_t max_bytes;
323
324 max_bytes = s->granularity * s->max_iov;
325
326
327 op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
328 assert(op->bytes);
329 assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
330 *op->bytes_handled = op->bytes;
331
332 if (s->cow_bitmap) {
333 *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
334 }
335
336 assert(*op->bytes_handled <= UINT_MAX);
337 assert(op->bytes <= s->buf_size);
338
339
340
341 assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
342
343 assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
344 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
345
346 while (s->buf_free_count < nb_chunks) {
347 trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
348 mirror_wait_for_free_in_flight_slot(s);
349 }
350
351
352
353
354 qemu_iovec_init(&op->qiov, nb_chunks);
355 while (nb_chunks-- > 0) {
356 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
357 size_t remaining = op->bytes - op->qiov.size;
358
359 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
360 s->buf_free_count--;
361 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
362 }
363
364
365 s->in_flight++;
366 s->bytes_in_flight += op->bytes;
367 trace_mirror_one_iteration(s, op->offset, op->bytes);
368
369 ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
370 &op->qiov, 0);
371 mirror_read_complete(op, ret);
372}
373
374static void coroutine_fn mirror_co_zero(void *opaque)
375{
376 MirrorOp *op = opaque;
377 int ret;
378
379 op->s->in_flight++;
380 op->s->bytes_in_flight += op->bytes;
381 *op->bytes_handled = op->bytes;
382
383 ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
384 op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
385 mirror_write_complete(op, ret);
386}
387
388static void coroutine_fn mirror_co_discard(void *opaque)
389{
390 MirrorOp *op = opaque;
391 int ret;
392
393 op->s->in_flight++;
394 op->s->bytes_in_flight += op->bytes;
395 *op->bytes_handled = op->bytes;
396
397 ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
398 mirror_write_complete(op, ret);
399}
400
401static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
402 unsigned bytes, MirrorMethod mirror_method)
403{
404 MirrorOp *op;
405 Coroutine *co;
406 int64_t bytes_handled = -1;
407
408 op = g_new(MirrorOp, 1);
409 *op = (MirrorOp){
410 .s = s,
411 .offset = offset,
412 .bytes = bytes,
413 .bytes_handled = &bytes_handled,
414 };
415 qemu_co_queue_init(&op->waiting_requests);
416
417 switch (mirror_method) {
418 case MIRROR_METHOD_COPY:
419 co = qemu_coroutine_create(mirror_co_read, op);
420 break;
421 case MIRROR_METHOD_ZERO:
422 co = qemu_coroutine_create(mirror_co_zero, op);
423 break;
424 case MIRROR_METHOD_DISCARD:
425 co = qemu_coroutine_create(mirror_co_discard, op);
426 break;
427 default:
428 abort();
429 }
430
431 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
432 qemu_coroutine_enter(co);
433
434
435
436
437 assert(bytes_handled >= 0);
438
439
440
441
442 assert(bytes_handled <= UINT_MAX);
443 return bytes_handled;
444}
445
446static uint64_t coroutine_fn mirror_iteration(MirrorBlockJob *s)
447{
448 BlockDriverState *source = s->mirror_top_bs->backing->bs;
449 MirrorOp *pseudo_op;
450 int64_t offset;
451 uint64_t delay_ns = 0, ret = 0;
452
453 int nb_chunks = 1;
454 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
455 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
456
457 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
458 offset = bdrv_dirty_iter_next(s->dbi);
459 if (offset < 0) {
460 bdrv_set_dirty_iter(s->dbi, 0);
461 offset = bdrv_dirty_iter_next(s->dbi);
462 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
463 assert(offset >= 0);
464 }
465 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
466
467 mirror_wait_on_conflicts(NULL, s, offset, 1);
468
469 job_pause_point(&s->common.job);
470
471
472
473 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
474 while (nb_chunks * s->granularity < s->buf_size) {
475 int64_t next_dirty;
476 int64_t next_offset = offset + nb_chunks * s->granularity;
477 int64_t next_chunk = next_offset / s->granularity;
478 if (next_offset >= s->bdev_length ||
479 !bdrv_get_dirty_locked(source, s->dirty_bitmap, next_offset)) {
480 break;
481 }
482 if (test_bit(next_chunk, s->in_flight_bitmap)) {
483 break;
484 }
485
486 next_dirty = bdrv_dirty_iter_next(s->dbi);
487 if (next_dirty > next_offset || next_dirty < 0) {
488
489 bdrv_set_dirty_iter(s->dbi, next_offset);
490 next_dirty = bdrv_dirty_iter_next(s->dbi);
491 }
492 assert(next_dirty == next_offset);
493 nb_chunks++;
494 }
495
496
497
498
499
500 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
501 nb_chunks * s->granularity);
502 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
503
504
505
506
507
508
509
510 pseudo_op = g_new(MirrorOp, 1);
511 *pseudo_op = (MirrorOp){
512 .offset = offset,
513 .bytes = nb_chunks * s->granularity,
514 .is_pseudo_op = true,
515 };
516 qemu_co_queue_init(&pseudo_op->waiting_requests);
517 QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
518
519 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
520 while (nb_chunks > 0 && offset < s->bdev_length) {
521 int ret;
522 int64_t io_bytes;
523 int64_t io_bytes_acct;
524 MirrorMethod mirror_method = MIRROR_METHOD_COPY;
525
526 assert(!(offset % s->granularity));
527 ret = bdrv_block_status_above(source, NULL, offset,
528 nb_chunks * s->granularity,
529 &io_bytes, NULL, NULL);
530 if (ret < 0) {
531 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
532 } else if (ret & BDRV_BLOCK_DATA) {
533 io_bytes = MIN(io_bytes, max_io_bytes);
534 }
535
536 io_bytes -= io_bytes % s->granularity;
537 if (io_bytes < s->granularity) {
538 io_bytes = s->granularity;
539 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
540 int64_t target_offset;
541 int64_t target_bytes;
542 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
543 &target_offset, &target_bytes);
544 if (target_offset == offset &&
545 target_bytes == io_bytes) {
546 mirror_method = ret & BDRV_BLOCK_ZERO ?
547 MIRROR_METHOD_ZERO :
548 MIRROR_METHOD_DISCARD;
549 }
550 }
551
552 while (s->in_flight >= MAX_IN_FLIGHT) {
553 trace_mirror_yield_in_flight(s, offset, s->in_flight);
554 mirror_wait_for_free_in_flight_slot(s);
555 }
556
557 if (s->ret < 0) {
558 ret = 0;
559 goto fail;
560 }
561
562 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
563 io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
564 if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
565 io_bytes_acct = 0;
566 } else {
567 io_bytes_acct = io_bytes;
568 }
569 assert(io_bytes);
570 offset += io_bytes;
571 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
572 delay_ns = block_job_ratelimit_get_delay(&s->common, io_bytes_acct);
573 }
574
575 ret = delay_ns;
576fail:
577 QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
578 qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
579 g_free(pseudo_op);
580
581 return ret;
582}
583
584static void mirror_free_init(MirrorBlockJob *s)
585{
586 int granularity = s->granularity;
587 size_t buf_size = s->buf_size;
588 uint8_t *buf = s->buf;
589
590 assert(s->buf_free_count == 0);
591 QSIMPLEQ_INIT(&s->buf_free);
592 while (buf_size != 0) {
593 MirrorBuffer *cur = (MirrorBuffer *)buf;
594 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
595 s->buf_free_count++;
596 buf_size -= granularity;
597 buf += granularity;
598 }
599}
600
601
602
603
604
605static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
606{
607 while (s->in_flight > 0) {
608 mirror_wait_for_free_in_flight_slot(s);
609 }
610}
611
612
613
614
615
616
617static int mirror_exit_common(Job *job)
618{
619 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
620 BlockJob *bjob = &s->common;
621 MirrorBDSOpaque *bs_opaque = s->mirror_top_bs->opaque;
622 AioContext *replace_aio_context = NULL;
623 BlockDriverState *src = s->mirror_top_bs->backing->bs;
624 BlockDriverState *target_bs = blk_bs(s->target);
625 BlockDriverState *mirror_top_bs = s->mirror_top_bs;
626 Error *local_err = NULL;
627 bool abort = job->ret < 0;
628 int ret = 0;
629
630 if (s->prepared) {
631 return 0;
632 }
633 s->prepared = true;
634
635 if (bdrv_chain_contains(src, target_bs)) {
636 bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
637 }
638
639 bdrv_release_dirty_bitmap(src, s->dirty_bitmap);
640
641
642
643 bdrv_ref(src);
644 bdrv_ref(mirror_top_bs);
645 bdrv_ref(target_bs);
646
647
648
649
650
651
652
653
654 blk_set_perm(s->target, 0, BLK_PERM_ALL, &error_abort);
655 blk_unref(s->target);
656 s->target = NULL;
657
658
659
660 bs_opaque->stop = true;
661 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
662 &error_abort);
663 if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
664 BlockDriverState *backing = s->is_none_mode ? src : s->base;
665 if (backing_bs(target_bs) != backing) {
666 bdrv_set_backing_hd(target_bs, backing, &local_err);
667 if (local_err) {
668 error_report_err(local_err);
669 ret = -EPERM;
670 }
671 }
672 }
673
674 if (s->to_replace) {
675 replace_aio_context = bdrv_get_aio_context(s->to_replace);
676 aio_context_acquire(replace_aio_context);
677 }
678
679 if (s->should_complete && !abort) {
680 BlockDriverState *to_replace = s->to_replace ?: src;
681 bool ro = bdrv_is_read_only(to_replace);
682
683 if (ro != bdrv_is_read_only(target_bs)) {
684 bdrv_reopen_set_read_only(target_bs, ro, NULL);
685 }
686
687
688
689 assert(s->in_drain);
690 bdrv_drained_begin(target_bs);
691 bdrv_replace_node(to_replace, target_bs, &local_err);
692 bdrv_drained_end(target_bs);
693 if (local_err) {
694 error_report_err(local_err);
695 ret = -EPERM;
696 }
697 }
698 if (s->to_replace) {
699 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
700 error_free(s->replace_blocker);
701 bdrv_unref(s->to_replace);
702 }
703 if (replace_aio_context) {
704 aio_context_release(replace_aio_context);
705 }
706 g_free(s->replaces);
707 bdrv_unref(target_bs);
708
709
710
711
712
713
714 block_job_remove_all_bdrv(bjob);
715 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
716
717
718
719
720 blk_remove_bs(bjob->blk);
721 blk_set_perm(bjob->blk, 0, BLK_PERM_ALL, &error_abort);
722 blk_insert_bs(bjob->blk, mirror_top_bs, &error_abort);
723
724 bs_opaque->job = NULL;
725
726 bdrv_drained_end(src);
727 s->in_drain = false;
728 bdrv_unref(mirror_top_bs);
729 bdrv_unref(src);
730
731 return ret;
732}
733
734static int mirror_prepare(Job *job)
735{
736 return mirror_exit_common(job);
737}
738
739static void mirror_abort(Job *job)
740{
741 int ret = mirror_exit_common(job);
742 assert(ret == 0);
743}
744
745static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
746{
747 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
748
749 if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
750 s->last_pause_ns = now;
751 job_sleep_ns(&s->common.job, 0);
752 } else {
753 job_pause_point(&s->common.job);
754 }
755}
756
757static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
758{
759 int64_t offset;
760 BlockDriverState *base = s->base;
761 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
762 BlockDriverState *target_bs = blk_bs(s->target);
763 int ret;
764 int64_t count;
765
766 if (base == NULL && !bdrv_has_zero_init(target_bs)) {
767 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
768 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
769 return 0;
770 }
771
772 s->initial_zeroing_ongoing = true;
773 for (offset = 0; offset < s->bdev_length; ) {
774 int bytes = MIN(s->bdev_length - offset,
775 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
776
777 mirror_throttle(s);
778
779 if (job_is_cancelled(&s->common.job)) {
780 s->initial_zeroing_ongoing = false;
781 return 0;
782 }
783
784 if (s->in_flight >= MAX_IN_FLIGHT) {
785 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
786 s->in_flight);
787 mirror_wait_for_free_in_flight_slot(s);
788 continue;
789 }
790
791 mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
792 offset += bytes;
793 }
794
795 mirror_wait_for_all_io(s);
796 s->initial_zeroing_ongoing = false;
797 }
798
799
800 for (offset = 0; offset < s->bdev_length; ) {
801
802 int bytes = MIN(s->bdev_length - offset,
803 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
804
805 mirror_throttle(s);
806
807 if (job_is_cancelled(&s->common.job)) {
808 return 0;
809 }
810
811 ret = bdrv_is_allocated_above(bs, base, false, offset, bytes, &count);
812 if (ret < 0) {
813 return ret;
814 }
815
816 assert(count);
817 if (ret == 1) {
818 bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
819 }
820 offset += count;
821 }
822 return 0;
823}
824
825
826
827
828static int mirror_flush(MirrorBlockJob *s)
829{
830 int ret = blk_flush(s->target);
831 if (ret < 0) {
832 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
833 s->ret = ret;
834 }
835 }
836 return ret;
837}
838
839static int coroutine_fn mirror_run(Job *job, Error **errp)
840{
841 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
842 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
843 BlockDriverState *target_bs = blk_bs(s->target);
844 bool need_drain = true;
845 int64_t length;
846 BlockDriverInfo bdi;
847 char backing_filename[2];
848
849 int ret = 0;
850
851 if (job_is_cancelled(&s->common.job)) {
852 goto immediate_exit;
853 }
854
855 s->bdev_length = bdrv_getlength(bs);
856 if (s->bdev_length < 0) {
857 ret = s->bdev_length;
858 goto immediate_exit;
859 }
860
861
862
863 if (s->base == blk_bs(s->target)) {
864 int64_t base_length;
865
866 base_length = blk_getlength(s->target);
867 if (base_length < 0) {
868 ret = base_length;
869 goto immediate_exit;
870 }
871
872 if (s->bdev_length > base_length) {
873 ret = blk_truncate(s->target, s->bdev_length, PREALLOC_MODE_OFF,
874 NULL);
875 if (ret < 0) {
876 goto immediate_exit;
877 }
878 }
879 }
880
881 if (s->bdev_length == 0) {
882
883 job_transition_to_ready(&s->common.job);
884 s->synced = true;
885 s->actively_synced = true;
886 while (!job_is_cancelled(&s->common.job) && !s->should_complete) {
887 job_yield(&s->common.job);
888 }
889 s->common.job.cancelled = false;
890 goto immediate_exit;
891 }
892
893 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
894 s->in_flight_bitmap = bitmap_new(length);
895
896
897
898
899
900 bdrv_get_backing_filename(target_bs, backing_filename,
901 sizeof(backing_filename));
902 if (!bdrv_get_info(target_bs, &bdi) && bdi.cluster_size) {
903 s->target_cluster_size = bdi.cluster_size;
904 } else {
905 s->target_cluster_size = BDRV_SECTOR_SIZE;
906 }
907 if (backing_filename[0] && !target_bs->backing &&
908 s->granularity < s->target_cluster_size) {
909 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
910 s->cow_bitmap = bitmap_new(length);
911 }
912 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
913
914 s->buf = qemu_try_blockalign(bs, s->buf_size);
915 if (s->buf == NULL) {
916 ret = -ENOMEM;
917 goto immediate_exit;
918 }
919
920 mirror_free_init(s);
921
922 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
923 if (!s->is_none_mode) {
924 ret = mirror_dirty_init(s);
925 if (ret < 0 || job_is_cancelled(&s->common.job)) {
926 goto immediate_exit;
927 }
928 }
929
930 assert(!s->dbi);
931 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
932 for (;;) {
933 uint64_t delay_ns = 0;
934 int64_t cnt, delta;
935 bool should_complete;
936
937
938
939 while (s->in_active_write_counter) {
940 mirror_wait_for_any_operation(s, true);
941 }
942
943 if (s->ret < 0) {
944 ret = s->ret;
945 goto immediate_exit;
946 }
947
948 job_pause_point(&s->common.job);
949
950 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
951
952
953
954 job_progress_set_remaining(&s->common.job, s->bytes_in_flight + cnt);
955
956
957
958
959
960 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
961 if (delta < BLOCK_JOB_SLICE_TIME &&
962 s->common.iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
963 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
964 (cnt == 0 && s->in_flight > 0)) {
965 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
966 mirror_wait_for_free_in_flight_slot(s);
967 continue;
968 } else if (cnt != 0) {
969 delay_ns = mirror_iteration(s);
970 }
971 }
972
973 should_complete = false;
974 if (s->in_flight == 0 && cnt == 0) {
975 trace_mirror_before_flush(s);
976 if (!s->synced) {
977 if (mirror_flush(s) < 0) {
978
979 continue;
980 }
981
982
983
984
985
986 job_transition_to_ready(&s->common.job);
987 s->synced = true;
988 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
989 s->actively_synced = true;
990 }
991 }
992
993 should_complete = s->should_complete ||
994 job_is_cancelled(&s->common.job);
995 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
996 }
997
998 if (cnt == 0 && should_complete) {
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009 trace_mirror_before_drain(s, cnt);
1010
1011 s->in_drain = true;
1012 bdrv_drained_begin(bs);
1013 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1014 if (cnt > 0 || mirror_flush(s) < 0) {
1015 bdrv_drained_end(bs);
1016 s->in_drain = false;
1017 continue;
1018 }
1019
1020
1021
1022
1023 assert(QLIST_EMPTY(&bs->tracked_requests));
1024 s->common.job.cancelled = false;
1025 need_drain = false;
1026 break;
1027 }
1028
1029 ret = 0;
1030
1031 if (s->synced && !should_complete) {
1032 delay_ns = (s->in_flight == 0 &&
1033 cnt == 0 ? BLOCK_JOB_SLICE_TIME : 0);
1034 }
1035 trace_mirror_before_sleep(s, cnt, s->synced, delay_ns);
1036 job_sleep_ns(&s->common.job, delay_ns);
1037 if (job_is_cancelled(&s->common.job) &&
1038 (!s->synced || s->common.job.force_cancel))
1039 {
1040 break;
1041 }
1042 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1043 }
1044
1045immediate_exit:
1046 if (s->in_flight > 0) {
1047
1048
1049
1050
1051 assert(ret < 0 || ((s->common.job.force_cancel || !s->synced) &&
1052 job_is_cancelled(&s->common.job)));
1053 assert(need_drain);
1054 mirror_wait_for_all_io(s);
1055 }
1056
1057 assert(s->in_flight == 0);
1058 qemu_vfree(s->buf);
1059 g_free(s->cow_bitmap);
1060 g_free(s->in_flight_bitmap);
1061 bdrv_dirty_iter_free(s->dbi);
1062
1063 if (need_drain) {
1064 s->in_drain = true;
1065 bdrv_drained_begin(bs);
1066 }
1067
1068 return ret;
1069}
1070
1071static void mirror_complete(Job *job, Error **errp)
1072{
1073 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1074 BlockDriverState *target;
1075
1076 target = blk_bs(s->target);
1077
1078 if (!s->synced) {
1079 error_setg(errp, "The active block job '%s' cannot be completed",
1080 job->id);
1081 return;
1082 }
1083
1084 if (s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
1085 int ret;
1086
1087 assert(!target->backing);
1088 ret = bdrv_open_backing_file(target, NULL, "backing", errp);
1089 if (ret < 0) {
1090 return;
1091 }
1092 }
1093
1094
1095 if (s->replaces) {
1096 AioContext *replace_aio_context;
1097
1098 s->to_replace = bdrv_find_node(s->replaces);
1099 if (!s->to_replace) {
1100 error_setg(errp, "Node name '%s' not found", s->replaces);
1101 return;
1102 }
1103
1104 replace_aio_context = bdrv_get_aio_context(s->to_replace);
1105 aio_context_acquire(replace_aio_context);
1106
1107
1108
1109
1110
1111 error_setg(&s->replace_blocker,
1112 "block device is in use by block-job-complete");
1113 bdrv_op_block_all(s->to_replace, s->replace_blocker);
1114 bdrv_ref(s->to_replace);
1115
1116 aio_context_release(replace_aio_context);
1117 }
1118
1119 s->should_complete = true;
1120 job_enter(job);
1121}
1122
1123static void coroutine_fn mirror_pause(Job *job)
1124{
1125 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1126
1127 mirror_wait_for_all_io(s);
1128}
1129
1130static bool mirror_drained_poll(BlockJob *job)
1131{
1132 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1133
1134
1135
1136
1137
1138
1139 if (!s->common.job.paused && !s->common.job.cancelled && !s->in_drain) {
1140 return true;
1141 }
1142
1143 return !!s->in_flight;
1144}
1145
1146static void mirror_drain(BlockJob *job)
1147{
1148 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1149
1150
1151
1152
1153 if (s->target) {
1154 BlockBackend *target = s->target;
1155 blk_ref(target);
1156 blk_drain(target);
1157 blk_unref(target);
1158 }
1159}
1160
1161static const BlockJobDriver mirror_job_driver = {
1162 .job_driver = {
1163 .instance_size = sizeof(MirrorBlockJob),
1164 .job_type = JOB_TYPE_MIRROR,
1165 .free = block_job_free,
1166 .user_resume = block_job_user_resume,
1167 .drain = block_job_drain,
1168 .run = mirror_run,
1169 .prepare = mirror_prepare,
1170 .abort = mirror_abort,
1171 .pause = mirror_pause,
1172 .complete = mirror_complete,
1173 },
1174 .drained_poll = mirror_drained_poll,
1175 .drain = mirror_drain,
1176};
1177
1178static const BlockJobDriver commit_active_job_driver = {
1179 .job_driver = {
1180 .instance_size = sizeof(MirrorBlockJob),
1181 .job_type = JOB_TYPE_COMMIT,
1182 .free = block_job_free,
1183 .user_resume = block_job_user_resume,
1184 .drain = block_job_drain,
1185 .run = mirror_run,
1186 .prepare = mirror_prepare,
1187 .abort = mirror_abort,
1188 .pause = mirror_pause,
1189 .complete = mirror_complete,
1190 },
1191 .drained_poll = mirror_drained_poll,
1192 .drain = mirror_drain,
1193};
1194
1195static void coroutine_fn
1196do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1197 uint64_t offset, uint64_t bytes,
1198 QEMUIOVector *qiov, int flags)
1199{
1200 QEMUIOVector target_qiov;
1201 uint64_t dirty_offset = offset;
1202 uint64_t dirty_bytes;
1203
1204 if (qiov) {
1205 qemu_iovec_init(&target_qiov, qiov->niov);
1206 }
1207
1208 while (true) {
1209 bool valid_area;
1210 int ret;
1211
1212 bdrv_dirty_bitmap_lock(job->dirty_bitmap);
1213 dirty_bytes = MIN(offset + bytes - dirty_offset, INT_MAX);
1214 valid_area = bdrv_dirty_bitmap_next_dirty_area(job->dirty_bitmap,
1215 &dirty_offset,
1216 &dirty_bytes);
1217 if (!valid_area) {
1218 bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1219 break;
1220 }
1221
1222 bdrv_reset_dirty_bitmap_locked(job->dirty_bitmap,
1223 dirty_offset, dirty_bytes);
1224 bdrv_dirty_bitmap_unlock(job->dirty_bitmap);
1225
1226 job_progress_increase_remaining(&job->common.job, dirty_bytes);
1227
1228 assert(dirty_offset - offset <= SIZE_MAX);
1229 if (qiov) {
1230 qemu_iovec_reset(&target_qiov);
1231 qemu_iovec_concat(&target_qiov, qiov,
1232 dirty_offset - offset, dirty_bytes);
1233 }
1234
1235 switch (method) {
1236 case MIRROR_METHOD_COPY:
1237 ret = blk_co_pwritev(job->target, dirty_offset, dirty_bytes,
1238 qiov ? &target_qiov : NULL, flags);
1239 break;
1240
1241 case MIRROR_METHOD_ZERO:
1242 assert(!qiov);
1243 ret = blk_co_pwrite_zeroes(job->target, dirty_offset, dirty_bytes,
1244 flags);
1245 break;
1246
1247 case MIRROR_METHOD_DISCARD:
1248 assert(!qiov);
1249 ret = blk_co_pdiscard(job->target, dirty_offset, dirty_bytes);
1250 break;
1251
1252 default:
1253 abort();
1254 }
1255
1256 if (ret >= 0) {
1257 job_progress_update(&job->common.job, dirty_bytes);
1258 } else {
1259 BlockErrorAction action;
1260
1261 bdrv_set_dirty_bitmap(job->dirty_bitmap, dirty_offset, dirty_bytes);
1262 job->actively_synced = false;
1263
1264 action = mirror_error_action(job, false, -ret);
1265 if (action == BLOCK_ERROR_ACTION_REPORT) {
1266 if (!job->ret) {
1267 job->ret = ret;
1268 }
1269 break;
1270 }
1271 }
1272
1273 dirty_offset += dirty_bytes;
1274 }
1275
1276 if (qiov) {
1277 qemu_iovec_destroy(&target_qiov);
1278 }
1279}
1280
1281static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1282 uint64_t offset,
1283 uint64_t bytes)
1284{
1285 MirrorOp *op;
1286 uint64_t start_chunk = offset / s->granularity;
1287 uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1288
1289 op = g_new(MirrorOp, 1);
1290 *op = (MirrorOp){
1291 .s = s,
1292 .offset = offset,
1293 .bytes = bytes,
1294 .is_active_write = true,
1295 };
1296 qemu_co_queue_init(&op->waiting_requests);
1297 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1298
1299 s->in_active_write_counter++;
1300
1301 mirror_wait_on_conflicts(op, s, offset, bytes);
1302
1303 bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1304
1305 return op;
1306}
1307
1308static void coroutine_fn active_write_settle(MirrorOp *op)
1309{
1310 uint64_t start_chunk = op->offset / op->s->granularity;
1311 uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1312 op->s->granularity);
1313
1314 if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1315 BdrvChild *source = op->s->mirror_top_bs->backing;
1316
1317 if (QLIST_FIRST(&source->bs->parents) == source &&
1318 QLIST_NEXT(source, next_parent) == NULL)
1319 {
1320
1321
1322
1323
1324 assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1325 }
1326 }
1327 bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1328 QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1329 qemu_co_queue_restart_all(&op->waiting_requests);
1330 g_free(op);
1331}
1332
1333static int coroutine_fn bdrv_mirror_top_preadv(BlockDriverState *bs,
1334 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1335{
1336 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1337}
1338
1339static int coroutine_fn bdrv_mirror_top_do_write(BlockDriverState *bs,
1340 MirrorMethod method, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1341 int flags)
1342{
1343 MirrorOp *op = NULL;
1344 MirrorBDSOpaque *s = bs->opaque;
1345 int ret = 0;
1346 bool copy_to_target;
1347
1348 copy_to_target = s->job->ret >= 0 &&
1349 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1350
1351 if (copy_to_target) {
1352 op = active_write_prepare(s->job, offset, bytes);
1353 }
1354
1355 switch (method) {
1356 case MIRROR_METHOD_COPY:
1357 ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1358 break;
1359
1360 case MIRROR_METHOD_ZERO:
1361 ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1362 break;
1363
1364 case MIRROR_METHOD_DISCARD:
1365 ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1366 break;
1367
1368 default:
1369 abort();
1370 }
1371
1372 if (ret < 0) {
1373 goto out;
1374 }
1375
1376 if (copy_to_target) {
1377 do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1378 }
1379
1380out:
1381 if (copy_to_target) {
1382 active_write_settle(op);
1383 }
1384 return ret;
1385}
1386
1387static int coroutine_fn bdrv_mirror_top_pwritev(BlockDriverState *bs,
1388 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags)
1389{
1390 MirrorBDSOpaque *s = bs->opaque;
1391 QEMUIOVector bounce_qiov;
1392 void *bounce_buf;
1393 int ret = 0;
1394 bool copy_to_target;
1395
1396 copy_to_target = s->job->ret >= 0 &&
1397 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1398
1399 if (copy_to_target) {
1400
1401
1402
1403
1404 bounce_buf = qemu_blockalign(bs, bytes);
1405 iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1406
1407 qemu_iovec_init(&bounce_qiov, 1);
1408 qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1409 qiov = &bounce_qiov;
1410 }
1411
1412 ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1413 flags);
1414
1415 if (copy_to_target) {
1416 qemu_iovec_destroy(&bounce_qiov);
1417 qemu_vfree(bounce_buf);
1418 }
1419
1420 return ret;
1421}
1422
1423static int coroutine_fn bdrv_mirror_top_flush(BlockDriverState *bs)
1424{
1425 if (bs->backing == NULL) {
1426
1427 return 0;
1428 }
1429 return bdrv_co_flush(bs->backing->bs);
1430}
1431
1432static int coroutine_fn bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs,
1433 int64_t offset, int bytes, BdrvRequestFlags flags)
1434{
1435 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1436 flags);
1437}
1438
1439static int coroutine_fn bdrv_mirror_top_pdiscard(BlockDriverState *bs,
1440 int64_t offset, int bytes)
1441{
1442 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1443 NULL, 0);
1444}
1445
1446static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1447{
1448 if (bs->backing == NULL) {
1449
1450
1451 return;
1452 }
1453 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1454 bs->backing->bs->filename);
1455}
1456
1457static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1458 const BdrvChildRole *role,
1459 BlockReopenQueue *reopen_queue,
1460 uint64_t perm, uint64_t shared,
1461 uint64_t *nperm, uint64_t *nshared)
1462{
1463 MirrorBDSOpaque *s = bs->opaque;
1464
1465 if (s->stop) {
1466
1467
1468
1469
1470 *nperm = 0;
1471 *nshared = BLK_PERM_ALL;
1472 return;
1473 }
1474
1475
1476 *nperm = 0;
1477 if (perm & BLK_PERM_WRITE) {
1478 *nperm |= BLK_PERM_WRITE;
1479 }
1480
1481 *nshared = BLK_PERM_ALL;
1482}
1483
1484static void bdrv_mirror_top_refresh_limits(BlockDriverState *bs, Error **errp)
1485{
1486 MirrorBDSOpaque *s = bs->opaque;
1487
1488 if (s && s->job && s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1489 bs->bl.request_alignment = s->job->granularity;
1490 }
1491}
1492
1493
1494
1495static BlockDriver bdrv_mirror_top = {
1496 .format_name = "mirror_top",
1497 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1498 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1499 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1500 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1501 .bdrv_co_flush = bdrv_mirror_top_flush,
1502 .bdrv_co_block_status = bdrv_co_block_status_from_backing,
1503 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1504 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1505 .bdrv_refresh_limits = bdrv_mirror_top_refresh_limits,
1506};
1507
1508static BlockJob *mirror_start_job(
1509 const char *job_id, BlockDriverState *bs,
1510 int creation_flags, BlockDriverState *target,
1511 const char *replaces, int64_t speed,
1512 uint32_t granularity, int64_t buf_size,
1513 BlockMirrorBackingMode backing_mode,
1514 BlockdevOnError on_source_error,
1515 BlockdevOnError on_target_error,
1516 bool unmap,
1517 BlockCompletionFunc *cb,
1518 void *opaque,
1519 const BlockJobDriver *driver,
1520 bool is_none_mode, BlockDriverState *base,
1521 bool auto_complete, const char *filter_node_name,
1522 bool is_mirror, MirrorCopyMode copy_mode,
1523 Error **errp)
1524{
1525 MirrorBlockJob *s;
1526 MirrorBDSOpaque *bs_opaque;
1527 BlockDriverState *mirror_top_bs;
1528 bool target_graph_mod;
1529 bool target_is_backing;
1530 Error *local_err = NULL;
1531 int ret;
1532
1533 if (granularity == 0) {
1534 granularity = bdrv_get_default_bitmap_granularity(target);
1535 }
1536
1537 assert(is_power_of_2(granularity));
1538
1539 if (buf_size < 0) {
1540 error_setg(errp, "Invalid parameter 'buf-size'");
1541 return NULL;
1542 }
1543
1544 if (buf_size == 0) {
1545 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1546 }
1547
1548 if (bs == target) {
1549 error_setg(errp, "Can't mirror node into itself");
1550 return NULL;
1551 }
1552
1553
1554
1555
1556 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1557 BDRV_O_RDWR, errp);
1558 if (mirror_top_bs == NULL) {
1559 return NULL;
1560 }
1561 if (!filter_node_name) {
1562 mirror_top_bs->implicit = true;
1563 }
1564
1565
1566 mirror_top_bs->never_freeze = true;
1567
1568 mirror_top_bs->total_sectors = bs->total_sectors;
1569 mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1570 mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
1571 BDRV_REQ_NO_FALLBACK;
1572 bs_opaque = g_new0(MirrorBDSOpaque, 1);
1573 mirror_top_bs->opaque = bs_opaque;
1574
1575
1576
1577 bdrv_ref(mirror_top_bs);
1578 bdrv_drained_begin(bs);
1579 bdrv_append(mirror_top_bs, bs, &local_err);
1580 bdrv_drained_end(bs);
1581
1582 if (local_err) {
1583 bdrv_unref(mirror_top_bs);
1584 error_propagate(errp, local_err);
1585 return NULL;
1586 }
1587
1588
1589 s = block_job_create(job_id, driver, NULL, mirror_top_bs,
1590 BLK_PERM_CONSISTENT_READ,
1591 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1592 BLK_PERM_WRITE | BLK_PERM_GRAPH_MOD, speed,
1593 creation_flags, cb, opaque, errp);
1594 if (!s) {
1595 goto fail;
1596 }
1597 bs_opaque->job = s;
1598
1599
1600 bdrv_unref(mirror_top_bs);
1601
1602 s->mirror_top_bs = mirror_top_bs;
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612 target_is_backing = bdrv_chain_contains(bs, target);
1613 target_graph_mod = (backing_mode != MIRROR_LEAVE_BACKING_CHAIN);
1614 s->target = blk_new(s->common.job.aio_context,
1615 BLK_PERM_WRITE | BLK_PERM_RESIZE |
1616 (target_graph_mod ? BLK_PERM_GRAPH_MOD : 0),
1617 BLK_PERM_WRITE_UNCHANGED |
1618 (target_is_backing ? BLK_PERM_CONSISTENT_READ |
1619 BLK_PERM_WRITE |
1620 BLK_PERM_GRAPH_MOD : 0));
1621 ret = blk_insert_bs(s->target, target, errp);
1622 if (ret < 0) {
1623 goto fail;
1624 }
1625 if (is_mirror) {
1626
1627
1628
1629
1630
1631
1632 blk_set_force_allow_inactivate(s->target);
1633 }
1634 blk_set_allow_aio_context_change(s->target, true);
1635
1636 s->replaces = g_strdup(replaces);
1637 s->on_source_error = on_source_error;
1638 s->on_target_error = on_target_error;
1639 s->is_none_mode = is_none_mode;
1640 s->backing_mode = backing_mode;
1641 s->copy_mode = copy_mode;
1642 s->base = base;
1643 s->granularity = granularity;
1644 s->buf_size = ROUND_UP(buf_size, granularity);
1645 s->unmap = unmap;
1646 if (auto_complete) {
1647 s->should_complete = true;
1648 }
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663 bdrv_refresh_limits(mirror_top_bs, &local_err);
1664 if (local_err) {
1665 error_propagate(errp, local_err);
1666 goto fail;
1667 }
1668
1669 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1670 if (!s->dirty_bitmap) {
1671 goto fail;
1672 }
1673
1674 ret = block_job_add_bdrv(&s->common, "source", bs, 0,
1675 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
1676 BLK_PERM_CONSISTENT_READ,
1677 errp);
1678 if (ret < 0) {
1679 goto fail;
1680 }
1681
1682
1683 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1684 &error_abort);
1685
1686
1687
1688 if (target_is_backing) {
1689 BlockDriverState *iter;
1690 for (iter = backing_bs(bs); iter != target; iter = backing_bs(iter)) {
1691
1692
1693
1694
1695 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1696 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE,
1697 errp);
1698 if (ret < 0) {
1699 goto fail;
1700 }
1701 }
1702
1703 if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1704 goto fail;
1705 }
1706 }
1707
1708 QTAILQ_INIT(&s->ops_in_flight);
1709
1710 trace_mirror_start(bs, s, opaque);
1711 job_start(&s->common.job);
1712
1713 return &s->common;
1714
1715fail:
1716 if (s) {
1717
1718
1719 bdrv_ref(mirror_top_bs);
1720
1721 g_free(s->replaces);
1722 blk_unref(s->target);
1723 bs_opaque->job = NULL;
1724 if (s->dirty_bitmap) {
1725 bdrv_release_dirty_bitmap(bs, s->dirty_bitmap);
1726 }
1727 job_early_fail(&s->common.job);
1728 }
1729
1730 bs_opaque->stop = true;
1731 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
1732 &error_abort);
1733 bdrv_replace_node(mirror_top_bs, backing_bs(mirror_top_bs), &error_abort);
1734
1735 bdrv_unref(mirror_top_bs);
1736
1737 return NULL;
1738}
1739
1740void mirror_start(const char *job_id, BlockDriverState *bs,
1741 BlockDriverState *target, const char *replaces,
1742 int creation_flags, int64_t speed,
1743 uint32_t granularity, int64_t buf_size,
1744 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1745 BlockdevOnError on_source_error,
1746 BlockdevOnError on_target_error,
1747 bool unmap, const char *filter_node_name,
1748 MirrorCopyMode copy_mode, Error **errp)
1749{
1750 bool is_none_mode;
1751 BlockDriverState *base;
1752
1753 if (mode == MIRROR_SYNC_MODE_INCREMENTAL) {
1754 error_setg(errp, "Sync mode 'incremental' not supported");
1755 return;
1756 }
1757 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1758 base = mode == MIRROR_SYNC_MODE_TOP ? backing_bs(bs) : NULL;
1759 mirror_start_job(job_id, bs, creation_flags, target, replaces,
1760 speed, granularity, buf_size, backing_mode,
1761 on_source_error, on_target_error, unmap, NULL, NULL,
1762 &mirror_job_driver, is_none_mode, base, false,
1763 filter_node_name, true, copy_mode, errp);
1764}
1765
1766BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
1767 BlockDriverState *base, int creation_flags,
1768 int64_t speed, BlockdevOnError on_error,
1769 const char *filter_node_name,
1770 BlockCompletionFunc *cb, void *opaque,
1771 bool auto_complete, Error **errp)
1772{
1773 bool base_read_only;
1774 Error *local_err = NULL;
1775 BlockJob *ret;
1776
1777 base_read_only = bdrv_is_read_only(base);
1778
1779 if (base_read_only) {
1780 if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1781 return NULL;
1782 }
1783 }
1784
1785 ret = mirror_start_job(
1786 job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1787 MIRROR_LEAVE_BACKING_CHAIN,
1788 on_error, on_error, true, cb, opaque,
1789 &commit_active_job_driver, false, base, auto_complete,
1790 filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1791 &local_err);
1792 if (local_err) {
1793 error_propagate(errp, local_err);
1794 goto error_restore_flags;
1795 }
1796
1797 return ret;
1798
1799error_restore_flags:
1800
1801
1802 if (base_read_only) {
1803 bdrv_reopen_set_read_only(base, true, NULL);
1804 }
1805 return NULL;
1806}
1807