1
2
3
4
5
6
7
8
9
10
11
12
13
14#include "qemu/osdep.h"
15#include "qemu/cutils.h"
16#include "qemu/coroutine.h"
17#include "qemu/range.h"
18#include "trace.h"
19#include "block/blockjob_int.h"
20#include "block/block_int.h"
21#include "block/dirty-bitmap.h"
22#include "sysemu/block-backend.h"
23#include "qapi/error.h"
24#include "qemu/ratelimit.h"
25#include "qemu/bitmap.h"
26#include "qemu/memalign.h"
27
28#define MAX_IN_FLIGHT 16
29#define MAX_IO_BYTES (1 << 20)
30#define DEFAULT_MIRROR_BUF_SIZE (MAX_IN_FLIGHT * MAX_IO_BYTES)
31
32
33
34
35typedef struct MirrorBuffer {
36 QSIMPLEQ_ENTRY(MirrorBuffer) next;
37} MirrorBuffer;
38
39typedef struct MirrorOp MirrorOp;
40
41typedef struct MirrorBlockJob {
42 BlockJob common;
43 BlockBackend *target;
44 BlockDriverState *mirror_top_bs;
45 BlockDriverState *base;
46 BlockDriverState *base_overlay;
47
48
49 char *replaces;
50
51 BlockDriverState *to_replace;
52
53 Error *replace_blocker;
54 bool is_none_mode;
55 BlockMirrorBackingMode backing_mode;
56
57 bool zero_target;
58 MirrorCopyMode copy_mode;
59 BlockdevOnError on_source_error, on_target_error;
60
61
62 bool actively_synced;
63 bool should_complete;
64 int64_t granularity;
65 size_t buf_size;
66 int64_t bdev_length;
67 unsigned long *cow_bitmap;
68 BdrvDirtyBitmap *dirty_bitmap;
69 BdrvDirtyBitmapIter *dbi;
70 uint8_t *buf;
71 QSIMPLEQ_HEAD(, MirrorBuffer) buf_free;
72 int buf_free_count;
73
74 uint64_t last_pause_ns;
75 unsigned long *in_flight_bitmap;
76 unsigned in_flight;
77 int64_t bytes_in_flight;
78 QTAILQ_HEAD(, MirrorOp) ops_in_flight;
79 int ret;
80 bool unmap;
81 int target_cluster_size;
82 int max_iov;
83 bool initial_zeroing_ongoing;
84 int in_active_write_counter;
85 int64_t active_write_bytes_in_flight;
86 bool prepared;
87 bool in_drain;
88} MirrorBlockJob;
89
90typedef struct MirrorBDSOpaque {
91 MirrorBlockJob *job;
92 bool stop;
93 bool is_commit;
94} MirrorBDSOpaque;
95
96struct MirrorOp {
97 MirrorBlockJob *s;
98 QEMUIOVector qiov;
99 int64_t offset;
100 uint64_t bytes;
101
102
103
104 int64_t *bytes_handled;
105
106 bool is_pseudo_op;
107 bool is_active_write;
108 bool is_in_flight;
109 CoQueue waiting_requests;
110 Coroutine *co;
111 MirrorOp *waiting_for_op;
112
113 QTAILQ_ENTRY(MirrorOp) next;
114};
115
116typedef enum MirrorMethod {
117 MIRROR_METHOD_COPY,
118 MIRROR_METHOD_ZERO,
119 MIRROR_METHOD_DISCARD,
120} MirrorMethod;
121
122static BlockErrorAction mirror_error_action(MirrorBlockJob *s, bool read,
123 int error)
124{
125 s->actively_synced = false;
126 if (read) {
127 return block_job_error_action(&s->common, s->on_source_error,
128 true, error);
129 } else {
130 return block_job_error_action(&s->common, s->on_target_error,
131 false, error);
132 }
133}
134
135static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self,
136 MirrorBlockJob *s,
137 uint64_t offset,
138 uint64_t bytes)
139{
140 uint64_t self_start_chunk = offset / s->granularity;
141 uint64_t self_end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
142 uint64_t self_nb_chunks = self_end_chunk - self_start_chunk;
143
144 while (find_next_bit(s->in_flight_bitmap, self_end_chunk,
145 self_start_chunk) < self_end_chunk &&
146 s->ret >= 0)
147 {
148 MirrorOp *op;
149
150 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
151 uint64_t op_start_chunk = op->offset / s->granularity;
152 uint64_t op_nb_chunks = DIV_ROUND_UP(op->offset + op->bytes,
153 s->granularity) -
154 op_start_chunk;
155
156 if (op == self) {
157 continue;
158 }
159
160 if (ranges_overlap(self_start_chunk, self_nb_chunks,
161 op_start_chunk, op_nb_chunks))
162 {
163 if (self) {
164
165
166
167
168
169 if (op->waiting_for_op) {
170 continue;
171 }
172
173 self->waiting_for_op = op;
174 }
175
176 qemu_co_queue_wait(&op->waiting_requests, NULL);
177
178 if (self) {
179 self->waiting_for_op = NULL;
180 }
181
182 break;
183 }
184 }
185 }
186}
187
188static void coroutine_fn mirror_iteration_done(MirrorOp *op, int ret)
189{
190 MirrorBlockJob *s = op->s;
191 struct iovec *iov;
192 int64_t chunk_num;
193 int i, nb_chunks;
194
195 trace_mirror_iteration_done(s, op->offset, op->bytes, ret);
196
197 s->in_flight--;
198 s->bytes_in_flight -= op->bytes;
199 iov = op->qiov.iov;
200 for (i = 0; i < op->qiov.niov; i++) {
201 MirrorBuffer *buf = (MirrorBuffer *) iov[i].iov_base;
202 QSIMPLEQ_INSERT_TAIL(&s->buf_free, buf, next);
203 s->buf_free_count++;
204 }
205
206 chunk_num = op->offset / s->granularity;
207 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
208
209 bitmap_clear(s->in_flight_bitmap, chunk_num, nb_chunks);
210 QTAILQ_REMOVE(&s->ops_in_flight, op, next);
211 if (ret >= 0) {
212 if (s->cow_bitmap) {
213 bitmap_set(s->cow_bitmap, chunk_num, nb_chunks);
214 }
215 if (!s->initial_zeroing_ongoing) {
216 job_progress_update(&s->common.job, op->bytes);
217 }
218 }
219 qemu_iovec_destroy(&op->qiov);
220
221 qemu_co_queue_restart_all(&op->waiting_requests);
222 g_free(op);
223}
224
225static void coroutine_fn mirror_write_complete(MirrorOp *op, int ret)
226{
227 MirrorBlockJob *s = op->s;
228
229 if (ret < 0) {
230 BlockErrorAction action;
231
232 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
233 action = mirror_error_action(s, false, -ret);
234 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
235 s->ret = ret;
236 }
237 }
238
239 mirror_iteration_done(op, ret);
240}
241
242static void coroutine_fn mirror_read_complete(MirrorOp *op, int ret)
243{
244 MirrorBlockJob *s = op->s;
245
246 if (ret < 0) {
247 BlockErrorAction action;
248
249 bdrv_set_dirty_bitmap(s->dirty_bitmap, op->offset, op->bytes);
250 action = mirror_error_action(s, true, -ret);
251 if (action == BLOCK_ERROR_ACTION_REPORT && s->ret >= 0) {
252 s->ret = ret;
253 }
254
255 mirror_iteration_done(op, ret);
256 return;
257 }
258
259 ret = blk_co_pwritev(s->target, op->offset, op->qiov.size, &op->qiov, 0);
260 mirror_write_complete(op, ret);
261}
262
263
264static inline int64_t mirror_clip_bytes(MirrorBlockJob *s,
265 int64_t offset,
266 int64_t bytes)
267{
268 return MIN(bytes, s->bdev_length - offset);
269}
270
271
272
273static int coroutine_fn mirror_cow_align(MirrorBlockJob *s, int64_t *offset,
274 uint64_t *bytes)
275{
276 bool need_cow;
277 int ret = 0;
278 int64_t align_offset = *offset;
279 int64_t align_bytes = *bytes;
280 int max_bytes = s->granularity * s->max_iov;
281
282 need_cow = !test_bit(*offset / s->granularity, s->cow_bitmap);
283 need_cow |= !test_bit((*offset + *bytes - 1) / s->granularity,
284 s->cow_bitmap);
285 if (need_cow) {
286 bdrv_round_to_clusters(blk_bs(s->target), *offset, *bytes,
287 &align_offset, &align_bytes);
288 }
289
290 if (align_bytes > max_bytes) {
291 align_bytes = max_bytes;
292 if (need_cow) {
293 align_bytes = QEMU_ALIGN_DOWN(align_bytes, s->target_cluster_size);
294 }
295 }
296
297
298 align_bytes = mirror_clip_bytes(s, align_offset, align_bytes);
299
300 ret = align_offset + align_bytes - (*offset + *bytes);
301 *offset = align_offset;
302 *bytes = align_bytes;
303 assert(ret >= 0);
304 return ret;
305}
306
307static inline void coroutine_fn
308mirror_wait_for_free_in_flight_slot(MirrorBlockJob *s)
309{
310 MirrorOp *op;
311
312 QTAILQ_FOREACH(op, &s->ops_in_flight, next) {
313
314
315
316
317
318
319
320
321
322 if (!op->is_pseudo_op && op->is_in_flight && !op->is_active_write) {
323 qemu_co_queue_wait(&op->waiting_requests, NULL);
324 return;
325 }
326 }
327 abort();
328}
329
330
331
332
333
334
335
336
337
338static void coroutine_fn mirror_co_read(void *opaque)
339{
340 MirrorOp *op = opaque;
341 MirrorBlockJob *s = op->s;
342 int nb_chunks;
343 uint64_t ret;
344 uint64_t max_bytes;
345
346 max_bytes = s->granularity * s->max_iov;
347
348
349 op->bytes = MIN(s->buf_size, MIN(max_bytes, op->bytes));
350 assert(op->bytes);
351 assert(op->bytes < BDRV_REQUEST_MAX_BYTES);
352 *op->bytes_handled = op->bytes;
353
354 if (s->cow_bitmap) {
355 *op->bytes_handled += mirror_cow_align(s, &op->offset, &op->bytes);
356 }
357
358 assert(*op->bytes_handled <= UINT_MAX);
359 assert(op->bytes <= s->buf_size);
360
361
362
363 assert(QEMU_IS_ALIGNED(op->offset, s->granularity));
364
365 assert(QEMU_IS_ALIGNED(op->bytes, BDRV_SECTOR_SIZE));
366 nb_chunks = DIV_ROUND_UP(op->bytes, s->granularity);
367
368 while (s->buf_free_count < nb_chunks) {
369 trace_mirror_yield_in_flight(s, op->offset, s->in_flight);
370 mirror_wait_for_free_in_flight_slot(s);
371 }
372
373
374
375
376 qemu_iovec_init(&op->qiov, nb_chunks);
377 while (nb_chunks-- > 0) {
378 MirrorBuffer *buf = QSIMPLEQ_FIRST(&s->buf_free);
379 size_t remaining = op->bytes - op->qiov.size;
380
381 QSIMPLEQ_REMOVE_HEAD(&s->buf_free, next);
382 s->buf_free_count--;
383 qemu_iovec_add(&op->qiov, buf, MIN(s->granularity, remaining));
384 }
385
386
387 s->in_flight++;
388 s->bytes_in_flight += op->bytes;
389 op->is_in_flight = true;
390 trace_mirror_one_iteration(s, op->offset, op->bytes);
391
392 WITH_GRAPH_RDLOCK_GUARD() {
393 ret = bdrv_co_preadv(s->mirror_top_bs->backing, op->offset, op->bytes,
394 &op->qiov, 0);
395 }
396 mirror_read_complete(op, ret);
397}
398
399static void coroutine_fn mirror_co_zero(void *opaque)
400{
401 MirrorOp *op = opaque;
402 int ret;
403
404 op->s->in_flight++;
405 op->s->bytes_in_flight += op->bytes;
406 *op->bytes_handled = op->bytes;
407 op->is_in_flight = true;
408
409 ret = blk_co_pwrite_zeroes(op->s->target, op->offset, op->bytes,
410 op->s->unmap ? BDRV_REQ_MAY_UNMAP : 0);
411 mirror_write_complete(op, ret);
412}
413
414static void coroutine_fn mirror_co_discard(void *opaque)
415{
416 MirrorOp *op = opaque;
417 int ret;
418
419 op->s->in_flight++;
420 op->s->bytes_in_flight += op->bytes;
421 *op->bytes_handled = op->bytes;
422 op->is_in_flight = true;
423
424 ret = blk_co_pdiscard(op->s->target, op->offset, op->bytes);
425 mirror_write_complete(op, ret);
426}
427
428static unsigned mirror_perform(MirrorBlockJob *s, int64_t offset,
429 unsigned bytes, MirrorMethod mirror_method)
430{
431 MirrorOp *op;
432 Coroutine *co;
433 int64_t bytes_handled = -1;
434
435 op = g_new(MirrorOp, 1);
436 *op = (MirrorOp){
437 .s = s,
438 .offset = offset,
439 .bytes = bytes,
440 .bytes_handled = &bytes_handled,
441 };
442 qemu_co_queue_init(&op->waiting_requests);
443
444 switch (mirror_method) {
445 case MIRROR_METHOD_COPY:
446 co = qemu_coroutine_create(mirror_co_read, op);
447 break;
448 case MIRROR_METHOD_ZERO:
449 co = qemu_coroutine_create(mirror_co_zero, op);
450 break;
451 case MIRROR_METHOD_DISCARD:
452 co = qemu_coroutine_create(mirror_co_discard, op);
453 break;
454 default:
455 abort();
456 }
457 op->co = co;
458
459 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
460 qemu_coroutine_enter(co);
461
462
463
464
465 assert(bytes_handled >= 0);
466
467
468
469
470 assert(bytes_handled <= UINT_MAX);
471 return bytes_handled;
472}
473
474static void coroutine_fn mirror_iteration(MirrorBlockJob *s)
475{
476 BlockDriverState *source = s->mirror_top_bs->backing->bs;
477 MirrorOp *pseudo_op;
478 int64_t offset;
479
480 int nb_chunks = 1;
481 bool write_zeroes_ok = bdrv_can_write_zeroes_with_unmap(blk_bs(s->target));
482 int max_io_bytes = MAX(s->buf_size / MAX_IN_FLIGHT, MAX_IO_BYTES);
483
484 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
485 offset = bdrv_dirty_iter_next(s->dbi);
486 if (offset < 0) {
487 bdrv_set_dirty_iter(s->dbi, 0);
488 offset = bdrv_dirty_iter_next(s->dbi);
489 trace_mirror_restart_iter(s, bdrv_get_dirty_count(s->dirty_bitmap));
490 assert(offset >= 0);
491 }
492 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
493
494
495
496
497
498
499
500
501 mirror_wait_on_conflicts(NULL, s, offset, 1);
502
503 job_pause_point(&s->common.job);
504
505
506
507 bdrv_dirty_bitmap_lock(s->dirty_bitmap);
508 while (nb_chunks * s->granularity < s->buf_size) {
509 int64_t next_dirty;
510 int64_t next_offset = offset + nb_chunks * s->granularity;
511 int64_t next_chunk = next_offset / s->granularity;
512 if (next_offset >= s->bdev_length ||
513 !bdrv_dirty_bitmap_get_locked(s->dirty_bitmap, next_offset)) {
514 break;
515 }
516 if (test_bit(next_chunk, s->in_flight_bitmap)) {
517 break;
518 }
519
520 next_dirty = bdrv_dirty_iter_next(s->dbi);
521 if (next_dirty > next_offset || next_dirty < 0) {
522
523 bdrv_set_dirty_iter(s->dbi, next_offset);
524 next_dirty = bdrv_dirty_iter_next(s->dbi);
525 }
526 assert(next_dirty == next_offset);
527 nb_chunks++;
528 }
529
530
531
532
533
534 bdrv_reset_dirty_bitmap_locked(s->dirty_bitmap, offset,
535 nb_chunks * s->granularity);
536 bdrv_dirty_bitmap_unlock(s->dirty_bitmap);
537
538
539
540
541
542
543
544 pseudo_op = g_new(MirrorOp, 1);
545 *pseudo_op = (MirrorOp){
546 .offset = offset,
547 .bytes = nb_chunks * s->granularity,
548 .is_pseudo_op = true,
549 };
550 qemu_co_queue_init(&pseudo_op->waiting_requests);
551 QTAILQ_INSERT_TAIL(&s->ops_in_flight, pseudo_op, next);
552
553 bitmap_set(s->in_flight_bitmap, offset / s->granularity, nb_chunks);
554 while (nb_chunks > 0 && offset < s->bdev_length) {
555 int ret;
556 int64_t io_bytes;
557 int64_t io_bytes_acct;
558 MirrorMethod mirror_method = MIRROR_METHOD_COPY;
559
560 assert(!(offset % s->granularity));
561 WITH_GRAPH_RDLOCK_GUARD() {
562 ret = bdrv_block_status_above(source, NULL, offset,
563 nb_chunks * s->granularity,
564 &io_bytes, NULL, NULL);
565 }
566 if (ret < 0) {
567 io_bytes = MIN(nb_chunks * s->granularity, max_io_bytes);
568 } else if (ret & BDRV_BLOCK_DATA) {
569 io_bytes = MIN(io_bytes, max_io_bytes);
570 }
571
572 io_bytes -= io_bytes % s->granularity;
573 if (io_bytes < s->granularity) {
574 io_bytes = s->granularity;
575 } else if (ret >= 0 && !(ret & BDRV_BLOCK_DATA)) {
576 int64_t target_offset;
577 int64_t target_bytes;
578 WITH_GRAPH_RDLOCK_GUARD() {
579 bdrv_round_to_clusters(blk_bs(s->target), offset, io_bytes,
580 &target_offset, &target_bytes);
581 }
582 if (target_offset == offset &&
583 target_bytes == io_bytes) {
584 mirror_method = ret & BDRV_BLOCK_ZERO ?
585 MIRROR_METHOD_ZERO :
586 MIRROR_METHOD_DISCARD;
587 }
588 }
589
590 while (s->in_flight >= MAX_IN_FLIGHT) {
591 trace_mirror_yield_in_flight(s, offset, s->in_flight);
592 mirror_wait_for_free_in_flight_slot(s);
593 }
594
595 if (s->ret < 0) {
596 ret = 0;
597 goto fail;
598 }
599
600 io_bytes = mirror_clip_bytes(s, offset, io_bytes);
601 io_bytes = mirror_perform(s, offset, io_bytes, mirror_method);
602 if (mirror_method != MIRROR_METHOD_COPY && write_zeroes_ok) {
603 io_bytes_acct = 0;
604 } else {
605 io_bytes_acct = io_bytes;
606 }
607 assert(io_bytes);
608 offset += io_bytes;
609 nb_chunks -= DIV_ROUND_UP(io_bytes, s->granularity);
610 block_job_ratelimit_processed_bytes(&s->common, io_bytes_acct);
611 }
612
613fail:
614 QTAILQ_REMOVE(&s->ops_in_flight, pseudo_op, next);
615 qemu_co_queue_restart_all(&pseudo_op->waiting_requests);
616 g_free(pseudo_op);
617}
618
619static void mirror_free_init(MirrorBlockJob *s)
620{
621 int granularity = s->granularity;
622 size_t buf_size = s->buf_size;
623 uint8_t *buf = s->buf;
624
625 assert(s->buf_free_count == 0);
626 QSIMPLEQ_INIT(&s->buf_free);
627 while (buf_size != 0) {
628 MirrorBuffer *cur = (MirrorBuffer *)buf;
629 QSIMPLEQ_INSERT_TAIL(&s->buf_free, cur, next);
630 s->buf_free_count++;
631 buf_size -= granularity;
632 buf += granularity;
633 }
634}
635
636
637
638
639
640static void coroutine_fn mirror_wait_for_all_io(MirrorBlockJob *s)
641{
642 while (s->in_flight > 0) {
643 mirror_wait_for_free_in_flight_slot(s);
644 }
645}
646
647
648
649
650
651
652static int mirror_exit_common(Job *job)
653{
654 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
655 BlockJob *bjob = &s->common;
656 MirrorBDSOpaque *bs_opaque;
657 AioContext *replace_aio_context = NULL;
658 BlockDriverState *src;
659 BlockDriverState *target_bs;
660 BlockDriverState *mirror_top_bs;
661 Error *local_err = NULL;
662 bool abort = job->ret < 0;
663 int ret = 0;
664
665 GLOBAL_STATE_CODE();
666
667 if (s->prepared) {
668 return 0;
669 }
670 s->prepared = true;
671
672 aio_context_acquire(qemu_get_aio_context());
673
674 mirror_top_bs = s->mirror_top_bs;
675 bs_opaque = mirror_top_bs->opaque;
676 src = mirror_top_bs->backing->bs;
677 target_bs = blk_bs(s->target);
678
679 if (bdrv_chain_contains(src, target_bs)) {
680 bdrv_unfreeze_backing_chain(mirror_top_bs, target_bs);
681 }
682
683 bdrv_release_dirty_bitmap(s->dirty_bitmap);
684
685
686
687 bdrv_ref(src);
688 bdrv_ref(mirror_top_bs);
689 bdrv_ref(target_bs);
690
691
692
693
694
695
696 blk_unref(s->target);
697 s->target = NULL;
698
699
700
701
702
703 bdrv_drained_begin(mirror_top_bs);
704 bs_opaque->stop = true;
705 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
706 &error_abort);
707 if (!abort && s->backing_mode == MIRROR_SOURCE_BACKING_CHAIN) {
708 BlockDriverState *backing = s->is_none_mode ? src : s->base;
709 BlockDriverState *unfiltered_target = bdrv_skip_filters(target_bs);
710
711 if (bdrv_cow_bs(unfiltered_target) != backing) {
712 bdrv_set_backing_hd(unfiltered_target, backing, &local_err);
713 if (local_err) {
714 error_report_err(local_err);
715 local_err = NULL;
716 ret = -EPERM;
717 }
718 }
719 } else if (!abort && s->backing_mode == MIRROR_OPEN_BACKING_CHAIN) {
720 assert(!bdrv_backing_chain_next(target_bs));
721 ret = bdrv_open_backing_file(bdrv_skip_filters(target_bs), NULL,
722 "backing", &local_err);
723 if (ret < 0) {
724 error_report_err(local_err);
725 local_err = NULL;
726 }
727 }
728
729 if (s->to_replace) {
730 replace_aio_context = bdrv_get_aio_context(s->to_replace);
731 aio_context_acquire(replace_aio_context);
732 }
733
734 if (s->should_complete && !abort) {
735 BlockDriverState *to_replace = s->to_replace ?: src;
736 bool ro = bdrv_is_read_only(to_replace);
737
738 if (ro != bdrv_is_read_only(target_bs)) {
739 bdrv_reopen_set_read_only(target_bs, ro, NULL);
740 }
741
742
743
744 assert(s->in_drain);
745 bdrv_drained_begin(target_bs);
746
747
748
749
750
751
752
753 bdrv_graph_rdlock_main_loop();
754 if (bdrv_recurse_can_replace(src, to_replace)) {
755 bdrv_replace_node(to_replace, target_bs, &local_err);
756 } else {
757 error_setg(&local_err, "Can no longer replace '%s' by '%s', "
758 "because it can no longer be guaranteed that doing so "
759 "would not lead to an abrupt change of visible data",
760 to_replace->node_name, target_bs->node_name);
761 }
762 bdrv_graph_rdunlock_main_loop();
763 bdrv_drained_end(target_bs);
764 if (local_err) {
765 error_report_err(local_err);
766 ret = -EPERM;
767 }
768 }
769 if (s->to_replace) {
770 bdrv_op_unblock_all(s->to_replace, s->replace_blocker);
771 error_free(s->replace_blocker);
772 bdrv_unref(s->to_replace);
773 }
774 if (replace_aio_context) {
775 aio_context_release(replace_aio_context);
776 }
777 g_free(s->replaces);
778 bdrv_unref(target_bs);
779
780
781
782
783
784
785 block_job_remove_all_bdrv(bjob);
786 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
787
788 bs_opaque->job = NULL;
789
790 bdrv_drained_end(src);
791 bdrv_drained_end(mirror_top_bs);
792 s->in_drain = false;
793 bdrv_unref(mirror_top_bs);
794 bdrv_unref(src);
795
796 aio_context_release(qemu_get_aio_context());
797
798 return ret;
799}
800
801static int mirror_prepare(Job *job)
802{
803 return mirror_exit_common(job);
804}
805
806static void mirror_abort(Job *job)
807{
808 int ret = mirror_exit_common(job);
809 assert(ret == 0);
810}
811
812static void coroutine_fn mirror_throttle(MirrorBlockJob *s)
813{
814 int64_t now = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
815
816 if (now - s->last_pause_ns > BLOCK_JOB_SLICE_TIME) {
817 s->last_pause_ns = now;
818 job_sleep_ns(&s->common.job, 0);
819 } else {
820 job_pause_point(&s->common.job);
821 }
822}
823
824static int coroutine_fn mirror_dirty_init(MirrorBlockJob *s)
825{
826 int64_t offset;
827 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
828 BlockDriverState *target_bs = blk_bs(s->target);
829 int ret;
830 int64_t count;
831
832 if (s->zero_target) {
833 if (!bdrv_can_write_zeroes_with_unmap(target_bs)) {
834 bdrv_set_dirty_bitmap(s->dirty_bitmap, 0, s->bdev_length);
835 return 0;
836 }
837
838 s->initial_zeroing_ongoing = true;
839 for (offset = 0; offset < s->bdev_length; ) {
840 int bytes = MIN(s->bdev_length - offset,
841 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
842
843 mirror_throttle(s);
844
845 if (job_is_cancelled(&s->common.job)) {
846 s->initial_zeroing_ongoing = false;
847 return 0;
848 }
849
850 if (s->in_flight >= MAX_IN_FLIGHT) {
851 trace_mirror_yield(s, UINT64_MAX, s->buf_free_count,
852 s->in_flight);
853 mirror_wait_for_free_in_flight_slot(s);
854 continue;
855 }
856
857 mirror_perform(s, offset, bytes, MIRROR_METHOD_ZERO);
858 offset += bytes;
859 }
860
861 mirror_wait_for_all_io(s);
862 s->initial_zeroing_ongoing = false;
863 }
864
865
866 for (offset = 0; offset < s->bdev_length; ) {
867
868 int bytes = MIN(s->bdev_length - offset,
869 QEMU_ALIGN_DOWN(INT_MAX, s->granularity));
870
871 mirror_throttle(s);
872
873 if (job_is_cancelled(&s->common.job)) {
874 return 0;
875 }
876
877 WITH_GRAPH_RDLOCK_GUARD() {
878 ret = bdrv_is_allocated_above(bs, s->base_overlay, true, offset,
879 bytes, &count);
880 }
881 if (ret < 0) {
882 return ret;
883 }
884
885 assert(count);
886 if (ret > 0) {
887 bdrv_set_dirty_bitmap(s->dirty_bitmap, offset, count);
888 }
889 offset += count;
890 }
891 return 0;
892}
893
894
895
896
897static int coroutine_fn mirror_flush(MirrorBlockJob *s)
898{
899 int ret = blk_co_flush(s->target);
900 if (ret < 0) {
901 if (mirror_error_action(s, false, -ret) == BLOCK_ERROR_ACTION_REPORT) {
902 s->ret = ret;
903 }
904 }
905 return ret;
906}
907
908static int coroutine_fn mirror_run(Job *job, Error **errp)
909{
910 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
911 BlockDriverState *bs = s->mirror_top_bs->backing->bs;
912 MirrorBDSOpaque *mirror_top_opaque = s->mirror_top_bs->opaque;
913 BlockDriverState *target_bs = blk_bs(s->target);
914 bool need_drain = true;
915 BlockDeviceIoStatus iostatus;
916 int64_t length;
917 int64_t target_length;
918 BlockDriverInfo bdi;
919 char backing_filename[2];
920
921 int ret = 0;
922
923 if (job_is_cancelled(&s->common.job)) {
924 goto immediate_exit;
925 }
926
927 bdrv_graph_co_rdlock();
928 s->bdev_length = bdrv_co_getlength(bs);
929 bdrv_graph_co_rdunlock();
930
931 if (s->bdev_length < 0) {
932 ret = s->bdev_length;
933 goto immediate_exit;
934 }
935
936 target_length = blk_co_getlength(s->target);
937 if (target_length < 0) {
938 ret = target_length;
939 goto immediate_exit;
940 }
941
942
943
944 if (s->base == blk_bs(s->target)) {
945 if (s->bdev_length > target_length) {
946 ret = blk_co_truncate(s->target, s->bdev_length, false,
947 PREALLOC_MODE_OFF, 0, NULL);
948 if (ret < 0) {
949 goto immediate_exit;
950 }
951 }
952 } else if (s->bdev_length != target_length) {
953 error_setg(errp, "Source and target image have different sizes");
954 ret = -EINVAL;
955 goto immediate_exit;
956 }
957
958 if (s->bdev_length == 0) {
959
960 job_transition_to_ready(&s->common.job);
961 s->actively_synced = true;
962 while (!job_cancel_requested(&s->common.job) && !s->should_complete) {
963 job_yield(&s->common.job);
964 }
965 goto immediate_exit;
966 }
967
968 length = DIV_ROUND_UP(s->bdev_length, s->granularity);
969 s->in_flight_bitmap = bitmap_new(length);
970
971
972
973
974
975 bdrv_get_backing_filename(target_bs, backing_filename,
976 sizeof(backing_filename));
977 bdrv_graph_co_rdlock();
978 if (!bdrv_co_get_info(target_bs, &bdi) && bdi.cluster_size) {
979 s->target_cluster_size = bdi.cluster_size;
980 } else {
981 s->target_cluster_size = BDRV_SECTOR_SIZE;
982 }
983 bdrv_graph_co_rdunlock();
984 if (backing_filename[0] && !bdrv_backing_chain_next(target_bs) &&
985 s->granularity < s->target_cluster_size) {
986 s->buf_size = MAX(s->buf_size, s->target_cluster_size);
987 s->cow_bitmap = bitmap_new(length);
988 }
989 s->max_iov = MIN(bs->bl.max_iov, target_bs->bl.max_iov);
990
991 s->buf = qemu_try_blockalign(bs, s->buf_size);
992 if (s->buf == NULL) {
993 ret = -ENOMEM;
994 goto immediate_exit;
995 }
996
997 mirror_free_init(s);
998
999 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1000 if (!s->is_none_mode) {
1001 ret = mirror_dirty_init(s);
1002 if (ret < 0 || job_is_cancelled(&s->common.job)) {
1003 goto immediate_exit;
1004 }
1005 }
1006
1007
1008
1009
1010
1011 mirror_top_opaque->job = s;
1012
1013 assert(!s->dbi);
1014 s->dbi = bdrv_dirty_iter_new(s->dirty_bitmap);
1015 for (;;) {
1016 int64_t cnt, delta;
1017 bool should_complete;
1018
1019 if (s->ret < 0) {
1020 ret = s->ret;
1021 goto immediate_exit;
1022 }
1023
1024 job_pause_point(&s->common.job);
1025
1026 if (job_is_cancelled(&s->common.job)) {
1027 ret = 0;
1028 goto immediate_exit;
1029 }
1030
1031 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1032
1033
1034
1035 job_progress_set_remaining(&s->common.job,
1036 s->bytes_in_flight + cnt +
1037 s->active_write_bytes_in_flight);
1038
1039
1040
1041
1042
1043 delta = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - s->last_pause_ns;
1044 WITH_JOB_LOCK_GUARD() {
1045 iostatus = s->common.iostatus;
1046 }
1047 if (delta < BLOCK_JOB_SLICE_TIME &&
1048 iostatus == BLOCK_DEVICE_IO_STATUS_OK) {
1049 if (s->in_flight >= MAX_IN_FLIGHT || s->buf_free_count == 0 ||
1050 (cnt == 0 && s->in_flight > 0)) {
1051 trace_mirror_yield(s, cnt, s->buf_free_count, s->in_flight);
1052 mirror_wait_for_free_in_flight_slot(s);
1053 continue;
1054 } else if (cnt != 0) {
1055 mirror_iteration(s);
1056 }
1057 }
1058
1059 should_complete = false;
1060 if (s->in_flight == 0 && cnt == 0) {
1061 trace_mirror_before_flush(s);
1062 if (!job_is_ready(&s->common.job)) {
1063 if (mirror_flush(s) < 0) {
1064
1065 continue;
1066 }
1067
1068
1069
1070
1071
1072 job_transition_to_ready(&s->common.job);
1073 if (s->copy_mode != MIRROR_COPY_MODE_BACKGROUND) {
1074 s->actively_synced = true;
1075 }
1076 }
1077
1078 should_complete = s->should_complete ||
1079 job_cancel_requested(&s->common.job);
1080 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1081 }
1082
1083 if (cnt == 0 && should_complete) {
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094 trace_mirror_before_drain(s, cnt);
1095
1096 s->in_drain = true;
1097 bdrv_drained_begin(bs);
1098
1099
1100 assert(s->in_active_write_counter == 0);
1101
1102 cnt = bdrv_get_dirty_count(s->dirty_bitmap);
1103 if (cnt > 0 || mirror_flush(s) < 0) {
1104 bdrv_drained_end(bs);
1105 s->in_drain = false;
1106 continue;
1107 }
1108
1109
1110
1111
1112 assert(QLIST_EMPTY(&bs->tracked_requests));
1113 need_drain = false;
1114 break;
1115 }
1116
1117 if (job_is_ready(&s->common.job) && !should_complete) {
1118 if (s->in_flight == 0 && cnt == 0) {
1119 trace_mirror_before_sleep(s, cnt, job_is_ready(&s->common.job),
1120 BLOCK_JOB_SLICE_TIME);
1121 job_sleep_ns(&s->common.job, BLOCK_JOB_SLICE_TIME);
1122 }
1123 } else {
1124 block_job_ratelimit_sleep(&s->common);
1125 }
1126 s->last_pause_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
1127 }
1128
1129immediate_exit:
1130 if (s->in_flight > 0) {
1131
1132
1133
1134
1135 assert(ret < 0 || job_is_cancelled(&s->common.job));
1136 assert(need_drain);
1137 mirror_wait_for_all_io(s);
1138 }
1139
1140 assert(s->in_flight == 0);
1141 qemu_vfree(s->buf);
1142 g_free(s->cow_bitmap);
1143 g_free(s->in_flight_bitmap);
1144 bdrv_dirty_iter_free(s->dbi);
1145
1146 if (need_drain) {
1147 s->in_drain = true;
1148 bdrv_drained_begin(bs);
1149 }
1150
1151 return ret;
1152}
1153
1154static void mirror_complete(Job *job, Error **errp)
1155{
1156 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1157
1158 if (!job_is_ready(job)) {
1159 error_setg(errp, "The active block job '%s' cannot be completed",
1160 job->id);
1161 return;
1162 }
1163
1164
1165 if (s->replaces) {
1166 AioContext *replace_aio_context;
1167
1168 s->to_replace = bdrv_find_node(s->replaces);
1169 if (!s->to_replace) {
1170 error_setg(errp, "Node name '%s' not found", s->replaces);
1171 return;
1172 }
1173
1174 replace_aio_context = bdrv_get_aio_context(s->to_replace);
1175 aio_context_acquire(replace_aio_context);
1176
1177
1178 error_setg(&s->replace_blocker,
1179 "block device is in use by block-job-complete");
1180 bdrv_op_block_all(s->to_replace, s->replace_blocker);
1181 bdrv_ref(s->to_replace);
1182
1183 aio_context_release(replace_aio_context);
1184 }
1185
1186 s->should_complete = true;
1187
1188
1189 WITH_JOB_LOCK_GUARD() {
1190 if (!job->paused) {
1191 job_enter_cond_locked(job, NULL);
1192 }
1193 }
1194}
1195
1196static void coroutine_fn mirror_pause(Job *job)
1197{
1198 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1199
1200 mirror_wait_for_all_io(s);
1201}
1202
1203static bool mirror_drained_poll(BlockJob *job)
1204{
1205 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common);
1206
1207
1208
1209
1210
1211
1212 WITH_JOB_LOCK_GUARD() {
1213 if (!s->common.job.paused && !job_is_cancelled_locked(&job->job)
1214 && !s->in_drain) {
1215 return true;
1216 }
1217 }
1218
1219 return !!s->in_flight;
1220}
1221
1222static bool mirror_cancel(Job *job, bool force)
1223{
1224 MirrorBlockJob *s = container_of(job, MirrorBlockJob, common.job);
1225 BlockDriverState *target = blk_bs(s->target);
1226
1227
1228
1229
1230
1231 force = force || !job_is_ready(job);
1232
1233 if (force) {
1234 bdrv_cancel_in_flight(target);
1235 }
1236 return force;
1237}
1238
1239static bool commit_active_cancel(Job *job, bool force)
1240{
1241
1242 return force || !job_is_ready(job);
1243}
1244
1245static const BlockJobDriver mirror_job_driver = {
1246 .job_driver = {
1247 .instance_size = sizeof(MirrorBlockJob),
1248 .job_type = JOB_TYPE_MIRROR,
1249 .free = block_job_free,
1250 .user_resume = block_job_user_resume,
1251 .run = mirror_run,
1252 .prepare = mirror_prepare,
1253 .abort = mirror_abort,
1254 .pause = mirror_pause,
1255 .complete = mirror_complete,
1256 .cancel = mirror_cancel,
1257 },
1258 .drained_poll = mirror_drained_poll,
1259};
1260
1261static const BlockJobDriver commit_active_job_driver = {
1262 .job_driver = {
1263 .instance_size = sizeof(MirrorBlockJob),
1264 .job_type = JOB_TYPE_COMMIT,
1265 .free = block_job_free,
1266 .user_resume = block_job_user_resume,
1267 .run = mirror_run,
1268 .prepare = mirror_prepare,
1269 .abort = mirror_abort,
1270 .pause = mirror_pause,
1271 .complete = mirror_complete,
1272 .cancel = commit_active_cancel,
1273 },
1274 .drained_poll = mirror_drained_poll,
1275};
1276
1277static void coroutine_fn
1278do_sync_target_write(MirrorBlockJob *job, MirrorMethod method,
1279 uint64_t offset, uint64_t bytes,
1280 QEMUIOVector *qiov, int flags)
1281{
1282 int ret;
1283 size_t qiov_offset = 0;
1284 int64_t bitmap_offset, bitmap_end;
1285
1286 if (!QEMU_IS_ALIGNED(offset, job->granularity) &&
1287 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset))
1288 {
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306 qiov_offset = QEMU_ALIGN_UP(offset, job->granularity) - offset;
1307 if (bytes <= qiov_offset) {
1308
1309 return;
1310 }
1311 offset += qiov_offset;
1312 bytes -= qiov_offset;
1313 }
1314
1315 if (!QEMU_IS_ALIGNED(offset + bytes, job->granularity) &&
1316 bdrv_dirty_bitmap_get(job->dirty_bitmap, offset + bytes - 1))
1317 {
1318 uint64_t tail = (offset + bytes) % job->granularity;
1319
1320 if (bytes <= tail) {
1321
1322 return;
1323 }
1324 bytes -= tail;
1325 }
1326
1327
1328
1329
1330
1331 bitmap_offset = QEMU_ALIGN_UP(offset, job->granularity);
1332 bitmap_end = QEMU_ALIGN_DOWN(offset + bytes, job->granularity);
1333 if (bitmap_offset < bitmap_end) {
1334 bdrv_reset_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1335 bitmap_end - bitmap_offset);
1336 }
1337
1338 job_progress_increase_remaining(&job->common.job, bytes);
1339 job->active_write_bytes_in_flight += bytes;
1340
1341 switch (method) {
1342 case MIRROR_METHOD_COPY:
1343 ret = blk_co_pwritev_part(job->target, offset, bytes,
1344 qiov, qiov_offset, flags);
1345 break;
1346
1347 case MIRROR_METHOD_ZERO:
1348 assert(!qiov);
1349 ret = blk_co_pwrite_zeroes(job->target, offset, bytes, flags);
1350 break;
1351
1352 case MIRROR_METHOD_DISCARD:
1353 assert(!qiov);
1354 ret = blk_co_pdiscard(job->target, offset, bytes);
1355 break;
1356
1357 default:
1358 abort();
1359 }
1360
1361 job->active_write_bytes_in_flight -= bytes;
1362 if (ret >= 0) {
1363 job_progress_update(&job->common.job, bytes);
1364 } else {
1365 BlockErrorAction action;
1366
1367
1368
1369
1370
1371
1372
1373 bitmap_offset = QEMU_ALIGN_DOWN(offset, job->granularity);
1374 bitmap_end = QEMU_ALIGN_UP(offset + bytes, job->granularity);
1375 bdrv_set_dirty_bitmap(job->dirty_bitmap, bitmap_offset,
1376 bitmap_end - bitmap_offset);
1377 job->actively_synced = false;
1378
1379 action = mirror_error_action(job, false, -ret);
1380 if (action == BLOCK_ERROR_ACTION_REPORT) {
1381 if (!job->ret) {
1382 job->ret = ret;
1383 }
1384 }
1385 }
1386}
1387
1388static MirrorOp *coroutine_fn active_write_prepare(MirrorBlockJob *s,
1389 uint64_t offset,
1390 uint64_t bytes)
1391{
1392 MirrorOp *op;
1393 uint64_t start_chunk = offset / s->granularity;
1394 uint64_t end_chunk = DIV_ROUND_UP(offset + bytes, s->granularity);
1395
1396 op = g_new(MirrorOp, 1);
1397 *op = (MirrorOp){
1398 .s = s,
1399 .offset = offset,
1400 .bytes = bytes,
1401 .is_active_write = true,
1402 .is_in_flight = true,
1403 .co = qemu_coroutine_self(),
1404 };
1405 qemu_co_queue_init(&op->waiting_requests);
1406 QTAILQ_INSERT_TAIL(&s->ops_in_flight, op, next);
1407
1408 s->in_active_write_counter++;
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423 mirror_wait_on_conflicts(op, s, offset, bytes);
1424
1425 bitmap_set(s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1426
1427 return op;
1428}
1429
1430static void coroutine_fn GRAPH_RDLOCK active_write_settle(MirrorOp *op)
1431{
1432 uint64_t start_chunk = op->offset / op->s->granularity;
1433 uint64_t end_chunk = DIV_ROUND_UP(op->offset + op->bytes,
1434 op->s->granularity);
1435
1436 if (!--op->s->in_active_write_counter && op->s->actively_synced) {
1437 BdrvChild *source = op->s->mirror_top_bs->backing;
1438
1439 if (QLIST_FIRST(&source->bs->parents) == source &&
1440 QLIST_NEXT(source, next_parent) == NULL)
1441 {
1442
1443
1444
1445
1446 assert(!bdrv_get_dirty_count(op->s->dirty_bitmap));
1447 }
1448 }
1449 bitmap_clear(op->s->in_flight_bitmap, start_chunk, end_chunk - start_chunk);
1450 QTAILQ_REMOVE(&op->s->ops_in_flight, op, next);
1451 qemu_co_queue_restart_all(&op->waiting_requests);
1452 g_free(op);
1453}
1454
1455static int coroutine_fn GRAPH_RDLOCK
1456bdrv_mirror_top_preadv(BlockDriverState *bs, int64_t offset, int64_t bytes,
1457 QEMUIOVector *qiov, BdrvRequestFlags flags)
1458{
1459 return bdrv_co_preadv(bs->backing, offset, bytes, qiov, flags);
1460}
1461
1462static int coroutine_fn GRAPH_RDLOCK
1463bdrv_mirror_top_do_write(BlockDriverState *bs, MirrorMethod method,
1464 uint64_t offset, uint64_t bytes, QEMUIOVector *qiov,
1465 int flags)
1466{
1467 MirrorOp *op = NULL;
1468 MirrorBDSOpaque *s = bs->opaque;
1469 int ret = 0;
1470 bool copy_to_target = false;
1471
1472 if (s->job) {
1473 copy_to_target = s->job->ret >= 0 &&
1474 !job_is_cancelled(&s->job->common.job) &&
1475 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1476 }
1477
1478 if (copy_to_target) {
1479 op = active_write_prepare(s->job, offset, bytes);
1480 }
1481
1482 switch (method) {
1483 case MIRROR_METHOD_COPY:
1484 ret = bdrv_co_pwritev(bs->backing, offset, bytes, qiov, flags);
1485 break;
1486
1487 case MIRROR_METHOD_ZERO:
1488 ret = bdrv_co_pwrite_zeroes(bs->backing, offset, bytes, flags);
1489 break;
1490
1491 case MIRROR_METHOD_DISCARD:
1492 ret = bdrv_co_pdiscard(bs->backing, offset, bytes);
1493 break;
1494
1495 default:
1496 abort();
1497 }
1498
1499 if (ret < 0) {
1500 goto out;
1501 }
1502
1503 if (copy_to_target) {
1504 do_sync_target_write(s->job, method, offset, bytes, qiov, flags);
1505 }
1506
1507out:
1508 if (copy_to_target) {
1509 active_write_settle(op);
1510 }
1511 return ret;
1512}
1513
1514static int coroutine_fn GRAPH_RDLOCK
1515bdrv_mirror_top_pwritev(BlockDriverState *bs, int64_t offset, int64_t bytes,
1516 QEMUIOVector *qiov, BdrvRequestFlags flags)
1517{
1518 MirrorBDSOpaque *s = bs->opaque;
1519 QEMUIOVector bounce_qiov;
1520 void *bounce_buf;
1521 int ret = 0;
1522 bool copy_to_target = false;
1523
1524 if (s->job) {
1525 copy_to_target = s->job->ret >= 0 &&
1526 !job_is_cancelled(&s->job->common.job) &&
1527 s->job->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING;
1528 }
1529
1530 if (copy_to_target) {
1531
1532
1533
1534
1535 bounce_buf = qemu_blockalign(bs, bytes);
1536 iov_to_buf_full(qiov->iov, qiov->niov, 0, bounce_buf, bytes);
1537
1538 qemu_iovec_init(&bounce_qiov, 1);
1539 qemu_iovec_add(&bounce_qiov, bounce_buf, bytes);
1540 qiov = &bounce_qiov;
1541
1542 flags &= ~BDRV_REQ_REGISTERED_BUF;
1543 }
1544
1545 ret = bdrv_mirror_top_do_write(bs, MIRROR_METHOD_COPY, offset, bytes, qiov,
1546 flags);
1547
1548 if (copy_to_target) {
1549 qemu_iovec_destroy(&bounce_qiov);
1550 qemu_vfree(bounce_buf);
1551 }
1552
1553 return ret;
1554}
1555
1556static int coroutine_fn GRAPH_RDLOCK bdrv_mirror_top_flush(BlockDriverState *bs)
1557{
1558 if (bs->backing == NULL) {
1559
1560 return 0;
1561 }
1562 return bdrv_co_flush(bs->backing->bs);
1563}
1564
1565static int coroutine_fn GRAPH_RDLOCK
1566bdrv_mirror_top_pwrite_zeroes(BlockDriverState *bs, int64_t offset,
1567 int64_t bytes, BdrvRequestFlags flags)
1568{
1569 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_ZERO, offset, bytes, NULL,
1570 flags);
1571}
1572
1573static int coroutine_fn GRAPH_RDLOCK
1574bdrv_mirror_top_pdiscard(BlockDriverState *bs, int64_t offset, int64_t bytes)
1575{
1576 return bdrv_mirror_top_do_write(bs, MIRROR_METHOD_DISCARD, offset, bytes,
1577 NULL, 0);
1578}
1579
1580static void bdrv_mirror_top_refresh_filename(BlockDriverState *bs)
1581{
1582 if (bs->backing == NULL) {
1583
1584
1585 return;
1586 }
1587 pstrcpy(bs->exact_filename, sizeof(bs->exact_filename),
1588 bs->backing->bs->filename);
1589}
1590
1591static void bdrv_mirror_top_child_perm(BlockDriverState *bs, BdrvChild *c,
1592 BdrvChildRole role,
1593 BlockReopenQueue *reopen_queue,
1594 uint64_t perm, uint64_t shared,
1595 uint64_t *nperm, uint64_t *nshared)
1596{
1597 MirrorBDSOpaque *s = bs->opaque;
1598
1599 if (s->stop) {
1600
1601
1602
1603
1604 *nperm = 0;
1605 *nshared = BLK_PERM_ALL;
1606 return;
1607 }
1608
1609 bdrv_default_perms(bs, c, role, reopen_queue,
1610 perm, shared, nperm, nshared);
1611
1612 if (s->is_commit) {
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627 *nperm &= ~BLK_PERM_CONSISTENT_READ;
1628 *nshared |= BLK_PERM_WRITE;
1629 }
1630}
1631
1632
1633
1634static BlockDriver bdrv_mirror_top = {
1635 .format_name = "mirror_top",
1636 .bdrv_co_preadv = bdrv_mirror_top_preadv,
1637 .bdrv_co_pwritev = bdrv_mirror_top_pwritev,
1638 .bdrv_co_pwrite_zeroes = bdrv_mirror_top_pwrite_zeroes,
1639 .bdrv_co_pdiscard = bdrv_mirror_top_pdiscard,
1640 .bdrv_co_flush = bdrv_mirror_top_flush,
1641 .bdrv_refresh_filename = bdrv_mirror_top_refresh_filename,
1642 .bdrv_child_perm = bdrv_mirror_top_child_perm,
1643
1644 .is_filter = true,
1645 .filtered_child_is_backing = true,
1646};
1647
1648static BlockJob *mirror_start_job(
1649 const char *job_id, BlockDriverState *bs,
1650 int creation_flags, BlockDriverState *target,
1651 const char *replaces, int64_t speed,
1652 uint32_t granularity, int64_t buf_size,
1653 BlockMirrorBackingMode backing_mode,
1654 bool zero_target,
1655 BlockdevOnError on_source_error,
1656 BlockdevOnError on_target_error,
1657 bool unmap,
1658 BlockCompletionFunc *cb,
1659 void *opaque,
1660 const BlockJobDriver *driver,
1661 bool is_none_mode, BlockDriverState *base,
1662 bool auto_complete, const char *filter_node_name,
1663 bool is_mirror, MirrorCopyMode copy_mode,
1664 Error **errp)
1665{
1666 MirrorBlockJob *s;
1667 MirrorBDSOpaque *bs_opaque;
1668 BlockDriverState *mirror_top_bs;
1669 bool target_is_backing;
1670 uint64_t target_perms, target_shared_perms;
1671 int ret;
1672
1673 if (granularity == 0) {
1674 granularity = bdrv_get_default_bitmap_granularity(target);
1675 }
1676
1677 assert(is_power_of_2(granularity));
1678
1679 if (buf_size < 0) {
1680 error_setg(errp, "Invalid parameter 'buf-size'");
1681 return NULL;
1682 }
1683
1684 if (buf_size == 0) {
1685 buf_size = DEFAULT_MIRROR_BUF_SIZE;
1686 }
1687
1688 if (bdrv_skip_filters(bs) == bdrv_skip_filters(target)) {
1689 error_setg(errp, "Can't mirror node into itself");
1690 return NULL;
1691 }
1692
1693 target_is_backing = bdrv_chain_contains(bs, target);
1694
1695
1696
1697
1698 mirror_top_bs = bdrv_new_open_driver(&bdrv_mirror_top, filter_node_name,
1699 BDRV_O_RDWR, errp);
1700 if (mirror_top_bs == NULL) {
1701 return NULL;
1702 }
1703 if (!filter_node_name) {
1704 mirror_top_bs->implicit = true;
1705 }
1706
1707
1708 mirror_top_bs->never_freeze = true;
1709
1710 mirror_top_bs->total_sectors = bs->total_sectors;
1711 mirror_top_bs->supported_write_flags = BDRV_REQ_WRITE_UNCHANGED;
1712 mirror_top_bs->supported_zero_flags = BDRV_REQ_WRITE_UNCHANGED |
1713 BDRV_REQ_NO_FALLBACK;
1714 bs_opaque = g_new0(MirrorBDSOpaque, 1);
1715 mirror_top_bs->opaque = bs_opaque;
1716
1717 bs_opaque->is_commit = target_is_backing;
1718
1719 bdrv_drained_begin(bs);
1720 ret = bdrv_append(mirror_top_bs, bs, errp);
1721 bdrv_drained_end(bs);
1722
1723 if (ret < 0) {
1724 bdrv_unref(mirror_top_bs);
1725 return NULL;
1726 }
1727
1728
1729 s = block_job_create(job_id, driver, NULL, mirror_top_bs,
1730 BLK_PERM_CONSISTENT_READ,
1731 BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE_UNCHANGED |
1732 BLK_PERM_WRITE, speed,
1733 creation_flags, cb, opaque, errp);
1734 if (!s) {
1735 goto fail;
1736 }
1737
1738
1739 bdrv_unref(mirror_top_bs);
1740
1741 s->mirror_top_bs = mirror_top_bs;
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752 target_perms = BLK_PERM_WRITE;
1753 target_shared_perms = BLK_PERM_WRITE_UNCHANGED;
1754
1755 if (target_is_backing) {
1756 int64_t bs_size, target_size;
1757 bs_size = bdrv_getlength(bs);
1758 if (bs_size < 0) {
1759 error_setg_errno(errp, -bs_size,
1760 "Could not inquire top image size");
1761 goto fail;
1762 }
1763
1764 target_size = bdrv_getlength(target);
1765 if (target_size < 0) {
1766 error_setg_errno(errp, -target_size,
1767 "Could not inquire base image size");
1768 goto fail;
1769 }
1770
1771 if (target_size < bs_size) {
1772 target_perms |= BLK_PERM_RESIZE;
1773 }
1774
1775 target_shared_perms |= BLK_PERM_CONSISTENT_READ | BLK_PERM_WRITE;
1776 } else if (bdrv_chain_contains(bs, bdrv_skip_filters(target))) {
1777
1778
1779
1780
1781 error_setg(errp, "Cannot mirror to a filter on top of a node in the "
1782 "source's backing chain");
1783 goto fail;
1784 }
1785
1786 s->target = blk_new(s->common.job.aio_context,
1787 target_perms, target_shared_perms);
1788 ret = blk_insert_bs(s->target, target, errp);
1789 if (ret < 0) {
1790 goto fail;
1791 }
1792 if (is_mirror) {
1793
1794
1795
1796
1797
1798
1799 blk_set_force_allow_inactivate(s->target);
1800 }
1801 blk_set_allow_aio_context_change(s->target, true);
1802 blk_set_disable_request_queuing(s->target, true);
1803
1804 s->replaces = g_strdup(replaces);
1805 s->on_source_error = on_source_error;
1806 s->on_target_error = on_target_error;
1807 s->is_none_mode = is_none_mode;
1808 s->backing_mode = backing_mode;
1809 s->zero_target = zero_target;
1810 s->copy_mode = copy_mode;
1811 s->base = base;
1812 s->base_overlay = bdrv_find_overlay(bs, base);
1813 s->granularity = granularity;
1814 s->buf_size = ROUND_UP(buf_size, granularity);
1815 s->unmap = unmap;
1816 if (auto_complete) {
1817 s->should_complete = true;
1818 }
1819
1820 s->dirty_bitmap = bdrv_create_dirty_bitmap(bs, granularity, NULL, errp);
1821 if (!s->dirty_bitmap) {
1822 goto fail;
1823 }
1824 if (s->copy_mode == MIRROR_COPY_MODE_WRITE_BLOCKING) {
1825 bdrv_disable_dirty_bitmap(s->dirty_bitmap);
1826 }
1827
1828 ret = block_job_add_bdrv(&s->common, "source", bs, 0,
1829 BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE |
1830 BLK_PERM_CONSISTENT_READ,
1831 errp);
1832 if (ret < 0) {
1833 goto fail;
1834 }
1835
1836
1837 block_job_add_bdrv(&s->common, "target", target, 0, BLK_PERM_ALL,
1838 &error_abort);
1839
1840
1841
1842 if (target_is_backing) {
1843 BlockDriverState *iter, *filtered_target;
1844 uint64_t iter_shared_perms;
1845
1846
1847
1848
1849
1850 filtered_target = bdrv_cow_bs(bdrv_find_overlay(bs, target));
1851
1852 assert(bdrv_skip_filters(filtered_target) ==
1853 bdrv_skip_filters(target));
1854
1855
1856
1857
1858
1859
1860
1861 iter_shared_perms = BLK_PERM_WRITE_UNCHANGED | BLK_PERM_WRITE;
1862
1863 for (iter = bdrv_filter_or_cow_bs(bs); iter != target;
1864 iter = bdrv_filter_or_cow_bs(iter))
1865 {
1866 if (iter == filtered_target) {
1867
1868
1869
1870
1871 iter_shared_perms |= BLK_PERM_CONSISTENT_READ;
1872 }
1873
1874 ret = block_job_add_bdrv(&s->common, "intermediate node", iter, 0,
1875 iter_shared_perms, errp);
1876 if (ret < 0) {
1877 goto fail;
1878 }
1879 }
1880
1881 if (bdrv_freeze_backing_chain(mirror_top_bs, target, errp) < 0) {
1882 goto fail;
1883 }
1884 }
1885
1886 QTAILQ_INIT(&s->ops_in_flight);
1887
1888 trace_mirror_start(bs, s, opaque);
1889 job_start(&s->common.job);
1890
1891 return &s->common;
1892
1893fail:
1894 if (s) {
1895
1896
1897 bdrv_ref(mirror_top_bs);
1898
1899 g_free(s->replaces);
1900 blk_unref(s->target);
1901 bs_opaque->job = NULL;
1902 if (s->dirty_bitmap) {
1903 bdrv_release_dirty_bitmap(s->dirty_bitmap);
1904 }
1905 job_early_fail(&s->common.job);
1906 }
1907
1908 bs_opaque->stop = true;
1909 bdrv_child_refresh_perms(mirror_top_bs, mirror_top_bs->backing,
1910 &error_abort);
1911 bdrv_replace_node(mirror_top_bs, mirror_top_bs->backing->bs, &error_abort);
1912
1913 bdrv_unref(mirror_top_bs);
1914
1915 return NULL;
1916}
1917
1918void mirror_start(const char *job_id, BlockDriverState *bs,
1919 BlockDriverState *target, const char *replaces,
1920 int creation_flags, int64_t speed,
1921 uint32_t granularity, int64_t buf_size,
1922 MirrorSyncMode mode, BlockMirrorBackingMode backing_mode,
1923 bool zero_target,
1924 BlockdevOnError on_source_error,
1925 BlockdevOnError on_target_error,
1926 bool unmap, const char *filter_node_name,
1927 MirrorCopyMode copy_mode, Error **errp)
1928{
1929 bool is_none_mode;
1930 BlockDriverState *base;
1931
1932 GLOBAL_STATE_CODE();
1933
1934 if ((mode == MIRROR_SYNC_MODE_INCREMENTAL) ||
1935 (mode == MIRROR_SYNC_MODE_BITMAP)) {
1936 error_setg(errp, "Sync mode '%s' not supported",
1937 MirrorSyncMode_str(mode));
1938 return;
1939 }
1940 is_none_mode = mode == MIRROR_SYNC_MODE_NONE;
1941 base = mode == MIRROR_SYNC_MODE_TOP ? bdrv_backing_chain_next(bs) : NULL;
1942 mirror_start_job(job_id, bs, creation_flags, target, replaces,
1943 speed, granularity, buf_size, backing_mode, zero_target,
1944 on_source_error, on_target_error, unmap, NULL, NULL,
1945 &mirror_job_driver, is_none_mode, base, false,
1946 filter_node_name, true, copy_mode, errp);
1947}
1948
1949BlockJob *commit_active_start(const char *job_id, BlockDriverState *bs,
1950 BlockDriverState *base, int creation_flags,
1951 int64_t speed, BlockdevOnError on_error,
1952 const char *filter_node_name,
1953 BlockCompletionFunc *cb, void *opaque,
1954 bool auto_complete, Error **errp)
1955{
1956 bool base_read_only;
1957 BlockJob *job;
1958
1959 GLOBAL_STATE_CODE();
1960
1961 base_read_only = bdrv_is_read_only(base);
1962
1963 if (base_read_only) {
1964 if (bdrv_reopen_set_read_only(base, false, errp) < 0) {
1965 return NULL;
1966 }
1967 }
1968
1969 job = mirror_start_job(
1970 job_id, bs, creation_flags, base, NULL, speed, 0, 0,
1971 MIRROR_LEAVE_BACKING_CHAIN, false,
1972 on_error, on_error, true, cb, opaque,
1973 &commit_active_job_driver, false, base, auto_complete,
1974 filter_node_name, false, MIRROR_COPY_MODE_BACKGROUND,
1975 errp);
1976 if (!job) {
1977 goto error_restore_flags;
1978 }
1979
1980 return job;
1981
1982error_restore_flags:
1983
1984
1985 if (base_read_only) {
1986 bdrv_reopen_set_read_only(base, true, NULL);
1987 }
1988 return NULL;
1989}
1990