1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61#include "qemu/osdep.h"
62#include "block/block.h"
63#include "block/block_int.h"
64#include "block/dirty-bitmap.h"
65#include "sysemu/block-backend.h"
66#include "sysemu/runstate.h"
67#include "qemu/main-loop.h"
68#include "qemu/error-report.h"
69#include "migration/misc.h"
70#include "migration/migration.h"
71#include "qemu-file.h"
72#include "migration/vmstate.h"
73#include "migration/register.h"
74#include "qemu/hbitmap.h"
75#include "qemu/cutils.h"
76#include "qemu/id.h"
77#include "qapi/error.h"
78#include "qapi/qapi-commands-migration.h"
79#include "qapi/qapi-visit-migration.h"
80#include "qapi/clone-visitor.h"
81#include "trace.h"
82#include "options.h"
83
84#define CHUNK_SIZE (1 << 10)
85
86
87
88
89
90
91
92
93
94#define DIRTY_BITMAP_MIG_FLAG_EOS 0x01
95#define DIRTY_BITMAP_MIG_FLAG_ZEROES 0x02
96#define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME 0x04
97#define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME 0x08
98#define DIRTY_BITMAP_MIG_FLAG_START 0x10
99#define DIRTY_BITMAP_MIG_FLAG_COMPLETE 0x20
100#define DIRTY_BITMAP_MIG_FLAG_BITS 0x40
101
102#define DIRTY_BITMAP_MIG_EXTRA_FLAGS 0x80
103
104#define DIRTY_BITMAP_MIG_START_FLAG_ENABLED 0x01
105#define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT 0x02
106
107#define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK 0xf8
108
109
110typedef struct SaveBitmapState {
111
112 BlockDriverState *bs;
113 char *node_alias;
114 char *bitmap_alias;
115 BdrvDirtyBitmap *bitmap;
116 uint64_t total_sectors;
117 uint64_t sectors_per_chunk;
118 QSIMPLEQ_ENTRY(SaveBitmapState) entry;
119 uint8_t flags;
120
121
122 bool bulk_completed;
123 uint64_t cur_sector;
124} SaveBitmapState;
125
126
127typedef struct DBMSaveState {
128 QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
129
130 bool bulk_completed;
131 bool no_bitmaps;
132
133
134 BlockDriverState *prev_bs;
135 BdrvDirtyBitmap *prev_bitmap;
136} DBMSaveState;
137
138typedef struct LoadBitmapState {
139 BlockDriverState *bs;
140 BdrvDirtyBitmap *bitmap;
141 bool migrated;
142 bool enabled;
143} LoadBitmapState;
144
145
146typedef struct DBMLoadState {
147 uint32_t flags;
148 char node_alias[256];
149 char bitmap_alias[256];
150 char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
151 BlockDriverState *bs;
152 BdrvDirtyBitmap *bitmap;
153
154 bool before_vm_start_handled;
155 BitmapMigrationBitmapAlias *bmap_inner;
156
157
158
159
160
161
162
163
164 bool cancelled;
165
166 GSList *bitmaps;
167 QemuMutex lock;
168} DBMLoadState;
169
170typedef struct DBMState {
171 DBMSaveState save;
172 DBMLoadState load;
173} DBMState;
174
175static DBMState dbm_state;
176
177
178typedef struct AliasMapInnerNode {
179 char *string;
180 GHashTable *subtree;
181} AliasMapInnerNode;
182
183static void free_alias_map_inner_node(void *amin_ptr)
184{
185 AliasMapInnerNode *amin = amin_ptr;
186
187 g_free(amin->string);
188 g_hash_table_unref(amin->subtree);
189 g_free(amin);
190}
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
217 bool name_to_alias,
218 Error **errp)
219{
220 GHashTable *alias_map;
221 size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
222
223 alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
224 g_free, free_alias_map_inner_node);
225
226 for (; bbm; bbm = bbm->next) {
227 const BitmapMigrationNodeAlias *bmna = bbm->value;
228 const BitmapMigrationBitmapAliasList *bmbal;
229 AliasMapInnerNode *amin;
230 GHashTable *bitmaps_map;
231 const char *node_map_from, *node_map_to;
232 GDestroyNotify gdn;
233
234 if (!id_wellformed(bmna->alias)) {
235 error_setg(errp, "The node alias '%s' is not well-formed",
236 bmna->alias);
237 goto fail;
238 }
239
240 if (strlen(bmna->alias) > UINT8_MAX) {
241 error_setg(errp, "The node alias '%s' is longer than %u bytes",
242 bmna->alias, UINT8_MAX);
243 goto fail;
244 }
245
246 if (strlen(bmna->node_name) > max_node_name_len) {
247 error_setg(errp, "The node name '%s' is longer than %zu bytes",
248 bmna->node_name, max_node_name_len);
249 goto fail;
250 }
251
252 if (name_to_alias) {
253 if (g_hash_table_contains(alias_map, bmna->node_name)) {
254 error_setg(errp, "The node name '%s' is mapped twice",
255 bmna->node_name);
256 goto fail;
257 }
258
259 node_map_from = bmna->node_name;
260 node_map_to = bmna->alias;
261 } else {
262 if (g_hash_table_contains(alias_map, bmna->alias)) {
263 error_setg(errp, "The node alias '%s' is used twice",
264 bmna->alias);
265 goto fail;
266 }
267
268 node_map_from = bmna->alias;
269 node_map_to = bmna->node_name;
270 }
271
272 gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
273 bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
274 gdn);
275
276 amin = g_new(AliasMapInnerNode, 1);
277 *amin = (AliasMapInnerNode){
278 .string = g_strdup(node_map_to),
279 .subtree = bitmaps_map,
280 };
281
282 g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
283
284 for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
285 const BitmapMigrationBitmapAlias *bmba = bmbal->value;
286 const char *bmap_map_from;
287
288 if (strlen(bmba->alias) > UINT8_MAX) {
289 error_setg(errp,
290 "The bitmap alias '%s' is longer than %u bytes",
291 bmba->alias, UINT8_MAX);
292 goto fail;
293 }
294
295 if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
296 error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
297 bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
298 goto fail;
299 }
300
301 if (name_to_alias) {
302 bmap_map_from = bmba->name;
303
304 if (g_hash_table_contains(bitmaps_map, bmba->name)) {
305 error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
306 bmna->node_name, bmba->name);
307 goto fail;
308 }
309 } else {
310 bmap_map_from = bmba->alias;
311
312 if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
313 error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
314 bmna->alias, bmba->alias);
315 goto fail;
316 }
317 }
318
319 g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
320 QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
321 }
322 }
323
324 return alias_map;
325
326fail:
327 g_hash_table_destroy(alias_map);
328 return NULL;
329}
330
331
332
333
334
335
336
337
338
339bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
340 Error **errp)
341{
342 GHashTable *alias_map;
343
344 alias_map = construct_alias_map(bbm, true, errp);
345 if (!alias_map) {
346 return false;
347 }
348 g_hash_table_destroy(alias_map);
349
350 alias_map = construct_alias_map(bbm, false, errp);
351 if (!alias_map) {
352 return false;
353 }
354 g_hash_table_destroy(alias_map);
355
356 return true;
357}
358
359static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
360{
361 uint8_t flags = qemu_get_byte(f);
362 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
363 flags = flags << 8 | qemu_get_byte(f);
364 if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
365 flags = flags << 16 | qemu_get_be16(f);
366 }
367 }
368
369 return flags;
370}
371
372static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
373{
374
375 assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
376
377 qemu_put_byte(f, flags);
378}
379
380static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
381 SaveBitmapState *dbms, uint32_t additional_flags)
382{
383 BlockDriverState *bs = dbms->bs;
384 BdrvDirtyBitmap *bitmap = dbms->bitmap;
385 uint32_t flags = additional_flags;
386 trace_send_bitmap_header_enter();
387
388 if (bs != s->prev_bs) {
389 s->prev_bs = bs;
390 flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
391 }
392
393 if (bitmap != s->prev_bitmap) {
394 s->prev_bitmap = bitmap;
395 flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
396 }
397
398 qemu_put_bitmap_flags(f, flags);
399
400 if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
401 qemu_put_counted_string(f, dbms->node_alias);
402 }
403
404 if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
405 qemu_put_counted_string(f, dbms->bitmap_alias);
406 }
407}
408
409static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
410 SaveBitmapState *dbms)
411{
412 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
413 qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
414 qemu_put_byte(f, dbms->flags);
415}
416
417static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
418 SaveBitmapState *dbms)
419{
420 send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
421}
422
423static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
424 SaveBitmapState *dbms,
425 uint64_t start_sector, uint32_t nr_sectors)
426{
427
428 uint64_t align = 4 * sizeof(long);
429 uint64_t unaligned_size =
430 bdrv_dirty_bitmap_serialization_size(
431 dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
432 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
433 uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
434 uint8_t *buf = g_malloc0(buf_size);
435 uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
436
437 bdrv_dirty_bitmap_serialize_part(
438 dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
439 (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
440
441 if (buffer_is_zero(buf, buf_size)) {
442 g_free(buf);
443 buf = NULL;
444 flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
445 }
446
447 trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
448
449 send_bitmap_header(f, s, dbms, flags);
450
451 qemu_put_be64(f, start_sector);
452 qemu_put_be32(f, nr_sectors);
453
454
455
456
457 if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
458 qemu_fflush(f);
459 } else {
460 qemu_put_be64(f, buf_size);
461 qemu_put_buffer(f, buf, buf_size);
462 }
463
464 g_free(buf);
465}
466
467
468static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
469{
470 SaveBitmapState *dbms;
471
472 while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
473 QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
474 bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
475 bdrv_unref(dbms->bs);
476 g_free(dbms->node_alias);
477 g_free(dbms->bitmap_alias);
478 g_free(dbms);
479 }
480}
481
482
483static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
484 const char *bs_name, GHashTable *alias_map)
485{
486 BdrvDirtyBitmap *bitmap;
487 SaveBitmapState *dbms;
488 GHashTable *bitmap_aliases;
489 const char *node_alias, *bitmap_name, *bitmap_alias;
490 Error *local_err = NULL;
491
492
493 assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
494
495 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
496 if (bdrv_dirty_bitmap_name(bitmap)) {
497 break;
498 }
499 }
500 if (!bitmap) {
501 return 0;
502 }
503
504 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
505
506 if (!bs_name || strcmp(bs_name, "") == 0) {
507 error_report("Bitmap '%s' in unnamed node can't be migrated",
508 bitmap_name);
509 return -1;
510 }
511
512 if (alias_map) {
513 const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
514
515 if (!amin) {
516
517 return 0;
518 }
519
520 node_alias = amin->string;
521 bitmap_aliases = amin->subtree;
522 } else {
523 node_alias = bs_name;
524 bitmap_aliases = NULL;
525 }
526
527 if (node_alias[0] == '#') {
528 error_report("Bitmap '%s' in a node with auto-generated "
529 "name '%s' can't be migrated",
530 bitmap_name, node_alias);
531 return -1;
532 }
533
534 FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
535 BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
536 bitmap_name = bdrv_dirty_bitmap_name(bitmap);
537 if (!bitmap_name) {
538 continue;
539 }
540
541 if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
542 error_report_err(local_err);
543 return -1;
544 }
545
546 if (bitmap_aliases) {
547 BitmapMigrationBitmapAlias *bmap_inner;
548
549 bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
550 if (!bmap_inner) {
551
552 continue;
553 }
554
555 bitmap_alias = bmap_inner->alias;
556 if (bmap_inner->transform) {
557 bitmap_transform = bmap_inner->transform;
558 }
559 } else {
560 if (strlen(bitmap_name) > UINT8_MAX) {
561 error_report("Cannot migrate bitmap '%s' on node '%s': "
562 "Name is longer than %u bytes",
563 bitmap_name, bs_name, UINT8_MAX);
564 return -1;
565 }
566 bitmap_alias = bitmap_name;
567 }
568
569 bdrv_ref(bs);
570 bdrv_dirty_bitmap_set_busy(bitmap, true);
571
572 dbms = g_new0(SaveBitmapState, 1);
573 dbms->bs = bs;
574 dbms->node_alias = g_strdup(node_alias);
575 dbms->bitmap_alias = g_strdup(bitmap_alias);
576 dbms->bitmap = bitmap;
577 dbms->total_sectors = bdrv_nb_sectors(bs);
578 dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
579 (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
580 assert(dbms->sectors_per_chunk != 0);
581 if (bdrv_dirty_bitmap_enabled(bitmap)) {
582 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
583 }
584 if (bitmap_transform &&
585 bitmap_transform->has_persistent) {
586 if (bitmap_transform->persistent) {
587 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
588 }
589 } else {
590 if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
591 dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
592 }
593 }
594
595 QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
596 }
597
598 return 0;
599}
600
601
602static int init_dirty_bitmap_migration(DBMSaveState *s)
603{
604 BlockDriverState *bs;
605 SaveBitmapState *dbms;
606 GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
607 BlockBackend *blk;
608 GHashTable *alias_map = NULL;
609
610 if (migrate_has_block_bitmap_mapping()) {
611 alias_map = construct_alias_map(migrate_block_bitmap_mapping(), true,
612 &error_abort);
613 }
614
615 s->bulk_completed = false;
616 s->prev_bs = NULL;
617 s->prev_bitmap = NULL;
618 s->no_bitmaps = false;
619
620 if (!alias_map) {
621
622
623
624
625 for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
626 const char *name = blk_name(blk);
627
628 if (!name || strcmp(name, "") == 0) {
629 continue;
630 }
631
632 bs = blk_bs(blk);
633
634
635 while (bs && bs->drv && bs->drv->is_filter &&
636 !bdrv_has_named_bitmaps(bs))
637 {
638 bs = bdrv_filter_bs(bs);
639 }
640
641 if (bs && bs->drv && !bs->drv->is_filter) {
642 if (add_bitmaps_to_list(s, bs, name, NULL)) {
643 goto fail;
644 }
645 g_hash_table_add(handled_by_blk, bs);
646 }
647 }
648 }
649
650 for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
651 if (g_hash_table_contains(handled_by_blk, bs)) {
652 continue;
653 }
654
655 if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
656 goto fail;
657 }
658 }
659
660
661 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
662 bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
663 }
664
665 if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
666 s->no_bitmaps = true;
667 }
668
669 g_hash_table_destroy(handled_by_blk);
670 if (alias_map) {
671 g_hash_table_destroy(alias_map);
672 }
673
674 return 0;
675
676fail:
677 g_hash_table_destroy(handled_by_blk);
678 if (alias_map) {
679 g_hash_table_destroy(alias_map);
680 }
681 dirty_bitmap_do_save_cleanup(s);
682
683 return -1;
684}
685
686
687static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
688 SaveBitmapState *dbms)
689{
690 uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
691 dbms->sectors_per_chunk);
692
693 send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
694
695 dbms->cur_sector += nr_sectors;
696 if (dbms->cur_sector >= dbms->total_sectors) {
697 dbms->bulk_completed = true;
698 }
699}
700
701
702static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
703{
704 SaveBitmapState *dbms;
705
706 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
707 while (!dbms->bulk_completed) {
708 bulk_phase_send_chunk(f, s, dbms);
709 if (limit && migration_rate_exceeded(f)) {
710 return;
711 }
712 }
713 }
714
715 s->bulk_completed = true;
716}
717
718
719static void dirty_bitmap_save_cleanup(void *opaque)
720{
721 DBMSaveState *s = &((DBMState *)opaque)->save;
722
723 dirty_bitmap_do_save_cleanup(s);
724}
725
726static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
727{
728 DBMSaveState *s = &((DBMState *)opaque)->save;
729
730 trace_dirty_bitmap_save_iterate(migration_in_postcopy());
731
732 if (migration_in_postcopy() && !s->bulk_completed) {
733 bulk_phase(f, s, true);
734 }
735
736 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
737
738 return s->bulk_completed;
739}
740
741
742
743static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
744{
745 DBMSaveState *s = &((DBMState *)opaque)->save;
746 SaveBitmapState *dbms;
747 trace_dirty_bitmap_save_complete_enter();
748
749 if (!s->bulk_completed) {
750 bulk_phase(f, s, false);
751 }
752
753 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
754 send_bitmap_complete(f, s, dbms);
755 }
756
757 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
758
759 trace_dirty_bitmap_save_complete_finish();
760
761 dirty_bitmap_save_cleanup(opaque);
762 return 0;
763}
764
765static void dirty_bitmap_state_pending(void *opaque,
766 uint64_t *must_precopy,
767 uint64_t *can_postcopy)
768{
769 DBMSaveState *s = &((DBMState *)opaque)->save;
770 SaveBitmapState *dbms;
771 uint64_t pending = 0;
772
773 qemu_mutex_lock_iothread();
774
775 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
776 uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
777 uint64_t sectors = dbms->bulk_completed ? 0 :
778 dbms->total_sectors - dbms->cur_sector;
779
780 pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
781 }
782
783 qemu_mutex_unlock_iothread();
784
785 trace_dirty_bitmap_state_pending(pending);
786
787 *can_postcopy += pending;
788}
789
790
791static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
792{
793 Error *local_err = NULL;
794 uint32_t granularity = qemu_get_be32(f);
795 uint8_t flags = qemu_get_byte(f);
796 LoadBitmapState *b;
797 bool persistent;
798
799 if (s->cancelled) {
800 return 0;
801 }
802
803 if (s->bitmap) {
804 error_report("Bitmap with the same name ('%s') already exists on "
805 "destination", bdrv_dirty_bitmap_name(s->bitmap));
806 return -EINVAL;
807 } else {
808 s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
809 s->bitmap_name, &local_err);
810 if (!s->bitmap) {
811 error_report_err(local_err);
812 return -EINVAL;
813 }
814 }
815
816 if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
817 error_report("Unknown flags in migrated dirty bitmap header: %x",
818 flags);
819 return -EINVAL;
820 }
821
822 if (s->bmap_inner &&
823 s->bmap_inner->transform &&
824 s->bmap_inner->transform->has_persistent) {
825 persistent = s->bmap_inner->transform->persistent;
826 } else {
827 persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
828 }
829
830 if (persistent) {
831 bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
832 }
833
834 bdrv_disable_dirty_bitmap(s->bitmap);
835 if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
836 bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
837 if (local_err) {
838 error_report_err(local_err);
839 return -EINVAL;
840 }
841 } else {
842 bdrv_dirty_bitmap_set_busy(s->bitmap, true);
843 }
844
845 b = g_new(LoadBitmapState, 1);
846 b->bs = s->bs;
847 b->bitmap = s->bitmap;
848 b->migrated = false;
849 b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
850
851 s->bitmaps = g_slist_prepend(s->bitmaps, b);
852
853 return 0;
854}
855
856
857
858
859
860
861
862
863
864static void before_vm_start_handle_item(void *item, void *opaque)
865{
866 DBMLoadState *s = opaque;
867 LoadBitmapState *b = item;
868
869 if (b->enabled) {
870 if (b->migrated) {
871 bdrv_enable_dirty_bitmap(b->bitmap);
872 } else {
873 bdrv_dirty_bitmap_enable_successor(b->bitmap);
874 }
875 }
876
877 if (b->migrated) {
878 s->bitmaps = g_slist_remove(s->bitmaps, b);
879 g_free(b);
880 }
881}
882
883void dirty_bitmap_mig_before_vm_start(void)
884{
885 DBMLoadState *s = &dbm_state.load;
886 qemu_mutex_lock(&s->lock);
887
888 assert(!s->before_vm_start_handled);
889 g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
890 s->before_vm_start_handled = true;
891
892 qemu_mutex_unlock(&s->lock);
893}
894
895static void cancel_incoming_locked(DBMLoadState *s)
896{
897 GSList *item;
898
899 if (s->cancelled) {
900 return;
901 }
902
903 s->cancelled = true;
904 s->bs = NULL;
905 s->bitmap = NULL;
906
907
908 for (item = s->bitmaps; item; item = g_slist_next(item)) {
909 LoadBitmapState *b = item->data;
910
911
912
913
914
915 assert(!s->before_vm_start_handled || !b->migrated);
916 if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
917 bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
918 } else {
919 bdrv_dirty_bitmap_set_busy(b->bitmap, false);
920 }
921 bdrv_release_dirty_bitmap(b->bitmap);
922 }
923
924 g_slist_free_full(s->bitmaps, g_free);
925 s->bitmaps = NULL;
926}
927
928void dirty_bitmap_mig_cancel_outgoing(void)
929{
930 dirty_bitmap_do_save_cleanup(&dbm_state.save);
931}
932
933void dirty_bitmap_mig_cancel_incoming(void)
934{
935 DBMLoadState *s = &dbm_state.load;
936
937 qemu_mutex_lock(&s->lock);
938
939 cancel_incoming_locked(s);
940
941 qemu_mutex_unlock(&s->lock);
942}
943
944static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
945{
946 GSList *item;
947 trace_dirty_bitmap_load_complete();
948
949 if (s->cancelled) {
950 return;
951 }
952
953 bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
954
955 if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
956 bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
957 } else {
958 bdrv_dirty_bitmap_set_busy(s->bitmap, false);
959 }
960
961 for (item = s->bitmaps; item; item = g_slist_next(item)) {
962 LoadBitmapState *b = item->data;
963
964 if (b->bitmap == s->bitmap) {
965 b->migrated = true;
966 if (s->before_vm_start_handled) {
967 s->bitmaps = g_slist_remove(s->bitmaps, b);
968 g_free(b);
969 }
970 break;
971 }
972 }
973}
974
975static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
976{
977 uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
978 uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
979 trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
980 nr_bytes >> BDRV_SECTOR_BITS);
981
982 if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
983 trace_dirty_bitmap_load_bits_zeroes();
984 if (!s->cancelled) {
985 bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
986 nr_bytes, false);
987 }
988 } else {
989 size_t ret;
990 g_autofree uint8_t *buf = NULL;
991 uint64_t buf_size = qemu_get_be64(f);
992 uint64_t needed_size;
993
994
995
996
997
998
999
1000
1001
1002
1003
1004 if (buf_size > 10 * CHUNK_SIZE) {
1005 error_report("Bitmap migration stream buffer allocation request "
1006 "is too large");
1007 return -EIO;
1008 }
1009
1010 buf = g_malloc(buf_size);
1011 ret = qemu_get_buffer(f, buf, buf_size);
1012 if (ret != buf_size) {
1013 error_report("Failed to read bitmap bits");
1014 return -EIO;
1015 }
1016
1017 if (s->cancelled) {
1018 return 0;
1019 }
1020
1021 needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1022 first_byte,
1023 nr_bytes);
1024
1025 if (needed_size > buf_size ||
1026 buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1027
1028 ) {
1029 error_report("Migrated bitmap granularity doesn't "
1030 "match the destination bitmap '%s' granularity",
1031 bdrv_dirty_bitmap_name(s->bitmap));
1032 cancel_incoming_locked(s);
1033 return 0;
1034 }
1035
1036 bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1037 false);
1038 }
1039
1040 return 0;
1041}
1042
1043static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1044 GHashTable *alias_map)
1045{
1046 GHashTable *bitmap_alias_map = NULL;
1047 Error *local_err = NULL;
1048 bool nothing;
1049 s->flags = qemu_get_bitmap_flags(f);
1050 trace_dirty_bitmap_load_header(s->flags);
1051
1052 nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1053
1054 if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1055 if (!qemu_get_counted_string(f, s->node_alias)) {
1056 error_report("Unable to read node alias string");
1057 return -EINVAL;
1058 }
1059
1060 if (!s->cancelled) {
1061 if (alias_map) {
1062 const AliasMapInnerNode *amin;
1063
1064 amin = g_hash_table_lookup(alias_map, s->node_alias);
1065 if (!amin) {
1066 error_setg(&local_err, "Error: Unknown node alias '%s'",
1067 s->node_alias);
1068 s->bs = NULL;
1069 } else {
1070 bitmap_alias_map = amin->subtree;
1071 s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1072 }
1073 } else {
1074 s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1075 &local_err);
1076 }
1077 if (!s->bs) {
1078 error_report_err(local_err);
1079 cancel_incoming_locked(s);
1080 }
1081 }
1082 } else if (s->bs) {
1083 if (alias_map) {
1084 const AliasMapInnerNode *amin;
1085
1086
1087 amin = g_hash_table_lookup(alias_map, s->node_alias);
1088 assert(amin != NULL);
1089
1090 bitmap_alias_map = amin->subtree;
1091 }
1092 } else if (!nothing && !s->cancelled) {
1093 error_report("Error: block device name is not set");
1094 cancel_incoming_locked(s);
1095 }
1096
1097 assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1098
1099 if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1100 const char *bitmap_name;
1101
1102 if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1103 error_report("Unable to read bitmap alias string");
1104 return -EINVAL;
1105 }
1106
1107 bitmap_name = s->bitmap_alias;
1108 if (!s->cancelled && bitmap_alias_map) {
1109 BitmapMigrationBitmapAlias *bmap_inner;
1110
1111 bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1112 if (!bmap_inner) {
1113 error_report("Error: Unknown bitmap alias '%s' on node "
1114 "'%s' (alias '%s')", s->bitmap_alias,
1115 s->bs->node_name, s->node_alias);
1116 cancel_incoming_locked(s);
1117 } else {
1118 bitmap_name = bmap_inner->name;
1119 }
1120
1121 s->bmap_inner = bmap_inner;
1122 }
1123
1124 if (!s->cancelled) {
1125 g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1126 s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1127
1128
1129
1130
1131
1132 if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1133 error_report("Error: unknown dirty bitmap "
1134 "'%s' for block device '%s'",
1135 s->bitmap_name, s->bs->node_name);
1136 cancel_incoming_locked(s);
1137 }
1138 }
1139 } else if (!s->bitmap && !nothing && !s->cancelled) {
1140 error_report("Error: block device name is not set");
1141 cancel_incoming_locked(s);
1142 }
1143
1144 return 0;
1145}
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1158{
1159 GHashTable *alias_map = NULL;
1160 DBMLoadState *s = &((DBMState *)opaque)->load;
1161 int ret = 0;
1162
1163 trace_dirty_bitmap_load_enter();
1164
1165 if (version_id != 1) {
1166 QEMU_LOCK_GUARD(&s->lock);
1167 cancel_incoming_locked(s);
1168 return -EINVAL;
1169 }
1170
1171 if (migrate_has_block_bitmap_mapping()) {
1172 alias_map = construct_alias_map(migrate_block_bitmap_mapping(), false,
1173 &error_abort);
1174 }
1175
1176 do {
1177 QEMU_LOCK_GUARD(&s->lock);
1178
1179 ret = dirty_bitmap_load_header(f, s, alias_map);
1180 if (ret < 0) {
1181 cancel_incoming_locked(s);
1182 goto fail;
1183 }
1184
1185 if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1186 ret = dirty_bitmap_load_start(f, s);
1187 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1188 dirty_bitmap_load_complete(f, s);
1189 } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1190 ret = dirty_bitmap_load_bits(f, s);
1191 }
1192
1193 if (!ret) {
1194 ret = qemu_file_get_error(f);
1195 }
1196
1197 if (ret) {
1198 cancel_incoming_locked(s);
1199 goto fail;
1200 }
1201 } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1202
1203 trace_dirty_bitmap_load_success();
1204 ret = 0;
1205fail:
1206 if (alias_map) {
1207 g_hash_table_destroy(alias_map);
1208 }
1209 return ret;
1210}
1211
1212static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1213{
1214 DBMSaveState *s = &((DBMState *)opaque)->save;
1215 SaveBitmapState *dbms = NULL;
1216
1217 qemu_mutex_lock_iothread();
1218 if (init_dirty_bitmap_migration(s) < 0) {
1219 qemu_mutex_unlock_iothread();
1220 return -1;
1221 }
1222
1223 QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1224 send_bitmap_start(f, s, dbms);
1225 }
1226 qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1227 qemu_mutex_unlock_iothread();
1228 return 0;
1229}
1230
1231static bool dirty_bitmap_is_active(void *opaque)
1232{
1233 DBMSaveState *s = &((DBMState *)opaque)->save;
1234
1235 return migrate_dirty_bitmaps() && !s->no_bitmaps;
1236}
1237
1238static bool dirty_bitmap_is_active_iterate(void *opaque)
1239{
1240 return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1241}
1242
1243static bool dirty_bitmap_has_postcopy(void *opaque)
1244{
1245 return true;
1246}
1247
1248static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1249 .save_setup = dirty_bitmap_save_setup,
1250 .save_live_complete_postcopy = dirty_bitmap_save_complete,
1251 .save_live_complete_precopy = dirty_bitmap_save_complete,
1252 .has_postcopy = dirty_bitmap_has_postcopy,
1253 .state_pending_exact = dirty_bitmap_state_pending,
1254 .state_pending_estimate = dirty_bitmap_state_pending,
1255 .save_live_iterate = dirty_bitmap_save_iterate,
1256 .is_active_iterate = dirty_bitmap_is_active_iterate,
1257 .load_state = dirty_bitmap_load,
1258 .save_cleanup = dirty_bitmap_save_cleanup,
1259 .is_active = dirty_bitmap_is_active,
1260};
1261
1262void dirty_bitmap_mig_init(void)
1263{
1264 QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1265 qemu_mutex_init(&dbm_state.load.lock);
1266
1267 register_savevm_live("dirty-bitmap", 0, 1,
1268 &savevm_dirty_bitmap_handlers,
1269 &dbm_state);
1270}
1271