qemu/migration/block-dirty-bitmap.c
<<
>>
Prefs
   1/*
   2 * Block dirty bitmap postcopy migration
   3 *
   4 * Copyright IBM, Corp. 2009
   5 * Copyright (c) 2016-2017 Virtuozzo International GmbH. All rights reserved.
   6 *
   7 * Authors:
   8 *  Liran Schour   <lirans@il.ibm.com>
   9 *  Vladimir Sementsov-Ogievskiy <vsementsov@virtuozzo.com>
  10 *
  11 * This work is licensed under the terms of the GNU GPL, version 2.  See
  12 * the COPYING file in the top-level directory.
  13 * This file is derived from migration/block.c, so it's author and IBM copyright
  14 * are here, although content is quite different.
  15 *
  16 * Contributions after 2012-01-13 are licensed under the terms of the
  17 * GNU GPL, version 2 or (at your option) any later version.
  18 *
  19 *                                ***
  20 *
  21 * Here postcopy migration of dirty bitmaps is realized. Only QMP-addressable
  22 * bitmaps are migrated.
  23 *
  24 * Bitmap migration implies creating bitmap with the same name and granularity
  25 * in destination QEMU. If the bitmap with the same name (for the same node)
  26 * already exists on destination an error will be generated.
  27 *
  28 * format of migration:
  29 *
  30 * # Header (shared for different chunk types)
  31 * 1, 2 or 4 bytes: flags (see qemu_{put,put}_flags)
  32 * [ 1 byte: node alias size ] \  flags & DEVICE_NAME
  33 * [ n bytes: node alias     ] /
  34 * [ 1 byte: bitmap alias size ] \  flags & BITMAP_NAME
  35 * [ n bytes: bitmap alias     ] /
  36 *
  37 * # Start of bitmap migration (flags & START)
  38 * header
  39 * be64: granularity
  40 * 1 byte: bitmap flags (corresponds to BdrvDirtyBitmap)
  41 *   bit 0    -  bitmap is enabled
  42 *   bit 1    -  bitmap is persistent
  43 *   bit 2    -  bitmap is autoloading
  44 *   bits 3-7 - reserved, must be zero
  45 *
  46 * # Complete of bitmap migration (flags & COMPLETE)
  47 * header
  48 *
  49 * # Data chunk of bitmap migration
  50 * header
  51 * be64: start sector
  52 * be32: number of sectors
  53 * [ be64: buffer size  ] \ ! (flags & ZEROES)
  54 * [ n bytes: buffer    ] /
  55 *
  56 * The last chunk in stream should contain flags & EOS. The chunk may skip
  57 * device and/or bitmap names, assuming them to be the same with the previous
  58 * chunk.
  59 */
  60
  61#include "qemu/osdep.h"
  62#include "block/block.h"
  63#include "block/block_int.h"
  64#include "sysemu/block-backend.h"
  65#include "sysemu/runstate.h"
  66#include "qemu/main-loop.h"
  67#include "qemu/error-report.h"
  68#include "migration/misc.h"
  69#include "migration/migration.h"
  70#include "qemu-file.h"
  71#include "migration/vmstate.h"
  72#include "migration/register.h"
  73#include "qemu/hbitmap.h"
  74#include "qemu/cutils.h"
  75#include "qemu/id.h"
  76#include "qapi/error.h"
  77#include "qapi/qapi-commands-migration.h"
  78#include "qapi/qapi-visit-migration.h"
  79#include "qapi/clone-visitor.h"
  80#include "trace.h"
  81
  82#define CHUNK_SIZE     (1 << 10)
  83
  84/* Flags occupy one, two or four bytes (Big Endian). The size is determined as
  85 * follows:
  86 * in first (most significant) byte bit 8 is clear  -->  one byte
  87 * in first byte bit 8 is set    -->  two or four bytes, depending on second
  88 *                                    byte:
  89 *    | in second byte bit 8 is clear  -->  two bytes
  90 *    | in second byte bit 8 is set    -->  four bytes
  91 */
  92#define DIRTY_BITMAP_MIG_FLAG_EOS           0x01
  93#define DIRTY_BITMAP_MIG_FLAG_ZEROES        0x02
  94#define DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME   0x04
  95#define DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME   0x08
  96#define DIRTY_BITMAP_MIG_FLAG_START         0x10
  97#define DIRTY_BITMAP_MIG_FLAG_COMPLETE      0x20
  98#define DIRTY_BITMAP_MIG_FLAG_BITS          0x40
  99
 100#define DIRTY_BITMAP_MIG_EXTRA_FLAGS        0x80
 101
 102#define DIRTY_BITMAP_MIG_START_FLAG_ENABLED          0x01
 103#define DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT       0x02
 104/* 0x04 was "AUTOLOAD" flags on older versions, now it is ignored */
 105#define DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK    0xf8
 106
 107/* State of one bitmap during save process */
 108typedef struct SaveBitmapState {
 109    /* Written during setup phase. */
 110    BlockDriverState *bs;
 111    char *node_alias;
 112    char *bitmap_alias;
 113    BdrvDirtyBitmap *bitmap;
 114    uint64_t total_sectors;
 115    uint64_t sectors_per_chunk;
 116    QSIMPLEQ_ENTRY(SaveBitmapState) entry;
 117    uint8_t flags;
 118
 119    /* For bulk phase. */
 120    bool bulk_completed;
 121    uint64_t cur_sector;
 122} SaveBitmapState;
 123
 124/* State of the dirty bitmap migration (DBM) during save process */
 125typedef struct DBMSaveState {
 126    QSIMPLEQ_HEAD(, SaveBitmapState) dbms_list;
 127
 128    bool bulk_completed;
 129    bool no_bitmaps;
 130
 131    /* for send_bitmap_bits() */
 132    BlockDriverState *prev_bs;
 133    BdrvDirtyBitmap *prev_bitmap;
 134} DBMSaveState;
 135
 136typedef struct LoadBitmapState {
 137    BlockDriverState *bs;
 138    BdrvDirtyBitmap *bitmap;
 139    bool migrated;
 140    bool enabled;
 141} LoadBitmapState;
 142
 143/* State of the dirty bitmap migration (DBM) during load process */
 144typedef struct DBMLoadState {
 145    uint32_t flags;
 146    char node_alias[256];
 147    char bitmap_alias[256];
 148    char bitmap_name[BDRV_BITMAP_MAX_NAME_SIZE + 1];
 149    BlockDriverState *bs;
 150    BdrvDirtyBitmap *bitmap;
 151
 152    bool before_vm_start_handled; /* set in dirty_bitmap_mig_before_vm_start */
 153    BitmapMigrationBitmapAlias *bmap_inner;
 154
 155    /*
 156     * cancelled
 157     * Incoming migration is cancelled for some reason. That means that we
 158     * still should read our chunks from migration stream, to not affect other
 159     * migration objects (like RAM), but just ignore them and do not touch any
 160     * bitmaps or nodes.
 161     */
 162    bool cancelled;
 163
 164    GSList *bitmaps;
 165    QemuMutex lock; /* protect bitmaps */
 166} DBMLoadState;
 167
 168typedef struct DBMState {
 169    DBMSaveState save;
 170    DBMLoadState load;
 171} DBMState;
 172
 173static DBMState dbm_state;
 174
 175/* For hash tables that map node/bitmap names to aliases */
 176typedef struct AliasMapInnerNode {
 177    char *string;
 178    GHashTable *subtree;
 179} AliasMapInnerNode;
 180
 181static void free_alias_map_inner_node(void *amin_ptr)
 182{
 183    AliasMapInnerNode *amin = amin_ptr;
 184
 185    g_free(amin->string);
 186    g_hash_table_unref(amin->subtree);
 187    g_free(amin);
 188}
 189
 190/**
 191 * Construct an alias map based on the given QMP structure.
 192 *
 193 * (Note that we cannot store such maps in the MigrationParameters
 194 * object, because that struct is defined by the QAPI schema, which
 195 * makes it basically impossible to have dicts with arbitrary keys.
 196 * Therefore, we instead have to construct these maps when migration
 197 * starts.)
 198 *
 199 * @bbm is the block_bitmap_mapping from the migration parameters.
 200 *
 201 * If @name_to_alias is true, the returned hash table will map node
 202 * and bitmap names to their respective aliases (for outgoing
 203 * migration).
 204 *
 205 * If @name_to_alias is false, the returned hash table will map node
 206 * and bitmap aliases to their respective names (for incoming
 207 * migration).
 208 *
 209 * The hash table maps node names/aliases to AliasMapInnerNode
 210 * objects, whose .string is the respective node alias/name, and whose
 211 * .subtree table maps bitmap names/aliases to the respective bitmap
 212 * alias/name.
 213 */
 214static GHashTable *construct_alias_map(const BitmapMigrationNodeAliasList *bbm,
 215                                       bool name_to_alias,
 216                                       Error **errp)
 217{
 218    GHashTable *alias_map;
 219    size_t max_node_name_len = sizeof_field(BlockDriverState, node_name) - 1;
 220
 221    alias_map = g_hash_table_new_full(g_str_hash, g_str_equal,
 222                                      g_free, free_alias_map_inner_node);
 223
 224    for (; bbm; bbm = bbm->next) {
 225        const BitmapMigrationNodeAlias *bmna = bbm->value;
 226        const BitmapMigrationBitmapAliasList *bmbal;
 227        AliasMapInnerNode *amin;
 228        GHashTable *bitmaps_map;
 229        const char *node_map_from, *node_map_to;
 230        GDestroyNotify gdn;
 231
 232        if (!id_wellformed(bmna->alias)) {
 233            error_setg(errp, "The node alias '%s' is not well-formed",
 234                       bmna->alias);
 235            goto fail;
 236        }
 237
 238        if (strlen(bmna->alias) > UINT8_MAX) {
 239            error_setg(errp, "The node alias '%s' is longer than %u bytes",
 240                       bmna->alias, UINT8_MAX);
 241            goto fail;
 242        }
 243
 244        if (strlen(bmna->node_name) > max_node_name_len) {
 245            error_setg(errp, "The node name '%s' is longer than %zu bytes",
 246                       bmna->node_name, max_node_name_len);
 247            goto fail;
 248        }
 249
 250        if (name_to_alias) {
 251            if (g_hash_table_contains(alias_map, bmna->node_name)) {
 252                error_setg(errp, "The node name '%s' is mapped twice",
 253                           bmna->node_name);
 254                goto fail;
 255            }
 256
 257            node_map_from = bmna->node_name;
 258            node_map_to = bmna->alias;
 259        } else {
 260            if (g_hash_table_contains(alias_map, bmna->alias)) {
 261                error_setg(errp, "The node alias '%s' is used twice",
 262                           bmna->alias);
 263                goto fail;
 264            }
 265
 266            node_map_from = bmna->alias;
 267            node_map_to = bmna->node_name;
 268        }
 269
 270        gdn = (GDestroyNotify) qapi_free_BitmapMigrationBitmapAlias;
 271        bitmaps_map = g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
 272                                            gdn);
 273
 274        amin = g_new(AliasMapInnerNode, 1);
 275        *amin = (AliasMapInnerNode){
 276            .string = g_strdup(node_map_to),
 277            .subtree = bitmaps_map,
 278        };
 279
 280        g_hash_table_insert(alias_map, g_strdup(node_map_from), amin);
 281
 282        for (bmbal = bmna->bitmaps; bmbal; bmbal = bmbal->next) {
 283            const BitmapMigrationBitmapAlias *bmba = bmbal->value;
 284            const char *bmap_map_from;
 285
 286            if (strlen(bmba->alias) > UINT8_MAX) {
 287                error_setg(errp,
 288                           "The bitmap alias '%s' is longer than %u bytes",
 289                           bmba->alias, UINT8_MAX);
 290                goto fail;
 291            }
 292
 293            if (strlen(bmba->name) > BDRV_BITMAP_MAX_NAME_SIZE) {
 294                error_setg(errp, "The bitmap name '%s' is longer than %d bytes",
 295                           bmba->name, BDRV_BITMAP_MAX_NAME_SIZE);
 296                goto fail;
 297            }
 298
 299            if (name_to_alias) {
 300                bmap_map_from = bmba->name;
 301
 302                if (g_hash_table_contains(bitmaps_map, bmba->name)) {
 303                    error_setg(errp, "The bitmap '%s'/'%s' is mapped twice",
 304                               bmna->node_name, bmba->name);
 305                    goto fail;
 306                }
 307            } else {
 308                bmap_map_from = bmba->alias;
 309
 310                if (g_hash_table_contains(bitmaps_map, bmba->alias)) {
 311                    error_setg(errp, "The bitmap alias '%s'/'%s' is used twice",
 312                               bmna->alias, bmba->alias);
 313                    goto fail;
 314                }
 315            }
 316
 317            g_hash_table_insert(bitmaps_map, g_strdup(bmap_map_from),
 318                                QAPI_CLONE(BitmapMigrationBitmapAlias, bmba));
 319        }
 320    }
 321
 322    return alias_map;
 323
 324fail:
 325    g_hash_table_destroy(alias_map);
 326    return NULL;
 327}
 328
 329/**
 330 * Run construct_alias_map() in both directions to check whether @bbm
 331 * is valid.
 332 * (This function is to be used by migration/migration.c to validate
 333 * the user-specified block-bitmap-mapping migration parameter.)
 334 *
 335 * Returns true if and only if the mapping is valid.
 336 */
 337bool check_dirty_bitmap_mig_alias_map(const BitmapMigrationNodeAliasList *bbm,
 338                                      Error **errp)
 339{
 340    GHashTable *alias_map;
 341
 342    alias_map = construct_alias_map(bbm, true, errp);
 343    if (!alias_map) {
 344        return false;
 345    }
 346    g_hash_table_destroy(alias_map);
 347
 348    alias_map = construct_alias_map(bbm, false, errp);
 349    if (!alias_map) {
 350        return false;
 351    }
 352    g_hash_table_destroy(alias_map);
 353
 354    return true;
 355}
 356
 357static uint32_t qemu_get_bitmap_flags(QEMUFile *f)
 358{
 359    uint8_t flags = qemu_get_byte(f);
 360    if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
 361        flags = flags << 8 | qemu_get_byte(f);
 362        if (flags & DIRTY_BITMAP_MIG_EXTRA_FLAGS) {
 363            flags = flags << 16 | qemu_get_be16(f);
 364        }
 365    }
 366
 367    return flags;
 368}
 369
 370static void qemu_put_bitmap_flags(QEMUFile *f, uint32_t flags)
 371{
 372    /* The code currently does not send flags as more than one byte */
 373    assert(!(flags & (0xffffff00 | DIRTY_BITMAP_MIG_EXTRA_FLAGS)));
 374
 375    qemu_put_byte(f, flags);
 376}
 377
 378static void send_bitmap_header(QEMUFile *f, DBMSaveState *s,
 379                               SaveBitmapState *dbms, uint32_t additional_flags)
 380{
 381    BlockDriverState *bs = dbms->bs;
 382    BdrvDirtyBitmap *bitmap = dbms->bitmap;
 383    uint32_t flags = additional_flags;
 384    trace_send_bitmap_header_enter();
 385
 386    if (bs != s->prev_bs) {
 387        s->prev_bs = bs;
 388        flags |= DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME;
 389    }
 390
 391    if (bitmap != s->prev_bitmap) {
 392        s->prev_bitmap = bitmap;
 393        flags |= DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME;
 394    }
 395
 396    qemu_put_bitmap_flags(f, flags);
 397
 398    if (flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
 399        qemu_put_counted_string(f, dbms->node_alias);
 400    }
 401
 402    if (flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
 403        qemu_put_counted_string(f, dbms->bitmap_alias);
 404    }
 405}
 406
 407static void send_bitmap_start(QEMUFile *f, DBMSaveState *s,
 408                              SaveBitmapState *dbms)
 409{
 410    send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_START);
 411    qemu_put_be32(f, bdrv_dirty_bitmap_granularity(dbms->bitmap));
 412    qemu_put_byte(f, dbms->flags);
 413}
 414
 415static void send_bitmap_complete(QEMUFile *f, DBMSaveState *s,
 416                                 SaveBitmapState *dbms)
 417{
 418    send_bitmap_header(f, s, dbms, DIRTY_BITMAP_MIG_FLAG_COMPLETE);
 419}
 420
 421static void send_bitmap_bits(QEMUFile *f, DBMSaveState *s,
 422                             SaveBitmapState *dbms,
 423                             uint64_t start_sector, uint32_t nr_sectors)
 424{
 425    /* align for buffer_is_zero() */
 426    uint64_t align = 4 * sizeof(long);
 427    uint64_t unaligned_size =
 428        bdrv_dirty_bitmap_serialization_size(
 429            dbms->bitmap, start_sector << BDRV_SECTOR_BITS,
 430            (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
 431    uint64_t buf_size = QEMU_ALIGN_UP(unaligned_size, align);
 432    uint8_t *buf = g_malloc0(buf_size);
 433    uint32_t flags = DIRTY_BITMAP_MIG_FLAG_BITS;
 434
 435    bdrv_dirty_bitmap_serialize_part(
 436        dbms->bitmap, buf, start_sector << BDRV_SECTOR_BITS,
 437        (uint64_t)nr_sectors << BDRV_SECTOR_BITS);
 438
 439    if (buffer_is_zero(buf, buf_size)) {
 440        g_free(buf);
 441        buf = NULL;
 442        flags |= DIRTY_BITMAP_MIG_FLAG_ZEROES;
 443    }
 444
 445    trace_send_bitmap_bits(flags, start_sector, nr_sectors, buf_size);
 446
 447    send_bitmap_header(f, s, dbms, flags);
 448
 449    qemu_put_be64(f, start_sector);
 450    qemu_put_be32(f, nr_sectors);
 451
 452    /* if a block is zero we need to flush here since the network
 453     * bandwidth is now a lot higher than the storage device bandwidth.
 454     * thus if we queue zero blocks we slow down the migration. */
 455    if (flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
 456        qemu_fflush(f);
 457    } else {
 458        qemu_put_be64(f, buf_size);
 459        qemu_put_buffer(f, buf, buf_size);
 460    }
 461
 462    g_free(buf);
 463}
 464
 465/* Called with iothread lock taken.  */
 466static void dirty_bitmap_do_save_cleanup(DBMSaveState *s)
 467{
 468    SaveBitmapState *dbms;
 469
 470    while ((dbms = QSIMPLEQ_FIRST(&s->dbms_list)) != NULL) {
 471        QSIMPLEQ_REMOVE_HEAD(&s->dbms_list, entry);
 472        bdrv_dirty_bitmap_set_busy(dbms->bitmap, false);
 473        bdrv_unref(dbms->bs);
 474        g_free(dbms->node_alias);
 475        g_free(dbms->bitmap_alias);
 476        g_free(dbms);
 477    }
 478}
 479
 480/* Called with iothread lock taken. */
 481static int add_bitmaps_to_list(DBMSaveState *s, BlockDriverState *bs,
 482                               const char *bs_name, GHashTable *alias_map)
 483{
 484    BdrvDirtyBitmap *bitmap;
 485    SaveBitmapState *dbms;
 486    GHashTable *bitmap_aliases;
 487    const char *node_alias, *bitmap_name, *bitmap_alias;
 488    Error *local_err = NULL;
 489
 490    /* When an alias map is given, @bs_name must be @bs's node name */
 491    assert(!alias_map || !strcmp(bs_name, bdrv_get_node_name(bs)));
 492
 493    FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 494        if (bdrv_dirty_bitmap_name(bitmap)) {
 495            break;
 496        }
 497    }
 498    if (!bitmap) {
 499        return 0;
 500    }
 501
 502    bitmap_name = bdrv_dirty_bitmap_name(bitmap);
 503
 504    if (!bs_name || strcmp(bs_name, "") == 0) {
 505        error_report("Bitmap '%s' in unnamed node can't be migrated",
 506                     bitmap_name);
 507        return -1;
 508    }
 509
 510    if (alias_map) {
 511        const AliasMapInnerNode *amin = g_hash_table_lookup(alias_map, bs_name);
 512
 513        if (!amin) {
 514            /* Skip bitmaps on nodes with no alias */
 515            return 0;
 516        }
 517
 518        node_alias = amin->string;
 519        bitmap_aliases = amin->subtree;
 520    } else {
 521        node_alias = bs_name;
 522        bitmap_aliases = NULL;
 523    }
 524
 525    if (node_alias[0] == '#') {
 526        error_report("Bitmap '%s' in a node with auto-generated "
 527                     "name '%s' can't be migrated",
 528                     bitmap_name, node_alias);
 529        return -1;
 530    }
 531
 532    FOR_EACH_DIRTY_BITMAP(bs, bitmap) {
 533        BitmapMigrationBitmapAliasTransform *bitmap_transform = NULL;
 534        bitmap_name = bdrv_dirty_bitmap_name(bitmap);
 535        if (!bitmap_name) {
 536            continue;
 537        }
 538
 539        if (bdrv_dirty_bitmap_check(bitmap, BDRV_BITMAP_DEFAULT, &local_err)) {
 540            error_report_err(local_err);
 541            return -1;
 542        }
 543
 544        if (bitmap_aliases) {
 545            BitmapMigrationBitmapAlias *bmap_inner;
 546
 547            bmap_inner = g_hash_table_lookup(bitmap_aliases, bitmap_name);
 548            if (!bmap_inner) {
 549                /* Skip bitmaps with no alias */
 550                continue;
 551            }
 552
 553            bitmap_alias = bmap_inner->alias;
 554            if (bmap_inner->has_transform) {
 555                bitmap_transform = bmap_inner->transform;
 556            }
 557        } else {
 558            if (strlen(bitmap_name) > UINT8_MAX) {
 559                error_report("Cannot migrate bitmap '%s' on node '%s': "
 560                             "Name is longer than %u bytes",
 561                             bitmap_name, bs_name, UINT8_MAX);
 562                return -1;
 563            }
 564            bitmap_alias = bitmap_name;
 565        }
 566
 567        bdrv_ref(bs);
 568        bdrv_dirty_bitmap_set_busy(bitmap, true);
 569
 570        dbms = g_new0(SaveBitmapState, 1);
 571        dbms->bs = bs;
 572        dbms->node_alias = g_strdup(node_alias);
 573        dbms->bitmap_alias = g_strdup(bitmap_alias);
 574        dbms->bitmap = bitmap;
 575        dbms->total_sectors = bdrv_nb_sectors(bs);
 576        dbms->sectors_per_chunk = CHUNK_SIZE * 8LLU *
 577            (bdrv_dirty_bitmap_granularity(bitmap) >> BDRV_SECTOR_BITS);
 578        assert(dbms->sectors_per_chunk != 0);
 579        if (bdrv_dirty_bitmap_enabled(bitmap)) {
 580            dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
 581        }
 582        if (bitmap_transform &&
 583            bitmap_transform->has_persistent) {
 584            if (bitmap_transform->persistent) {
 585                dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
 586            }
 587        } else {
 588            if (bdrv_dirty_bitmap_get_persistence(bitmap)) {
 589                dbms->flags |= DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
 590            }
 591        }
 592
 593        QSIMPLEQ_INSERT_TAIL(&s->dbms_list, dbms, entry);
 594    }
 595
 596    return 0;
 597}
 598
 599/* Called with iothread lock taken. */
 600static int init_dirty_bitmap_migration(DBMSaveState *s)
 601{
 602    BlockDriverState *bs;
 603    SaveBitmapState *dbms;
 604    GHashTable *handled_by_blk = g_hash_table_new(NULL, NULL);
 605    BlockBackend *blk;
 606    const MigrationParameters *mig_params = &migrate_get_current()->parameters;
 607    GHashTable *alias_map = NULL;
 608
 609    if (mig_params->has_block_bitmap_mapping) {
 610        alias_map = construct_alias_map(mig_params->block_bitmap_mapping, true,
 611                                        &error_abort);
 612    }
 613
 614    s->bulk_completed = false;
 615    s->prev_bs = NULL;
 616    s->prev_bitmap = NULL;
 617    s->no_bitmaps = false;
 618
 619    if (!alias_map) {
 620        /*
 621         * Use blockdevice name for direct (or filtered) children of named block
 622         * backends.
 623         */
 624        for (blk = blk_next(NULL); blk; blk = blk_next(blk)) {
 625            const char *name = blk_name(blk);
 626
 627            if (!name || strcmp(name, "") == 0) {
 628                continue;
 629            }
 630
 631            bs = blk_bs(blk);
 632
 633            /* Skip filters without bitmaps */
 634            while (bs && bs->drv && bs->drv->is_filter &&
 635                   !bdrv_has_named_bitmaps(bs))
 636            {
 637                bs = bdrv_filter_bs(bs);
 638            }
 639
 640            if (bs && bs->drv && !bs->drv->is_filter) {
 641                if (add_bitmaps_to_list(s, bs, name, NULL)) {
 642                    goto fail;
 643                }
 644                g_hash_table_add(handled_by_blk, bs);
 645            }
 646        }
 647    }
 648
 649    for (bs = bdrv_next_all_states(NULL); bs; bs = bdrv_next_all_states(bs)) {
 650        if (g_hash_table_contains(handled_by_blk, bs)) {
 651            continue;
 652        }
 653
 654        if (add_bitmaps_to_list(s, bs, bdrv_get_node_name(bs), alias_map)) {
 655            goto fail;
 656        }
 657    }
 658
 659    /* unset migration flags here, to not roll back it */
 660    QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
 661        bdrv_dirty_bitmap_skip_store(dbms->bitmap, true);
 662    }
 663
 664    if (QSIMPLEQ_EMPTY(&s->dbms_list)) {
 665        s->no_bitmaps = true;
 666    }
 667
 668    g_hash_table_destroy(handled_by_blk);
 669    if (alias_map) {
 670        g_hash_table_destroy(alias_map);
 671    }
 672
 673    return 0;
 674
 675fail:
 676    g_hash_table_destroy(handled_by_blk);
 677    if (alias_map) {
 678        g_hash_table_destroy(alias_map);
 679    }
 680    dirty_bitmap_do_save_cleanup(s);
 681
 682    return -1;
 683}
 684
 685/* Called with no lock taken.  */
 686static void bulk_phase_send_chunk(QEMUFile *f, DBMSaveState *s,
 687                                  SaveBitmapState *dbms)
 688{
 689    uint32_t nr_sectors = MIN(dbms->total_sectors - dbms->cur_sector,
 690                             dbms->sectors_per_chunk);
 691
 692    send_bitmap_bits(f, s, dbms, dbms->cur_sector, nr_sectors);
 693
 694    dbms->cur_sector += nr_sectors;
 695    if (dbms->cur_sector >= dbms->total_sectors) {
 696        dbms->bulk_completed = true;
 697    }
 698}
 699
 700/* Called with no lock taken.  */
 701static void bulk_phase(QEMUFile *f, DBMSaveState *s, bool limit)
 702{
 703    SaveBitmapState *dbms;
 704
 705    QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
 706        while (!dbms->bulk_completed) {
 707            bulk_phase_send_chunk(f, s, dbms);
 708            if (limit && qemu_file_rate_limit(f)) {
 709                return;
 710            }
 711        }
 712    }
 713
 714    s->bulk_completed = true;
 715}
 716
 717/* for SaveVMHandlers */
 718static void dirty_bitmap_save_cleanup(void *opaque)
 719{
 720    DBMSaveState *s = &((DBMState *)opaque)->save;
 721
 722    dirty_bitmap_do_save_cleanup(s);
 723}
 724
 725static int dirty_bitmap_save_iterate(QEMUFile *f, void *opaque)
 726{
 727    DBMSaveState *s = &((DBMState *)opaque)->save;
 728
 729    trace_dirty_bitmap_save_iterate(migration_in_postcopy());
 730
 731    if (migration_in_postcopy() && !s->bulk_completed) {
 732        bulk_phase(f, s, true);
 733    }
 734
 735    qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
 736
 737    return s->bulk_completed;
 738}
 739
 740/* Called with iothread lock taken.  */
 741
 742static int dirty_bitmap_save_complete(QEMUFile *f, void *opaque)
 743{
 744    DBMSaveState *s = &((DBMState *)opaque)->save;
 745    SaveBitmapState *dbms;
 746    trace_dirty_bitmap_save_complete_enter();
 747
 748    if (!s->bulk_completed) {
 749        bulk_phase(f, s, false);
 750    }
 751
 752    QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
 753        send_bitmap_complete(f, s, dbms);
 754    }
 755
 756    qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
 757
 758    trace_dirty_bitmap_save_complete_finish();
 759
 760    dirty_bitmap_save_cleanup(opaque);
 761    return 0;
 762}
 763
 764static void dirty_bitmap_save_pending(QEMUFile *f, void *opaque,
 765                                      uint64_t max_size,
 766                                      uint64_t *res_precopy_only,
 767                                      uint64_t *res_compatible,
 768                                      uint64_t *res_postcopy_only)
 769{
 770    DBMSaveState *s = &((DBMState *)opaque)->save;
 771    SaveBitmapState *dbms;
 772    uint64_t pending = 0;
 773
 774    qemu_mutex_lock_iothread();
 775
 776    QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
 777        uint64_t gran = bdrv_dirty_bitmap_granularity(dbms->bitmap);
 778        uint64_t sectors = dbms->bulk_completed ? 0 :
 779                           dbms->total_sectors - dbms->cur_sector;
 780
 781        pending += DIV_ROUND_UP(sectors * BDRV_SECTOR_SIZE, gran);
 782    }
 783
 784    qemu_mutex_unlock_iothread();
 785
 786    trace_dirty_bitmap_save_pending(pending, max_size);
 787
 788    *res_postcopy_only += pending;
 789}
 790
 791/* First occurrence of this bitmap. It should be created if doesn't exist */
 792static int dirty_bitmap_load_start(QEMUFile *f, DBMLoadState *s)
 793{
 794    Error *local_err = NULL;
 795    uint32_t granularity = qemu_get_be32(f);
 796    uint8_t flags = qemu_get_byte(f);
 797    LoadBitmapState *b;
 798    bool persistent;
 799
 800    if (s->cancelled) {
 801        return 0;
 802    }
 803
 804    if (s->bitmap) {
 805        error_report("Bitmap with the same name ('%s') already exists on "
 806                     "destination", bdrv_dirty_bitmap_name(s->bitmap));
 807        return -EINVAL;
 808    } else {
 809        s->bitmap = bdrv_create_dirty_bitmap(s->bs, granularity,
 810                                             s->bitmap_name, &local_err);
 811        if (!s->bitmap) {
 812            error_report_err(local_err);
 813            return -EINVAL;
 814        }
 815    }
 816
 817    if (flags & DIRTY_BITMAP_MIG_START_FLAG_RESERVED_MASK) {
 818        error_report("Unknown flags in migrated dirty bitmap header: %x",
 819                     flags);
 820        return -EINVAL;
 821    }
 822
 823    if (s->bmap_inner &&
 824        s->bmap_inner->has_transform &&
 825        s->bmap_inner->transform->has_persistent) {
 826        persistent = s->bmap_inner->transform->persistent;
 827    } else {
 828        persistent = flags & DIRTY_BITMAP_MIG_START_FLAG_PERSISTENT;
 829    }
 830
 831    if (persistent) {
 832        bdrv_dirty_bitmap_set_persistence(s->bitmap, true);
 833    }
 834
 835    bdrv_disable_dirty_bitmap(s->bitmap);
 836    if (flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED) {
 837        bdrv_dirty_bitmap_create_successor(s->bitmap, &local_err);
 838        if (local_err) {
 839            error_report_err(local_err);
 840            return -EINVAL;
 841        }
 842    } else {
 843        bdrv_dirty_bitmap_set_busy(s->bitmap, true);
 844    }
 845
 846    b = g_new(LoadBitmapState, 1);
 847    b->bs = s->bs;
 848    b->bitmap = s->bitmap;
 849    b->migrated = false;
 850    b->enabled = flags & DIRTY_BITMAP_MIG_START_FLAG_ENABLED;
 851
 852    s->bitmaps = g_slist_prepend(s->bitmaps, b);
 853
 854    return 0;
 855}
 856
 857/*
 858 * before_vm_start_handle_item
 859 *
 860 * g_slist_foreach helper
 861 *
 862 * item is LoadBitmapState*
 863 * opaque is DBMLoadState*
 864 */
 865static void before_vm_start_handle_item(void *item, void *opaque)
 866{
 867    DBMLoadState *s = opaque;
 868    LoadBitmapState *b = item;
 869
 870    if (b->enabled) {
 871        if (b->migrated) {
 872            bdrv_enable_dirty_bitmap(b->bitmap);
 873        } else {
 874            bdrv_dirty_bitmap_enable_successor(b->bitmap);
 875        }
 876    }
 877
 878    if (b->migrated) {
 879        s->bitmaps = g_slist_remove(s->bitmaps, b);
 880        g_free(b);
 881    }
 882}
 883
 884void dirty_bitmap_mig_before_vm_start(void)
 885{
 886    DBMLoadState *s = &dbm_state.load;
 887    qemu_mutex_lock(&s->lock);
 888
 889    assert(!s->before_vm_start_handled);
 890    g_slist_foreach(s->bitmaps, before_vm_start_handle_item, s);
 891    s->before_vm_start_handled = true;
 892
 893    qemu_mutex_unlock(&s->lock);
 894}
 895
 896static void cancel_incoming_locked(DBMLoadState *s)
 897{
 898    GSList *item;
 899
 900    if (s->cancelled) {
 901        return;
 902    }
 903
 904    s->cancelled = true;
 905    s->bs = NULL;
 906    s->bitmap = NULL;
 907
 908    /* Drop all unfinished bitmaps */
 909    for (item = s->bitmaps; item; item = g_slist_next(item)) {
 910        LoadBitmapState *b = item->data;
 911
 912        /*
 913         * Bitmap must be unfinished, as finished bitmaps should already be
 914         * removed from the list.
 915         */
 916        assert(!s->before_vm_start_handled || !b->migrated);
 917        if (bdrv_dirty_bitmap_has_successor(b->bitmap)) {
 918            bdrv_reclaim_dirty_bitmap(b->bitmap, &error_abort);
 919        } else {
 920            bdrv_dirty_bitmap_set_busy(b->bitmap, false);
 921        }
 922        bdrv_release_dirty_bitmap(b->bitmap);
 923    }
 924
 925    g_slist_free_full(s->bitmaps, g_free);
 926    s->bitmaps = NULL;
 927}
 928
 929void dirty_bitmap_mig_cancel_outgoing(void)
 930{
 931    dirty_bitmap_do_save_cleanup(&dbm_state.save);
 932}
 933
 934void dirty_bitmap_mig_cancel_incoming(void)
 935{
 936    DBMLoadState *s = &dbm_state.load;
 937
 938    qemu_mutex_lock(&s->lock);
 939
 940    cancel_incoming_locked(s);
 941
 942    qemu_mutex_unlock(&s->lock);
 943}
 944
 945static void dirty_bitmap_load_complete(QEMUFile *f, DBMLoadState *s)
 946{
 947    GSList *item;
 948    trace_dirty_bitmap_load_complete();
 949
 950    if (s->cancelled) {
 951        return;
 952    }
 953
 954    bdrv_dirty_bitmap_deserialize_finish(s->bitmap);
 955
 956    if (bdrv_dirty_bitmap_has_successor(s->bitmap)) {
 957        bdrv_reclaim_dirty_bitmap(s->bitmap, &error_abort);
 958    } else {
 959        bdrv_dirty_bitmap_set_busy(s->bitmap, false);
 960    }
 961
 962    for (item = s->bitmaps; item; item = g_slist_next(item)) {
 963        LoadBitmapState *b = item->data;
 964
 965        if (b->bitmap == s->bitmap) {
 966            b->migrated = true;
 967            if (s->before_vm_start_handled) {
 968                s->bitmaps = g_slist_remove(s->bitmaps, b);
 969                g_free(b);
 970            }
 971            break;
 972        }
 973    }
 974}
 975
 976static int dirty_bitmap_load_bits(QEMUFile *f, DBMLoadState *s)
 977{
 978    uint64_t first_byte = qemu_get_be64(f) << BDRV_SECTOR_BITS;
 979    uint64_t nr_bytes = (uint64_t)qemu_get_be32(f) << BDRV_SECTOR_BITS;
 980    trace_dirty_bitmap_load_bits_enter(first_byte >> BDRV_SECTOR_BITS,
 981                                       nr_bytes >> BDRV_SECTOR_BITS);
 982
 983    if (s->flags & DIRTY_BITMAP_MIG_FLAG_ZEROES) {
 984        trace_dirty_bitmap_load_bits_zeroes();
 985        if (!s->cancelled) {
 986            bdrv_dirty_bitmap_deserialize_zeroes(s->bitmap, first_byte,
 987                                                 nr_bytes, false);
 988        }
 989    } else {
 990        size_t ret;
 991        g_autofree uint8_t *buf = NULL;
 992        uint64_t buf_size = qemu_get_be64(f);
 993        uint64_t needed_size;
 994
 995        /*
 996         * The actual check for buf_size is done a bit later. We can't do it in
 997         * cancelled mode as we don't have the bitmap to check the constraints
 998         * (so, we allocate a buffer and read prior to the check). On the other
 999         * hand, we shouldn't blindly g_malloc the number from the stream.
1000         * Actually one chunk should not be larger than CHUNK_SIZE. Let's allow
1001         * a bit larger (which means that bitmap migration will fail anyway and
1002         * the whole migration will most probably fail soon due to broken
1003         * stream).
1004         */
1005        if (buf_size > 10 * CHUNK_SIZE) {
1006            error_report("Bitmap migration stream buffer allocation request "
1007                         "is too large");
1008            return -EIO;
1009        }
1010
1011        buf = g_malloc(buf_size);
1012        ret = qemu_get_buffer(f, buf, buf_size);
1013        if (ret != buf_size) {
1014            error_report("Failed to read bitmap bits");
1015            return -EIO;
1016        }
1017
1018        if (s->cancelled) {
1019            return 0;
1020        }
1021
1022        needed_size = bdrv_dirty_bitmap_serialization_size(s->bitmap,
1023                                                           first_byte,
1024                                                           nr_bytes);
1025
1026        if (needed_size > buf_size ||
1027            buf_size > QEMU_ALIGN_UP(needed_size, 4 * sizeof(long))
1028             /* Here used same alignment as in send_bitmap_bits */
1029        ) {
1030            error_report("Migrated bitmap granularity doesn't "
1031                         "match the destination bitmap '%s' granularity",
1032                         bdrv_dirty_bitmap_name(s->bitmap));
1033            cancel_incoming_locked(s);
1034            return 0;
1035        }
1036
1037        bdrv_dirty_bitmap_deserialize_part(s->bitmap, buf, first_byte, nr_bytes,
1038                                           false);
1039    }
1040
1041    return 0;
1042}
1043
1044static int dirty_bitmap_load_header(QEMUFile *f, DBMLoadState *s,
1045                                    GHashTable *alias_map)
1046{
1047    GHashTable *bitmap_alias_map = NULL;
1048    Error *local_err = NULL;
1049    bool nothing;
1050    s->flags = qemu_get_bitmap_flags(f);
1051    trace_dirty_bitmap_load_header(s->flags);
1052
1053    nothing = s->flags == (s->flags & DIRTY_BITMAP_MIG_FLAG_EOS);
1054
1055    if (s->flags & DIRTY_BITMAP_MIG_FLAG_DEVICE_NAME) {
1056        if (!qemu_get_counted_string(f, s->node_alias)) {
1057            error_report("Unable to read node alias string");
1058            return -EINVAL;
1059        }
1060
1061        if (!s->cancelled) {
1062            if (alias_map) {
1063                const AliasMapInnerNode *amin;
1064
1065                amin = g_hash_table_lookup(alias_map, s->node_alias);
1066                if (!amin) {
1067                    error_setg(&local_err, "Error: Unknown node alias '%s'",
1068                               s->node_alias);
1069                    s->bs = NULL;
1070                } else {
1071                    bitmap_alias_map = amin->subtree;
1072                    s->bs = bdrv_lookup_bs(NULL, amin->string, &local_err);
1073                }
1074            } else {
1075                s->bs = bdrv_lookup_bs(s->node_alias, s->node_alias,
1076                                       &local_err);
1077            }
1078            if (!s->bs) {
1079                error_report_err(local_err);
1080                cancel_incoming_locked(s);
1081            }
1082        }
1083    } else if (s->bs) {
1084        if (alias_map) {
1085            const AliasMapInnerNode *amin;
1086
1087            /* Must be present in the map, or s->bs would not be set */
1088            amin = g_hash_table_lookup(alias_map, s->node_alias);
1089            assert(amin != NULL);
1090
1091            bitmap_alias_map = amin->subtree;
1092        }
1093    } else if (!nothing && !s->cancelled) {
1094        error_report("Error: block device name is not set");
1095        cancel_incoming_locked(s);
1096    }
1097
1098    assert(nothing || s->cancelled || !!alias_map == !!bitmap_alias_map);
1099
1100    if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITMAP_NAME) {
1101        const char *bitmap_name;
1102
1103        if (!qemu_get_counted_string(f, s->bitmap_alias)) {
1104            error_report("Unable to read bitmap alias string");
1105            return -EINVAL;
1106        }
1107
1108        bitmap_name = s->bitmap_alias;
1109        if (!s->cancelled && bitmap_alias_map) {
1110            BitmapMigrationBitmapAlias *bmap_inner;
1111
1112            bmap_inner = g_hash_table_lookup(bitmap_alias_map, s->bitmap_alias);
1113            if (!bmap_inner) {
1114                error_report("Error: Unknown bitmap alias '%s' on node "
1115                             "'%s' (alias '%s')", s->bitmap_alias,
1116                             s->bs->node_name, s->node_alias);
1117                cancel_incoming_locked(s);
1118            } else {
1119                bitmap_name = bmap_inner->name;
1120            }
1121
1122            s->bmap_inner = bmap_inner;
1123        }
1124
1125        if (!s->cancelled) {
1126            g_strlcpy(s->bitmap_name, bitmap_name, sizeof(s->bitmap_name));
1127            s->bitmap = bdrv_find_dirty_bitmap(s->bs, s->bitmap_name);
1128
1129            /*
1130             * bitmap may be NULL here, it wouldn't be an error if it is the
1131             * first occurrence of the bitmap
1132             */
1133            if (!s->bitmap && !(s->flags & DIRTY_BITMAP_MIG_FLAG_START)) {
1134                error_report("Error: unknown dirty bitmap "
1135                             "'%s' for block device '%s'",
1136                             s->bitmap_name, s->bs->node_name);
1137                cancel_incoming_locked(s);
1138            }
1139        }
1140    } else if (!s->bitmap && !nothing && !s->cancelled) {
1141        error_report("Error: block device name is not set");
1142        cancel_incoming_locked(s);
1143    }
1144
1145    return 0;
1146}
1147
1148/*
1149 * dirty_bitmap_load
1150 *
1151 * Load sequence of dirty bitmap chunks. Return error only on fatal io stream
1152 * violations. On other errors just cancel bitmaps incoming migration and return
1153 * 0.
1154 *
1155 * Note, than when incoming bitmap migration is canceled, we still must read all
1156 * our chunks (and just ignore them), to not affect other migration objects.
1157 */
1158static int dirty_bitmap_load(QEMUFile *f, void *opaque, int version_id)
1159{
1160    GHashTable *alias_map = NULL;
1161    const MigrationParameters *mig_params = &migrate_get_current()->parameters;
1162    DBMLoadState *s = &((DBMState *)opaque)->load;
1163    int ret = 0;
1164
1165    trace_dirty_bitmap_load_enter();
1166
1167    if (version_id != 1) {
1168        QEMU_LOCK_GUARD(&s->lock);
1169        cancel_incoming_locked(s);
1170        return -EINVAL;
1171    }
1172
1173    if (mig_params->has_block_bitmap_mapping) {
1174        alias_map = construct_alias_map(mig_params->block_bitmap_mapping,
1175                                        false, &error_abort);
1176    }
1177
1178    do {
1179        QEMU_LOCK_GUARD(&s->lock);
1180
1181        ret = dirty_bitmap_load_header(f, s, alias_map);
1182        if (ret < 0) {
1183            cancel_incoming_locked(s);
1184            goto fail;
1185        }
1186
1187        if (s->flags & DIRTY_BITMAP_MIG_FLAG_START) {
1188            ret = dirty_bitmap_load_start(f, s);
1189        } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_COMPLETE) {
1190            dirty_bitmap_load_complete(f, s);
1191        } else if (s->flags & DIRTY_BITMAP_MIG_FLAG_BITS) {
1192            ret = dirty_bitmap_load_bits(f, s);
1193        }
1194
1195        if (!ret) {
1196            ret = qemu_file_get_error(f);
1197        }
1198
1199        if (ret) {
1200            cancel_incoming_locked(s);
1201            goto fail;
1202        }
1203    } while (!(s->flags & DIRTY_BITMAP_MIG_FLAG_EOS));
1204
1205    trace_dirty_bitmap_load_success();
1206    ret = 0;
1207fail:
1208    if (alias_map) {
1209        g_hash_table_destroy(alias_map);
1210    }
1211    return ret;
1212}
1213
1214static int dirty_bitmap_save_setup(QEMUFile *f, void *opaque)
1215{
1216    DBMSaveState *s = &((DBMState *)opaque)->save;
1217    SaveBitmapState *dbms = NULL;
1218    if (init_dirty_bitmap_migration(s) < 0) {
1219        return -1;
1220    }
1221
1222    QSIMPLEQ_FOREACH(dbms, &s->dbms_list, entry) {
1223        send_bitmap_start(f, s, dbms);
1224    }
1225    qemu_put_bitmap_flags(f, DIRTY_BITMAP_MIG_FLAG_EOS);
1226
1227    return 0;
1228}
1229
1230static bool dirty_bitmap_is_active(void *opaque)
1231{
1232    DBMSaveState *s = &((DBMState *)opaque)->save;
1233
1234    return migrate_dirty_bitmaps() && !s->no_bitmaps;
1235}
1236
1237static bool dirty_bitmap_is_active_iterate(void *opaque)
1238{
1239    return dirty_bitmap_is_active(opaque) && !runstate_is_running();
1240}
1241
1242static bool dirty_bitmap_has_postcopy(void *opaque)
1243{
1244    return true;
1245}
1246
1247static SaveVMHandlers savevm_dirty_bitmap_handlers = {
1248    .save_setup = dirty_bitmap_save_setup,
1249    .save_live_complete_postcopy = dirty_bitmap_save_complete,
1250    .save_live_complete_precopy = dirty_bitmap_save_complete,
1251    .has_postcopy = dirty_bitmap_has_postcopy,
1252    .save_live_pending = dirty_bitmap_save_pending,
1253    .save_live_iterate = dirty_bitmap_save_iterate,
1254    .is_active_iterate = dirty_bitmap_is_active_iterate,
1255    .load_state = dirty_bitmap_load,
1256    .save_cleanup = dirty_bitmap_save_cleanup,
1257    .is_active = dirty_bitmap_is_active,
1258};
1259
1260void dirty_bitmap_mig_init(void)
1261{
1262    QSIMPLEQ_INIT(&dbm_state.save.dbms_list);
1263    qemu_mutex_init(&dbm_state.load.lock);
1264
1265    register_savevm_live("dirty-bitmap", 0, 1,
1266                         &savevm_dirty_bitmap_handlers,
1267                         &dbm_state);
1268}
1269