linux/drivers/md/dm-cache-metadata.c
<<
>>
Prefs
   1/*
   2 * Copyright (C) 2012 Red Hat, Inc.
   3 *
   4 * This file is released under the GPL.
   5 */
   6
   7#include "dm-cache-metadata.h"
   8
   9#include "persistent-data/dm-array.h"
  10#include "persistent-data/dm-bitset.h"
  11#include "persistent-data/dm-space-map.h"
  12#include "persistent-data/dm-space-map-disk.h"
  13#include "persistent-data/dm-transaction-manager.h"
  14
  15#include <linux/device-mapper.h>
  16#include <linux/refcount.h>
  17
  18/*----------------------------------------------------------------*/
  19
  20#define DM_MSG_PREFIX   "cache metadata"
  21
  22#define CACHE_SUPERBLOCK_MAGIC 06142003
  23#define CACHE_SUPERBLOCK_LOCATION 0
  24
  25/*
  26 * defines a range of metadata versions that this module can handle.
  27 */
  28#define MIN_CACHE_VERSION 1
  29#define MAX_CACHE_VERSION 2
  30
  31/*
  32 *  3 for btree insert +
  33 *  2 for btree lookup used within space map
  34 */
  35#define CACHE_MAX_CONCURRENT_LOCKS 5
  36#define SPACE_MAP_ROOT_SIZE 128
  37
  38enum superblock_flag_bits {
  39        /* for spotting crashes that would invalidate the dirty bitset */
  40        CLEAN_SHUTDOWN,
  41        /* metadata must be checked using the tools */
  42        NEEDS_CHECK,
  43};
  44
  45/*
  46 * Each mapping from cache block -> origin block carries a set of flags.
  47 */
  48enum mapping_bits {
  49        /*
  50         * A valid mapping.  Because we're using an array we clear this
  51         * flag for an non existant mapping.
  52         */
  53        M_VALID = 1,
  54
  55        /*
  56         * The data on the cache is different from that on the origin.
  57         * This flag is only used by metadata format 1.
  58         */
  59        M_DIRTY = 2
  60};
  61
  62struct cache_disk_superblock {
  63        __le32 csum;
  64        __le32 flags;
  65        __le64 blocknr;
  66
  67        __u8 uuid[16];
  68        __le64 magic;
  69        __le32 version;
  70
  71        __u8 policy_name[CACHE_POLICY_NAME_SIZE];
  72        __le32 policy_hint_size;
  73
  74        __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  75        __le64 mapping_root;
  76        __le64 hint_root;
  77
  78        __le64 discard_root;
  79        __le64 discard_block_size;
  80        __le64 discard_nr_blocks;
  81
  82        __le32 data_block_size;
  83        __le32 metadata_block_size;
  84        __le32 cache_blocks;
  85
  86        __le32 compat_flags;
  87        __le32 compat_ro_flags;
  88        __le32 incompat_flags;
  89
  90        __le32 read_hits;
  91        __le32 read_misses;
  92        __le32 write_hits;
  93        __le32 write_misses;
  94
  95        __le32 policy_version[CACHE_POLICY_VERSION_SIZE];
  96
  97        /*
  98         * Metadata format 2 fields.
  99         */
 100        __le64 dirty_root;
 101} __packed;
 102
 103struct dm_cache_metadata {
 104        refcount_t ref_count;
 105        struct list_head list;
 106
 107        unsigned version;
 108        struct block_device *bdev;
 109        struct dm_block_manager *bm;
 110        struct dm_space_map *metadata_sm;
 111        struct dm_transaction_manager *tm;
 112
 113        struct dm_array_info info;
 114        struct dm_array_info hint_info;
 115        struct dm_disk_bitset discard_info;
 116
 117        struct rw_semaphore root_lock;
 118        unsigned long flags;
 119        dm_block_t root;
 120        dm_block_t hint_root;
 121        dm_block_t discard_root;
 122
 123        sector_t discard_block_size;
 124        dm_dblock_t discard_nr_blocks;
 125
 126        sector_t data_block_size;
 127        dm_cblock_t cache_blocks;
 128        bool changed:1;
 129        bool clean_when_opened:1;
 130
 131        char policy_name[CACHE_POLICY_NAME_SIZE];
 132        unsigned policy_version[CACHE_POLICY_VERSION_SIZE];
 133        size_t policy_hint_size;
 134        struct dm_cache_statistics stats;
 135
 136        /*
 137         * Reading the space map root can fail, so we read it into this
 138         * buffer before the superblock is locked and updated.
 139         */
 140        __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
 141
 142        /*
 143         * Set if a transaction has to be aborted but the attempt to roll
 144         * back to the previous (good) transaction failed.  The only
 145         * metadata operation permissible in this state is the closing of
 146         * the device.
 147         */
 148        bool fail_io:1;
 149
 150        /*
 151         * Metadata format 2 fields.
 152         */
 153        dm_block_t dirty_root;
 154        struct dm_disk_bitset dirty_info;
 155
 156        /*
 157         * These structures are used when loading metadata.  They're too
 158         * big to put on the stack.
 159         */
 160        struct dm_array_cursor mapping_cursor;
 161        struct dm_array_cursor hint_cursor;
 162        struct dm_bitset_cursor dirty_cursor;
 163};
 164
 165/*-------------------------------------------------------------------
 166 * superblock validator
 167 *-----------------------------------------------------------------*/
 168
 169#define SUPERBLOCK_CSUM_XOR 9031977
 170
 171static void sb_prepare_for_write(struct dm_block_validator *v,
 172                                 struct dm_block *b,
 173                                 size_t sb_block_size)
 174{
 175        struct cache_disk_superblock *disk_super = dm_block_data(b);
 176
 177        disk_super->blocknr = cpu_to_le64(dm_block_location(b));
 178        disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 179                                                      sb_block_size - sizeof(__le32),
 180                                                      SUPERBLOCK_CSUM_XOR));
 181}
 182
 183static int check_metadata_version(struct cache_disk_superblock *disk_super)
 184{
 185        uint32_t metadata_version = le32_to_cpu(disk_super->version);
 186
 187        if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {
 188                DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
 189                      metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);
 190                return -EINVAL;
 191        }
 192
 193        return 0;
 194}
 195
 196static int sb_check(struct dm_block_validator *v,
 197                    struct dm_block *b,
 198                    size_t sb_block_size)
 199{
 200        struct cache_disk_superblock *disk_super = dm_block_data(b);
 201        __le32 csum_le;
 202
 203        if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
 204                DMERR("sb_check failed: blocknr %llu: wanted %llu",
 205                      le64_to_cpu(disk_super->blocknr),
 206                      (unsigned long long)dm_block_location(b));
 207                return -ENOTBLK;
 208        }
 209
 210        if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
 211                DMERR("sb_check failed: magic %llu: wanted %llu",
 212                      le64_to_cpu(disk_super->magic),
 213                      (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
 214                return -EILSEQ;
 215        }
 216
 217        csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
 218                                             sb_block_size - sizeof(__le32),
 219                                             SUPERBLOCK_CSUM_XOR));
 220        if (csum_le != disk_super->csum) {
 221                DMERR("sb_check failed: csum %u: wanted %u",
 222                      le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
 223                return -EILSEQ;
 224        }
 225
 226        return check_metadata_version(disk_super);
 227}
 228
 229static struct dm_block_validator sb_validator = {
 230        .name = "superblock",
 231        .prepare_for_write = sb_prepare_for_write,
 232        .check = sb_check
 233};
 234
 235/*----------------------------------------------------------------*/
 236
 237static int superblock_read_lock(struct dm_cache_metadata *cmd,
 238                                struct dm_block **sblock)
 239{
 240        return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 241                               &sb_validator, sblock);
 242}
 243
 244static int superblock_lock_zero(struct dm_cache_metadata *cmd,
 245                                struct dm_block **sblock)
 246{
 247        return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 248                                     &sb_validator, sblock);
 249}
 250
 251static int superblock_lock(struct dm_cache_metadata *cmd,
 252                           struct dm_block **sblock)
 253{
 254        return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 255                                &sb_validator, sblock);
 256}
 257
 258/*----------------------------------------------------------------*/
 259
 260static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
 261{
 262        int r;
 263        unsigned i;
 264        struct dm_block *b;
 265        __le64 *data_le, zero = cpu_to_le64(0);
 266        unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
 267
 268        /*
 269         * We can't use a validator here - it may be all zeroes.
 270         */
 271        r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
 272        if (r)
 273                return r;
 274
 275        data_le = dm_block_data(b);
 276        *result = true;
 277        for (i = 0; i < sb_block_size; i++) {
 278                if (data_le[i] != zero) {
 279                        *result = false;
 280                        break;
 281                }
 282        }
 283
 284        dm_bm_unlock(b);
 285
 286        return 0;
 287}
 288
 289static void __setup_mapping_info(struct dm_cache_metadata *cmd)
 290{
 291        struct dm_btree_value_type vt;
 292
 293        vt.context = NULL;
 294        vt.size = sizeof(__le64);
 295        vt.inc = NULL;
 296        vt.dec = NULL;
 297        vt.equal = NULL;
 298        dm_array_info_init(&cmd->info, cmd->tm, &vt);
 299
 300        if (cmd->policy_hint_size) {
 301                vt.size = sizeof(__le32);
 302                dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
 303        }
 304}
 305
 306static int __save_sm_root(struct dm_cache_metadata *cmd)
 307{
 308        int r;
 309        size_t metadata_len;
 310
 311        r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
 312        if (r < 0)
 313                return r;
 314
 315        return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,
 316                               metadata_len);
 317}
 318
 319static void __copy_sm_root(struct dm_cache_metadata *cmd,
 320                           struct cache_disk_superblock *disk_super)
 321{
 322        memcpy(&disk_super->metadata_space_map_root,
 323               &cmd->metadata_space_map_root,
 324               sizeof(cmd->metadata_space_map_root));
 325}
 326
 327static bool separate_dirty_bits(struct dm_cache_metadata *cmd)
 328{
 329        return cmd->version >= 2;
 330}
 331
 332static int __write_initial_superblock(struct dm_cache_metadata *cmd)
 333{
 334        int r;
 335        struct dm_block *sblock;
 336        struct cache_disk_superblock *disk_super;
 337        sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT;
 338
 339        /* FIXME: see if we can lose the max sectors limit */
 340        if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
 341                bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
 342
 343        r = dm_tm_pre_commit(cmd->tm);
 344        if (r < 0)
 345                return r;
 346
 347        /*
 348         * dm_sm_copy_root() can fail.  So we need to do it before we start
 349         * updating the superblock.
 350         */
 351        r = __save_sm_root(cmd);
 352        if (r)
 353                return r;
 354
 355        r = superblock_lock_zero(cmd, &sblock);
 356        if (r)
 357                return r;
 358
 359        disk_super = dm_block_data(sblock);
 360        disk_super->flags = 0;
 361        memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
 362        disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
 363        disk_super->version = cpu_to_le32(cmd->version);
 364        memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
 365        memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
 366        disk_super->policy_hint_size = cpu_to_le32(0);
 367
 368        __copy_sm_root(cmd, disk_super);
 369
 370        disk_super->mapping_root = cpu_to_le64(cmd->root);
 371        disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 372        disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 373        disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 374        disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 375        disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
 376        disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
 377        disk_super->cache_blocks = cpu_to_le32(0);
 378
 379        disk_super->read_hits = cpu_to_le32(0);
 380        disk_super->read_misses = cpu_to_le32(0);
 381        disk_super->write_hits = cpu_to_le32(0);
 382        disk_super->write_misses = cpu_to_le32(0);
 383
 384        if (separate_dirty_bits(cmd))
 385                disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
 386
 387        return dm_tm_commit(cmd->tm, sblock);
 388}
 389
 390static int __format_metadata(struct dm_cache_metadata *cmd)
 391{
 392        int r;
 393
 394        r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 395                                 &cmd->tm, &cmd->metadata_sm);
 396        if (r < 0) {
 397                DMERR("tm_create_with_sm failed");
 398                return r;
 399        }
 400
 401        __setup_mapping_info(cmd);
 402
 403        r = dm_array_empty(&cmd->info, &cmd->root);
 404        if (r < 0)
 405                goto bad;
 406
 407        if (separate_dirty_bits(cmd)) {
 408                dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
 409                r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);
 410                if (r < 0)
 411                        goto bad;
 412        }
 413
 414        dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 415        r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
 416        if (r < 0)
 417                goto bad;
 418
 419        cmd->discard_block_size = 0;
 420        cmd->discard_nr_blocks = 0;
 421
 422        r = __write_initial_superblock(cmd);
 423        if (r)
 424                goto bad;
 425
 426        cmd->clean_when_opened = true;
 427        return 0;
 428
 429bad:
 430        dm_tm_destroy(cmd->tm);
 431        dm_sm_destroy(cmd->metadata_sm);
 432
 433        return r;
 434}
 435
 436static int __check_incompat_features(struct cache_disk_superblock *disk_super,
 437                                     struct dm_cache_metadata *cmd)
 438{
 439        uint32_t incompat_flags, features;
 440
 441        incompat_flags = le32_to_cpu(disk_super->incompat_flags);
 442        features = incompat_flags & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
 443        if (features) {
 444                DMERR("could not access metadata due to unsupported optional features (%lx).",
 445                      (unsigned long)features);
 446                return -EINVAL;
 447        }
 448
 449        /*
 450         * Check for read-only metadata to skip the following RDWR checks.
 451         */
 452        if (get_disk_ro(cmd->bdev->bd_disk))
 453                return 0;
 454
 455        features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
 456        if (features) {
 457                DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
 458                      (unsigned long)features);
 459                return -EINVAL;
 460        }
 461
 462        return 0;
 463}
 464
 465static int __open_metadata(struct dm_cache_metadata *cmd)
 466{
 467        int r;
 468        struct dm_block *sblock;
 469        struct cache_disk_superblock *disk_super;
 470        unsigned long sb_flags;
 471
 472        r = superblock_read_lock(cmd, &sblock);
 473        if (r < 0) {
 474                DMERR("couldn't read lock superblock");
 475                return r;
 476        }
 477
 478        disk_super = dm_block_data(sblock);
 479
 480        /* Verify the data block size hasn't changed */
 481        if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
 482                DMERR("changing the data block size (from %u to %llu) is not supported",
 483                      le32_to_cpu(disk_super->data_block_size),
 484                      (unsigned long long)cmd->data_block_size);
 485                r = -EINVAL;
 486                goto bad;
 487        }
 488
 489        r = __check_incompat_features(disk_super, cmd);
 490        if (r < 0)
 491                goto bad;
 492
 493        r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
 494                               disk_super->metadata_space_map_root,
 495                               sizeof(disk_super->metadata_space_map_root),
 496                               &cmd->tm, &cmd->metadata_sm);
 497        if (r < 0) {
 498                DMERR("tm_open_with_sm failed");
 499                goto bad;
 500        }
 501
 502        __setup_mapping_info(cmd);
 503        dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
 504        dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
 505        sb_flags = le32_to_cpu(disk_super->flags);
 506        cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
 507        dm_bm_unlock(sblock);
 508
 509        return 0;
 510
 511bad:
 512        dm_bm_unlock(sblock);
 513        return r;
 514}
 515
 516static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
 517                                     bool format_device)
 518{
 519        int r;
 520        bool unformatted = false;
 521
 522        r = __superblock_all_zeroes(cmd->bm, &unformatted);
 523        if (r)
 524                return r;
 525
 526        if (unformatted)
 527                return format_device ? __format_metadata(cmd) : -EPERM;
 528
 529        return __open_metadata(cmd);
 530}
 531
 532static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
 533                                            bool may_format_device)
 534{
 535        int r;
 536        cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
 537                                          CACHE_MAX_CONCURRENT_LOCKS);
 538        if (IS_ERR(cmd->bm)) {
 539                DMERR("could not create block manager");
 540                r = PTR_ERR(cmd->bm);
 541                cmd->bm = NULL;
 542                return r;
 543        }
 544
 545        r = __open_or_format_metadata(cmd, may_format_device);
 546        if (r) {
 547                dm_block_manager_destroy(cmd->bm);
 548                cmd->bm = NULL;
 549        }
 550
 551        return r;
 552}
 553
 554static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd)
 555{
 556        dm_sm_destroy(cmd->metadata_sm);
 557        dm_tm_destroy(cmd->tm);
 558        dm_block_manager_destroy(cmd->bm);
 559}
 560
 561typedef unsigned long (*flags_mutator)(unsigned long);
 562
 563static void update_flags(struct cache_disk_superblock *disk_super,
 564                         flags_mutator mutator)
 565{
 566        uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
 567        disk_super->flags = cpu_to_le32(sb_flags);
 568}
 569
 570static unsigned long set_clean_shutdown(unsigned long flags)
 571{
 572        set_bit(CLEAN_SHUTDOWN, &flags);
 573        return flags;
 574}
 575
 576static unsigned long clear_clean_shutdown(unsigned long flags)
 577{
 578        clear_bit(CLEAN_SHUTDOWN, &flags);
 579        return flags;
 580}
 581
 582static void read_superblock_fields(struct dm_cache_metadata *cmd,
 583                                   struct cache_disk_superblock *disk_super)
 584{
 585        cmd->version = le32_to_cpu(disk_super->version);
 586        cmd->flags = le32_to_cpu(disk_super->flags);
 587        cmd->root = le64_to_cpu(disk_super->mapping_root);
 588        cmd->hint_root = le64_to_cpu(disk_super->hint_root);
 589        cmd->discard_root = le64_to_cpu(disk_super->discard_root);
 590        cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
 591        cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
 592        cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
 593        cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
 594        strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
 595        cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
 596        cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
 597        cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
 598        cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
 599
 600        cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
 601        cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
 602        cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
 603        cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
 604
 605        if (separate_dirty_bits(cmd))
 606                cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);
 607
 608        cmd->changed = false;
 609}
 610
 611/*
 612 * The mutator updates the superblock flags.
 613 */
 614static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
 615                                     flags_mutator mutator)
 616{
 617        int r;
 618        struct cache_disk_superblock *disk_super;
 619        struct dm_block *sblock;
 620
 621        r = superblock_lock(cmd, &sblock);
 622        if (r)
 623                return r;
 624
 625        disk_super = dm_block_data(sblock);
 626        update_flags(disk_super, mutator);
 627        read_superblock_fields(cmd, disk_super);
 628        dm_bm_unlock(sblock);
 629
 630        return dm_bm_flush(cmd->bm);
 631}
 632
 633static int __begin_transaction(struct dm_cache_metadata *cmd)
 634{
 635        int r;
 636        struct cache_disk_superblock *disk_super;
 637        struct dm_block *sblock;
 638
 639        /*
 640         * We re-read the superblock every time.  Shouldn't need to do this
 641         * really.
 642         */
 643        r = superblock_read_lock(cmd, &sblock);
 644        if (r)
 645                return r;
 646
 647        disk_super = dm_block_data(sblock);
 648        read_superblock_fields(cmd, disk_super);
 649        dm_bm_unlock(sblock);
 650
 651        return 0;
 652}
 653
 654static int __commit_transaction(struct dm_cache_metadata *cmd,
 655                                flags_mutator mutator)
 656{
 657        int r;
 658        struct cache_disk_superblock *disk_super;
 659        struct dm_block *sblock;
 660
 661        /*
 662         * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
 663         */
 664        BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
 665
 666        if (separate_dirty_bits(cmd)) {
 667                r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,
 668                                    &cmd->dirty_root);
 669                if (r)
 670                        return r;
 671        }
 672
 673        r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
 674                            &cmd->discard_root);
 675        if (r)
 676                return r;
 677
 678        r = dm_tm_pre_commit(cmd->tm);
 679        if (r < 0)
 680                return r;
 681
 682        r = __save_sm_root(cmd);
 683        if (r)
 684                return r;
 685
 686        r = superblock_lock(cmd, &sblock);
 687        if (r)
 688                return r;
 689
 690        disk_super = dm_block_data(sblock);
 691
 692        disk_super->flags = cpu_to_le32(cmd->flags);
 693        if (mutator)
 694                update_flags(disk_super, mutator);
 695
 696        disk_super->mapping_root = cpu_to_le64(cmd->root);
 697        if (separate_dirty_bits(cmd))
 698                disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
 699        disk_super->hint_root = cpu_to_le64(cmd->hint_root);
 700        disk_super->discard_root = cpu_to_le64(cmd->discard_root);
 701        disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
 702        disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
 703        disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
 704        strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
 705        disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
 706        disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
 707        disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
 708        disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);
 709
 710        disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
 711        disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
 712        disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
 713        disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
 714        __copy_sm_root(cmd, disk_super);
 715
 716        return dm_tm_commit(cmd->tm, sblock);
 717}
 718
 719/*----------------------------------------------------------------*/
 720
 721/*
 722 * The mappings are held in a dm-array that has 64-bit values stored in
 723 * little-endian format.  The index is the cblock, the high 48bits of the
 724 * value are the oblock and the low 16 bit the flags.
 725 */
 726#define FLAGS_MASK ((1 << 16) - 1)
 727
 728static __le64 pack_value(dm_oblock_t block, unsigned flags)
 729{
 730        uint64_t value = from_oblock(block);
 731        value <<= 16;
 732        value = value | (flags & FLAGS_MASK);
 733        return cpu_to_le64(value);
 734}
 735
 736static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
 737{
 738        uint64_t value = le64_to_cpu(value_le);
 739        uint64_t b = value >> 16;
 740        *block = to_oblock(b);
 741        *flags = value & FLAGS_MASK;
 742}
 743
 744/*----------------------------------------------------------------*/
 745
 746static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
 747                                               sector_t data_block_size,
 748                                               bool may_format_device,
 749                                               size_t policy_hint_size,
 750                                               unsigned metadata_version)
 751{
 752        int r;
 753        struct dm_cache_metadata *cmd;
 754
 755        cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
 756        if (!cmd) {
 757                DMERR("could not allocate metadata struct");
 758                return ERR_PTR(-ENOMEM);
 759        }
 760
 761        cmd->version = metadata_version;
 762        refcount_set(&cmd->ref_count, 1);
 763        init_rwsem(&cmd->root_lock);
 764        cmd->bdev = bdev;
 765        cmd->data_block_size = data_block_size;
 766        cmd->cache_blocks = 0;
 767        cmd->policy_hint_size = policy_hint_size;
 768        cmd->changed = true;
 769        cmd->fail_io = false;
 770
 771        r = __create_persistent_data_objects(cmd, may_format_device);
 772        if (r) {
 773                kfree(cmd);
 774                return ERR_PTR(r);
 775        }
 776
 777        r = __begin_transaction_flags(cmd, clear_clean_shutdown);
 778        if (r < 0) {
 779                dm_cache_metadata_close(cmd);
 780                return ERR_PTR(r);
 781        }
 782
 783        return cmd;
 784}
 785
 786/*
 787 * We keep a little list of ref counted metadata objects to prevent two
 788 * different target instances creating separate bufio instances.  This is
 789 * an issue if a table is reloaded before the suspend.
 790 */
 791static DEFINE_MUTEX(table_lock);
 792static LIST_HEAD(table);
 793
 794static struct dm_cache_metadata *lookup(struct block_device *bdev)
 795{
 796        struct dm_cache_metadata *cmd;
 797
 798        list_for_each_entry(cmd, &table, list)
 799                if (cmd->bdev == bdev) {
 800                        refcount_inc(&cmd->ref_count);
 801                        return cmd;
 802                }
 803
 804        return NULL;
 805}
 806
 807static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
 808                                                sector_t data_block_size,
 809                                                bool may_format_device,
 810                                                size_t policy_hint_size,
 811                                                unsigned metadata_version)
 812{
 813        struct dm_cache_metadata *cmd, *cmd2;
 814
 815        mutex_lock(&table_lock);
 816        cmd = lookup(bdev);
 817        mutex_unlock(&table_lock);
 818
 819        if (cmd)
 820                return cmd;
 821
 822        cmd = metadata_open(bdev, data_block_size, may_format_device,
 823                            policy_hint_size, metadata_version);
 824        if (!IS_ERR(cmd)) {
 825                mutex_lock(&table_lock);
 826                cmd2 = lookup(bdev);
 827                if (cmd2) {
 828                        mutex_unlock(&table_lock);
 829                        __destroy_persistent_data_objects(cmd);
 830                        kfree(cmd);
 831                        return cmd2;
 832                }
 833                list_add(&cmd->list, &table);
 834                mutex_unlock(&table_lock);
 835        }
 836
 837        return cmd;
 838}
 839
 840static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
 841{
 842        if (cmd->data_block_size != data_block_size) {
 843                DMERR("data_block_size (%llu) different from that in metadata (%llu)",
 844                      (unsigned long long) data_block_size,
 845                      (unsigned long long) cmd->data_block_size);
 846                return false;
 847        }
 848
 849        return true;
 850}
 851
 852struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
 853                                                 sector_t data_block_size,
 854                                                 bool may_format_device,
 855                                                 size_t policy_hint_size,
 856                                                 unsigned metadata_version)
 857{
 858        struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,
 859                                                       policy_hint_size, metadata_version);
 860
 861        if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
 862                dm_cache_metadata_close(cmd);
 863                return ERR_PTR(-EINVAL);
 864        }
 865
 866        return cmd;
 867}
 868
 869void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
 870{
 871        if (refcount_dec_and_test(&cmd->ref_count)) {
 872                mutex_lock(&table_lock);
 873                list_del(&cmd->list);
 874                mutex_unlock(&table_lock);
 875
 876                if (!cmd->fail_io)
 877                        __destroy_persistent_data_objects(cmd);
 878                kfree(cmd);
 879        }
 880}
 881
 882/*
 883 * Checks that the given cache block is either unmapped or clean.
 884 */
 885static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,
 886                                      bool *result)
 887{
 888        int r;
 889        __le64 value;
 890        dm_oblock_t ob;
 891        unsigned flags;
 892
 893        r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);
 894        if (r)
 895                return r;
 896
 897        unpack_value(value, &ob, &flags);
 898        *result = !((flags & M_VALID) && (flags & M_DIRTY));
 899
 900        return 0;
 901}
 902
 903static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,
 904                                           dm_cblock_t begin, dm_cblock_t end,
 905                                           bool *result)
 906{
 907        int r;
 908        *result = true;
 909
 910        while (begin != end) {
 911                r = block_clean_combined_dirty(cmd, begin, result);
 912                if (r) {
 913                        DMERR("block_clean_combined_dirty failed");
 914                        return r;
 915                }
 916
 917                if (!*result) {
 918                        DMERR("cache block %llu is dirty",
 919                              (unsigned long long) from_cblock(begin));
 920                        return 0;
 921                }
 922
 923                begin = to_cblock(from_cblock(begin) + 1);
 924        }
 925
 926        return 0;
 927}
 928
 929static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,
 930                                           dm_cblock_t begin, dm_cblock_t end,
 931                                           bool *result)
 932{
 933        int r;
 934        bool dirty_flag;
 935        *result = true;
 936
 937        if (from_cblock(cmd->cache_blocks) == 0)
 938                /* Nothing to do */
 939                return 0;
 940
 941        r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
 942                                   from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);
 943        if (r) {
 944                DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__);
 945                return r;
 946        }
 947
 948        r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));
 949        if (r) {
 950                DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__);
 951                dm_bitset_cursor_end(&cmd->dirty_cursor);
 952                return r;
 953        }
 954
 955        while (begin != end) {
 956                /*
 957                 * We assume that unmapped blocks have their dirty bit
 958                 * cleared.
 959                 */
 960                dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);
 961                if (dirty_flag) {
 962                        DMERR("%s: cache block %llu is dirty", __func__,
 963                              (unsigned long long) from_cblock(begin));
 964                        dm_bitset_cursor_end(&cmd->dirty_cursor);
 965                        *result = false;
 966                        return 0;
 967                }
 968
 969                begin = to_cblock(from_cblock(begin) + 1);
 970                if (begin == end)
 971                        break;
 972
 973                r = dm_bitset_cursor_next(&cmd->dirty_cursor);
 974                if (r) {
 975                        DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__);
 976                        dm_bitset_cursor_end(&cmd->dirty_cursor);
 977                        return r;
 978                }
 979        }
 980
 981        dm_bitset_cursor_end(&cmd->dirty_cursor);
 982
 983        return 0;
 984}
 985
 986static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,
 987                                        dm_cblock_t begin, dm_cblock_t end,
 988                                        bool *result)
 989{
 990        if (separate_dirty_bits(cmd))
 991                return blocks_are_clean_separate_dirty(cmd, begin, end, result);
 992        else
 993                return blocks_are_clean_combined_dirty(cmd, begin, end, result);
 994}
 995
 996static bool cmd_write_lock(struct dm_cache_metadata *cmd)
 997{
 998        down_write(&cmd->root_lock);
 999        if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {
1000                up_write(&cmd->root_lock);
1001                return false;
1002        }
1003        return true;
1004}
1005
1006#define WRITE_LOCK(cmd)                         \
1007        do {                                    \
1008                if (!cmd_write_lock((cmd)))     \
1009                        return -EINVAL;         \
1010        } while(0)
1011
1012#define WRITE_LOCK_VOID(cmd)                    \
1013        do {                                    \
1014                if (!cmd_write_lock((cmd)))     \
1015                        return;                 \
1016        } while(0)
1017
1018#define WRITE_UNLOCK(cmd) \
1019        up_write(&(cmd)->root_lock)
1020
1021static bool cmd_read_lock(struct dm_cache_metadata *cmd)
1022{
1023        down_read(&cmd->root_lock);
1024        if (cmd->fail_io) {
1025                up_read(&cmd->root_lock);
1026                return false;
1027        }
1028        return true;
1029}
1030
1031#define READ_LOCK(cmd)                          \
1032        do {                                    \
1033                if (!cmd_read_lock((cmd)))      \
1034                        return -EINVAL;         \
1035        } while(0)
1036
1037#define READ_LOCK_VOID(cmd)                     \
1038        do {                                    \
1039                if (!cmd_read_lock((cmd)))      \
1040                        return;                 \
1041        } while(0)
1042
1043#define READ_UNLOCK(cmd) \
1044        up_read(&(cmd)->root_lock)
1045
1046int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
1047{
1048        int r;
1049        bool clean;
1050        __le64 null_mapping = pack_value(0, 0);
1051
1052        WRITE_LOCK(cmd);
1053        __dm_bless_for_disk(&null_mapping);
1054
1055        if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {
1056                r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);
1057                if (r) {
1058                        __dm_unbless_for_disk(&null_mapping);
1059                        goto out;
1060                }
1061
1062                if (!clean) {
1063                        DMERR("unable to shrink cache due to dirty blocks");
1064                        r = -EINVAL;
1065                        __dm_unbless_for_disk(&null_mapping);
1066                        goto out;
1067                }
1068        }
1069
1070        r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
1071                            from_cblock(new_cache_size),
1072                            &null_mapping, &cmd->root);
1073        if (r)
1074                goto out;
1075
1076        if (separate_dirty_bits(cmd)) {
1077                r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,
1078                                     from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),
1079                                     false, &cmd->dirty_root);
1080                if (r)
1081                        goto out;
1082        }
1083
1084        cmd->cache_blocks = new_cache_size;
1085        cmd->changed = true;
1086
1087out:
1088        WRITE_UNLOCK(cmd);
1089
1090        return r;
1091}
1092
1093int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
1094                                   sector_t discard_block_size,
1095                                   dm_dblock_t new_nr_entries)
1096{
1097        int r;
1098
1099        WRITE_LOCK(cmd);
1100        r = dm_bitset_resize(&cmd->discard_info,
1101                             cmd->discard_root,
1102                             from_dblock(cmd->discard_nr_blocks),
1103                             from_dblock(new_nr_entries),
1104                             false, &cmd->discard_root);
1105        if (!r) {
1106                cmd->discard_block_size = discard_block_size;
1107                cmd->discard_nr_blocks = new_nr_entries;
1108        }
1109
1110        cmd->changed = true;
1111        WRITE_UNLOCK(cmd);
1112
1113        return r;
1114}
1115
1116static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1117{
1118        return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
1119                                 from_dblock(b), &cmd->discard_root);
1120}
1121
1122static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
1123{
1124        return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
1125                                   from_dblock(b), &cmd->discard_root);
1126}
1127
1128static int __discard(struct dm_cache_metadata *cmd,
1129                     dm_dblock_t dblock, bool discard)
1130{
1131        int r;
1132
1133        r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
1134        if (r)
1135                return r;
1136
1137        cmd->changed = true;
1138        return 0;
1139}
1140
1141int dm_cache_set_discard(struct dm_cache_metadata *cmd,
1142                         dm_dblock_t dblock, bool discard)
1143{
1144        int r;
1145
1146        WRITE_LOCK(cmd);
1147        r = __discard(cmd, dblock, discard);
1148        WRITE_UNLOCK(cmd);
1149
1150        return r;
1151}
1152
1153static int __load_discards(struct dm_cache_metadata *cmd,
1154                           load_discard_fn fn, void *context)
1155{
1156        int r = 0;
1157        uint32_t b;
1158        struct dm_bitset_cursor c;
1159
1160        if (from_dblock(cmd->discard_nr_blocks) == 0)
1161                /* nothing to do */
1162                return 0;
1163
1164        if (cmd->clean_when_opened) {
1165                r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);
1166                if (r)
1167                        return r;
1168
1169                r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,
1170                                           from_dblock(cmd->discard_nr_blocks), &c);
1171                if (r)
1172                        return r;
1173
1174                for (b = 0; ; b++) {
1175                        r = fn(context, cmd->discard_block_size, to_dblock(b),
1176                               dm_bitset_cursor_get_value(&c));
1177                        if (r)
1178                                break;
1179
1180                        if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))
1181                                break;
1182
1183                        r = dm_bitset_cursor_next(&c);
1184                        if (r)
1185                                break;
1186                }
1187
1188                dm_bitset_cursor_end(&c);
1189
1190        } else {
1191                for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
1192                        r = fn(context, cmd->discard_block_size, to_dblock(b), false);
1193                        if (r)
1194                                return r;
1195                }
1196        }
1197
1198        return r;
1199}
1200
1201int dm_cache_load_discards(struct dm_cache_metadata *cmd,
1202                           load_discard_fn fn, void *context)
1203{
1204        int r;
1205
1206        READ_LOCK(cmd);
1207        r = __load_discards(cmd, fn, context);
1208        READ_UNLOCK(cmd);
1209
1210        return r;
1211}
1212
1213int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)
1214{
1215        READ_LOCK(cmd);
1216        *result = cmd->cache_blocks;
1217        READ_UNLOCK(cmd);
1218
1219        return 0;
1220}
1221
1222static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1223{
1224        int r;
1225        __le64 value = pack_value(0, 0);
1226
1227        __dm_bless_for_disk(&value);
1228        r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1229                               &value, &cmd->root);
1230        if (r)
1231                return r;
1232
1233        cmd->changed = true;
1234        return 0;
1235}
1236
1237int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
1238{
1239        int r;
1240
1241        WRITE_LOCK(cmd);
1242        r = __remove(cmd, cblock);
1243        WRITE_UNLOCK(cmd);
1244
1245        return r;
1246}
1247
1248static int __insert(struct dm_cache_metadata *cmd,
1249                    dm_cblock_t cblock, dm_oblock_t oblock)
1250{
1251        int r;
1252        __le64 value = pack_value(oblock, M_VALID);
1253        __dm_bless_for_disk(&value);
1254
1255        r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1256                               &value, &cmd->root);
1257        if (r)
1258                return r;
1259
1260        cmd->changed = true;
1261        return 0;
1262}
1263
1264int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
1265                            dm_cblock_t cblock, dm_oblock_t oblock)
1266{
1267        int r;
1268
1269        WRITE_LOCK(cmd);
1270        r = __insert(cmd, cblock, oblock);
1271        WRITE_UNLOCK(cmd);
1272
1273        return r;
1274}
1275
1276struct thunk {
1277        load_mapping_fn fn;
1278        void *context;
1279
1280        struct dm_cache_metadata *cmd;
1281        bool respect_dirty_flags;
1282        bool hints_valid;
1283};
1284
1285static bool policy_unchanged(struct dm_cache_metadata *cmd,
1286                             struct dm_cache_policy *policy)
1287{
1288        const char *policy_name = dm_cache_policy_get_name(policy);
1289        const unsigned *policy_version = dm_cache_policy_get_version(policy);
1290        size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
1291
1292        /*
1293         * Ensure policy names match.
1294         */
1295        if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
1296                return false;
1297
1298        /*
1299         * Ensure policy major versions match.
1300         */
1301        if (cmd->policy_version[0] != policy_version[0])
1302                return false;
1303
1304        /*
1305         * Ensure policy hint sizes match.
1306         */
1307        if (cmd->policy_hint_size != policy_hint_size)
1308                return false;
1309
1310        return true;
1311}
1312
1313static bool hints_array_initialized(struct dm_cache_metadata *cmd)
1314{
1315        return cmd->hint_root && cmd->policy_hint_size;
1316}
1317
1318static bool hints_array_available(struct dm_cache_metadata *cmd,
1319                                  struct dm_cache_policy *policy)
1320{
1321        return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
1322                hints_array_initialized(cmd);
1323}
1324
1325static int __load_mapping_v1(struct dm_cache_metadata *cmd,
1326                             uint64_t cb, bool hints_valid,
1327                             struct dm_array_cursor *mapping_cursor,
1328                             struct dm_array_cursor *hint_cursor,
1329                             load_mapping_fn fn, void *context)
1330{
1331        int r = 0;
1332
1333        __le64 mapping;
1334        __le32 hint = 0;
1335
1336        __le64 *mapping_value_le;
1337        __le32 *hint_value_le;
1338
1339        dm_oblock_t oblock;
1340        unsigned flags;
1341        bool dirty = true;
1342
1343        dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1344        memcpy(&mapping, mapping_value_le, sizeof(mapping));
1345        unpack_value(mapping, &oblock, &flags);
1346
1347        if (flags & M_VALID) {
1348                if (hints_valid) {
1349                        dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1350                        memcpy(&hint, hint_value_le, sizeof(hint));
1351                }
1352                if (cmd->clean_when_opened)
1353                        dirty = flags & M_DIRTY;
1354
1355                r = fn(context, oblock, to_cblock(cb), dirty,
1356                       le32_to_cpu(hint), hints_valid);
1357                if (r) {
1358                        DMERR("policy couldn't load cache block %llu",
1359                              (unsigned long long) from_cblock(to_cblock(cb)));
1360                }
1361        }
1362
1363        return r;
1364}
1365
1366static int __load_mapping_v2(struct dm_cache_metadata *cmd,
1367                             uint64_t cb, bool hints_valid,
1368                             struct dm_array_cursor *mapping_cursor,
1369                             struct dm_array_cursor *hint_cursor,
1370                             struct dm_bitset_cursor *dirty_cursor,
1371                             load_mapping_fn fn, void *context)
1372{
1373        int r = 0;
1374
1375        __le64 mapping;
1376        __le32 hint = 0;
1377
1378        __le64 *mapping_value_le;
1379        __le32 *hint_value_le;
1380
1381        dm_oblock_t oblock;
1382        unsigned flags;
1383        bool dirty = true;
1384
1385        dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
1386        memcpy(&mapping, mapping_value_le, sizeof(mapping));
1387        unpack_value(mapping, &oblock, &flags);
1388
1389        if (flags & M_VALID) {
1390                if (hints_valid) {
1391                        dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
1392                        memcpy(&hint, hint_value_le, sizeof(hint));
1393                }
1394                if (cmd->clean_when_opened)
1395                        dirty = dm_bitset_cursor_get_value(dirty_cursor);
1396
1397                r = fn(context, oblock, to_cblock(cb), dirty,
1398                       le32_to_cpu(hint), hints_valid);
1399                if (r) {
1400                        DMERR("policy couldn't load cache block %llu",
1401                              (unsigned long long) from_cblock(to_cblock(cb)));
1402                }
1403        }
1404
1405        return r;
1406}
1407
1408static int __load_mappings(struct dm_cache_metadata *cmd,
1409                           struct dm_cache_policy *policy,
1410                           load_mapping_fn fn, void *context)
1411{
1412        int r;
1413        uint64_t cb;
1414
1415        bool hints_valid = hints_array_available(cmd, policy);
1416
1417        if (from_cblock(cmd->cache_blocks) == 0)
1418                /* Nothing to do */
1419                return 0;
1420
1421        r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);
1422        if (r)
1423                return r;
1424
1425        if (hints_valid) {
1426                r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);
1427                if (r) {
1428                        dm_array_cursor_end(&cmd->mapping_cursor);
1429                        return r;
1430                }
1431        }
1432
1433        if (separate_dirty_bits(cmd)) {
1434                r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
1435                                           from_cblock(cmd->cache_blocks),
1436                                           &cmd->dirty_cursor);
1437                if (r) {
1438                        dm_array_cursor_end(&cmd->hint_cursor);
1439                        dm_array_cursor_end(&cmd->mapping_cursor);
1440                        return r;
1441                }
1442        }
1443
1444        for (cb = 0; ; cb++) {
1445                if (separate_dirty_bits(cmd))
1446                        r = __load_mapping_v2(cmd, cb, hints_valid,
1447                                              &cmd->mapping_cursor,
1448                                              &cmd->hint_cursor,
1449                                              &cmd->dirty_cursor,
1450                                              fn, context);
1451                else
1452                        r = __load_mapping_v1(cmd, cb, hints_valid,
1453                                              &cmd->mapping_cursor, &cmd->hint_cursor,
1454                                              fn, context);
1455                if (r)
1456                        goto out;
1457
1458                /*
1459                 * We need to break out before we move the cursors.
1460                 */
1461                if (cb >= (from_cblock(cmd->cache_blocks) - 1))
1462                        break;
1463
1464                r = dm_array_cursor_next(&cmd->mapping_cursor);
1465                if (r) {
1466                        DMERR("dm_array_cursor_next for mapping failed");
1467                        goto out;
1468                }
1469
1470                if (hints_valid) {
1471                        r = dm_array_cursor_next(&cmd->hint_cursor);
1472                        if (r) {
1473                                dm_array_cursor_end(&cmd->hint_cursor);
1474                                hints_valid = false;
1475                        }
1476                }
1477
1478                if (separate_dirty_bits(cmd)) {
1479                        r = dm_bitset_cursor_next(&cmd->dirty_cursor);
1480                        if (r) {
1481                                DMERR("dm_bitset_cursor_next for dirty failed");
1482                                goto out;
1483                        }
1484                }
1485        }
1486out:
1487        dm_array_cursor_end(&cmd->mapping_cursor);
1488        if (hints_valid)
1489                dm_array_cursor_end(&cmd->hint_cursor);
1490
1491        if (separate_dirty_bits(cmd))
1492                dm_bitset_cursor_end(&cmd->dirty_cursor);
1493
1494        return r;
1495}
1496
1497int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
1498                           struct dm_cache_policy *policy,
1499                           load_mapping_fn fn, void *context)
1500{
1501        int r;
1502
1503        READ_LOCK(cmd);
1504        r = __load_mappings(cmd, policy, fn, context);
1505        READ_UNLOCK(cmd);
1506
1507        return r;
1508}
1509
1510static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
1511{
1512        int r = 0;
1513        __le64 value;
1514        dm_oblock_t oblock;
1515        unsigned flags;
1516
1517        memcpy(&value, leaf, sizeof(value));
1518        unpack_value(value, &oblock, &flags);
1519
1520        return r;
1521}
1522
1523static int __dump_mappings(struct dm_cache_metadata *cmd)
1524{
1525        return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
1526}
1527
1528void dm_cache_dump(struct dm_cache_metadata *cmd)
1529{
1530        READ_LOCK_VOID(cmd);
1531        __dump_mappings(cmd);
1532        READ_UNLOCK(cmd);
1533}
1534
1535int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
1536{
1537        int r;
1538
1539        READ_LOCK(cmd);
1540        r = cmd->changed;
1541        READ_UNLOCK(cmd);
1542
1543        return r;
1544}
1545
1546static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
1547{
1548        int r;
1549        unsigned flags;
1550        dm_oblock_t oblock;
1551        __le64 value;
1552
1553        r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
1554        if (r)
1555                return r;
1556
1557        unpack_value(value, &oblock, &flags);
1558
1559        if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
1560                /* nothing to be done */
1561                return 0;
1562
1563        value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
1564        __dm_bless_for_disk(&value);
1565
1566        r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
1567                               &value, &cmd->root);
1568        if (r)
1569                return r;
1570
1571        cmd->changed = true;
1572        return 0;
1573
1574}
1575
1576static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
1577{
1578        int r;
1579        unsigned i;
1580        for (i = 0; i < nr_bits; i++) {
1581                r = __dirty(cmd, to_cblock(i), test_bit(i, bits));
1582                if (r)
1583                        return r;
1584        }
1585
1586        return 0;
1587}
1588
1589static int is_dirty_callback(uint32_t index, bool *value, void *context)
1590{
1591        unsigned long *bits = context;
1592        *value = test_bit(index, bits);
1593        return 0;
1594}
1595
1596static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
1597{
1598        int r = 0;
1599
1600        /* nr_bits is really just a sanity check */
1601        if (nr_bits != from_cblock(cmd->cache_blocks)) {
1602                DMERR("dirty bitset is wrong size");
1603                return -EINVAL;
1604        }
1605
1606        r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);
1607        if (r)
1608                return r;
1609
1610        cmd->changed = true;
1611        return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);
1612}
1613
1614int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,
1615                            unsigned nr_bits,
1616                            unsigned long *bits)
1617{
1618        int r;
1619
1620        WRITE_LOCK(cmd);
1621        if (separate_dirty_bits(cmd))
1622                r = __set_dirty_bits_v2(cmd, nr_bits, bits);
1623        else
1624                r = __set_dirty_bits_v1(cmd, nr_bits, bits);
1625        WRITE_UNLOCK(cmd);
1626
1627        return r;
1628}
1629
1630void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
1631                                 struct dm_cache_statistics *stats)
1632{
1633        READ_LOCK_VOID(cmd);
1634        *stats = cmd->stats;
1635        READ_UNLOCK(cmd);
1636}
1637
1638void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
1639                                 struct dm_cache_statistics *stats)
1640{
1641        WRITE_LOCK_VOID(cmd);
1642        cmd->stats = *stats;
1643        WRITE_UNLOCK(cmd);
1644}
1645
1646int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
1647{
1648        int r = -EINVAL;
1649        flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
1650                                 clear_clean_shutdown);
1651
1652        WRITE_LOCK(cmd);
1653        if (cmd->fail_io)
1654                goto out;
1655
1656        r = __commit_transaction(cmd, mutator);
1657        if (r)
1658                goto out;
1659
1660        r = __begin_transaction(cmd);
1661out:
1662        WRITE_UNLOCK(cmd);
1663        return r;
1664}
1665
1666int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
1667                                           dm_block_t *result)
1668{
1669        int r = -EINVAL;
1670
1671        READ_LOCK(cmd);
1672        if (!cmd->fail_io)
1673                r = dm_sm_get_nr_free(cmd->metadata_sm, result);
1674        READ_UNLOCK(cmd);
1675
1676        return r;
1677}
1678
1679int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
1680                                   dm_block_t *result)
1681{
1682        int r = -EINVAL;
1683
1684        READ_LOCK(cmd);
1685        if (!cmd->fail_io)
1686                r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
1687        READ_UNLOCK(cmd);
1688
1689        return r;
1690}
1691
1692/*----------------------------------------------------------------*/
1693
1694static int get_hint(uint32_t index, void *value_le, void *context)
1695{
1696        uint32_t value;
1697        struct dm_cache_policy *policy = context;
1698
1699        value = policy_get_hint(policy, to_cblock(index));
1700        *((__le32 *) value_le) = cpu_to_le32(value);
1701
1702        return 0;
1703}
1704
1705/*
1706 * It's quicker to always delete the hint array, and recreate with
1707 * dm_array_new().
1708 */
1709static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1710{
1711        int r;
1712        size_t hint_size;
1713        const char *policy_name = dm_cache_policy_get_name(policy);
1714        const unsigned *policy_version = dm_cache_policy_get_version(policy);
1715
1716        if (!policy_name[0] ||
1717            (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
1718                return -EINVAL;
1719
1720        strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
1721        memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
1722
1723        hint_size = dm_cache_policy_get_hint_size(policy);
1724        if (!hint_size)
1725                return 0; /* short-circuit hints initialization */
1726        cmd->policy_hint_size = hint_size;
1727
1728        if (cmd->hint_root) {
1729                r = dm_array_del(&cmd->hint_info, cmd->hint_root);
1730                if (r)
1731                        return r;
1732        }
1733
1734        return dm_array_new(&cmd->hint_info, &cmd->hint_root,
1735                            from_cblock(cmd->cache_blocks),
1736                            get_hint, policy);
1737}
1738
1739int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
1740{
1741        int r;
1742
1743        WRITE_LOCK(cmd);
1744        r = write_hints(cmd, policy);
1745        WRITE_UNLOCK(cmd);
1746
1747        return r;
1748}
1749
1750int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
1751{
1752        int r;
1753
1754        READ_LOCK(cmd);
1755        r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
1756        READ_UNLOCK(cmd);
1757
1758        return r;
1759}
1760
1761void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
1762{
1763        WRITE_LOCK_VOID(cmd);
1764        dm_bm_set_read_only(cmd->bm);
1765        WRITE_UNLOCK(cmd);
1766}
1767
1768void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)
1769{
1770        WRITE_LOCK_VOID(cmd);
1771        dm_bm_set_read_write(cmd->bm);
1772        WRITE_UNLOCK(cmd);
1773}
1774
1775int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)
1776{
1777        int r;
1778        struct dm_block *sblock;
1779        struct cache_disk_superblock *disk_super;
1780
1781        WRITE_LOCK(cmd);
1782        set_bit(NEEDS_CHECK, &cmd->flags);
1783
1784        r = superblock_lock(cmd, &sblock);
1785        if (r) {
1786                DMERR("couldn't read superblock");
1787                goto out;
1788        }
1789
1790        disk_super = dm_block_data(sblock);
1791        disk_super->flags = cpu_to_le32(cmd->flags);
1792
1793        dm_bm_unlock(sblock);
1794
1795out:
1796        WRITE_UNLOCK(cmd);
1797        return r;
1798}
1799
1800int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)
1801{
1802        READ_LOCK(cmd);
1803        *result = !!test_bit(NEEDS_CHECK, &cmd->flags);
1804        READ_UNLOCK(cmd);
1805
1806        return 0;
1807}
1808
1809int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
1810{
1811        int r;
1812
1813        WRITE_LOCK(cmd);
1814        __destroy_persistent_data_objects(cmd);
1815        r = __create_persistent_data_objects(cmd, false);
1816        if (r)
1817                cmd->fail_io = true;
1818        WRITE_UNLOCK(cmd);
1819
1820        return r;
1821}
1822