LXR linux/fs/erofs/zmap.c

   1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2018-2019 HUAWEI, Inc.
   4 *             https://www.huawei.com/
   5 * Created by Gao Xiang <gaoxiang25@huawei.com>
   6 */
   7#include "internal.h"
   8#include <asm/unaligned.h>
   9#include <trace/events/erofs.h>
  10
  11int z_erofs_fill_inode(struct inode *inode)
  12{
  13        struct erofs_inode *const vi = EROFS_I(inode);
  14
  15        if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
  16                vi->z_advise = 0;
  17                vi->z_algorithmtype[0] = 0;
  18                vi->z_algorithmtype[1] = 0;
  19                vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
  20                vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits;
  21                vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits;
  22                set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
  23        }
  24
  25        inode->i_mapping->a_ops = &z_erofs_aops;
  26        return 0;
  27}
  28
  29static int z_erofs_fill_inode_lazy(struct inode *inode)
  30{
  31        struct erofs_inode *const vi = EROFS_I(inode);
  32        struct super_block *const sb = inode->i_sb;
  33        int err;
  34        erofs_off_t pos;
  35        struct page *page;
  36        void *kaddr;
  37        struct z_erofs_map_header *h;
  38
  39        if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
  40                return 0;
  41
  42        if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
  43                return -ERESTARTSYS;
  44
  45        err = 0;
  46        if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
  47                goto out_unlock;
  48
  49        DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
  50
  51        pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
  52                    vi->xattr_isize, 8);
  53        page = erofs_get_meta_page(sb, erofs_blknr(pos));
  54        if (IS_ERR(page)) {
  55                err = PTR_ERR(page);
  56                goto out_unlock;
  57        }
  58
  59        kaddr = kmap_atomic(page);
  60
  61        h = kaddr + erofs_blkoff(pos);
  62        vi->z_advise = le16_to_cpu(h->h_advise);
  63        vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
  64        vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
  65
  66        if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) {
  67                erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel",
  68                          vi->z_algorithmtype[0], vi->nid);
  69                err = -EOPNOTSUPP;
  70                goto unmap_done;
  71        }
  72
  73        vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
  74        vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits +
  75                                        ((h->h_clusterbits >> 3) & 3);
  76
  77        if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) {
  78                erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel",
  79                          vi->z_physical_clusterbits[0], vi->nid);
  80                err = -EOPNOTSUPP;
  81                goto unmap_done;
  82        }
  83
  84        vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits +
  85                                        ((h->h_clusterbits >> 5) & 7);
  86        set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
  87unmap_done:
  88        kunmap_atomic(kaddr);
  89        unlock_page(page);
  90        put_page(page);
  91out_unlock:
  92        clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
  93        return err;
  94}
  95
  96struct z_erofs_maprecorder {
  97        struct inode *inode;
  98        struct erofs_map_blocks *map;
  99        void *kaddr;
 100
 101        unsigned long lcn;
 102        /* compression extent information gathered */
 103        u8  type;
 104        u16 clusterofs;
 105        u16 delta[2];
 106        erofs_blk_t pblk;
 107};
 108
 109static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
 110                                  erofs_blk_t eblk)
 111{
 112        struct super_block *const sb = m->inode->i_sb;
 113        struct erofs_map_blocks *const map = m->map;
 114        struct page *mpage = map->mpage;
 115
 116        if (mpage) {
 117                if (mpage->index == eblk) {
 118                        if (!m->kaddr)
 119                                m->kaddr = kmap_atomic(mpage);
 120                        return 0;
 121                }
 122
 123                if (m->kaddr) {
 124                        kunmap_atomic(m->kaddr);
 125                        m->kaddr = NULL;
 126                }
 127                put_page(mpage);
 128        }
 129
 130        mpage = erofs_get_meta_page(sb, eblk);
 131        if (IS_ERR(mpage)) {
 132                map->mpage = NULL;
 133                return PTR_ERR(mpage);
 134        }
 135        m->kaddr = kmap_atomic(mpage);
 136        unlock_page(mpage);
 137        map->mpage = mpage;
 138        return 0;
 139}
 140
 141static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 142                                         unsigned long lcn)
 143{
 144        struct inode *const inode = m->inode;
 145        struct erofs_inode *const vi = EROFS_I(inode);
 146        const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid);
 147        const erofs_off_t pos =
 148                Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize +
 149                                               vi->xattr_isize) +
 150                lcn * sizeof(struct z_erofs_vle_decompressed_index);
 151        struct z_erofs_vle_decompressed_index *di;
 152        unsigned int advise, type;
 153        int err;
 154
 155        err = z_erofs_reload_indexes(m, erofs_blknr(pos));
 156        if (err)
 157                return err;
 158
 159        m->lcn = lcn;
 160        di = m->kaddr + erofs_blkoff(pos);
 161
 162        advise = le16_to_cpu(di->di_advise);
 163        type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
 164                ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
 165        switch (type) {
 166        case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 167                m->clusterofs = 1 << vi->z_logical_clusterbits;
 168                m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
 169                m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
 170                break;
 171        case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 172        case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
 173                m->clusterofs = le16_to_cpu(di->di_clusterofs);
 174                m->pblk = le32_to_cpu(di->di_u.blkaddr);
 175                break;
 176        default:
 177                DBG_BUGON(1);
 178                return -EOPNOTSUPP;
 179        }
 180        m->type = type;
 181        return 0;
 182}
 183
 184static unsigned int decode_compactedbits(unsigned int lobits,
 185                                         unsigned int lomask,
 186                                         u8 *in, unsigned int pos, u8 *type)
 187{
 188        const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
 189        const unsigned int lo = v & lomask;
 190
 191        *type = (v >> lobits) & 3;
 192        return lo;
 193}
 194
 195static int unpack_compacted_index(struct z_erofs_maprecorder *m,
 196                                  unsigned int amortizedshift,
 197                                  unsigned int eofs)
 198{
 199        struct erofs_inode *const vi = EROFS_I(m->inode);
 200        const unsigned int lclusterbits = vi->z_logical_clusterbits;
 201        const unsigned int lomask = (1 << lclusterbits) - 1;
 202        unsigned int vcnt, base, lo, encodebits, nblk;
 203        int i;
 204        u8 *in, type;
 205
 206        if (1 << amortizedshift == 4)
 207                vcnt = 2;
 208        else if (1 << amortizedshift == 2 && lclusterbits == 12)
 209                vcnt = 16;
 210        else
 211                return -EOPNOTSUPP;
 212
 213        encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
 214        base = round_down(eofs, vcnt << amortizedshift);
 215        in = m->kaddr + base;
 216
 217        i = (eofs - base) >> amortizedshift;
 218
 219        lo = decode_compactedbits(lclusterbits, lomask,
 220                                  in, encodebits * i, &type);
 221        m->type = type;
 222        if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
 223                m->clusterofs = 1 << lclusterbits;
 224                if (i + 1 != vcnt) {
 225                        m->delta[0] = lo;
 226                        return 0;
 227                }
 228                /*
 229                 * since the last lcluster in the pack is special,
 230                 * of which lo saves delta[1] rather than delta[0].
 231                 * Hence, get delta[0] by the previous lcluster indirectly.
 232                 */
 233                lo = decode_compactedbits(lclusterbits, lomask,
 234                                          in, encodebits * (i - 1), &type);
 235                if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
 236                        lo = 0;
 237                m->delta[0] = lo + 1;
 238                return 0;
 239        }
 240        m->clusterofs = lo;
 241        m->delta[0] = 0;
 242        /* figout out blkaddr (pblk) for HEAD lclusters */
 243        nblk = 1;
 244        while (i > 0) {
 245                --i;
 246                lo = decode_compactedbits(lclusterbits, lomask,
 247                                          in, encodebits * i, &type);
 248                if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
 249                        i -= lo;
 250
 251                if (i >= 0)
 252                        ++nblk;
 253        }
 254        in += (vcnt << amortizedshift) - sizeof(__le32);
 255        m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
 256        return 0;
 257}
 258
 259static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 260                                            unsigned long lcn)
 261{
 262        struct inode *const inode = m->inode;
 263        struct erofs_inode *const vi = EROFS_I(inode);
 264        const unsigned int lclusterbits = vi->z_logical_clusterbits;
 265        const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) +
 266                                        vi->inode_isize + vi->xattr_isize, 8) +
 267                sizeof(struct z_erofs_map_header);
 268        const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
 269        unsigned int compacted_4b_initial, compacted_2b;
 270        unsigned int amortizedshift;
 271        erofs_off_t pos;
 272        int err;
 273
 274        if (lclusterbits != 12)
 275                return -EOPNOTSUPP;
 276
 277        if (lcn >= totalidx)
 278                return -EINVAL;
 279
 280        m->lcn = lcn;
 281        /* used to align to 32-byte (compacted_2b) alignment */
 282        compacted_4b_initial = (32 - ebase % 32) / 4;
 283        if (compacted_4b_initial == 32 / 4)
 284                compacted_4b_initial = 0;
 285
 286        if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B)
 287                compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
 288        else
 289                compacted_2b = 0;
 290
 291        pos = ebase;
 292        if (lcn < compacted_4b_initial) {
 293                amortizedshift = 2;
 294                goto out;
 295        }
 296        pos += compacted_4b_initial * 4;
 297        lcn -= compacted_4b_initial;
 298
 299        if (lcn < compacted_2b) {
 300                amortizedshift = 1;
 301                goto out;
 302        }
 303        pos += compacted_2b * 2;
 304        lcn -= compacted_2b;
 305        amortizedshift = 2;
 306out:
 307        pos += lcn * (1 << amortizedshift);
 308        err = z_erofs_reload_indexes(m, erofs_blknr(pos));
 309        if (err)
 310                return err;
 311        return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos));
 312}
 313
 314static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 315                                          unsigned int lcn)
 316{
 317        const unsigned int datamode = EROFS_I(m->inode)->datalayout;
 318
 319        if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
 320                return legacy_load_cluster_from_disk(m, lcn);
 321
 322        if (datamode == EROFS_INODE_FLAT_COMPRESSION)
 323                return compacted_load_cluster_from_disk(m, lcn);
 324
 325        return -EINVAL;
 326}
 327
 328static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
 329                                   unsigned int lookback_distance)
 330{
 331        struct erofs_inode *const vi = EROFS_I(m->inode);
 332        struct erofs_map_blocks *const map = m->map;
 333        const unsigned int lclusterbits = vi->z_logical_clusterbits;
 334        unsigned long lcn = m->lcn;
 335        int err;
 336
 337        if (lcn < lookback_distance) {
 338                erofs_err(m->inode->i_sb,
 339                          "bogus lookback distance @ nid %llu", vi->nid);
 340                DBG_BUGON(1);
 341                return -EFSCORRUPTED;
 342        }
 343
 344        /* load extent head logical cluster if needed */
 345        lcn -= lookback_distance;
 346        err = z_erofs_load_cluster_from_disk(m, lcn);
 347        if (err)
 348                return err;
 349
 350        switch (m->type) {
 351        case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 352                if (!m->delta[0]) {
 353                        erofs_err(m->inode->i_sb,
 354                                  "invalid lookback distance 0 @ nid %llu",
 355                                  vi->nid);
 356                        DBG_BUGON(1);
 357                        return -EFSCORRUPTED;
 358                }
 359                return z_erofs_extent_lookback(m, m->delta[0]);
 360        case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 361                map->m_flags &= ~EROFS_MAP_ZIPPED;
 362                fallthrough;
 363        case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
 364                map->m_la = (lcn << lclusterbits) | m->clusterofs;
 365                break;
 366        default:
 367                erofs_err(m->inode->i_sb,
 368                          "unknown type %u @ lcn %lu of nid %llu",
 369                          m->type, lcn, vi->nid);
 370                DBG_BUGON(1);
 371                return -EOPNOTSUPP;
 372        }
 373        return 0;
 374}
 375
 376int z_erofs_map_blocks_iter(struct inode *inode,
 377                            struct erofs_map_blocks *map,
 378                            int flags)
 379{
 380        struct erofs_inode *const vi = EROFS_I(inode);
 381        struct z_erofs_maprecorder m = {
 382                .inode = inode,
 383                .map = map,
 384        };
 385        int err = 0;
 386        unsigned int lclusterbits, endoff;
 387        unsigned long long ofs, end;
 388
 389        trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
 390
 391        /* when trying to read beyond EOF, leave it unmapped */
 392        if (map->m_la >= inode->i_size) {
 393                map->m_llen = map->m_la + 1 - inode->i_size;
 394                map->m_la = inode->i_size;
 395                map->m_flags = 0;
 396                goto out;
 397        }
 398
 399        err = z_erofs_fill_inode_lazy(inode);
 400        if (err)
 401                goto out;
 402
 403        lclusterbits = vi->z_logical_clusterbits;
 404        ofs = map->m_la;
 405        m.lcn = ofs >> lclusterbits;
 406        endoff = ofs & ((1 << lclusterbits) - 1);
 407
 408        err = z_erofs_load_cluster_from_disk(&m, m.lcn);
 409        if (err)
 410                goto unmap_out;
 411
 412        map->m_flags = EROFS_MAP_ZIPPED;        /* by default, compressed */
 413        end = (m.lcn + 1ULL) << lclusterbits;
 414
 415        switch (m.type) {
 416        case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 417                if (endoff >= m.clusterofs)
 418                        map->m_flags &= ~EROFS_MAP_ZIPPED;
 419                fallthrough;
 420        case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
 421                if (endoff >= m.clusterofs) {
 422                        map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
 423                        break;
 424                }
 425                /* m.lcn should be >= 1 if endoff < m.clusterofs */
 426                if (!m.lcn) {
 427                        erofs_err(inode->i_sb,
 428                                  "invalid logical cluster 0 at nid %llu",
 429                                  vi->nid);
 430                        err = -EFSCORRUPTED;
 431                        goto unmap_out;
 432                }
 433                end = (m.lcn << lclusterbits) | m.clusterofs;
 434                map->m_flags |= EROFS_MAP_FULL_MAPPED;
 435                m.delta[0] = 1;
 436                fallthrough;
 437        case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 438                /* get the correspoinding first chunk */
 439                err = z_erofs_extent_lookback(&m, m.delta[0]);
 440                if (err)
 441                        goto unmap_out;
 442                break;
 443        default:
 444                erofs_err(inode->i_sb,
 445                          "unknown type %u @ offset %llu of nid %llu",
 446                          m.type, ofs, vi->nid);
 447                err = -EOPNOTSUPP;
 448                goto unmap_out;
 449        }
 450
 451        map->m_llen = end - map->m_la;
 452        map->m_plen = 1 << lclusterbits;
 453        map->m_pa = blknr_to_addr(m.pblk);
 454        map->m_flags |= EROFS_MAP_MAPPED;
 455
 456unmap_out:
 457        if (m.kaddr)
 458                kunmap_atomic(m.kaddr);
 459
 460out:
 461        erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
 462                  __func__, map->m_la, map->m_pa,
 463                  map->m_llen, map->m_plen, map->m_flags);
 464
 465        trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
 466
 467        /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
 468        DBG_BUGON(err < 0 && err != -ENOMEM);
 469        return err;
 470}
 471
 472