LXR linux/fs/btrfs/zstd.c

   1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Copyright (c) 2016-present, Facebook, Inc.
   4 * All rights reserved.
   5 *
   6 */
   7
   8#include <linux/bio.h>
   9#include <linux/err.h>
  10#include <linux/init.h>
  11#include <linux/kernel.h>
  12#include <linux/mm.h>
  13#include <linux/pagemap.h>
  14#include <linux/refcount.h>
  15#include <linux/sched.h>
  16#include <linux/slab.h>
  17#include <linux/zstd.h>
  18#include "compression.h"
  19
  20#define ZSTD_BTRFS_MAX_WINDOWLOG 17
  21#define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
  22#define ZSTD_BTRFS_DEFAULT_LEVEL 3
  23
  24static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
  25{
  26        ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
  27                                                src_len, 0);
  28
  29        if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
  30                params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
  31        WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
  32        return params;
  33}
  34
  35struct workspace {
  36        void *mem;
  37        size_t size;
  38        char *buf;
  39        struct list_head list;
  40        ZSTD_inBuffer in_buf;
  41        ZSTD_outBuffer out_buf;
  42};
  43
  44static void zstd_free_workspace(struct list_head *ws)
  45{
  46        struct workspace *workspace = list_entry(ws, struct workspace, list);
  47
  48        kvfree(workspace->mem);
  49        kfree(workspace->buf);
  50        kfree(workspace);
  51}
  52
  53static struct list_head *zstd_alloc_workspace(void)
  54{
  55        ZSTD_parameters params =
  56                        zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
  57        struct workspace *workspace;
  58
  59        workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
  60        if (!workspace)
  61                return ERR_PTR(-ENOMEM);
  62
  63        workspace->size = max_t(size_t,
  64                        ZSTD_CStreamWorkspaceBound(params.cParams),
  65                        ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
  66        workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
  67        workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  68        if (!workspace->mem || !workspace->buf)
  69                goto fail;
  70
  71        INIT_LIST_HEAD(&workspace->list);
  72
  73        return &workspace->list;
  74fail:
  75        zstd_free_workspace(&workspace->list);
  76        return ERR_PTR(-ENOMEM);
  77}
  78
  79static int zstd_compress_pages(struct list_head *ws,
  80                struct address_space *mapping,
  81                u64 start,
  82                struct page **pages,
  83                unsigned long *out_pages,
  84                unsigned long *total_in,
  85                unsigned long *total_out)
  86{
  87        struct workspace *workspace = list_entry(ws, struct workspace, list);
  88        ZSTD_CStream *stream;
  89        int ret = 0;
  90        int nr_pages = 0;
  91        struct page *in_page = NULL;  /* The current page to read */
  92        struct page *out_page = NULL; /* The current page to write to */
  93        unsigned long tot_in = 0;
  94        unsigned long tot_out = 0;
  95        unsigned long len = *total_out;
  96        const unsigned long nr_dest_pages = *out_pages;
  97        unsigned long max_out = nr_dest_pages * PAGE_SIZE;
  98        ZSTD_parameters params = zstd_get_btrfs_parameters(len);
  99
 100        *out_pages = 0;
 101        *total_out = 0;
 102        *total_in = 0;
 103
 104        /* Initialize the stream */
 105        stream = ZSTD_initCStream(params, len, workspace->mem,
 106                        workspace->size);
 107        if (!stream) {
 108                pr_warn("BTRFS: ZSTD_initCStream failed\n");
 109                ret = -EIO;
 110                goto out;
 111        }
 112
 113        /* map in the first page of input data */
 114        in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 115        workspace->in_buf.src = kmap(in_page);
 116        workspace->in_buf.pos = 0;
 117        workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 118
 119
 120        /* Allocate and map in the output buffer */
 121        out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 122        if (out_page == NULL) {
 123                ret = -ENOMEM;
 124                goto out;
 125        }
 126        pages[nr_pages++] = out_page;
 127        workspace->out_buf.dst = kmap(out_page);
 128        workspace->out_buf.pos = 0;
 129        workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 130
 131        while (1) {
 132                size_t ret2;
 133
 134                ret2 = ZSTD_compressStream(stream, &workspace->out_buf,
 135                                &workspace->in_buf);
 136                if (ZSTD_isError(ret2)) {
 137                        pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
 138                                        ZSTD_getErrorCode(ret2));
 139                        ret = -EIO;
 140                        goto out;
 141                }
 142
 143                /* Check to see if we are making it bigger */
 144                if (tot_in + workspace->in_buf.pos > 8192 &&
 145                                tot_in + workspace->in_buf.pos <
 146                                tot_out + workspace->out_buf.pos) {
 147                        ret = -E2BIG;
 148                        goto out;
 149                }
 150
 151                /* We've reached the end of our output range */
 152                if (workspace->out_buf.pos >= max_out) {
 153                        tot_out += workspace->out_buf.pos;
 154                        ret = -E2BIG;
 155                        goto out;
 156                }
 157
 158                /* Check if we need more output space */
 159                if (workspace->out_buf.pos == workspace->out_buf.size) {
 160                        tot_out += PAGE_SIZE;
 161                        max_out -= PAGE_SIZE;
 162                        kunmap(out_page);
 163                        if (nr_pages == nr_dest_pages) {
 164                                out_page = NULL;
 165                                ret = -E2BIG;
 166                                goto out;
 167                        }
 168                        out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 169                        if (out_page == NULL) {
 170                                ret = -ENOMEM;
 171                                goto out;
 172                        }
 173                        pages[nr_pages++] = out_page;
 174                        workspace->out_buf.dst = kmap(out_page);
 175                        workspace->out_buf.pos = 0;
 176                        workspace->out_buf.size = min_t(size_t, max_out,
 177                                                        PAGE_SIZE);
 178                }
 179
 180                /* We've reached the end of the input */
 181                if (workspace->in_buf.pos >= len) {
 182                        tot_in += workspace->in_buf.pos;
 183                        break;
 184                }
 185
 186                /* Check if we need more input */
 187                if (workspace->in_buf.pos == workspace->in_buf.size) {
 188                        tot_in += PAGE_SIZE;
 189                        kunmap(in_page);
 190                        put_page(in_page);
 191
 192                        start += PAGE_SIZE;
 193                        len -= PAGE_SIZE;
 194                        in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 195                        workspace->in_buf.src = kmap(in_page);
 196                        workspace->in_buf.pos = 0;
 197                        workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 198                }
 199        }
 200        while (1) {
 201                size_t ret2;
 202
 203                ret2 = ZSTD_endStream(stream, &workspace->out_buf);
 204                if (ZSTD_isError(ret2)) {
 205                        pr_debug("BTRFS: ZSTD_endStream returned %d\n",
 206                                        ZSTD_getErrorCode(ret2));
 207                        ret = -EIO;
 208                        goto out;
 209                }
 210                if (ret2 == 0) {
 211                        tot_out += workspace->out_buf.pos;
 212                        break;
 213                }
 214                if (workspace->out_buf.pos >= max_out) {
 215                        tot_out += workspace->out_buf.pos;
 216                        ret = -E2BIG;
 217                        goto out;
 218                }
 219
 220                tot_out += PAGE_SIZE;
 221                max_out -= PAGE_SIZE;
 222                kunmap(out_page);
 223                if (nr_pages == nr_dest_pages) {
 224                        out_page = NULL;
 225                        ret = -E2BIG;
 226                        goto out;
 227                }
 228                out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 229                if (out_page == NULL) {
 230                        ret = -ENOMEM;
 231                        goto out;
 232                }
 233                pages[nr_pages++] = out_page;
 234                workspace->out_buf.dst = kmap(out_page);
 235                workspace->out_buf.pos = 0;
 236                workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 237        }
 238
 239        if (tot_out >= tot_in) {
 240                ret = -E2BIG;
 241                goto out;
 242        }
 243
 244        ret = 0;
 245        *total_in = tot_in;
 246        *total_out = tot_out;
 247out:
 248        *out_pages = nr_pages;
 249        /* Cleanup */
 250        if (in_page) {
 251                kunmap(in_page);
 252                put_page(in_page);
 253        }
 254        if (out_page)
 255                kunmap(out_page);
 256        return ret;
 257}
 258
 259static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 260{
 261        struct workspace *workspace = list_entry(ws, struct workspace, list);
 262        struct page **pages_in = cb->compressed_pages;
 263        u64 disk_start = cb->start;
 264        struct bio *orig_bio = cb->orig_bio;
 265        size_t srclen = cb->compressed_len;
 266        ZSTD_DStream *stream;
 267        int ret = 0;
 268        unsigned long page_in_index = 0;
 269        unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
 270        unsigned long buf_start;
 271        unsigned long total_out = 0;
 272
 273        stream = ZSTD_initDStream(
 274                        ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 275        if (!stream) {
 276                pr_debug("BTRFS: ZSTD_initDStream failed\n");
 277                ret = -EIO;
 278                goto done;
 279        }
 280
 281        workspace->in_buf.src = kmap(pages_in[page_in_index]);
 282        workspace->in_buf.pos = 0;
 283        workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 284
 285        workspace->out_buf.dst = workspace->buf;
 286        workspace->out_buf.pos = 0;
 287        workspace->out_buf.size = PAGE_SIZE;
 288
 289        while (1) {
 290                size_t ret2;
 291
 292                ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 293                                &workspace->in_buf);
 294                if (ZSTD_isError(ret2)) {
 295                        pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 296                                        ZSTD_getErrorCode(ret2));
 297                        ret = -EIO;
 298                        goto done;
 299                }
 300                buf_start = total_out;
 301                total_out += workspace->out_buf.pos;
 302                workspace->out_buf.pos = 0;
 303
 304                ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
 305                                buf_start, total_out, disk_start, orig_bio);
 306                if (ret == 0)
 307                        break;
 308
 309                if (workspace->in_buf.pos >= srclen)
 310                        break;
 311
 312                /* Check if we've hit the end of a frame */
 313                if (ret2 == 0)
 314                        break;
 315
 316                if (workspace->in_buf.pos == workspace->in_buf.size) {
 317                        kunmap(pages_in[page_in_index++]);
 318                        if (page_in_index >= total_pages_in) {
 319                                workspace->in_buf.src = NULL;
 320                                ret = -EIO;
 321                                goto done;
 322                        }
 323                        srclen -= PAGE_SIZE;
 324                        workspace->in_buf.src = kmap(pages_in[page_in_index]);
 325                        workspace->in_buf.pos = 0;
 326                        workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 327                }
 328        }
 329        ret = 0;
 330        zero_fill_bio(orig_bio);
 331done:
 332        if (workspace->in_buf.src)
 333                kunmap(pages_in[page_in_index]);
 334        return ret;
 335}
 336
 337static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
 338                struct page *dest_page,
 339                unsigned long start_byte,
 340                size_t srclen, size_t destlen)
 341{
 342        struct workspace *workspace = list_entry(ws, struct workspace, list);
 343        ZSTD_DStream *stream;
 344        int ret = 0;
 345        size_t ret2;
 346        unsigned long total_out = 0;
 347        unsigned long pg_offset = 0;
 348        char *kaddr;
 349
 350        stream = ZSTD_initDStream(
 351                        ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 352        if (!stream) {
 353                pr_warn("BTRFS: ZSTD_initDStream failed\n");
 354                ret = -EIO;
 355                goto finish;
 356        }
 357
 358        destlen = min_t(size_t, destlen, PAGE_SIZE);
 359
 360        workspace->in_buf.src = data_in;
 361        workspace->in_buf.pos = 0;
 362        workspace->in_buf.size = srclen;
 363
 364        workspace->out_buf.dst = workspace->buf;
 365        workspace->out_buf.pos = 0;
 366        workspace->out_buf.size = PAGE_SIZE;
 367
 368        ret2 = 1;
 369        while (pg_offset < destlen
 370               && workspace->in_buf.pos < workspace->in_buf.size) {
 371                unsigned long buf_start;
 372                unsigned long buf_offset;
 373                unsigned long bytes;
 374
 375                /* Check if the frame is over and we still need more input */
 376                if (ret2 == 0) {
 377                        pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
 378                        ret = -EIO;
 379                        goto finish;
 380                }
 381                ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 382                                &workspace->in_buf);
 383                if (ZSTD_isError(ret2)) {
 384                        pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 385                                        ZSTD_getErrorCode(ret2));
 386                        ret = -EIO;
 387                        goto finish;
 388                }
 389
 390                buf_start = total_out;
 391                total_out += workspace->out_buf.pos;
 392                workspace->out_buf.pos = 0;
 393
 394                if (total_out <= start_byte)
 395                        continue;
 396
 397                if (total_out > start_byte && buf_start < start_byte)
 398                        buf_offset = start_byte - buf_start;
 399                else
 400                        buf_offset = 0;
 401
 402                bytes = min_t(unsigned long, destlen - pg_offset,
 403                                workspace->out_buf.size - buf_offset);
 404
 405                kaddr = kmap_atomic(dest_page);
 406                memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset,
 407                                bytes);
 408                kunmap_atomic(kaddr);
 409
 410                pg_offset += bytes;
 411        }
 412        ret = 0;
 413finish:
 414        if (pg_offset < destlen) {
 415                kaddr = kmap_atomic(dest_page);
 416                memset(kaddr + pg_offset, 0, destlen - pg_offset);
 417                kunmap_atomic(kaddr);
 418        }
 419        return ret;
 420}
 421
 422static void zstd_set_level(struct list_head *ws, unsigned int type)
 423{
 424}
 425
 426const struct btrfs_compress_op btrfs_zstd_compress = {
 427        .alloc_workspace = zstd_alloc_workspace,
 428        .free_workspace = zstd_free_workspace,
 429        .compress_pages = zstd_compress_pages,
 430        .decompress_bio = zstd_decompress_bio,
 431        .decompress = zstd_decompress,
 432        .set_level = zstd_set_level,
 433};
 434