fs/btrfs/zstd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-present, Facebook, Inc.
   4  * All rights reserved.
   5  *
   6  */
   7
   8 #include <linux/bio.h>
   9 #include <linux/err.h>
  10 #include <linux/init.h>
  11 #include <linux/kernel.h>
  12 #include <linux/mm.h>
  13 #include <linux/pagemap.h>
  14 #include <linux/refcount.h>
  15 #include <linux/sched.h>
  16 #include <linux/slab.h>
  17 #include <linux/zstd.h>
  18 #include "compression.h"
  19
  20 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
  21 #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
  22 #define ZSTD_BTRFS_DEFAULT_LEVEL 3
  23
  24 static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
  25 {
  26         ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
  27                                                 src_len, 0);
  28
  29         if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
  30                 params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
  31         WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
  32         return params;
  33 }
  34
  35 struct workspace {
  36         void *mem;
  37         size_t size;
  38         char *buf;
  39         struct list_head list;
  40         ZSTD_inBuffer in_buf;
  41         ZSTD_outBuffer out_buf;
  42 };
  43
  44 static void zstd_free_workspace(struct list_head *ws)
  45 {
  46         struct workspace *workspace = list_entry(ws, struct workspace, list);
  47
  48         kvfree(workspace->mem);
  49         kfree(workspace->buf);
  50         kfree(workspace);
  51 }
  52
  53 static struct list_head *zstd_alloc_workspace(void)
  54 {
  55         ZSTD_parameters params =
  56                         zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
  57         struct workspace *workspace;
  58
  59         workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
  60         if (!workspace)
  61                 return ERR_PTR(-ENOMEM);
  62
  63         workspace->size = max_t(size_t,
  64                         ZSTD_CStreamWorkspaceBound(params.cParams),
  65                         ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
  66         workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
  67         workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  68         if (!workspace->mem || !workspace->buf)
  69                 goto fail;
  70
  71         INIT_LIST_HEAD(&workspace->list);
  72
  73         return &workspace->list;
  74 fail:
  75         zstd_free_workspace(&workspace->list);
  76         return ERR_PTR(-ENOMEM);
  77 }
  78
  79 static int zstd_compress_pages(struct list_head *ws,
  80                 struct address_space *mapping,
  81                 u64 start,
  82                 struct page **pages,
  83                 unsigned long *out_pages,
  84                 unsigned long *total_in,
  85                 unsigned long *total_out)
  86 {
  87         struct workspace *workspace = list_entry(ws, struct workspace, list);
  88         ZSTD_CStream *stream;
  89         int ret = 0;
  90         int nr_pages = 0;
  91         struct page *in_page = NULL;  /* The current page to read */
  92         struct page *out_page = NULL; /* The current page to write to */
  93         unsigned long tot_in = 0;
  94         unsigned long tot_out = 0;
  95         unsigned long len = *total_out;
  96         const unsigned long nr_dest_pages = *out_pages;
  97         unsigned long max_out = nr_dest_pages * PAGE_SIZE;
  98         ZSTD_parameters params = zstd_get_btrfs_parameters(len);
  99
 100         *out_pages = 0;
 101         *total_out = 0;
 102         *total_in = 0;
 103
 104         /* Initialize the stream */
 105         stream = ZSTD_initCStream(params, len, workspace->mem,
 106                         workspace->size);
 107         if (!stream) {
 108                 pr_warn("BTRFS: ZSTD_initCStream failed\n");
 109                 ret = -EIO;
 110                 goto out;
 111         }
 112
 113         /* map in the first page of input data */
 114         in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 115         workspace->in_buf.src = kmap(in_page);
 116         workspace->in_buf.pos = 0;
 117         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 118
 119
 120         /* Allocate and map in the output buffer */
 121         out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 122         if (out_page == NULL) {
 123                 ret = -ENOMEM;
 124                 goto out;
 125         }
 126         pages[nr_pages++] = out_page;
 127         workspace->out_buf.dst = kmap(out_page);
 128         workspace->out_buf.pos = 0;
 129         workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 130
 131         while (1) {
 132                 size_t ret2;
 133
 134                 ret2 = ZSTD_compressStream(stream, &workspace->out_buf,
 135                                 &workspace->in_buf);
 136                 if (ZSTD_isError(ret2)) {
 137                         pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
 138                                         ZSTD_getErrorCode(ret2));
 139                         ret = -EIO;
 140                         goto out;
 141                 }
 142
 143                 /* Check to see if we are making it bigger */
 144                 if (tot_in + workspace->in_buf.pos > 8192 &&
 145                                 tot_in + workspace->in_buf.pos <
 146                                 tot_out + workspace->out_buf.pos) {
 147                         ret = -E2BIG;
 148                         goto out;
 149                 }
 150
 151                 /* We've reached the end of our output range */
 152                 if (workspace->out_buf.pos >= max_out) {
 153                         tot_out += workspace->out_buf.pos;
 154                         ret = -E2BIG;
 155                         goto out;
 156                 }
 157
 158                 /* Check if we need more output space */
 159                 if (workspace->out_buf.pos == workspace->out_buf.size) {
 160                         tot_out += PAGE_SIZE;
 161                         max_out -= PAGE_SIZE;
 162                         kunmap(out_page);
 163                         if (nr_pages == nr_dest_pages) {
 164                                 out_page = NULL;
 165                                 ret = -E2BIG;
 166                                 goto out;
 167                         }
 168                         out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 169                         if (out_page == NULL) {
 170                                 ret = -ENOMEM;
 171                                 goto out;
 172                         }
 173                         pages[nr_pages++] = out_page;
 174                         workspace->out_buf.dst = kmap(out_page);
 175                         workspace->out_buf.pos = 0;
 176                         workspace->out_buf.size = min_t(size_t, max_out,
 177                                                         PAGE_SIZE);
 178                 }
 179
 180                 /* We've reached the end of the input */
 181                 if (workspace->in_buf.pos >= len) {
 182                         tot_in += workspace->in_buf.pos;
 183                         break;
 184                 }
 185
 186                 /* Check if we need more input */
 187                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 188                         tot_in += PAGE_SIZE;
 189                         kunmap(in_page);
 190                         put_page(in_page);
 191
 192                         start += PAGE_SIZE;
 193                         len -= PAGE_SIZE;
 194                         in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 195                         workspace->in_buf.src = kmap(in_page);
 196                         workspace->in_buf.pos = 0;
 197                         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 198                 }
 199         }
 200         while (1) {
 201                 size_t ret2;
 202
 203                 ret2 = ZSTD_endStream(stream, &workspace->out_buf);
 204                 if (ZSTD_isError(ret2)) {
 205                         pr_debug("BTRFS: ZSTD_endStream returned %d\n",
 206                                         ZSTD_getErrorCode(ret2));
 207                         ret = -EIO;
 208                         goto out;
 209                 }
 210                 if (ret2 == 0) {
 211                         tot_out += workspace->out_buf.pos;
 212                         break;
 213                 }
 214                 if (workspace->out_buf.pos >= max_out) {
 215                         tot_out += workspace->out_buf.pos;
 216                         ret = -E2BIG;
 217                         goto out;
 218                 }
 219
 220                 tot_out += PAGE_SIZE;
 221                 max_out -= PAGE_SIZE;
 222                 kunmap(out_page);
 223                 if (nr_pages == nr_dest_pages) {
 224                         out_page = NULL;
 225                         ret = -E2BIG;
 226                         goto out;
 227                 }
 228                 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 229                 if (out_page == NULL) {
 230                         ret = -ENOMEM;
 231                         goto out;
 232                 }
 233                 pages[nr_pages++] = out_page;
 234                 workspace->out_buf.dst = kmap(out_page);
 235                 workspace->out_buf.pos = 0;
 236                 workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 237         }
 238
 239         if (tot_out >= tot_in) {
 240                 ret = -E2BIG;
 241                 goto out;
 242         }
 243
 244         ret = 0;
 245         *total_in = tot_in;
 246         *total_out = tot_out;
 247 out:
 248         *out_pages = nr_pages;
 249         /* Cleanup */
 250         if (in_page) {
 251                 kunmap(in_page);
 252                 put_page(in_page);
 253         }
 254         if (out_page)
 255                 kunmap(out_page);
 256         return ret;
 257 }
 258
 259 static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 260 {
 261         struct workspace *workspace = list_entry(ws, struct workspace, list);
 262         struct page **pages_in = cb->compressed_pages;
 263         u64 disk_start = cb->start;
 264         struct bio *orig_bio = cb->orig_bio;
 265         size_t srclen = cb->compressed_len;
 266         ZSTD_DStream *stream;
 267         int ret = 0;
 268         unsigned long page_in_index = 0;
 269         unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
 270         unsigned long buf_start;
 271         unsigned long total_out = 0;
 272
 273         stream = ZSTD_initDStream(
 274                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 275         if (!stream) {
 276                 pr_debug("BTRFS: ZSTD_initDStream failed\n");
 277                 ret = -EIO;
 278                 goto done;
 279         }
 280
 281         workspace->in_buf.src = kmap(pages_in[page_in_index]);
 282         workspace->in_buf.pos = 0;
 283         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 284
 285         workspace->out_buf.dst = workspace->buf;
 286         workspace->out_buf.pos = 0;
 287         workspace->out_buf.size = PAGE_SIZE;
 288
 289         while (1) {
 290                 size_t ret2;
 291
 292                 ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 293                                 &workspace->in_buf);
 294                 if (ZSTD_isError(ret2)) {
 295                         pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 296                                         ZSTD_getErrorCode(ret2));
 297                         ret = -EIO;
 298                         goto done;
 299                 }
 300                 buf_start = total_out;
 301                 total_out += workspace->out_buf.pos;
 302                 workspace->out_buf.pos = 0;
 303
 304                 ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
 305                                 buf_start, total_out, disk_start, orig_bio);
 306                 if (ret == 0)
 307                         break;
 308
 309                 if (workspace->in_buf.pos >= srclen)
 310                         break;
 311
 312                 /* Check if we've hit the end of a frame */
 313                 if (ret2 == 0)
 314                         break;
 315
 316                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 317                         kunmap(pages_in[page_in_index++]);
 318                         if (page_in_index >= total_pages_in) {
 319                                 workspace->in_buf.src = NULL;
 320                                 ret = -EIO;
 321                                 goto done;
 322                         }
 323                         srclen -= PAGE_SIZE;
 324                         workspace->in_buf.src = kmap(pages_in[page_in_index]);
 325                         workspace->in_buf.pos = 0;
 326                         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 327                 }
 328         }
 329         ret = 0;
 330         zero_fill_bio(orig_bio);
 331 done:
 332         if (workspace->in_buf.src)
 333                 kunmap(pages_in[page_in_index]);
 334         return ret;
 335 }
 336
 337 static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
 338                 struct page *dest_page,
 339                 unsigned long start_byte,
 340                 size_t srclen, size_t destlen)
 341 {
 342         struct workspace *workspace = list_entry(ws, struct workspace, list);
 343         ZSTD_DStream *stream;
 344         int ret = 0;
 345         size_t ret2;
 346         unsigned long total_out = 0;
 347         unsigned long pg_offset = 0;
 348         char *kaddr;
 349
 350         stream = ZSTD_initDStream(
 351                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 352         if (!stream) {
 353                 pr_warn("BTRFS: ZSTD_initDStream failed\n");
 354                 ret = -EIO;
 355                 goto finish;
 356         }
 357
 358         destlen = min_t(size_t, destlen, PAGE_SIZE);
 359
 360         workspace->in_buf.src = data_in;
 361         workspace->in_buf.pos = 0;
 362         workspace->in_buf.size = srclen;
 363
 364         workspace->out_buf.dst = workspace->buf;
 365         workspace->out_buf.pos = 0;
 366         workspace->out_buf.size = PAGE_SIZE;
 367
 368         ret2 = 1;
 369         while (pg_offset < destlen
 370                && workspace->in_buf.pos < workspace->in_buf.size) {
 371                 unsigned long buf_start;
 372                 unsigned long buf_offset;
 373                 unsigned long bytes;
 374
 375                 /* Check if the frame is over and we still need more input */
 376                 if (ret2 == 0) {
 377                         pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
 378                         ret = -EIO;
 379                         goto finish;
 380                 }
 381                 ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 382                                 &workspace->in_buf);
 383                 if (ZSTD_isError(ret2)) {
 384                         pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 385                                         ZSTD_getErrorCode(ret2));
 386                         ret = -EIO;
 387                         goto finish;
 388                 }
 389
 390                 buf_start = total_out;
 391                 total_out += workspace->out_buf.pos;
 392                 workspace->out_buf.pos = 0;
 393
 394                 if (total_out <= start_byte)
 395                         continue;
 396
 397                 if (total_out > start_byte && buf_start < start_byte)
 398                         buf_offset = start_byte - buf_start;
 399                 else
 400                         buf_offset = 0;
 401
 402                 bytes = min_t(unsigned long, destlen - pg_offset,
 403                                 workspace->out_buf.size - buf_offset);
 404
 405                 kaddr = kmap_atomic(dest_page);
 406                 memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset,
 407                                 bytes);
 408                 kunmap_atomic(kaddr);
 409
 410                 pg_offset += bytes;
 411         }
 412         ret = 0;
 413 finish:
 414         if (pg_offset < destlen) {
 415                 kaddr = kmap_atomic(dest_page);
 416                 memset(kaddr + pg_offset, 0, destlen - pg_offset);
 417                 kunmap_atomic(kaddr);
 418         }
 419         return ret;
 420 }
 421
 422 static void zstd_set_level(struct list_head *ws, unsigned int type)
 423 {
 424 }
 425
 426 const struct btrfs_compress_op btrfs_zstd_compress = {
 427         .alloc_workspace = zstd_alloc_workspace,
 428         .free_workspace = zstd_free_workspace,
 429         .compress_pages = zstd_compress_pages,
 430         .decompress_bio = zstd_decompress_bio,
 431         .decompress = zstd_decompress,
 432         .set_level = zstd_set_level,
 433 };