fs/btrfs/zstd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-present, Facebook, Inc.
   4  * All rights reserved.
   5  *
   6  */
   7
   8 #include <linux/bio.h>
   9 #include <linux/err.h>
  10 #include <linux/init.h>
  11 #include <linux/kernel.h>
  12 #include <linux/mm.h>
  13 #include <linux/pagemap.h>
  14 #include <linux/refcount.h>
  15 #include <linux/sched.h>
  16 #include <linux/slab.h>
  17 #include <linux/zstd.h>
  18 #include "compression.h"
  19
  20 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
  21 #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
  22 #define ZSTD_BTRFS_DEFAULT_LEVEL 3
  23
  24 static ZSTD_parameters zstd_get_btrfs_parameters(size_t src_len)
  25 {
  26         ZSTD_parameters params = ZSTD_getParams(ZSTD_BTRFS_DEFAULT_LEVEL,
  27                                                 src_len, 0);
  28
  29         if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
  30                 params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
  31         WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
  32         return params;
  33 }
  34
  35 struct workspace {
  36         void *mem;
  37         size_t size;
  38         char *buf;
  39         struct list_head list;
  40         ZSTD_inBuffer in_buf;
  41         ZSTD_outBuffer out_buf;
  42 };
  43
  44 static struct workspace_manager wsm;
  45
  46 static void zstd_init_workspace_manager(void)
  47 {
  48         btrfs_init_workspace_manager(&wsm, &btrfs_zstd_compress);
  49 }
  50
  51 static void zstd_cleanup_workspace_manager(void)
  52 {
  53         btrfs_cleanup_workspace_manager(&wsm);
  54 }
  55
  56 static struct list_head *zstd_get_workspace(unsigned int level)
  57 {
  58         return btrfs_get_workspace(&wsm, level);
  59 }
  60
  61 static void zstd_put_workspace(struct list_head *ws)
  62 {
  63         btrfs_put_workspace(&wsm, ws);
  64 }
  65
  66 static void zstd_free_workspace(struct list_head *ws)
  67 {
  68         struct workspace *workspace = list_entry(ws, struct workspace, list);
  69
  70         kvfree(workspace->mem);
  71         kfree(workspace->buf);
  72         kfree(workspace);
  73 }
  74
  75 static struct list_head *zstd_alloc_workspace(unsigned int level)
  76 {
  77         ZSTD_parameters params =
  78                         zstd_get_btrfs_parameters(ZSTD_BTRFS_MAX_INPUT);
  79         struct workspace *workspace;
  80
  81         workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
  82         if (!workspace)
  83                 return ERR_PTR(-ENOMEM);
  84
  85         workspace->size = max_t(size_t,
  86                         ZSTD_CStreamWorkspaceBound(params.cParams),
  87                         ZSTD_DStreamWorkspaceBound(ZSTD_BTRFS_MAX_INPUT));
  88         workspace->mem = kvmalloc(workspace->size, GFP_KERNEL);
  89         workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  90         if (!workspace->mem || !workspace->buf)
  91                 goto fail;
  92
  93         INIT_LIST_HEAD(&workspace->list);
  94
  95         return &workspace->list;
  96 fail:
  97         zstd_free_workspace(&workspace->list);
  98         return ERR_PTR(-ENOMEM);
  99 }
 100
 101 static int zstd_compress_pages(struct list_head *ws,
 102                 struct address_space *mapping,
 103                 u64 start,
 104                 struct page **pages,
 105                 unsigned long *out_pages,
 106                 unsigned long *total_in,
 107                 unsigned long *total_out)
 108 {
 109         struct workspace *workspace = list_entry(ws, struct workspace, list);
 110         ZSTD_CStream *stream;
 111         int ret = 0;
 112         int nr_pages = 0;
 113         struct page *in_page = NULL;  /* The current page to read */
 114         struct page *out_page = NULL; /* The current page to write to */
 115         unsigned long tot_in = 0;
 116         unsigned long tot_out = 0;
 117         unsigned long len = *total_out;
 118         const unsigned long nr_dest_pages = *out_pages;
 119         unsigned long max_out = nr_dest_pages * PAGE_SIZE;
 120         ZSTD_parameters params = zstd_get_btrfs_parameters(len);
 121
 122         *out_pages = 0;
 123         *total_out = 0;
 124         *total_in = 0;
 125
 126         /* Initialize the stream */
 127         stream = ZSTD_initCStream(params, len, workspace->mem,
 128                         workspace->size);
 129         if (!stream) {
 130                 pr_warn("BTRFS: ZSTD_initCStream failed\n");
 131                 ret = -EIO;
 132                 goto out;
 133         }
 134
 135         /* map in the first page of input data */
 136         in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 137         workspace->in_buf.src = kmap(in_page);
 138         workspace->in_buf.pos = 0;
 139         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 140
 141
 142         /* Allocate and map in the output buffer */
 143         out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 144         if (out_page == NULL) {
 145                 ret = -ENOMEM;
 146                 goto out;
 147         }
 148         pages[nr_pages++] = out_page;
 149         workspace->out_buf.dst = kmap(out_page);
 150         workspace->out_buf.pos = 0;
 151         workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 152
 153         while (1) {
 154                 size_t ret2;
 155
 156                 ret2 = ZSTD_compressStream(stream, &workspace->out_buf,
 157                                 &workspace->in_buf);
 158                 if (ZSTD_isError(ret2)) {
 159                         pr_debug("BTRFS: ZSTD_compressStream returned %d\n",
 160                                         ZSTD_getErrorCode(ret2));
 161                         ret = -EIO;
 162                         goto out;
 163                 }
 164
 165                 /* Check to see if we are making it bigger */
 166                 if (tot_in + workspace->in_buf.pos > 8192 &&
 167                                 tot_in + workspace->in_buf.pos <
 168                                 tot_out + workspace->out_buf.pos) {
 169                         ret = -E2BIG;
 170                         goto out;
 171                 }
 172
 173                 /* We've reached the end of our output range */
 174                 if (workspace->out_buf.pos >= max_out) {
 175                         tot_out += workspace->out_buf.pos;
 176                         ret = -E2BIG;
 177                         goto out;
 178                 }
 179
 180                 /* Check if we need more output space */
 181                 if (workspace->out_buf.pos == workspace->out_buf.size) {
 182                         tot_out += PAGE_SIZE;
 183                         max_out -= PAGE_SIZE;
 184                         kunmap(out_page);
 185                         if (nr_pages == nr_dest_pages) {
 186                                 out_page = NULL;
 187                                 ret = -E2BIG;
 188                                 goto out;
 189                         }
 190                         out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 191                         if (out_page == NULL) {
 192                                 ret = -ENOMEM;
 193                                 goto out;
 194                         }
 195                         pages[nr_pages++] = out_page;
 196                         workspace->out_buf.dst = kmap(out_page);
 197                         workspace->out_buf.pos = 0;
 198                         workspace->out_buf.size = min_t(size_t, max_out,
 199                                                         PAGE_SIZE);
 200                 }
 201
 202                 /* We've reached the end of the input */
 203                 if (workspace->in_buf.pos >= len) {
 204                         tot_in += workspace->in_buf.pos;
 205                         break;
 206                 }
 207
 208                 /* Check if we need more input */
 209                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 210                         tot_in += PAGE_SIZE;
 211                         kunmap(in_page);
 212                         put_page(in_page);
 213
 214                         start += PAGE_SIZE;
 215                         len -= PAGE_SIZE;
 216                         in_page = find_get_page(mapping, start >> PAGE_SHIFT);
 217                         workspace->in_buf.src = kmap(in_page);
 218                         workspace->in_buf.pos = 0;
 219                         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 220                 }
 221         }
 222         while (1) {
 223                 size_t ret2;
 224
 225                 ret2 = ZSTD_endStream(stream, &workspace->out_buf);
 226                 if (ZSTD_isError(ret2)) {
 227                         pr_debug("BTRFS: ZSTD_endStream returned %d\n",
 228                                         ZSTD_getErrorCode(ret2));
 229                         ret = -EIO;
 230                         goto out;
 231                 }
 232                 if (ret2 == 0) {
 233                         tot_out += workspace->out_buf.pos;
 234                         break;
 235                 }
 236                 if (workspace->out_buf.pos >= max_out) {
 237                         tot_out += workspace->out_buf.pos;
 238                         ret = -E2BIG;
 239                         goto out;
 240                 }
 241
 242                 tot_out += PAGE_SIZE;
 243                 max_out -= PAGE_SIZE;
 244                 kunmap(out_page);
 245                 if (nr_pages == nr_dest_pages) {
 246                         out_page = NULL;
 247                         ret = -E2BIG;
 248                         goto out;
 249                 }
 250                 out_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
 251                 if (out_page == NULL) {
 252                         ret = -ENOMEM;
 253                         goto out;
 254                 }
 255                 pages[nr_pages++] = out_page;
 256                 workspace->out_buf.dst = kmap(out_page);
 257                 workspace->out_buf.pos = 0;
 258                 workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 259         }
 260
 261         if (tot_out >= tot_in) {
 262                 ret = -E2BIG;
 263                 goto out;
 264         }
 265
 266         ret = 0;
 267         *total_in = tot_in;
 268         *total_out = tot_out;
 269 out:
 270         *out_pages = nr_pages;
 271         /* Cleanup */
 272         if (in_page) {
 273                 kunmap(in_page);
 274                 put_page(in_page);
 275         }
 276         if (out_page)
 277                 kunmap(out_page);
 278         return ret;
 279 }
 280
 281 static int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 282 {
 283         struct workspace *workspace = list_entry(ws, struct workspace, list);
 284         struct page **pages_in = cb->compressed_pages;
 285         u64 disk_start = cb->start;
 286         struct bio *orig_bio = cb->orig_bio;
 287         size_t srclen = cb->compressed_len;
 288         ZSTD_DStream *stream;
 289         int ret = 0;
 290         unsigned long page_in_index = 0;
 291         unsigned long total_pages_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
 292         unsigned long buf_start;
 293         unsigned long total_out = 0;
 294
 295         stream = ZSTD_initDStream(
 296                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 297         if (!stream) {
 298                 pr_debug("BTRFS: ZSTD_initDStream failed\n");
 299                 ret = -EIO;
 300                 goto done;
 301         }
 302
 303         workspace->in_buf.src = kmap(pages_in[page_in_index]);
 304         workspace->in_buf.pos = 0;
 305         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 306
 307         workspace->out_buf.dst = workspace->buf;
 308         workspace->out_buf.pos = 0;
 309         workspace->out_buf.size = PAGE_SIZE;
 310
 311         while (1) {
 312                 size_t ret2;
 313
 314                 ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 315                                 &workspace->in_buf);
 316                 if (ZSTD_isError(ret2)) {
 317                         pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 318                                         ZSTD_getErrorCode(ret2));
 319                         ret = -EIO;
 320                         goto done;
 321                 }
 322                 buf_start = total_out;
 323                 total_out += workspace->out_buf.pos;
 324                 workspace->out_buf.pos = 0;
 325
 326                 ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
 327                                 buf_start, total_out, disk_start, orig_bio);
 328                 if (ret == 0)
 329                         break;
 330
 331                 if (workspace->in_buf.pos >= srclen)
 332                         break;
 333
 334                 /* Check if we've hit the end of a frame */
 335                 if (ret2 == 0)
 336                         break;
 337
 338                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 339                         kunmap(pages_in[page_in_index++]);
 340                         if (page_in_index >= total_pages_in) {
 341                                 workspace->in_buf.src = NULL;
 342                                 ret = -EIO;
 343                                 goto done;
 344                         }
 345                         srclen -= PAGE_SIZE;
 346                         workspace->in_buf.src = kmap(pages_in[page_in_index]);
 347                         workspace->in_buf.pos = 0;
 348                         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 349                 }
 350         }
 351         ret = 0;
 352         zero_fill_bio(orig_bio);
 353 done:
 354         if (workspace->in_buf.src)
 355                 kunmap(pages_in[page_in_index]);
 356         return ret;
 357 }
 358
 359 static int zstd_decompress(struct list_head *ws, unsigned char *data_in,
 360                 struct page *dest_page,
 361                 unsigned long start_byte,
 362                 size_t srclen, size_t destlen)
 363 {
 364         struct workspace *workspace = list_entry(ws, struct workspace, list);
 365         ZSTD_DStream *stream;
 366         int ret = 0;
 367         size_t ret2;
 368         unsigned long total_out = 0;
 369         unsigned long pg_offset = 0;
 370         char *kaddr;
 371
 372         stream = ZSTD_initDStream(
 373                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 374         if (!stream) {
 375                 pr_warn("BTRFS: ZSTD_initDStream failed\n");
 376                 ret = -EIO;
 377                 goto finish;
 378         }
 379
 380         destlen = min_t(size_t, destlen, PAGE_SIZE);
 381
 382         workspace->in_buf.src = data_in;
 383         workspace->in_buf.pos = 0;
 384         workspace->in_buf.size = srclen;
 385
 386         workspace->out_buf.dst = workspace->buf;
 387         workspace->out_buf.pos = 0;
 388         workspace->out_buf.size = PAGE_SIZE;
 389
 390         ret2 = 1;
 391         while (pg_offset < destlen
 392                && workspace->in_buf.pos < workspace->in_buf.size) {
 393                 unsigned long buf_start;
 394                 unsigned long buf_offset;
 395                 unsigned long bytes;
 396
 397                 /* Check if the frame is over and we still need more input */
 398                 if (ret2 == 0) {
 399                         pr_debug("BTRFS: ZSTD_decompressStream ended early\n");
 400                         ret = -EIO;
 401                         goto finish;
 402                 }
 403                 ret2 = ZSTD_decompressStream(stream, &workspace->out_buf,
 404                                 &workspace->in_buf);
 405                 if (ZSTD_isError(ret2)) {
 406                         pr_debug("BTRFS: ZSTD_decompressStream returned %d\n",
 407                                         ZSTD_getErrorCode(ret2));
 408                         ret = -EIO;
 409                         goto finish;
 410                 }
 411
 412                 buf_start = total_out;
 413                 total_out += workspace->out_buf.pos;
 414                 workspace->out_buf.pos = 0;
 415
 416                 if (total_out <= start_byte)
 417                         continue;
 418
 419                 if (total_out > start_byte && buf_start < start_byte)
 420                         buf_offset = start_byte - buf_start;
 421                 else
 422                         buf_offset = 0;
 423
 424                 bytes = min_t(unsigned long, destlen - pg_offset,
 425                                 workspace->out_buf.size - buf_offset);
 426
 427                 kaddr = kmap_atomic(dest_page);
 428                 memcpy(kaddr + pg_offset, workspace->out_buf.dst + buf_offset,
 429                                 bytes);
 430                 kunmap_atomic(kaddr);
 431
 432                 pg_offset += bytes;
 433         }
 434         ret = 0;
 435 finish:
 436         if (pg_offset < destlen) {
 437                 kaddr = kmap_atomic(dest_page);
 438                 memset(kaddr + pg_offset, 0, destlen - pg_offset);
 439                 kunmap_atomic(kaddr);
 440         }
 441         return ret;
 442 }
 443
 444 static unsigned int zstd_set_level(unsigned int level)
 445 {
 446         return ZSTD_BTRFS_DEFAULT_LEVEL;
 447 }
 448
 449 const struct btrfs_compress_op btrfs_zstd_compress = {
 450         .init_workspace_manager = zstd_init_workspace_manager,
 451         .cleanup_workspace_manager = zstd_cleanup_workspace_manager,
 452         .get_workspace = zstd_get_workspace,
 453         .put_workspace = zstd_put_workspace,
 454         .alloc_workspace = zstd_alloc_workspace,
 455         .free_workspace = zstd_free_workspace,
 456         .compress_pages = zstd_compress_pages,
 457         .decompress_bio = zstd_decompress_bio,
 458         .decompress = zstd_decompress,
 459         .set_level = zstd_set_level,
 460 };