fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/blkdev.h>
  20 #include <linux/ratelimit.h>
  21 #include "ctree.h"
  22 #include "volumes.h"
  23 #include "disk-io.h"
  24 #include "ordered-data.h"
  25 #include "transaction.h"
  26 #include "backref.h"
  27 #include "extent_io.h"
  28 #include "dev-replace.h"
  29 #include "check-integrity.h"
  30 #include "rcu-string.h"
  31 #include "raid56.h"
  32
  33 /*
  34  * This is only the first step towards a full-features scrub. It reads all
  35  * extent and super block and verifies the checksums. In case a bad checksum
  36  * is found or the extent cannot be read, good data will be written back if
  37  * any can be found.
  38  *
  39  * Future enhancements:
  40  *  - In case an unrepairable extent is encountered, track which files are
  41  *    affected and report them
  42  *  - track and record media errors, throw out bad devices
  43  *  - add a mode to also read unallocated space
  44  */
  45
  46 struct scrub_block;
  47 struct scrub_ctx;
  48
  49 /*
  50  * the following three values only influence the performance.
  51  * The last one configures the number of parallel and outstanding I/O
  52  * operations. The first two values configure an upper limit for the number
  53  * of (dynamically allocated) pages that are added to a bio.
  54  */
  55 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  56 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  57 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  58
  59 /*
  60  * the following value times PAGE_SIZE needs to be large enough to match the
  61  * largest node/leaf/sector size that shall be supported.
  62  * Values larger than BTRFS_STRIPE_LEN are not supported.
  63  */
  64 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  65
  66 struct scrub_recover {
  67         atomic_t                refs;
  68         struct btrfs_bio        *bbio;
  69         u64                     map_length;
  70 };
  71
  72 struct scrub_page {
  73         struct scrub_block      *sblock;
  74         struct page             *page;
  75         struct btrfs_device     *dev;
  76         struct list_head        list;
  77         u64                     flags;  /* extent flags */
  78         u64                     generation;
  79         u64                     logical;
  80         u64                     physical;
  81         u64                     physical_for_dev_replace;
  82         atomic_t                refs;
  83         struct {
  84                 unsigned int    mirror_num:8;
  85                 unsigned int    have_csum:1;
  86                 unsigned int    io_error:1;
  87         };
  88         u8                      csum[BTRFS_CSUM_SIZE];
  89
  90         struct scrub_recover    *recover;
  91 };
  92
  93 struct scrub_bio {
  94         int                     index;
  95         struct scrub_ctx        *sctx;
  96         struct btrfs_device     *dev;
  97         struct bio              *bio;
  98         int                     err;
  99         u64                     logical;
 100         u64                     physical;
 101 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
 102         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
 103 #else
 104         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
 105 #endif
 106         int                     page_count;
 107         int                     next_free;
 108         struct btrfs_work       work;
 109 };
 110
 111 struct scrub_block {
 112         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 113         int                     page_count;
 114         atomic_t                outstanding_pages;
 115         atomic_t                refs; /* free mem on transition to zero */
 116         struct scrub_ctx        *sctx;
 117         struct scrub_parity     *sparity;
 118         struct {
 119                 unsigned int    header_error:1;
 120                 unsigned int    checksum_error:1;
 121                 unsigned int    no_io_error_seen:1;
 122                 unsigned int    generation_error:1; /* also sets header_error */
 123
 124                 /* The following is for the data used to check parity */
 125                 /* It is for the data with checksum */
 126                 unsigned int    data_corrected:1;
 127         };
 128         struct btrfs_work       work;
 129 };
 130
 131 /* Used for the chunks with parity stripe such RAID5/6 */
 132 struct scrub_parity {
 133         struct scrub_ctx        *sctx;
 134
 135         struct btrfs_device     *scrub_dev;
 136
 137         u64                     logic_start;
 138
 139         u64                     logic_end;
 140
 141         int                     nsectors;
 142
 143         int                     stripe_len;
 144
 145         atomic_t                refs;
 146
 147         struct list_head        spages;
 148
 149         /* Work of parity check and repair */
 150         struct btrfs_work       work;
 151
 152         /* Mark the parity blocks which have data */
 153         unsigned long           *dbitmap;
 154
 155         /*
 156          * Mark the parity blocks which have data, but errors happen when
 157          * read data or check data
 158          */
 159         unsigned long           *ebitmap;
 160
 161         unsigned long           bitmap[0];
 162 };
 163
 164 struct scrub_wr_ctx {
 165         struct scrub_bio *wr_curr_bio;
 166         struct btrfs_device *tgtdev;
 167         int pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 168         atomic_t flush_all_writes;
 169         struct mutex wr_lock;
 170 };
 171
 172 struct scrub_ctx {
 173         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 174         struct btrfs_root       *dev_root;
 175         int                     first_free;
 176         int                     curr;
 177         atomic_t                bios_in_flight;
 178         atomic_t                workers_pending;
 179         spinlock_t              list_lock;
 180         wait_queue_head_t       list_wait;
 181         u16                     csum_size;
 182         struct list_head        csum_list;
 183         atomic_t                cancel_req;
 184         int                     readonly;
 185         int                     pages_per_rd_bio;
 186         u32                     sectorsize;
 187         u32                     nodesize;
 188
 189         int                     is_dev_replace;
 190         struct scrub_wr_ctx     wr_ctx;
 191
 192         /*
 193          * statistics
 194          */
 195         struct btrfs_scrub_progress stat;
 196         spinlock_t              stat_lock;
 197
 198         /*
 199          * Use a ref counter to avoid use-after-free issues. Scrub workers
 200          * decrement bios_in_flight and workers_pending and then do a wakeup
 201          * on the list_wait wait queue. We must ensure the main scrub task
 202          * doesn't free the scrub context before or while the workers are
 203          * doing the wakeup() call.
 204          */
 205         atomic_t                refs;
 206 };
 207
 208 struct scrub_fixup_nodatasum {
 209         struct scrub_ctx        *sctx;
 210         struct btrfs_device     *dev;
 211         u64                     logical;
 212         struct btrfs_root       *root;
 213         struct btrfs_work       work;
 214         int                     mirror_num;
 215 };
 216
 217 struct scrub_nocow_inode {
 218         u64                     inum;
 219         u64                     offset;
 220         u64                     root;
 221         struct list_head        list;
 222 };
 223
 224 struct scrub_copy_nocow_ctx {
 225         struct scrub_ctx        *sctx;
 226         u64                     logical;
 227         u64                     len;
 228         int                     mirror_num;
 229         u64                     physical_for_dev_replace;
 230         struct list_head        inodes;
 231         struct btrfs_work       work;
 232 };
 233
 234 struct scrub_warning {
 235         struct btrfs_path       *path;
 236         u64                     extent_item_size;
 237         const char              *errstr;
 238         sector_t                sector;
 239         u64                     logical;
 240         struct btrfs_device     *dev;
 241 };
 242
 243 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 244 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 245 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx);
 246 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx);
 247 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 248 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
 249                                      struct scrub_block *sblocks_for_recheck);
 250 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 251                                 struct scrub_block *sblock, int is_metadata,
 252                                 int have_csum, u8 *csum, u64 generation,
 253                                 u16 csum_size, int retry_failed_mirror);
 254 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
 255                                          struct scrub_block *sblock,
 256                                          int is_metadata, int have_csum,
 257                                          const u8 *csum, u64 generation,
 258                                          u16 csum_size);
 259 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 260                                              struct scrub_block *sblock_good);
 261 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 262                                             struct scrub_block *sblock_good,
 263                                             int page_num, int force_write);
 264 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 265 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 266                                            int page_num);
 267 static int scrub_checksum_data(struct scrub_block *sblock);
 268 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 269 static int scrub_checksum_super(struct scrub_block *sblock);
 270 static void scrub_block_get(struct scrub_block *sblock);
 271 static void scrub_block_put(struct scrub_block *sblock);
 272 static void scrub_page_get(struct scrub_page *spage);
 273 static void scrub_page_put(struct scrub_page *spage);
 274 static void scrub_parity_get(struct scrub_parity *sparity);
 275 static void scrub_parity_put(struct scrub_parity *sparity);
 276 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 277                                     struct scrub_page *spage);
 278 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 279                        u64 physical, struct btrfs_device *dev, u64 flags,
 280                        u64 gen, int mirror_num, u8 *csum, int force,
 281                        u64 physical_for_dev_replace);
 282 static void scrub_bio_end_io(struct bio *bio, int err);
 283 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 284 static void scrub_block_complete(struct scrub_block *sblock);
 285 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 286                                u64 extent_logical, u64 extent_len,
 287                                u64 *extent_physical,
 288                                struct btrfs_device **extent_dev,
 289                                int *extent_mirror_num);
 290 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
 291                               struct scrub_wr_ctx *wr_ctx,
 292                               struct btrfs_fs_info *fs_info,
 293                               struct btrfs_device *dev,
 294                               int is_dev_replace);
 295 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx);
 296 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 297                                     struct scrub_page *spage);
 298 static void scrub_wr_submit(struct scrub_ctx *sctx);
 299 static void scrub_wr_bio_end_io(struct bio *bio, int err);
 300 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 301 static int write_page_nocow(struct scrub_ctx *sctx,
 302                             u64 physical_for_dev_replace, struct page *page);
 303 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
 304                                       struct scrub_copy_nocow_ctx *ctx);
 305 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
 306                             int mirror_num, u64 physical_for_dev_replace);
 307 static void copy_nocow_pages_worker(struct btrfs_work *work);
 308 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 309 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 310 static void scrub_put_ctx(struct scrub_ctx *sctx);
 311
 312
 313 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 314 {
 315         atomic_inc(&sctx->refs);
 316         atomic_inc(&sctx->bios_in_flight);
 317 }
 318
 319 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 320 {
 321         atomic_dec(&sctx->bios_in_flight);
 322         wake_up(&sctx->list_wait);
 323         scrub_put_ctx(sctx);
 324 }
 325
 326 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 327 {
 328         while (atomic_read(&fs_info->scrub_pause_req)) {
 329                 mutex_unlock(&fs_info->scrub_lock);
 330                 wait_event(fs_info->scrub_pause_wait,
 331                    atomic_read(&fs_info->scrub_pause_req) == 0);
 332                 mutex_lock(&fs_info->scrub_lock);
 333         }
 334 }
 335
 336 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
 337 {
 338         atomic_inc(&fs_info->scrubs_paused);
 339         wake_up(&fs_info->scrub_pause_wait);
 340 }
 341
 342 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
 343 {
 344         mutex_lock(&fs_info->scrub_lock);
 345         __scrub_blocked_if_needed(fs_info);
 346         atomic_dec(&fs_info->scrubs_paused);
 347         mutex_unlock(&fs_info->scrub_lock);
 348
 349         wake_up(&fs_info->scrub_pause_wait);
 350 }
 351
 352 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 353 {
 354         scrub_pause_on(fs_info);
 355         scrub_pause_off(fs_info);
 356 }
 357
 358 /*
 359  * used for workers that require transaction commits (i.e., for the
 360  * NOCOW case)
 361  */
 362 static void scrub_pending_trans_workers_inc(struct scrub_ctx *sctx)
 363 {
 364         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 365
 366         atomic_inc(&sctx->refs);
 367         /*
 368          * increment scrubs_running to prevent cancel requests from
 369          * completing as long as a worker is running. we must also
 370          * increment scrubs_paused to prevent deadlocking on pause
 371          * requests used for transactions commits (as the worker uses a
 372          * transaction context). it is safe to regard the worker
 373          * as paused for all matters practical. effectively, we only
 374          * avoid cancellation requests from completing.
 375          */
 376         mutex_lock(&fs_info->scrub_lock);
 377         atomic_inc(&fs_info->scrubs_running);
 378         atomic_inc(&fs_info->scrubs_paused);
 379         mutex_unlock(&fs_info->scrub_lock);
 380
 381         /*
 382          * check if @scrubs_running=@scrubs_paused condition
 383          * inside wait_event() is not an atomic operation.
 384          * which means we may inc/dec @scrub_running/paused
 385          * at any time. Let's wake up @scrub_pause_wait as
 386          * much as we can to let commit transaction blocked less.
 387          */
 388         wake_up(&fs_info->scrub_pause_wait);
 389
 390         atomic_inc(&sctx->workers_pending);
 391 }
 392
 393 /* used for workers that require transaction commits */
 394 static void scrub_pending_trans_workers_dec(struct scrub_ctx *sctx)
 395 {
 396         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
 397
 398         /*
 399          * see scrub_pending_trans_workers_inc() why we're pretending
 400          * to be paused in the scrub counters
 401          */
 402         mutex_lock(&fs_info->scrub_lock);
 403         atomic_dec(&fs_info->scrubs_running);
 404         atomic_dec(&fs_info->scrubs_paused);
 405         mutex_unlock(&fs_info->scrub_lock);
 406         atomic_dec(&sctx->workers_pending);
 407         wake_up(&fs_info->scrub_pause_wait);
 408         wake_up(&sctx->list_wait);
 409         scrub_put_ctx(sctx);
 410 }
 411
 412 static void scrub_free_csums(struct scrub_ctx *sctx)
 413 {
 414         while (!list_empty(&sctx->csum_list)) {
 415                 struct btrfs_ordered_sum *sum;
 416                 sum = list_first_entry(&sctx->csum_list,
 417                                        struct btrfs_ordered_sum, list);
 418                 list_del(&sum->list);
 419                 kfree(sum);
 420         }
 421 }
 422
 423 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 424 {
 425         int i;
 426
 427         if (!sctx)
 428                 return;
 429
 430         scrub_free_wr_ctx(&sctx->wr_ctx);
 431
 432         /* this can happen when scrub is cancelled */
 433         if (sctx->curr != -1) {
 434                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 435
 436                 for (i = 0; i < sbio->page_count; i++) {
 437                         WARN_ON(!sbio->pagev[i]->page);
 438                         scrub_block_put(sbio->pagev[i]->sblock);
 439                 }
 440                 bio_put(sbio->bio);
 441         }
 442
 443         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 444                 struct scrub_bio *sbio = sctx->bios[i];
 445
 446                 if (!sbio)
 447                         break;
 448                 kfree(sbio);
 449         }
 450
 451         scrub_free_csums(sctx);
 452         kfree(sctx);
 453 }
 454
 455 static void scrub_put_ctx(struct scrub_ctx *sctx)
 456 {
 457         if (atomic_dec_and_test(&sctx->refs))
 458                 scrub_free_ctx(sctx);
 459 }
 460
 461 static noinline_for_stack
 462 struct scrub_ctx *scrub_setup_ctx(struct btrfs_device *dev, int is_dev_replace)
 463 {
 464         struct scrub_ctx *sctx;
 465         int             i;
 466         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 467         int pages_per_rd_bio;
 468         int ret;
 469
 470         /*
 471          * the setting of pages_per_rd_bio is correct for scrub but might
 472          * be wrong for the dev_replace code where we might read from
 473          * different devices in the initial huge bios. However, that
 474          * code is able to correctly handle the case when adding a page
 475          * to a bio fails.
 476          */
 477         if (dev->bdev)
 478                 pages_per_rd_bio = min_t(int, SCRUB_PAGES_PER_RD_BIO,
 479                                          bio_get_nr_vecs(dev->bdev));
 480         else
 481                 pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 482         sctx = kzalloc(sizeof(*sctx), GFP_NOFS);
 483         if (!sctx)
 484                 goto nomem;
 485         atomic_set(&sctx->refs, 1);
 486         sctx->is_dev_replace = is_dev_replace;
 487         sctx->pages_per_rd_bio = pages_per_rd_bio;
 488         sctx->curr = -1;
 489         sctx->dev_root = dev->dev_root;
 490         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 491                 struct scrub_bio *sbio;
 492
 493                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 494                 if (!sbio)
 495                         goto nomem;
 496                 sctx->bios[i] = sbio;
 497
 498                 sbio->index = i;
 499                 sbio->sctx = sctx;
 500                 sbio->page_count = 0;
 501                 btrfs_init_work(&sbio->work, btrfs_scrub_helper,
 502                                 scrub_bio_end_io_worker, NULL, NULL);
 503
 504                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 505                         sctx->bios[i]->next_free = i + 1;
 506                 else
 507                         sctx->bios[i]->next_free = -1;
 508         }
 509         sctx->first_free = 0;
 510         sctx->nodesize = dev->dev_root->nodesize;
 511         sctx->sectorsize = dev->dev_root->sectorsize;
 512         atomic_set(&sctx->bios_in_flight, 0);
 513         atomic_set(&sctx->workers_pending, 0);
 514         atomic_set(&sctx->cancel_req, 0);
 515         sctx->csum_size = btrfs_super_csum_size(fs_info->super_copy);
 516         INIT_LIST_HEAD(&sctx->csum_list);
 517
 518         spin_lock_init(&sctx->list_lock);
 519         spin_lock_init(&sctx->stat_lock);
 520         init_waitqueue_head(&sctx->list_wait);
 521
 522         ret = scrub_setup_wr_ctx(sctx, &sctx->wr_ctx, fs_info,
 523                                  fs_info->dev_replace.tgtdev, is_dev_replace);
 524         if (ret) {
 525                 scrub_free_ctx(sctx);
 526                 return ERR_PTR(ret);
 527         }
 528         return sctx;
 529
 530 nomem:
 531         scrub_free_ctx(sctx);
 532         return ERR_PTR(-ENOMEM);
 533 }
 534
 535 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 536                                      void *warn_ctx)
 537 {
 538         u64 isize;
 539         u32 nlink;
 540         int ret;
 541         int i;
 542         struct extent_buffer *eb;
 543         struct btrfs_inode_item *inode_item;
 544         struct scrub_warning *swarn = warn_ctx;
 545         struct btrfs_fs_info *fs_info = swarn->dev->dev_root->fs_info;
 546         struct inode_fs_paths *ipath = NULL;
 547         struct btrfs_root *local_root;
 548         struct btrfs_key root_key;
 549         struct btrfs_key key;
 550
 551         root_key.objectid = root;
 552         root_key.type = BTRFS_ROOT_ITEM_KEY;
 553         root_key.offset = (u64)-1;
 554         local_root = btrfs_read_fs_root_no_name(fs_info, &root_key);
 555         if (IS_ERR(local_root)) {
 556                 ret = PTR_ERR(local_root);
 557                 goto err;
 558         }
 559
 560         /*
 561          * this makes the path point to (inum INODE_ITEM ioff)
 562          */
 563         key.objectid = inum;
 564         key.type = BTRFS_INODE_ITEM_KEY;
 565         key.offset = 0;
 566
 567         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
 568         if (ret) {
 569                 btrfs_release_path(swarn->path);
 570                 goto err;
 571         }
 572
 573         eb = swarn->path->nodes[0];
 574         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 575                                         struct btrfs_inode_item);
 576         isize = btrfs_inode_size(eb, inode_item);
 577         nlink = btrfs_inode_nlink(eb, inode_item);
 578         btrfs_release_path(swarn->path);
 579
 580         ipath = init_ipath(4096, local_root, swarn->path);
 581         if (IS_ERR(ipath)) {
 582                 ret = PTR_ERR(ipath);
 583                 ipath = NULL;
 584                 goto err;
 585         }
 586         ret = paths_from_inode(inum, ipath);
 587
 588         if (ret < 0)
 589                 goto err;
 590
 591         /*
 592          * we deliberately ignore the bit ipath might have been too small to
 593          * hold all of the paths here
 594          */
 595         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 596                 printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
 597                         "%s, sector %llu, root %llu, inode %llu, offset %llu, "
 598                         "length %llu, links %u (path: %s)\n", swarn->errstr,
 599                         swarn->logical, rcu_str_deref(swarn->dev->name),
 600                         (unsigned long long)swarn->sector, root, inum, offset,
 601                         min(isize - offset, (u64)PAGE_SIZE), nlink,
 602                         (char *)(unsigned long)ipath->fspath->val[i]);
 603
 604         free_ipath(ipath);
 605         return 0;
 606
 607 err:
 608         printk_in_rcu(KERN_WARNING "BTRFS: %s at logical %llu on dev "
 609                 "%s, sector %llu, root %llu, inode %llu, offset %llu: path "
 610                 "resolving failed with ret=%d\n", swarn->errstr,
 611                 swarn->logical, rcu_str_deref(swarn->dev->name),
 612                 (unsigned long long)swarn->sector, root, inum, offset, ret);
 613
 614         free_ipath(ipath);
 615         return 0;
 616 }
 617
 618 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 619 {
 620         struct btrfs_device *dev;
 621         struct btrfs_fs_info *fs_info;
 622         struct btrfs_path *path;
 623         struct btrfs_key found_key;
 624         struct extent_buffer *eb;
 625         struct btrfs_extent_item *ei;
 626         struct scrub_warning swarn;
 627         unsigned long ptr = 0;
 628         u64 extent_item_pos;
 629         u64 flags = 0;
 630         u64 ref_root;
 631         u32 item_size;
 632         u8 ref_level;
 633         int ret;
 634
 635         WARN_ON(sblock->page_count < 1);
 636         dev = sblock->pagev[0]->dev;
 637         fs_info = sblock->sctx->dev_root->fs_info;
 638
 639         path = btrfs_alloc_path();
 640         if (!path)
 641                 return;
 642
 643         swarn.sector = (sblock->pagev[0]->physical) >> 9;
 644         swarn.logical = sblock->pagev[0]->logical;
 645         swarn.errstr = errstr;
 646         swarn.dev = NULL;
 647
 648         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 649                                   &flags);
 650         if (ret < 0)
 651                 goto out;
 652
 653         extent_item_pos = swarn.logical - found_key.objectid;
 654         swarn.extent_item_size = found_key.offset;
 655
 656         eb = path->nodes[0];
 657         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 658         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 659
 660         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 661                 do {
 662                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
 663                                                       item_size, &ref_root,
 664                                                       &ref_level);
 665                         printk_in_rcu(KERN_WARNING
 666                                 "BTRFS: %s at logical %llu on dev %s, "
 667                                 "sector %llu: metadata %s (level %d) in tree "
 668                                 "%llu\n", errstr, swarn.logical,
 669                                 rcu_str_deref(dev->name),
 670                                 (unsigned long long)swarn.sector,
 671                                 ref_level ? "node" : "leaf",
 672                                 ret < 0 ? -1 : ref_level,
 673                                 ret < 0 ? -1 : ref_root);
 674                 } while (ret != 1);
 675                 btrfs_release_path(path);
 676         } else {
 677                 btrfs_release_path(path);
 678                 swarn.path = path;
 679                 swarn.dev = dev;
 680                 iterate_extent_inodes(fs_info, found_key.objectid,
 681                                         extent_item_pos, 1,
 682                                         scrub_print_warning_inode, &swarn);
 683         }
 684
 685 out:
 686         btrfs_free_path(path);
 687 }
 688
 689 static int scrub_fixup_readpage(u64 inum, u64 offset, u64 root, void *fixup_ctx)
 690 {
 691         struct page *page = NULL;
 692         unsigned long index;
 693         struct scrub_fixup_nodatasum *fixup = fixup_ctx;
 694         int ret;
 695         int corrected = 0;
 696         struct btrfs_key key;
 697         struct inode *inode = NULL;
 698         struct btrfs_fs_info *fs_info;
 699         u64 end = offset + PAGE_SIZE - 1;
 700         struct btrfs_root *local_root;
 701         int srcu_index;
 702
 703         key.objectid = root;
 704         key.type = BTRFS_ROOT_ITEM_KEY;
 705         key.offset = (u64)-1;
 706
 707         fs_info = fixup->root->fs_info;
 708         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
 709
 710         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
 711         if (IS_ERR(local_root)) {
 712                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 713                 return PTR_ERR(local_root);
 714         }
 715
 716         key.type = BTRFS_INODE_ITEM_KEY;
 717         key.objectid = inum;
 718         key.offset = 0;
 719         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
 720         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
 721         if (IS_ERR(inode))
 722                 return PTR_ERR(inode);
 723
 724         index = offset >> PAGE_CACHE_SHIFT;
 725
 726         page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
 727         if (!page) {
 728                 ret = -ENOMEM;
 729                 goto out;
 730         }
 731
 732         if (PageUptodate(page)) {
 733                 if (PageDirty(page)) {
 734                         /*
 735                          * we need to write the data to the defect sector. the
 736                          * data that was in that sector is not in memory,
 737                          * because the page was modified. we must not write the
 738                          * modified page to that sector.
 739                          *
 740                          * TODO: what could be done here: wait for the delalloc
 741                          *       runner to write out that page (might involve
 742                          *       COW) and see whether the sector is still
 743                          *       referenced afterwards.
 744                          *
 745                          * For the meantime, we'll treat this error
 746                          * incorrectable, although there is a chance that a
 747                          * later scrub will find the bad sector again and that
 748                          * there's no dirty page in memory, then.
 749                          */
 750                         ret = -EIO;
 751                         goto out;
 752                 }
 753                 ret = repair_io_failure(inode, offset, PAGE_SIZE,
 754                                         fixup->logical, page,
 755                                         offset - page_offset(page),
 756                                         fixup->mirror_num);
 757                 unlock_page(page);
 758                 corrected = !ret;
 759         } else {
 760                 /*
 761                  * we need to get good data first. the general readpage path
 762                  * will call repair_io_failure for us, we just have to make
 763                  * sure we read the bad mirror.
 764                  */
 765                 ret = set_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 766                                         EXTENT_DAMAGED, GFP_NOFS);
 767                 if (ret) {
 768                         /* set_extent_bits should give proper error */
 769                         WARN_ON(ret > 0);
 770                         if (ret > 0)
 771                                 ret = -EFAULT;
 772                         goto out;
 773                 }
 774
 775                 ret = extent_read_full_page(&BTRFS_I(inode)->io_tree, page,
 776                                                 btrfs_get_extent,
 777                                                 fixup->mirror_num);
 778                 wait_on_page_locked(page);
 779
 780                 corrected = !test_range_bit(&BTRFS_I(inode)->io_tree, offset,
 781                                                 end, EXTENT_DAMAGED, 0, NULL);
 782                 if (!corrected)
 783                         clear_extent_bits(&BTRFS_I(inode)->io_tree, offset, end,
 784                                                 EXTENT_DAMAGED, GFP_NOFS);
 785         }
 786
 787 out:
 788         if (page)
 789                 put_page(page);
 790
 791         iput(inode);
 792
 793         if (ret < 0)
 794                 return ret;
 795
 796         if (ret == 0 && corrected) {
 797                 /*
 798                  * we only need to call readpage for one of the inodes belonging
 799                  * to this extent. so make iterate_extent_inodes stop
 800                  */
 801                 return 1;
 802         }
 803
 804         return -EIO;
 805 }
 806
 807 static void scrub_fixup_nodatasum(struct btrfs_work *work)
 808 {
 809         int ret;
 810         struct scrub_fixup_nodatasum *fixup;
 811         struct scrub_ctx *sctx;
 812         struct btrfs_trans_handle *trans = NULL;
 813         struct btrfs_path *path;
 814         int uncorrectable = 0;
 815
 816         fixup = container_of(work, struct scrub_fixup_nodatasum, work);
 817         sctx = fixup->sctx;
 818
 819         path = btrfs_alloc_path();
 820         if (!path) {
 821                 spin_lock(&sctx->stat_lock);
 822                 ++sctx->stat.malloc_errors;
 823                 spin_unlock(&sctx->stat_lock);
 824                 uncorrectable = 1;
 825                 goto out;
 826         }
 827
 828         trans = btrfs_join_transaction(fixup->root);
 829         if (IS_ERR(trans)) {
 830                 uncorrectable = 1;
 831                 goto out;
 832         }
 833
 834         /*
 835          * the idea is to trigger a regular read through the standard path. we
 836          * read a page from the (failed) logical address by specifying the
 837          * corresponding copynum of the failed sector. thus, that readpage is
 838          * expected to fail.
 839          * that is the point where on-the-fly error correction will kick in
 840          * (once it's finished) and rewrite the failed sector if a good copy
 841          * can be found.
 842          */
 843         ret = iterate_inodes_from_logical(fixup->logical, fixup->root->fs_info,
 844                                                 path, scrub_fixup_readpage,
 845                                                 fixup);
 846         if (ret < 0) {
 847                 uncorrectable = 1;
 848                 goto out;
 849         }
 850         WARN_ON(ret != 1);
 851
 852         spin_lock(&sctx->stat_lock);
 853         ++sctx->stat.corrected_errors;
 854         spin_unlock(&sctx->stat_lock);
 855
 856 out:
 857         if (trans && !IS_ERR(trans))
 858                 btrfs_end_transaction(trans, fixup->root);
 859         if (uncorrectable) {
 860                 spin_lock(&sctx->stat_lock);
 861                 ++sctx->stat.uncorrectable_errors;
 862                 spin_unlock(&sctx->stat_lock);
 863                 btrfs_dev_replace_stats_inc(
 864                         &sctx->dev_root->fs_info->dev_replace.
 865                         num_uncorrectable_read_errors);
 866                 printk_ratelimited_in_rcu(KERN_ERR "BTRFS: "
 867                     "unable to fixup (nodatasum) error at logical %llu on dev %s\n",
 868                         fixup->logical, rcu_str_deref(fixup->dev->name));
 869         }
 870
 871         btrfs_free_path(path);
 872         kfree(fixup);
 873
 874         scrub_pending_trans_workers_dec(sctx);
 875 }
 876
 877 static inline void scrub_get_recover(struct scrub_recover *recover)
 878 {
 879         atomic_inc(&recover->refs);
 880 }
 881
 882 static inline void scrub_put_recover(struct scrub_recover *recover)
 883 {
 884         if (atomic_dec_and_test(&recover->refs)) {
 885                 btrfs_put_bbio(recover->bbio);
 886                 kfree(recover);
 887         }
 888 }
 889
 890 /*
 891  * scrub_handle_errored_block gets called when either verification of the
 892  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 893  * case, this function handles all pages in the bio, even though only one
 894  * may be bad.
 895  * The goal of this function is to repair the errored block by using the
 896  * contents of one of the mirrors.
 897  */
 898 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 899 {
 900         struct scrub_ctx *sctx = sblock_to_check->sctx;
 901         struct btrfs_device *dev;
 902         struct btrfs_fs_info *fs_info;
 903         u64 length;
 904         u64 logical;
 905         u64 generation;
 906         unsigned int failed_mirror_index;
 907         unsigned int is_metadata;
 908         unsigned int have_csum;
 909         u8 *csum;
 910         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 911         struct scrub_block *sblock_bad;
 912         int ret;
 913         int mirror_index;
 914         int page_num;
 915         int success;
 916         static DEFINE_RATELIMIT_STATE(_rs, DEFAULT_RATELIMIT_INTERVAL,
 917                                       DEFAULT_RATELIMIT_BURST);
 918
 919         BUG_ON(sblock_to_check->page_count < 1);
 920         fs_info = sctx->dev_root->fs_info;
 921         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 922                 /*
 923                  * if we find an error in a super block, we just report it.
 924                  * They will get written with the next transaction commit
 925                  * anyway
 926                  */
 927                 spin_lock(&sctx->stat_lock);
 928                 ++sctx->stat.super_errors;
 929                 spin_unlock(&sctx->stat_lock);
 930                 return 0;
 931         }
 932         length = sblock_to_check->page_count * PAGE_SIZE;
 933         logical = sblock_to_check->pagev[0]->logical;
 934         generation = sblock_to_check->pagev[0]->generation;
 935         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 936         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 937         is_metadata = !(sblock_to_check->pagev[0]->flags &
 938                         BTRFS_EXTENT_FLAG_DATA);
 939         have_csum = sblock_to_check->pagev[0]->have_csum;
 940         csum = sblock_to_check->pagev[0]->csum;
 941         dev = sblock_to_check->pagev[0]->dev;
 942
 943         if (sctx->is_dev_replace && !is_metadata && !have_csum) {
 944                 sblocks_for_recheck = NULL;
 945                 goto nodatasum_case;
 946         }
 947
 948         /*
 949          * read all mirrors one after the other. This includes to
 950          * re-read the extent or metadata block that failed (that was
 951          * the cause that this fixup code is called) another time,
 952          * page by page this time in order to know which pages
 953          * caused I/O errors and which ones are good (for all mirrors).
 954          * It is the goal to handle the situation when more than one
 955          * mirror contains I/O errors, but the errors do not
 956          * overlap, i.e. the data can be repaired by selecting the
 957          * pages from those mirrors without I/O error on the
 958          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 959          * would be that mirror #1 has an I/O error on the first page,
 960          * the second page is good, and mirror #2 has an I/O error on
 961          * the second page, but the first page is good.
 962          * Then the first page of the first mirror can be repaired by
 963          * taking the first page of the second mirror, and the
 964          * second page of the second mirror can be repaired by
 965          * copying the contents of the 2nd page of the 1st mirror.
 966          * One more note: if the pages of one mirror contain I/O
 967          * errors, the checksum cannot be verified. In order to get
 968          * the best data for repairing, the first attempt is to find
 969          * a mirror without I/O errors and with a validated checksum.
 970          * Only if this is not possible, the pages are picked from
 971          * mirrors with I/O errors without considering the checksum.
 972          * If the latter is the case, at the end, the checksum of the
 973          * repaired area is verified in order to correctly maintain
 974          * the statistics.
 975          */
 976
 977         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
 978                                       sizeof(*sblocks_for_recheck), GFP_NOFS);
 979         if (!sblocks_for_recheck) {
 980                 spin_lock(&sctx->stat_lock);
 981                 sctx->stat.malloc_errors++;
 982                 sctx->stat.read_errors++;
 983                 sctx->stat.uncorrectable_errors++;
 984                 spin_unlock(&sctx->stat_lock);
 985                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 986                 goto out;
 987         }
 988
 989         /* setup the context, map the logical blocks and alloc the pages */
 990         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
 991         if (ret) {
 992                 spin_lock(&sctx->stat_lock);
 993                 sctx->stat.read_errors++;
 994                 sctx->stat.uncorrectable_errors++;
 995                 spin_unlock(&sctx->stat_lock);
 996                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 997                 goto out;
 998         }
 999         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
1000         sblock_bad = sblocks_for_recheck + failed_mirror_index;
1001
1002         /* build and submit the bios for the failed mirror, check checksums */
1003         scrub_recheck_block(fs_info, sblock_bad, is_metadata, have_csum,
1004                             csum, generation, sctx->csum_size, 1);
1005
1006         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
1007             sblock_bad->no_io_error_seen) {
1008                 /*
1009                  * the error disappeared after reading page by page, or
1010                  * the area was part of a huge bio and other parts of the
1011                  * bio caused I/O errors, or the block layer merged several
1012                  * read requests into one and the error is caused by a
1013                  * different bio (usually one of the two latter cases is
1014                  * the cause)
1015                  */
1016                 spin_lock(&sctx->stat_lock);
1017                 sctx->stat.unverified_errors++;
1018                 sblock_to_check->data_corrected = 1;
1019                 spin_unlock(&sctx->stat_lock);
1020
1021                 if (sctx->is_dev_replace)
1022                         scrub_write_block_to_dev_replace(sblock_bad);
1023                 goto out;
1024         }
1025
1026         if (!sblock_bad->no_io_error_seen) {
1027                 spin_lock(&sctx->stat_lock);
1028                 sctx->stat.read_errors++;
1029                 spin_unlock(&sctx->stat_lock);
1030                 if (__ratelimit(&_rs))
1031                         scrub_print_warning("i/o error", sblock_to_check);
1032                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
1033         } else if (sblock_bad->checksum_error) {
1034                 spin_lock(&sctx->stat_lock);
1035                 sctx->stat.csum_errors++;
1036                 spin_unlock(&sctx->stat_lock);
1037                 if (__ratelimit(&_rs))
1038                         scrub_print_warning("checksum error", sblock_to_check);
1039                 btrfs_dev_stat_inc_and_print(dev,
1040                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
1041         } else if (sblock_bad->header_error) {
1042                 spin_lock(&sctx->stat_lock);
1043                 sctx->stat.verify_errors++;
1044                 spin_unlock(&sctx->stat_lock);
1045                 if (__ratelimit(&_rs))
1046                         scrub_print_warning("checksum/header error",
1047                                             sblock_to_check);
1048                 if (sblock_bad->generation_error)
1049                         btrfs_dev_stat_inc_and_print(dev,
1050                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1051                 else
1052                         btrfs_dev_stat_inc_and_print(dev,
1053                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1054         }
1055
1056         if (sctx->readonly) {
1057                 ASSERT(!sctx->is_dev_replace);
1058                 goto out;
1059         }
1060
1061         if (!is_metadata && !have_csum) {
1062                 struct scrub_fixup_nodatasum *fixup_nodatasum;
1063
1064                 WARN_ON(sctx->is_dev_replace);
1065
1066 nodatasum_case:
1067
1068                 /*
1069                  * !is_metadata and !have_csum, this means that the data
1070                  * might not be COW'ed, that it might be modified
1071                  * concurrently. The general strategy to work on the
1072                  * commit root does not help in the case when COW is not
1073                  * used.
1074                  */
1075                 fixup_nodatasum = kzalloc(sizeof(*fixup_nodatasum), GFP_NOFS);
1076                 if (!fixup_nodatasum)
1077                         goto did_not_correct_error;
1078                 fixup_nodatasum->sctx = sctx;
1079                 fixup_nodatasum->dev = dev;
1080                 fixup_nodatasum->logical = logical;
1081                 fixup_nodatasum->root = fs_info->extent_root;
1082                 fixup_nodatasum->mirror_num = failed_mirror_index + 1;
1083                 scrub_pending_trans_workers_inc(sctx);
1084                 btrfs_init_work(&fixup_nodatasum->work, btrfs_scrub_helper,
1085                                 scrub_fixup_nodatasum, NULL, NULL);
1086                 btrfs_queue_work(fs_info->scrub_workers,
1087                                  &fixup_nodatasum->work);
1088                 goto out;
1089         }
1090
1091         /*
1092          * now build and submit the bios for the other mirrors, check
1093          * checksums.
1094          * First try to pick the mirror which is completely without I/O
1095          * errors and also does not have a checksum error.
1096          * If one is found, and if a checksum is present, the full block
1097          * that is known to contain an error is rewritten. Afterwards
1098          * the block is known to be corrected.
1099          * If a mirror is found which is completely correct, and no
1100          * checksum is present, only those pages are rewritten that had
1101          * an I/O error in the block to be repaired, since it cannot be
1102          * determined, which copy of the other pages is better (and it
1103          * could happen otherwise that a correct page would be
1104          * overwritten by a bad one).
1105          */
1106         for (mirror_index = 0;
1107              mirror_index < BTRFS_MAX_MIRRORS &&
1108              sblocks_for_recheck[mirror_index].page_count > 0;
1109              mirror_index++) {
1110                 struct scrub_block *sblock_other;
1111
1112                 if (mirror_index == failed_mirror_index)
1113                         continue;
1114                 sblock_other = sblocks_for_recheck + mirror_index;
1115
1116                 /* build and submit the bios, check checksums */
1117                 scrub_recheck_block(fs_info, sblock_other, is_metadata,
1118                                     have_csum, csum, generation,
1119                                     sctx->csum_size, 0);
1120
1121                 if (!sblock_other->header_error &&
1122                     !sblock_other->checksum_error &&
1123                     sblock_other->no_io_error_seen) {
1124                         if (sctx->is_dev_replace) {
1125                                 scrub_write_block_to_dev_replace(sblock_other);
1126                                 goto corrected_error;
1127                         } else {
1128                                 ret = scrub_repair_block_from_good_copy(
1129                                                 sblock_bad, sblock_other);
1130                                 if (!ret)
1131                                         goto corrected_error;
1132                         }
1133                 }
1134         }
1135
1136         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1137                 goto did_not_correct_error;
1138
1139         /*
1140          * In case of I/O errors in the area that is supposed to be
1141          * repaired, continue by picking good copies of those pages.
1142          * Select the good pages from mirrors to rewrite bad pages from
1143          * the area to fix. Afterwards verify the checksum of the block
1144          * that is supposed to be repaired. This verification step is
1145          * only done for the purpose of statistic counting and for the
1146          * final scrub report, whether errors remain.
1147          * A perfect algorithm could make use of the checksum and try
1148          * all possible combinations of pages from the different mirrors
1149          * until the checksum verification succeeds. For example, when
1150          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1151          * of mirror #2 is readable but the final checksum test fails,
1152          * then the 2nd page of mirror #3 could be tried, whether now
1153          * the final checksum succeedes. But this would be a rare
1154          * exception and is therefore not implemented. At least it is
1155          * avoided that the good copy is overwritten.
1156          * A more useful improvement would be to pick the sectors
1157          * without I/O error based on sector sizes (512 bytes on legacy
1158          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1159          * mirror could be repaired by taking 512 byte of a different
1160          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1161          * area are unreadable.
1162          */
1163         success = 1;
1164         for (page_num = 0; page_num < sblock_bad->page_count;
1165              page_num++) {
1166                 struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1167                 struct scrub_block *sblock_other = NULL;
1168
1169                 /* skip no-io-error page in scrub */
1170                 if (!page_bad->io_error && !sctx->is_dev_replace)
1171                         continue;
1172
1173                 /* try to find no-io-error page in mirrors */
1174                 if (page_bad->io_error) {
1175                         for (mirror_index = 0;
1176                              mirror_index < BTRFS_MAX_MIRRORS &&
1177                              sblocks_for_recheck[mirror_index].page_count > 0;
1178                              mirror_index++) {
1179                                 if (!sblocks_for_recheck[mirror_index].
1180                                     pagev[page_num]->io_error) {
1181                                         sblock_other = sblocks_for_recheck +
1182                                                        mirror_index;
1183                                         break;
1184                                 }
1185                         }
1186                         if (!sblock_other)
1187                                 success = 0;
1188                 }
1189
1190                 if (sctx->is_dev_replace) {
1191                         /*
1192                          * did not find a mirror to fetch the page
1193                          * from. scrub_write_page_to_dev_replace()
1194                          * handles this case (page->io_error), by
1195                          * filling the block with zeros before
1196                          * submitting the write request
1197                          */
1198                         if (!sblock_other)
1199                                 sblock_other = sblock_bad;
1200
1201                         if (scrub_write_page_to_dev_replace(sblock_other,
1202                                                             page_num) != 0) {
1203                                 btrfs_dev_replace_stats_inc(
1204                                         &sctx->dev_root->
1205                                         fs_info->dev_replace.
1206                                         num_write_errors);
1207                                 success = 0;
1208                         }
1209                 } else if (sblock_other) {
1210                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1211                                                                sblock_other,
1212                                                                page_num, 0);
1213                         if (0 == ret)
1214                                 page_bad->io_error = 0;
1215                         else
1216                                 success = 0;
1217                 }
1218         }
1219
1220         if (success && !sctx->is_dev_replace) {
1221                 if (is_metadata || have_csum) {
1222                         /*
1223                          * need to verify the checksum now that all
1224                          * sectors on disk are repaired (the write
1225                          * request for data to be repaired is on its way).
1226                          * Just be lazy and use scrub_recheck_block()
1227                          * which re-reads the data before the checksum
1228                          * is verified, but most likely the data comes out
1229                          * of the page cache.
1230                          */
1231                         scrub_recheck_block(fs_info, sblock_bad,
1232                                             is_metadata, have_csum, csum,
1233                                             generation, sctx->csum_size, 1);
1234                         if (!sblock_bad->header_error &&
1235                             !sblock_bad->checksum_error &&
1236                             sblock_bad->no_io_error_seen)
1237                                 goto corrected_error;
1238                         else
1239                                 goto did_not_correct_error;
1240                 } else {
1241 corrected_error:
1242                         spin_lock(&sctx->stat_lock);
1243                         sctx->stat.corrected_errors++;
1244                         sblock_to_check->data_corrected = 1;
1245                         spin_unlock(&sctx->stat_lock);
1246                         printk_ratelimited_in_rcu(KERN_ERR
1247                                 "BTRFS: fixed up error at logical %llu on dev %s\n",
1248                                 logical, rcu_str_deref(dev->name));
1249                 }
1250         } else {
1251 did_not_correct_error:
1252                 spin_lock(&sctx->stat_lock);
1253                 sctx->stat.uncorrectable_errors++;
1254                 spin_unlock(&sctx->stat_lock);
1255                 printk_ratelimited_in_rcu(KERN_ERR
1256                         "BTRFS: unable to fixup (regular) error at logical %llu on dev %s\n",
1257                         logical, rcu_str_deref(dev->name));
1258         }
1259
1260 out:
1261         if (sblocks_for_recheck) {
1262                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1263                      mirror_index++) {
1264                         struct scrub_block *sblock = sblocks_for_recheck +
1265                                                      mirror_index;
1266                         struct scrub_recover *recover;
1267                         int page_index;
1268
1269                         for (page_index = 0; page_index < sblock->page_count;
1270                              page_index++) {
1271                                 sblock->pagev[page_index]->sblock = NULL;
1272                                 recover = sblock->pagev[page_index]->recover;
1273                                 if (recover) {
1274                                         scrub_put_recover(recover);
1275                                         sblock->pagev[page_index]->recover =
1276                                                                         NULL;
1277                                 }
1278                                 scrub_page_put(sblock->pagev[page_index]);
1279                         }
1280                 }
1281                 kfree(sblocks_for_recheck);
1282         }
1283
1284         return 0;
1285 }
1286
1287 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1288 {
1289         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1290                 return 2;
1291         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1292                 return 3;
1293         else
1294                 return (int)bbio->num_stripes;
1295 }
1296
1297 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1298                                                  u64 *raid_map,
1299                                                  u64 mapped_length,
1300                                                  int nstripes, int mirror,
1301                                                  int *stripe_index,
1302                                                  u64 *stripe_offset)
1303 {
1304         int i;
1305
1306         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1307                 /* RAID5/6 */
1308                 for (i = 0; i < nstripes; i++) {
1309                         if (raid_map[i] == RAID6_Q_STRIPE ||
1310                             raid_map[i] == RAID5_P_STRIPE)
1311                                 continue;
1312
1313                         if (logical >= raid_map[i] &&
1314                             logical < raid_map[i] + mapped_length)
1315                                 break;
1316                 }
1317
1318                 *stripe_index = i;
1319                 *stripe_offset = logical - raid_map[i];
1320         } else {
1321                 /* The other RAID type */
1322                 *stripe_index = mirror;
1323                 *stripe_offset = 0;
1324         }
1325 }
1326
1327 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1328                                      struct scrub_block *sblocks_for_recheck)
1329 {
1330         struct scrub_ctx *sctx = original_sblock->sctx;
1331         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
1332         u64 length = original_sblock->page_count * PAGE_SIZE;
1333         u64 logical = original_sblock->pagev[0]->logical;
1334         struct scrub_recover *recover;
1335         struct btrfs_bio *bbio;
1336         u64 sublen;
1337         u64 mapped_length;
1338         u64 stripe_offset;
1339         int stripe_index;
1340         int page_index = 0;
1341         int mirror_index;
1342         int nmirrors;
1343         int ret;
1344
1345         /*
1346          * note: the two members refs and outstanding_pages
1347          * are not used (and not set) in the blocks that are used for
1348          * the recheck procedure
1349          */
1350
1351         while (length > 0) {
1352                 sublen = min_t(u64, length, PAGE_SIZE);
1353                 mapped_length = sublen;
1354                 bbio = NULL;
1355
1356                 /*
1357                  * with a length of PAGE_SIZE, each returned stripe
1358                  * represents one mirror
1359                  */
1360                 ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical,
1361                                        &mapped_length, &bbio, 0, 1);
1362                 if (ret || !bbio || mapped_length < sublen) {
1363                         btrfs_put_bbio(bbio);
1364                         return -EIO;
1365                 }
1366
1367                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1368                 if (!recover) {
1369                         btrfs_put_bbio(bbio);
1370                         return -ENOMEM;
1371                 }
1372
1373                 atomic_set(&recover->refs, 1);
1374                 recover->bbio = bbio;
1375                 recover->map_length = mapped_length;
1376
1377                 BUG_ON(page_index >= SCRUB_PAGES_PER_RD_BIO);
1378
1379                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1380
1381                 for (mirror_index = 0; mirror_index < nmirrors;
1382                      mirror_index++) {
1383                         struct scrub_block *sblock;
1384                         struct scrub_page *page;
1385
1386                         sblock = sblocks_for_recheck + mirror_index;
1387                         sblock->sctx = sctx;
1388                         page = kzalloc(sizeof(*page), GFP_NOFS);
1389                         if (!page) {
1390 leave_nomem:
1391                                 spin_lock(&sctx->stat_lock);
1392                                 sctx->stat.malloc_errors++;
1393                                 spin_unlock(&sctx->stat_lock);
1394                                 scrub_put_recover(recover);
1395                                 return -ENOMEM;
1396                         }
1397                         scrub_page_get(page);
1398                         sblock->pagev[page_index] = page;
1399                         page->logical = logical;
1400
1401                         scrub_stripe_index_and_offset(logical,
1402                                                       bbio->map_type,
1403                                                       bbio->raid_map,
1404                                                       mapped_length,
1405                                                       bbio->num_stripes -
1406                                                       bbio->num_tgtdevs,
1407                                                       mirror_index,
1408                                                       &stripe_index,
1409                                                       &stripe_offset);
1410                         page->physical = bbio->stripes[stripe_index].physical +
1411                                          stripe_offset;
1412                         page->dev = bbio->stripes[stripe_index].dev;
1413
1414                         BUG_ON(page_index >= original_sblock->page_count);
1415                         page->physical_for_dev_replace =
1416                                 original_sblock->pagev[page_index]->
1417                                 physical_for_dev_replace;
1418                         /* for missing devices, dev->bdev is NULL */
1419                         page->mirror_num = mirror_index + 1;
1420                         sblock->page_count++;
1421                         page->page = alloc_page(GFP_NOFS);
1422                         if (!page->page)
1423                                 goto leave_nomem;
1424
1425                         scrub_get_recover(recover);
1426                         page->recover = recover;
1427                 }
1428                 scrub_put_recover(recover);
1429                 length -= sublen;
1430                 logical += sublen;
1431                 page_index++;
1432         }
1433
1434         return 0;
1435 }
1436
1437 struct scrub_bio_ret {
1438         struct completion event;
1439         int error;
1440 };
1441
1442 static void scrub_bio_wait_endio(struct bio *bio, int error)
1443 {
1444         struct scrub_bio_ret *ret = bio->bi_private;
1445
1446         ret->error = error;
1447         complete(&ret->event);
1448 }
1449
1450 static inline int scrub_is_page_on_raid56(struct scrub_page *page)
1451 {
1452         return page->recover &&
1453                (page->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
1454 }
1455
1456 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1457                                         struct bio *bio,
1458                                         struct scrub_page *page)
1459 {
1460         struct scrub_bio_ret done;
1461         int ret;
1462
1463         init_completion(&done.event);
1464         done.error = 0;
1465         bio->bi_iter.bi_sector = page->logical >> 9;
1466         bio->bi_private = &done;
1467         bio->bi_end_io = scrub_bio_wait_endio;
1468
1469         ret = raid56_parity_recover(fs_info->fs_root, bio, page->recover->bbio,
1470                                     page->recover->map_length,
1471                                     page->mirror_num, 0);
1472         if (ret)
1473                 return ret;
1474
1475         wait_for_completion(&done.event);
1476         if (done.error)
1477                 return -EIO;
1478
1479         return 0;
1480 }
1481
1482 /*
1483  * this function will check the on disk data for checksum errors, header
1484  * errors and read I/O errors. If any I/O errors happen, the exact pages
1485  * which are errored are marked as being bad. The goal is to enable scrub
1486  * to take those pages that are not errored from all the mirrors so that
1487  * the pages that are errored in the just handled mirror can be repaired.
1488  */
1489 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1490                                 struct scrub_block *sblock, int is_metadata,
1491                                 int have_csum, u8 *csum, u64 generation,
1492                                 u16 csum_size, int retry_failed_mirror)
1493 {
1494         int page_num;
1495
1496         sblock->no_io_error_seen = 1;
1497         sblock->header_error = 0;
1498         sblock->checksum_error = 0;
1499
1500         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1501                 struct bio *bio;
1502                 struct scrub_page *page = sblock->pagev[page_num];
1503
1504                 if (page->dev->bdev == NULL) {
1505                         page->io_error = 1;
1506                         sblock->no_io_error_seen = 0;
1507                         continue;
1508                 }
1509
1510                 WARN_ON(!page->page);
1511                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1512                 if (!bio) {
1513                         page->io_error = 1;
1514                         sblock->no_io_error_seen = 0;
1515                         continue;
1516                 }
1517                 bio->bi_bdev = page->dev->bdev;
1518
1519                 bio_add_page(bio, page->page, PAGE_SIZE, 0);
1520                 if (!retry_failed_mirror && scrub_is_page_on_raid56(page)) {
1521                         if (scrub_submit_raid56_bio_wait(fs_info, bio, page))
1522                                 sblock->no_io_error_seen = 0;
1523                 } else {
1524                         bio->bi_iter.bi_sector = page->physical >> 9;
1525
1526                         if (btrfsic_submit_bio_wait(READ, bio))
1527                                 sblock->no_io_error_seen = 0;
1528                 }
1529
1530                 bio_put(bio);
1531         }
1532
1533         if (sblock->no_io_error_seen)
1534                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
1535                                              have_csum, csum, generation,
1536                                              csum_size);
1537
1538         return;
1539 }
1540
1541 static inline int scrub_check_fsid(u8 fsid[],
1542                                    struct scrub_page *spage)
1543 {
1544         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1545         int ret;
1546
1547         ret = memcmp(fsid, fs_devices->fsid, BTRFS_UUID_SIZE);
1548         return !ret;
1549 }
1550
1551 static void scrub_recheck_block_checksum(struct btrfs_fs_info *fs_info,
1552                                          struct scrub_block *sblock,
1553                                          int is_metadata, int have_csum,
1554                                          const u8 *csum, u64 generation,
1555                                          u16 csum_size)
1556 {
1557         int page_num;
1558         u8 calculated_csum[BTRFS_CSUM_SIZE];
1559         u32 crc = ~(u32)0;
1560         void *mapped_buffer;
1561
1562         WARN_ON(!sblock->pagev[0]->page);
1563         if (is_metadata) {
1564                 struct btrfs_header *h;
1565
1566                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1567                 h = (struct btrfs_header *)mapped_buffer;
1568
1569                 if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h) ||
1570                     !scrub_check_fsid(h->fsid, sblock->pagev[0]) ||
1571                     memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1572                            BTRFS_UUID_SIZE)) {
1573                         sblock->header_error = 1;
1574                 } else if (generation != btrfs_stack_header_generation(h)) {
1575                         sblock->header_error = 1;
1576                         sblock->generation_error = 1;
1577                 }
1578                 csum = h->csum;
1579         } else {
1580                 if (!have_csum)
1581                         return;
1582
1583                 mapped_buffer = kmap_atomic(sblock->pagev[0]->page);
1584         }
1585
1586         for (page_num = 0;;) {
1587                 if (page_num == 0 && is_metadata)
1588                         crc = btrfs_csum_data(
1589                                 ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE,
1590                                 crc, PAGE_SIZE - BTRFS_CSUM_SIZE);
1591                 else
1592                         crc = btrfs_csum_data(mapped_buffer, crc, PAGE_SIZE);
1593
1594                 kunmap_atomic(mapped_buffer);
1595                 page_num++;
1596                 if (page_num >= sblock->page_count)
1597                         break;
1598                 WARN_ON(!sblock->pagev[page_num]->page);
1599
1600                 mapped_buffer = kmap_atomic(sblock->pagev[page_num]->page);
1601         }
1602
1603         btrfs_csum_final(crc, calculated_csum);
1604         if (memcmp(calculated_csum, csum, csum_size))
1605                 sblock->checksum_error = 1;
1606 }
1607
1608 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1609                                              struct scrub_block *sblock_good)
1610 {
1611         int page_num;
1612         int ret = 0;
1613
1614         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1615                 int ret_sub;
1616
1617                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1618                                                            sblock_good,
1619                                                            page_num, 1);
1620                 if (ret_sub)
1621                         ret = ret_sub;
1622         }
1623
1624         return ret;
1625 }
1626
1627 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1628                                             struct scrub_block *sblock_good,
1629                                             int page_num, int force_write)
1630 {
1631         struct scrub_page *page_bad = sblock_bad->pagev[page_num];
1632         struct scrub_page *page_good = sblock_good->pagev[page_num];
1633
1634         BUG_ON(page_bad->page == NULL);
1635         BUG_ON(page_good->page == NULL);
1636         if (force_write || sblock_bad->header_error ||
1637             sblock_bad->checksum_error || page_bad->io_error) {
1638                 struct bio *bio;
1639                 int ret;
1640
1641                 if (!page_bad->dev->bdev) {
1642                         printk_ratelimited(KERN_WARNING "BTRFS: "
1643                                 "scrub_repair_page_from_good_copy(bdev == NULL) "
1644                                 "is unexpected!\n");
1645                         return -EIO;
1646                 }
1647
1648                 bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
1649                 if (!bio)
1650                         return -EIO;
1651                 bio->bi_bdev = page_bad->dev->bdev;
1652                 bio->bi_iter.bi_sector = page_bad->physical >> 9;
1653
1654                 ret = bio_add_page(bio, page_good->page, PAGE_SIZE, 0);
1655                 if (PAGE_SIZE != ret) {
1656                         bio_put(bio);
1657                         return -EIO;
1658                 }
1659
1660                 if (btrfsic_submit_bio_wait(WRITE, bio)) {
1661                         btrfs_dev_stat_inc_and_print(page_bad->dev,
1662                                 BTRFS_DEV_STAT_WRITE_ERRS);
1663                         btrfs_dev_replace_stats_inc(
1664                                 &sblock_bad->sctx->dev_root->fs_info->
1665                                 dev_replace.num_write_errors);
1666                         bio_put(bio);
1667                         return -EIO;
1668                 }
1669                 bio_put(bio);
1670         }
1671
1672         return 0;
1673 }
1674
1675 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1676 {
1677         int page_num;
1678
1679         /*
1680          * This block is used for the check of the parity on the source device,
1681          * so the data needn't be written into the destination device.
1682          */
1683         if (sblock->sparity)
1684                 return;
1685
1686         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1687                 int ret;
1688
1689                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1690                 if (ret)
1691                         btrfs_dev_replace_stats_inc(
1692                                 &sblock->sctx->dev_root->fs_info->dev_replace.
1693                                 num_write_errors);
1694         }
1695 }
1696
1697 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1698                                            int page_num)
1699 {
1700         struct scrub_page *spage = sblock->pagev[page_num];
1701
1702         BUG_ON(spage->page == NULL);
1703         if (spage->io_error) {
1704                 void *mapped_buffer = kmap_atomic(spage->page);
1705
1706                 memset(mapped_buffer, 0, PAGE_CACHE_SIZE);
1707                 flush_dcache_page(spage->page);
1708                 kunmap_atomic(mapped_buffer);
1709         }
1710         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1711 }
1712
1713 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1714                                     struct scrub_page *spage)
1715 {
1716         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1717         struct scrub_bio *sbio;
1718         int ret;
1719
1720         mutex_lock(&wr_ctx->wr_lock);
1721 again:
1722         if (!wr_ctx->wr_curr_bio) {
1723                 wr_ctx->wr_curr_bio = kzalloc(sizeof(*wr_ctx->wr_curr_bio),
1724                                               GFP_NOFS);
1725                 if (!wr_ctx->wr_curr_bio) {
1726                         mutex_unlock(&wr_ctx->wr_lock);
1727                         return -ENOMEM;
1728                 }
1729                 wr_ctx->wr_curr_bio->sctx = sctx;
1730                 wr_ctx->wr_curr_bio->page_count = 0;
1731         }
1732         sbio = wr_ctx->wr_curr_bio;
1733         if (sbio->page_count == 0) {
1734                 struct bio *bio;
1735
1736                 sbio->physical = spage->physical_for_dev_replace;
1737                 sbio->logical = spage->logical;
1738                 sbio->dev = wr_ctx->tgtdev;
1739                 bio = sbio->bio;
1740                 if (!bio) {
1741                         bio = btrfs_io_bio_alloc(GFP_NOFS, wr_ctx->pages_per_wr_bio);
1742                         if (!bio) {
1743                                 mutex_unlock(&wr_ctx->wr_lock);
1744                                 return -ENOMEM;
1745                         }
1746                         sbio->bio = bio;
1747                 }
1748
1749                 bio->bi_private = sbio;
1750                 bio->bi_end_io = scrub_wr_bio_end_io;
1751                 bio->bi_bdev = sbio->dev->bdev;
1752                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1753                 sbio->err = 0;
1754         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1755                    spage->physical_for_dev_replace ||
1756                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1757                    spage->logical) {
1758                 scrub_wr_submit(sctx);
1759                 goto again;
1760         }
1761
1762         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1763         if (ret != PAGE_SIZE) {
1764                 if (sbio->page_count < 1) {
1765                         bio_put(sbio->bio);
1766                         sbio->bio = NULL;
1767                         mutex_unlock(&wr_ctx->wr_lock);
1768                         return -EIO;
1769                 }
1770                 scrub_wr_submit(sctx);
1771                 goto again;
1772         }
1773
1774         sbio->pagev[sbio->page_count] = spage;
1775         scrub_page_get(spage);
1776         sbio->page_count++;
1777         if (sbio->page_count == wr_ctx->pages_per_wr_bio)
1778                 scrub_wr_submit(sctx);
1779         mutex_unlock(&wr_ctx->wr_lock);
1780
1781         return 0;
1782 }
1783
1784 static void scrub_wr_submit(struct scrub_ctx *sctx)
1785 {
1786         struct scrub_wr_ctx *wr_ctx = &sctx->wr_ctx;
1787         struct scrub_bio *sbio;
1788
1789         if (!wr_ctx->wr_curr_bio)
1790                 return;
1791
1792         sbio = wr_ctx->wr_curr_bio;
1793         wr_ctx->wr_curr_bio = NULL;
1794         WARN_ON(!sbio->bio->bi_bdev);
1795         scrub_pending_bio_inc(sctx);
1796         /* process all writes in a single worker thread. Then the block layer
1797          * orders the requests before sending them to the driver which
1798          * doubled the write performance on spinning disks when measured
1799          * with Linux 3.5 */
1800         btrfsic_submit_bio(WRITE, sbio->bio);
1801 }
1802
1803 static void scrub_wr_bio_end_io(struct bio *bio, int err)
1804 {
1805         struct scrub_bio *sbio = bio->bi_private;
1806         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
1807
1808         sbio->err = err;
1809         sbio->bio = bio;
1810
1811         btrfs_init_work(&sbio->work, btrfs_scrubwrc_helper,
1812                          scrub_wr_bio_end_io_worker, NULL, NULL);
1813         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1814 }
1815
1816 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1817 {
1818         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1819         struct scrub_ctx *sctx = sbio->sctx;
1820         int i;
1821
1822         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1823         if (sbio->err) {
1824                 struct btrfs_dev_replace *dev_replace =
1825                         &sbio->sctx->dev_root->fs_info->dev_replace;
1826
1827                 for (i = 0; i < sbio->page_count; i++) {
1828                         struct scrub_page *spage = sbio->pagev[i];
1829
1830                         spage->io_error = 1;
1831                         btrfs_dev_replace_stats_inc(&dev_replace->
1832                                                     num_write_errors);
1833                 }
1834         }
1835
1836         for (i = 0; i < sbio->page_count; i++)
1837                 scrub_page_put(sbio->pagev[i]);
1838
1839         bio_put(sbio->bio);
1840         kfree(sbio);
1841         scrub_pending_bio_dec(sctx);
1842 }
1843
1844 static int scrub_checksum(struct scrub_block *sblock)
1845 {
1846         u64 flags;
1847         int ret;
1848
1849         WARN_ON(sblock->page_count < 1);
1850         flags = sblock->pagev[0]->flags;
1851         ret = 0;
1852         if (flags & BTRFS_EXTENT_FLAG_DATA)
1853                 ret = scrub_checksum_data(sblock);
1854         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1855                 ret = scrub_checksum_tree_block(sblock);
1856         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1857                 (void)scrub_checksum_super(sblock);
1858         else
1859                 WARN_ON(1);
1860         if (ret)
1861                 scrub_handle_errored_block(sblock);
1862
1863         return ret;
1864 }
1865
1866 static int scrub_checksum_data(struct scrub_block *sblock)
1867 {
1868         struct scrub_ctx *sctx = sblock->sctx;
1869         u8 csum[BTRFS_CSUM_SIZE];
1870         u8 *on_disk_csum;
1871         struct page *page;
1872         void *buffer;
1873         u32 crc = ~(u32)0;
1874         int fail = 0;
1875         u64 len;
1876         int index;
1877
1878         BUG_ON(sblock->page_count < 1);
1879         if (!sblock->pagev[0]->have_csum)
1880                 return 0;
1881
1882         on_disk_csum = sblock->pagev[0]->csum;
1883         page = sblock->pagev[0]->page;
1884         buffer = kmap_atomic(page);
1885
1886         len = sctx->sectorsize;
1887         index = 0;
1888         for (;;) {
1889                 u64 l = min_t(u64, len, PAGE_SIZE);
1890
1891                 crc = btrfs_csum_data(buffer, crc, l);
1892                 kunmap_atomic(buffer);
1893                 len -= l;
1894                 if (len == 0)
1895                         break;
1896                 index++;
1897                 BUG_ON(index >= sblock->page_count);
1898                 BUG_ON(!sblock->pagev[index]->page);
1899                 page = sblock->pagev[index]->page;
1900                 buffer = kmap_atomic(page);
1901         }
1902
1903         btrfs_csum_final(crc, csum);
1904         if (memcmp(csum, on_disk_csum, sctx->csum_size))
1905                 fail = 1;
1906
1907         return fail;
1908 }
1909
1910 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1911 {
1912         struct scrub_ctx *sctx = sblock->sctx;
1913         struct btrfs_header *h;
1914         struct btrfs_root *root = sctx->dev_root;
1915         struct btrfs_fs_info *fs_info = root->fs_info;
1916         u8 calculated_csum[BTRFS_CSUM_SIZE];
1917         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1918         struct page *page;
1919         void *mapped_buffer;
1920         u64 mapped_size;
1921         void *p;
1922         u32 crc = ~(u32)0;
1923         int fail = 0;
1924         int crc_fail = 0;
1925         u64 len;
1926         int index;
1927
1928         BUG_ON(sblock->page_count < 1);
1929         page = sblock->pagev[0]->page;
1930         mapped_buffer = kmap_atomic(page);
1931         h = (struct btrfs_header *)mapped_buffer;
1932         memcpy(on_disk_csum, h->csum, sctx->csum_size);
1933
1934         /*
1935          * we don't use the getter functions here, as we
1936          * a) don't have an extent buffer and
1937          * b) the page is already kmapped
1938          */
1939
1940         if (sblock->pagev[0]->logical != btrfs_stack_header_bytenr(h))
1941                 ++fail;
1942
1943         if (sblock->pagev[0]->generation != btrfs_stack_header_generation(h))
1944                 ++fail;
1945
1946         if (!scrub_check_fsid(h->fsid, sblock->pagev[0]))
1947                 ++fail;
1948
1949         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1950                    BTRFS_UUID_SIZE))
1951                 ++fail;
1952
1953         len = sctx->nodesize - BTRFS_CSUM_SIZE;
1954         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
1955         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
1956         index = 0;
1957         for (;;) {
1958                 u64 l = min_t(u64, len, mapped_size);
1959
1960                 crc = btrfs_csum_data(p, crc, l);
1961                 kunmap_atomic(mapped_buffer);
1962                 len -= l;
1963                 if (len == 0)
1964                         break;
1965                 index++;
1966                 BUG_ON(index >= sblock->page_count);
1967                 BUG_ON(!sblock->pagev[index]->page);
1968                 page = sblock->pagev[index]->page;
1969                 mapped_buffer = kmap_atomic(page);
1970                 mapped_size = PAGE_SIZE;
1971                 p = mapped_buffer;
1972         }
1973
1974         btrfs_csum_final(crc, calculated_csum);
1975         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
1976                 ++crc_fail;
1977
1978         return fail || crc_fail;
1979 }
1980
1981 static int scrub_checksum_super(struct scrub_block *sblock)
1982 {
1983         struct btrfs_super_block *s;
1984         struct scrub_ctx *sctx = sblock->sctx;
1985         u8 calculated_csum[BTRFS_CSUM_SIZE];
1986         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1987         struct page *page;
1988         void *mapped_buffer;
1989         u64 mapped_size;
1990         void *p;
1991         u32 crc = ~(u32)0;
1992         int fail_gen = 0;
1993         int fail_cor = 0;
1994         u64 len;
1995         int index;
1996
1997         BUG_ON(sblock->page_count < 1);
1998         page = sblock->pagev[0]->page;
1999         mapped_buffer = kmap_atomic(page);
2000         s = (struct btrfs_super_block *)mapped_buffer;
2001         memcpy(on_disk_csum, s->csum, sctx->csum_size);
2002
2003         if (sblock->pagev[0]->logical != btrfs_super_bytenr(s))
2004                 ++fail_cor;
2005
2006         if (sblock->pagev[0]->generation != btrfs_super_generation(s))
2007                 ++fail_gen;
2008
2009         if (!scrub_check_fsid(s->fsid, sblock->pagev[0]))
2010                 ++fail_cor;
2011
2012         len = BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE;
2013         mapped_size = PAGE_SIZE - BTRFS_CSUM_SIZE;
2014         p = ((u8 *)mapped_buffer) + BTRFS_CSUM_SIZE;
2015         index = 0;
2016         for (;;) {
2017                 u64 l = min_t(u64, len, mapped_size);
2018
2019                 crc = btrfs_csum_data(p, crc, l);
2020                 kunmap_atomic(mapped_buffer);
2021                 len -= l;
2022                 if (len == 0)
2023                         break;
2024                 index++;
2025                 BUG_ON(index >= sblock->page_count);
2026                 BUG_ON(!sblock->pagev[index]->page);
2027                 page = sblock->pagev[index]->page;
2028                 mapped_buffer = kmap_atomic(page);
2029                 mapped_size = PAGE_SIZE;
2030                 p = mapped_buffer;
2031         }
2032
2033         btrfs_csum_final(crc, calculated_csum);
2034         if (memcmp(calculated_csum, on_disk_csum, sctx->csum_size))
2035                 ++fail_cor;
2036
2037         if (fail_cor + fail_gen) {
2038                 /*
2039                  * if we find an error in a super block, we just report it.
2040                  * They will get written with the next transaction commit
2041                  * anyway
2042                  */
2043                 spin_lock(&sctx->stat_lock);
2044                 ++sctx->stat.super_errors;
2045                 spin_unlock(&sctx->stat_lock);
2046                 if (fail_cor)
2047                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2048                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
2049                 else
2050                         btrfs_dev_stat_inc_and_print(sblock->pagev[0]->dev,
2051                                 BTRFS_DEV_STAT_GENERATION_ERRS);
2052         }
2053
2054         return fail_cor + fail_gen;
2055 }
2056
2057 static void scrub_block_get(struct scrub_block *sblock)
2058 {
2059         atomic_inc(&sblock->refs);
2060 }
2061
2062 static void scrub_block_put(struct scrub_block *sblock)
2063 {
2064         if (atomic_dec_and_test(&sblock->refs)) {
2065                 int i;
2066
2067                 if (sblock->sparity)
2068                         scrub_parity_put(sblock->sparity);
2069
2070                 for (i = 0; i < sblock->page_count; i++)
2071                         scrub_page_put(sblock->pagev[i]);
2072                 kfree(sblock);
2073         }
2074 }
2075
2076 static void scrub_page_get(struct scrub_page *spage)
2077 {
2078         atomic_inc(&spage->refs);
2079 }
2080
2081 static void scrub_page_put(struct scrub_page *spage)
2082 {
2083         if (atomic_dec_and_test(&spage->refs)) {
2084                 if (spage->page)
2085                         __free_page(spage->page);
2086                 kfree(spage);
2087         }
2088 }
2089
2090 static void scrub_submit(struct scrub_ctx *sctx)
2091 {
2092         struct scrub_bio *sbio;
2093
2094         if (sctx->curr == -1)
2095                 return;
2096
2097         sbio = sctx->bios[sctx->curr];
2098         sctx->curr = -1;
2099         scrub_pending_bio_inc(sctx);
2100         btrfsic_submit_bio(READ, sbio->bio);
2101 }
2102
2103 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
2104                                     struct scrub_page *spage)
2105 {
2106         struct scrub_block *sblock = spage->sblock;
2107         struct scrub_bio *sbio;
2108         int ret;
2109
2110 again:
2111         /*
2112          * grab a fresh bio or wait for one to become available
2113          */
2114         while (sctx->curr == -1) {
2115                 spin_lock(&sctx->list_lock);
2116                 sctx->curr = sctx->first_free;
2117                 if (sctx->curr != -1) {
2118                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
2119                         sctx->bios[sctx->curr]->next_free = -1;
2120                         sctx->bios[sctx->curr]->page_count = 0;
2121                         spin_unlock(&sctx->list_lock);
2122                 } else {
2123                         spin_unlock(&sctx->list_lock);
2124                         wait_event(sctx->list_wait, sctx->first_free != -1);
2125                 }
2126         }
2127         sbio = sctx->bios[sctx->curr];
2128         if (sbio->page_count == 0) {
2129                 struct bio *bio;
2130
2131                 sbio->physical = spage->physical;
2132                 sbio->logical = spage->logical;
2133                 sbio->dev = spage->dev;
2134                 bio = sbio->bio;
2135                 if (!bio) {
2136                         bio = btrfs_io_bio_alloc(GFP_NOFS, sctx->pages_per_rd_bio);
2137                         if (!bio)
2138                                 return -ENOMEM;
2139                         sbio->bio = bio;
2140                 }
2141
2142                 bio->bi_private = sbio;
2143                 bio->bi_end_io = scrub_bio_end_io;
2144                 bio->bi_bdev = sbio->dev->bdev;
2145                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2146                 sbio->err = 0;
2147         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2148                    spage->physical ||
2149                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2150                    spage->logical ||
2151                    sbio->dev != spage->dev) {
2152                 scrub_submit(sctx);
2153                 goto again;
2154         }
2155
2156         sbio->pagev[sbio->page_count] = spage;
2157         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2158         if (ret != PAGE_SIZE) {
2159                 if (sbio->page_count < 1) {
2160                         bio_put(sbio->bio);
2161                         sbio->bio = NULL;
2162                         return -EIO;
2163                 }
2164                 scrub_submit(sctx);
2165                 goto again;
2166         }
2167
2168         scrub_block_get(sblock); /* one for the page added to the bio */
2169         atomic_inc(&sblock->outstanding_pages);
2170         sbio->page_count++;
2171         if (sbio->page_count == sctx->pages_per_rd_bio)
2172                 scrub_submit(sctx);
2173
2174         return 0;
2175 }
2176
2177 static void scrub_missing_raid56_end_io(struct bio *bio, int error)
2178 {
2179         struct scrub_block *sblock = bio->bi_private;
2180         struct btrfs_fs_info *fs_info = sblock->sctx->dev_root->fs_info;
2181
2182         if (error)
2183                 sblock->no_io_error_seen = 0;
2184
2185         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2186 }
2187
2188 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2189 {
2190         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2191         struct scrub_ctx *sctx = sblock->sctx;
2192         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2193         unsigned int is_metadata;
2194         unsigned int have_csum;
2195         u8 *csum;
2196         u64 generation;
2197         u64 logical;
2198         struct btrfs_device *dev;
2199
2200         is_metadata = !(sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA);
2201         have_csum = sblock->pagev[0]->have_csum;
2202         csum = sblock->pagev[0]->csum;
2203         generation = sblock->pagev[0]->generation;
2204         logical = sblock->pagev[0]->logical;
2205         dev = sblock->pagev[0]->dev;
2206
2207         if (sblock->no_io_error_seen) {
2208                 scrub_recheck_block_checksum(fs_info, sblock, is_metadata,
2209                                              have_csum, csum, generation,
2210                                              sctx->csum_size);
2211         }
2212
2213         if (!sblock->no_io_error_seen) {
2214                 spin_lock(&sctx->stat_lock);
2215                 sctx->stat.read_errors++;
2216                 spin_unlock(&sctx->stat_lock);
2217                 printk_ratelimited_in_rcu(KERN_ERR
2218                         "BTRFS: I/O error rebulding logical %llu for dev %s\n",
2219                         logical, rcu_str_deref(dev->name));
2220         } else if (sblock->header_error || sblock->checksum_error) {
2221                 spin_lock(&sctx->stat_lock);
2222                 sctx->stat.uncorrectable_errors++;
2223                 spin_unlock(&sctx->stat_lock);
2224                 printk_ratelimited_in_rcu(KERN_ERR
2225                         "BTRFS: failed to rebuild valid logical %llu for dev %s\n",
2226                         logical, rcu_str_deref(dev->name));
2227         } else {
2228                 scrub_write_block_to_dev_replace(sblock);
2229         }
2230
2231         scrub_block_put(sblock);
2232
2233         if (sctx->is_dev_replace &&
2234             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2235                 mutex_lock(&sctx->wr_ctx.wr_lock);
2236                 scrub_wr_submit(sctx);
2237                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2238         }
2239
2240         scrub_pending_bio_dec(sctx);
2241 }
2242
2243 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2244 {
2245         struct scrub_ctx *sctx = sblock->sctx;
2246         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2247         u64 length = sblock->page_count * PAGE_SIZE;
2248         u64 logical = sblock->pagev[0]->logical;
2249         struct btrfs_bio *bbio;
2250         struct bio *bio;
2251         struct btrfs_raid_bio *rbio;
2252         int ret;
2253         int i;
2254
2255         ret = btrfs_map_sblock(fs_info, REQ_GET_READ_MIRRORS, logical, &length,
2256                                &bbio, 0, 1);
2257         if (ret || !bbio || !bbio->raid_map)
2258                 goto bbio_out;
2259
2260         if (WARN_ON(!sctx->is_dev_replace ||
2261                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2262                 /*
2263                  * We shouldn't be scrubbing a missing device. Even for dev
2264                  * replace, we should only get here for RAID 5/6. We either
2265                  * managed to mount something with no mirrors remaining or
2266                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2267                  */
2268                 goto bbio_out;
2269         }
2270
2271         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2272         if (!bio)
2273                 goto bbio_out;
2274
2275         bio->bi_iter.bi_sector = logical >> 9;
2276         bio->bi_private = sblock;
2277         bio->bi_end_io = scrub_missing_raid56_end_io;
2278
2279         rbio = raid56_alloc_missing_rbio(sctx->dev_root, bio, bbio, length);
2280         if (!rbio)
2281                 goto rbio_out;
2282
2283         for (i = 0; i < sblock->page_count; i++) {
2284                 struct scrub_page *spage = sblock->pagev[i];
2285
2286                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2287         }
2288
2289         btrfs_init_work(&sblock->work, btrfs_scrub_helper,
2290                         scrub_missing_raid56_worker, NULL, NULL);
2291         scrub_block_get(sblock);
2292         scrub_pending_bio_inc(sctx);
2293         raid56_submit_missing_rbio(rbio);
2294         return;
2295
2296 rbio_out:
2297         bio_put(bio);
2298 bbio_out:
2299         btrfs_put_bbio(bbio);
2300         spin_lock(&sctx->stat_lock);
2301         sctx->stat.malloc_errors++;
2302         spin_unlock(&sctx->stat_lock);
2303 }
2304
2305 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
2306                        u64 physical, struct btrfs_device *dev, u64 flags,
2307                        u64 gen, int mirror_num, u8 *csum, int force,
2308                        u64 physical_for_dev_replace)
2309 {
2310         struct scrub_block *sblock;
2311         int index;
2312
2313         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2314         if (!sblock) {
2315                 spin_lock(&sctx->stat_lock);
2316                 sctx->stat.malloc_errors++;
2317                 spin_unlock(&sctx->stat_lock);
2318                 return -ENOMEM;
2319         }
2320
2321         /* one ref inside this function, plus one for each page added to
2322          * a bio later on */
2323         atomic_set(&sblock->refs, 1);
2324         sblock->sctx = sctx;
2325         sblock->no_io_error_seen = 1;
2326
2327         for (index = 0; len > 0; index++) {
2328                 struct scrub_page *spage;
2329                 u64 l = min_t(u64, len, PAGE_SIZE);
2330
2331                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2332                 if (!spage) {
2333 leave_nomem:
2334                         spin_lock(&sctx->stat_lock);
2335                         sctx->stat.malloc_errors++;
2336                         spin_unlock(&sctx->stat_lock);
2337                         scrub_block_put(sblock);
2338                         return -ENOMEM;
2339                 }
2340                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2341                 scrub_page_get(spage);
2342                 sblock->pagev[index] = spage;
2343                 spage->sblock = sblock;
2344                 spage->dev = dev;
2345                 spage->flags = flags;
2346                 spage->generation = gen;
2347                 spage->logical = logical;
2348                 spage->physical = physical;
2349                 spage->physical_for_dev_replace = physical_for_dev_replace;
2350                 spage->mirror_num = mirror_num;
2351                 if (csum) {
2352                         spage->have_csum = 1;
2353                         memcpy(spage->csum, csum, sctx->csum_size);
2354                 } else {
2355                         spage->have_csum = 0;
2356                 }
2357                 sblock->page_count++;
2358                 spage->page = alloc_page(GFP_NOFS);
2359                 if (!spage->page)
2360                         goto leave_nomem;
2361                 len -= l;
2362                 logical += l;
2363                 physical += l;
2364                 physical_for_dev_replace += l;
2365         }
2366
2367         WARN_ON(sblock->page_count == 0);
2368         if (dev->missing) {
2369                 /*
2370                  * This case should only be hit for RAID 5/6 device replace. See
2371                  * the comment in scrub_missing_raid56_pages() for details.
2372                  */
2373                 scrub_missing_raid56_pages(sblock);
2374         } else {
2375                 for (index = 0; index < sblock->page_count; index++) {
2376                         struct scrub_page *spage = sblock->pagev[index];
2377                         int ret;
2378
2379                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2380                         if (ret) {
2381                                 scrub_block_put(sblock);
2382                                 return ret;
2383                         }
2384                 }
2385
2386                 if (force)
2387                         scrub_submit(sctx);
2388         }
2389
2390         /* last one frees, either here or in bio completion for last page */
2391         scrub_block_put(sblock);
2392         return 0;
2393 }
2394
2395 static void scrub_bio_end_io(struct bio *bio, int err)
2396 {
2397         struct scrub_bio *sbio = bio->bi_private;
2398         struct btrfs_fs_info *fs_info = sbio->dev->dev_root->fs_info;
2399
2400         sbio->err = err;
2401         sbio->bio = bio;
2402
2403         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2404 }
2405
2406 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2407 {
2408         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2409         struct scrub_ctx *sctx = sbio->sctx;
2410         int i;
2411
2412         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2413         if (sbio->err) {
2414                 for (i = 0; i < sbio->page_count; i++) {
2415                         struct scrub_page *spage = sbio->pagev[i];
2416
2417                         spage->io_error = 1;
2418                         spage->sblock->no_io_error_seen = 0;
2419                 }
2420         }
2421
2422         /* now complete the scrub_block items that have all pages completed */
2423         for (i = 0; i < sbio->page_count; i++) {
2424                 struct scrub_page *spage = sbio->pagev[i];
2425                 struct scrub_block *sblock = spage->sblock;
2426
2427                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2428                         scrub_block_complete(sblock);
2429                 scrub_block_put(sblock);
2430         }
2431
2432         bio_put(sbio->bio);
2433         sbio->bio = NULL;
2434         spin_lock(&sctx->list_lock);
2435         sbio->next_free = sctx->first_free;
2436         sctx->first_free = sbio->index;
2437         spin_unlock(&sctx->list_lock);
2438
2439         if (sctx->is_dev_replace &&
2440             atomic_read(&sctx->wr_ctx.flush_all_writes)) {
2441                 mutex_lock(&sctx->wr_ctx.wr_lock);
2442                 scrub_wr_submit(sctx);
2443                 mutex_unlock(&sctx->wr_ctx.wr_lock);
2444         }
2445
2446         scrub_pending_bio_dec(sctx);
2447 }
2448
2449 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2450                                        unsigned long *bitmap,
2451                                        u64 start, u64 len)
2452 {
2453         u32 offset;
2454         int nsectors;
2455         int sectorsize = sparity->sctx->dev_root->sectorsize;
2456
2457         if (len >= sparity->stripe_len) {
2458                 bitmap_set(bitmap, 0, sparity->nsectors);
2459                 return;
2460         }
2461
2462         start -= sparity->logic_start;
2463         start = div_u64_rem(start, sparity->stripe_len, &offset);
2464         offset /= sectorsize;
2465         nsectors = (int)len / sectorsize;
2466
2467         if (offset + nsectors <= sparity->nsectors) {
2468                 bitmap_set(bitmap, offset, nsectors);
2469                 return;
2470         }
2471
2472         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2473         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2474 }
2475
2476 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2477                                                    u64 start, u64 len)
2478 {
2479         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2480 }
2481
2482 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2483                                                   u64 start, u64 len)
2484 {
2485         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2486 }
2487
2488 static void scrub_block_complete(struct scrub_block *sblock)
2489 {
2490         int corrupted = 0;
2491
2492         if (!sblock->no_io_error_seen) {
2493                 corrupted = 1;
2494                 scrub_handle_errored_block(sblock);
2495         } else {
2496                 /*
2497                  * if has checksum error, write via repair mechanism in
2498                  * dev replace case, otherwise write here in dev replace
2499                  * case.
2500                  */
2501                 corrupted = scrub_checksum(sblock);
2502                 if (!corrupted && sblock->sctx->is_dev_replace)
2503                         scrub_write_block_to_dev_replace(sblock);
2504         }
2505
2506         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2507                 u64 start = sblock->pagev[0]->logical;
2508                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2509                           PAGE_SIZE;
2510
2511                 scrub_parity_mark_sectors_error(sblock->sparity,
2512                                                 start, end - start);
2513         }
2514 }
2515
2516 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u64 len,
2517                            u8 *csum)
2518 {
2519         struct btrfs_ordered_sum *sum = NULL;
2520         unsigned long index;
2521         unsigned long num_sectors;
2522
2523         while (!list_empty(&sctx->csum_list)) {
2524                 sum = list_first_entry(&sctx->csum_list,
2525                                        struct btrfs_ordered_sum, list);
2526                 if (sum->bytenr > logical)
2527                         return 0;
2528                 if (sum->bytenr + sum->len > logical)
2529                         break;
2530
2531                 ++sctx->stat.csum_discards;
2532                 list_del(&sum->list);
2533                 kfree(sum);
2534                 sum = NULL;
2535         }
2536         if (!sum)
2537                 return 0;
2538
2539         index = ((u32)(logical - sum->bytenr)) / sctx->sectorsize;
2540         num_sectors = sum->len / sctx->sectorsize;
2541         memcpy(csum, sum->sums + index, sctx->csum_size);
2542         if (index == num_sectors - 1) {
2543                 list_del(&sum->list);
2544                 kfree(sum);
2545         }
2546         return 1;
2547 }
2548
2549 /* scrub extent tries to collect up to 64 kB for each bio */
2550 static int scrub_extent(struct scrub_ctx *sctx, u64 logical, u64 len,
2551                         u64 physical, struct btrfs_device *dev, u64 flags,
2552                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2553 {
2554         int ret;
2555         u8 csum[BTRFS_CSUM_SIZE];
2556         u32 blocksize;
2557
2558         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2559                 blocksize = sctx->sectorsize;
2560                 spin_lock(&sctx->stat_lock);
2561                 sctx->stat.data_extents_scrubbed++;
2562                 sctx->stat.data_bytes_scrubbed += len;
2563                 spin_unlock(&sctx->stat_lock);
2564         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2565                 blocksize = sctx->nodesize;
2566                 spin_lock(&sctx->stat_lock);
2567                 sctx->stat.tree_extents_scrubbed++;
2568                 sctx->stat.tree_bytes_scrubbed += len;
2569                 spin_unlock(&sctx->stat_lock);
2570         } else {
2571                 blocksize = sctx->sectorsize;
2572                 WARN_ON(1);
2573         }
2574
2575         while (len) {
2576                 u64 l = min_t(u64, len, blocksize);
2577                 int have_csum = 0;
2578
2579                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2580                         /* push csums to sbio */
2581                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2582                         if (have_csum == 0)
2583                                 ++sctx->stat.no_csum;
2584                         if (sctx->is_dev_replace && !have_csum) {
2585                                 ret = copy_nocow_pages(sctx, logical, l,
2586                                                        mirror_num,
2587                                                       physical_for_dev_replace);
2588                                 goto behind_scrub_pages;
2589                         }
2590                 }
2591                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2592                                   mirror_num, have_csum ? csum : NULL, 0,
2593                                   physical_for_dev_replace);
2594 behind_scrub_pages:
2595                 if (ret)
2596                         return ret;
2597                 len -= l;
2598                 logical += l;
2599                 physical += l;
2600                 physical_for_dev_replace += l;
2601         }
2602         return 0;
2603 }
2604
2605 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2606                                   u64 logical, u64 len,
2607                                   u64 physical, struct btrfs_device *dev,
2608                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2609 {
2610         struct scrub_ctx *sctx = sparity->sctx;
2611         struct scrub_block *sblock;
2612         int index;
2613
2614         sblock = kzalloc(sizeof(*sblock), GFP_NOFS);
2615         if (!sblock) {
2616                 spin_lock(&sctx->stat_lock);
2617                 sctx->stat.malloc_errors++;
2618                 spin_unlock(&sctx->stat_lock);
2619                 return -ENOMEM;
2620         }
2621
2622         /* one ref inside this function, plus one for each page added to
2623          * a bio later on */
2624         atomic_set(&sblock->refs, 1);
2625         sblock->sctx = sctx;
2626         sblock->no_io_error_seen = 1;
2627         sblock->sparity = sparity;
2628         scrub_parity_get(sparity);
2629
2630         for (index = 0; len > 0; index++) {
2631                 struct scrub_page *spage;
2632                 u64 l = min_t(u64, len, PAGE_SIZE);
2633
2634                 spage = kzalloc(sizeof(*spage), GFP_NOFS);
2635                 if (!spage) {
2636 leave_nomem:
2637                         spin_lock(&sctx->stat_lock);
2638                         sctx->stat.malloc_errors++;
2639                         spin_unlock(&sctx->stat_lock);
2640                         scrub_block_put(sblock);
2641                         return -ENOMEM;
2642                 }
2643                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2644                 /* For scrub block */
2645                 scrub_page_get(spage);
2646                 sblock->pagev[index] = spage;
2647                 /* For scrub parity */
2648                 scrub_page_get(spage);
2649                 list_add_tail(&spage->list, &sparity->spages);
2650                 spage->sblock = sblock;
2651                 spage->dev = dev;
2652                 spage->flags = flags;
2653                 spage->generation = gen;
2654                 spage->logical = logical;
2655                 spage->physical = physical;
2656                 spage->mirror_num = mirror_num;
2657                 if (csum) {
2658                         spage->have_csum = 1;
2659                         memcpy(spage->csum, csum, sctx->csum_size);
2660                 } else {
2661                         spage->have_csum = 0;
2662                 }
2663                 sblock->page_count++;
2664                 spage->page = alloc_page(GFP_NOFS);
2665                 if (!spage->page)
2666                         goto leave_nomem;
2667                 len -= l;
2668                 logical += l;
2669                 physical += l;
2670         }
2671
2672         WARN_ON(sblock->page_count == 0);
2673         for (index = 0; index < sblock->page_count; index++) {
2674                 struct scrub_page *spage = sblock->pagev[index];
2675                 int ret;
2676
2677                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2678                 if (ret) {
2679                         scrub_block_put(sblock);
2680                         return ret;
2681                 }
2682         }
2683
2684         /* last one frees, either here or in bio completion for last page */
2685         scrub_block_put(sblock);
2686         return 0;
2687 }
2688
2689 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2690                                    u64 logical, u64 len,
2691                                    u64 physical, struct btrfs_device *dev,
2692                                    u64 flags, u64 gen, int mirror_num)
2693 {
2694         struct scrub_ctx *sctx = sparity->sctx;
2695         int ret;
2696         u8 csum[BTRFS_CSUM_SIZE];
2697         u32 blocksize;
2698
2699         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2700                 blocksize = sctx->sectorsize;
2701         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2702                 blocksize = sctx->nodesize;
2703         } else {
2704                 blocksize = sctx->sectorsize;
2705                 WARN_ON(1);
2706         }
2707
2708         while (len) {
2709                 u64 l = min_t(u64, len, blocksize);
2710                 int have_csum = 0;
2711
2712                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2713                         /* push csums to sbio */
2714                         have_csum = scrub_find_csum(sctx, logical, l, csum);
2715                         if (have_csum == 0)
2716                                 goto skip;
2717                 }
2718                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2719                                              flags, gen, mirror_num,
2720                                              have_csum ? csum : NULL);
2721                 if (ret)
2722                         return ret;
2723 skip:
2724                 len -= l;
2725                 logical += l;
2726                 physical += l;
2727         }
2728         return 0;
2729 }
2730
2731 /*
2732  * Given a physical address, this will calculate it's
2733  * logical offset. if this is a parity stripe, it will return
2734  * the most left data stripe's logical offset.
2735  *
2736  * return 0 if it is a data stripe, 1 means parity stripe.
2737  */
2738 static int get_raid56_logic_offset(u64 physical, int num,
2739                                    struct map_lookup *map, u64 *offset,
2740                                    u64 *stripe_start)
2741 {
2742         int i;
2743         int j = 0;
2744         u64 stripe_nr;
2745         u64 last_offset;
2746         u32 stripe_index;
2747         u32 rot;
2748
2749         last_offset = (physical - map->stripes[num].physical) *
2750                       nr_data_stripes(map);
2751         if (stripe_start)
2752                 *stripe_start = last_offset;
2753
2754         *offset = last_offset;
2755         for (i = 0; i < nr_data_stripes(map); i++) {
2756                 *offset = last_offset + i * map->stripe_len;
2757
2758                 stripe_nr = div_u64(*offset, map->stripe_len);
2759                 stripe_nr = div_u64(stripe_nr, nr_data_stripes(map));
2760
2761                 /* Work out the disk rotation on this stripe-set */
2762                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2763                 /* calculate which stripe this data locates */
2764                 rot += i;
2765                 stripe_index = rot % map->num_stripes;
2766                 if (stripe_index == num)
2767                         return 0;
2768                 if (stripe_index < num)
2769                         j++;
2770         }
2771         *offset = last_offset + j * map->stripe_len;
2772         return 1;
2773 }
2774
2775 static void scrub_free_parity(struct scrub_parity *sparity)
2776 {
2777         struct scrub_ctx *sctx = sparity->sctx;
2778         struct scrub_page *curr, *next;
2779         int nbits;
2780
2781         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2782         if (nbits) {
2783                 spin_lock(&sctx->stat_lock);
2784                 sctx->stat.read_errors += nbits;
2785                 sctx->stat.uncorrectable_errors += nbits;
2786                 spin_unlock(&sctx->stat_lock);
2787         }
2788
2789         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2790                 list_del_init(&curr->list);
2791                 scrub_page_put(curr);
2792         }
2793
2794         kfree(sparity);
2795 }
2796
2797 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2798 {
2799         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2800                                                     work);
2801         struct scrub_ctx *sctx = sparity->sctx;
2802
2803         scrub_free_parity(sparity);
2804         scrub_pending_bio_dec(sctx);
2805 }
2806
2807 static void scrub_parity_bio_endio(struct bio *bio, int error)
2808 {
2809         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2810
2811         if (error)
2812                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2813                           sparity->nsectors);
2814
2815         bio_put(bio);
2816
2817         btrfs_init_work(&sparity->work, btrfs_scrubparity_helper,
2818                         scrub_parity_bio_endio_worker, NULL, NULL);
2819         btrfs_queue_work(sparity->sctx->dev_root->fs_info->scrub_parity_workers,
2820                          &sparity->work);
2821 }
2822
2823 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2824 {
2825         struct scrub_ctx *sctx = sparity->sctx;
2826         struct bio *bio;
2827         struct btrfs_raid_bio *rbio;
2828         struct scrub_page *spage;
2829         struct btrfs_bio *bbio = NULL;
2830         u64 length;
2831         int ret;
2832
2833         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2834                            sparity->nsectors))
2835                 goto out;
2836
2837         length = sparity->logic_end - sparity->logic_start;
2838         ret = btrfs_map_sblock(sctx->dev_root->fs_info, WRITE,
2839                                sparity->logic_start,
2840                                &length, &bbio, 0, 1);
2841         if (ret || !bbio || !bbio->raid_map)
2842                 goto bbio_out;
2843
2844         bio = btrfs_io_bio_alloc(GFP_NOFS, 0);
2845         if (!bio)
2846                 goto bbio_out;
2847
2848         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2849         bio->bi_private = sparity;
2850         bio->bi_end_io = scrub_parity_bio_endio;
2851
2852         rbio = raid56_parity_alloc_scrub_rbio(sctx->dev_root, bio, bbio,
2853                                               length, sparity->scrub_dev,
2854                                               sparity->dbitmap,
2855                                               sparity->nsectors);
2856         if (!rbio)
2857                 goto rbio_out;
2858
2859         list_for_each_entry(spage, &sparity->spages, list)
2860                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2861
2862         scrub_pending_bio_inc(sctx);
2863         raid56_parity_submit_scrub_rbio(rbio);
2864         return;
2865
2866 rbio_out:
2867         bio_put(bio);
2868 bbio_out:
2869         btrfs_put_bbio(bbio);
2870         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2871                   sparity->nsectors);
2872         spin_lock(&sctx->stat_lock);
2873         sctx->stat.malloc_errors++;
2874         spin_unlock(&sctx->stat_lock);
2875 out:
2876         scrub_free_parity(sparity);
2877 }
2878
2879 static inline int scrub_calc_parity_bitmap_len(int nsectors)
2880 {
2881         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * (BITS_PER_LONG / 8);
2882 }
2883
2884 static void scrub_parity_get(struct scrub_parity *sparity)
2885 {
2886         atomic_inc(&sparity->refs);
2887 }
2888
2889 static void scrub_parity_put(struct scrub_parity *sparity)
2890 {
2891         if (!atomic_dec_and_test(&sparity->refs))
2892                 return;
2893
2894         scrub_parity_check_and_repair(sparity);
2895 }
2896
2897 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2898                                                   struct map_lookup *map,
2899                                                   struct btrfs_device *sdev,
2900                                                   struct btrfs_path *path,
2901                                                   u64 logic_start,
2902                                                   u64 logic_end)
2903 {
2904         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
2905         struct btrfs_root *root = fs_info->extent_root;
2906         struct btrfs_root *csum_root = fs_info->csum_root;
2907         struct btrfs_extent_item *extent;
2908         u64 flags;
2909         int ret;
2910         int slot;
2911         struct extent_buffer *l;
2912         struct btrfs_key key;
2913         u64 generation;
2914         u64 extent_logical;
2915         u64 extent_physical;
2916         u64 extent_len;
2917         struct btrfs_device *extent_dev;
2918         struct scrub_parity *sparity;
2919         int nsectors;
2920         int bitmap_len;
2921         int extent_mirror_num;
2922         int stop_loop = 0;
2923
2924         nsectors = map->stripe_len / root->sectorsize;
2925         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2926         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2927                           GFP_NOFS);
2928         if (!sparity) {
2929                 spin_lock(&sctx->stat_lock);
2930                 sctx->stat.malloc_errors++;
2931                 spin_unlock(&sctx->stat_lock);
2932                 return -ENOMEM;
2933         }
2934
2935         sparity->stripe_len = map->stripe_len;
2936         sparity->nsectors = nsectors;
2937         sparity->sctx = sctx;
2938         sparity->scrub_dev = sdev;
2939         sparity->logic_start = logic_start;
2940         sparity->logic_end = logic_end;
2941         atomic_set(&sparity->refs, 1);
2942         INIT_LIST_HEAD(&sparity->spages);
2943         sparity->dbitmap = sparity->bitmap;
2944         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2945
2946         ret = 0;
2947         while (logic_start < logic_end) {
2948                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2949                         key.type = BTRFS_METADATA_ITEM_KEY;
2950                 else
2951                         key.type = BTRFS_EXTENT_ITEM_KEY;
2952                 key.objectid = logic_start;
2953                 key.offset = (u64)-1;
2954
2955                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2956                 if (ret < 0)
2957                         goto out;
2958
2959                 if (ret > 0) {
2960                         ret = btrfs_previous_extent_item(root, path, 0);
2961                         if (ret < 0)
2962                                 goto out;
2963                         if (ret > 0) {
2964                                 btrfs_release_path(path);
2965                                 ret = btrfs_search_slot(NULL, root, &key,
2966                                                         path, 0, 0);
2967                                 if (ret < 0)
2968                                         goto out;
2969                         }
2970                 }
2971
2972                 stop_loop = 0;
2973                 while (1) {
2974                         u64 bytes;
2975
2976                         l = path->nodes[0];
2977                         slot = path->slots[0];
2978                         if (slot >= btrfs_header_nritems(l)) {
2979                                 ret = btrfs_next_leaf(root, path);
2980                                 if (ret == 0)
2981                                         continue;
2982                                 if (ret < 0)
2983                                         goto out;
2984
2985                                 stop_loop = 1;
2986                                 break;
2987                         }
2988                         btrfs_item_key_to_cpu(l, &key, slot);
2989
2990                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2991                             key.type != BTRFS_METADATA_ITEM_KEY)
2992                                 goto next;
2993
2994                         if (key.type == BTRFS_METADATA_ITEM_KEY)
2995                                 bytes = root->nodesize;
2996                         else
2997                                 bytes = key.offset;
2998
2999                         if (key.objectid + bytes <= logic_start)
3000                                 goto next;
3001
3002                         if (key.objectid >= logic_end) {
3003                                 stop_loop = 1;
3004                                 break;
3005                         }
3006
3007                         while (key.objectid >= logic_start + map->stripe_len)
3008                                 logic_start += map->stripe_len;
3009
3010                         extent = btrfs_item_ptr(l, slot,
3011                                                 struct btrfs_extent_item);
3012                         flags = btrfs_extent_flags(l, extent);
3013                         generation = btrfs_extent_generation(l, extent);
3014
3015                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3016                             (key.objectid < logic_start ||
3017                              key.objectid + bytes >
3018                              logic_start + map->stripe_len)) {
3019                                 btrfs_err(fs_info, "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3020                                           key.objectid, logic_start);
3021                                 goto next;
3022                         }
3023 again:
3024                         extent_logical = key.objectid;
3025                         extent_len = bytes;
3026
3027                         if (extent_logical < logic_start) {
3028                                 extent_len -= logic_start - extent_logical;
3029                                 extent_logical = logic_start;
3030                         }
3031
3032                         if (extent_logical + extent_len >
3033                             logic_start + map->stripe_len)
3034                                 extent_len = logic_start + map->stripe_len -
3035                                              extent_logical;
3036
3037                         scrub_parity_mark_sectors_data(sparity, extent_logical,
3038                                                        extent_len);
3039
3040                         scrub_remap_extent(fs_info, extent_logical,
3041                                            extent_len, &extent_physical,
3042                                            &extent_dev,
3043                                            &extent_mirror_num);
3044
3045                         ret = btrfs_lookup_csums_range(csum_root,
3046                                                 extent_logical,
3047                                                 extent_logical + extent_len - 1,
3048                                                 &sctx->csum_list, 1);
3049                         if (ret)
3050                                 goto out;
3051
3052                         ret = scrub_extent_for_parity(sparity, extent_logical,
3053                                                       extent_len,
3054                                                       extent_physical,
3055                                                       extent_dev, flags,
3056                                                       generation,
3057                                                       extent_mirror_num);
3058
3059                         scrub_free_csums(sctx);
3060
3061                         if (ret)
3062                                 goto out;
3063
3064                         if (extent_logical + extent_len <
3065                             key.objectid + bytes) {
3066                                 logic_start += map->stripe_len;
3067
3068                                 if (logic_start >= logic_end) {
3069                                         stop_loop = 1;
3070                                         break;
3071                                 }
3072
3073                                 if (logic_start < key.objectid + bytes) {
3074                                         cond_resched();
3075                                         goto again;
3076                                 }
3077                         }
3078 next:
3079                         path->slots[0]++;
3080                 }
3081
3082                 btrfs_release_path(path);
3083
3084                 if (stop_loop)
3085                         break;
3086
3087                 logic_start += map->stripe_len;
3088         }
3089 out:
3090         if (ret < 0)
3091                 scrub_parity_mark_sectors_error(sparity, logic_start,
3092                                                 logic_end - logic_start);
3093         scrub_parity_put(sparity);
3094         scrub_submit(sctx);
3095         mutex_lock(&sctx->wr_ctx.wr_lock);
3096         scrub_wr_submit(sctx);
3097         mutex_unlock(&sctx->wr_ctx.wr_lock);
3098
3099         btrfs_release_path(path);
3100         return ret < 0 ? ret : 0;
3101 }
3102
3103 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3104                                            struct map_lookup *map,
3105                                            struct btrfs_device *scrub_dev,
3106                                            int num, u64 base, u64 length,
3107                                            int is_dev_replace)
3108 {
3109         struct btrfs_path *path, *ppath;
3110         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
3111         struct btrfs_root *root = fs_info->extent_root;
3112         struct btrfs_root *csum_root = fs_info->csum_root;
3113         struct btrfs_extent_item *extent;
3114         struct blk_plug plug;
3115         u64 flags;
3116         int ret;
3117         int slot;
3118         u64 nstripes;
3119         struct extent_buffer *l;
3120         struct btrfs_key key;
3121         u64 physical;
3122         u64 logical;
3123         u64 logic_end;
3124         u64 physical_end;
3125         u64 generation;
3126         int mirror_num;
3127         struct reada_control *reada1;
3128         struct reada_control *reada2;
3129         struct btrfs_key key_start;
3130         struct btrfs_key key_end;
3131         u64 increment = map->stripe_len;
3132         u64 offset;
3133         u64 extent_logical;
3134         u64 extent_physical;
3135         u64 extent_len;
3136         u64 stripe_logical;
3137         u64 stripe_end;
3138         struct btrfs_device *extent_dev;
3139         int extent_mirror_num;
3140         int stop_loop = 0;
3141
3142         physical = map->stripes[num].physical;
3143         offset = 0;
3144         nstripes = div_u64(length, map->stripe_len);
3145         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3146                 offset = map->stripe_len * num;
3147                 increment = map->stripe_len * map->num_stripes;
3148                 mirror_num = 1;
3149         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3150                 int factor = map->num_stripes / map->sub_stripes;
3151                 offset = map->stripe_len * (num / map->sub_stripes);
3152                 increment = map->stripe_len * factor;
3153                 mirror_num = num % map->sub_stripes + 1;
3154         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
3155                 increment = map->stripe_len;
3156                 mirror_num = num % map->num_stripes + 1;
3157         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3158                 increment = map->stripe_len;
3159                 mirror_num = num % map->num_stripes + 1;
3160         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3161                 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3162                 increment = map->stripe_len * nr_data_stripes(map);
3163                 mirror_num = 1;
3164         } else {
3165                 increment = map->stripe_len;
3166                 mirror_num = 1;
3167         }
3168
3169         path = btrfs_alloc_path();
3170         if (!path)
3171                 return -ENOMEM;
3172
3173         ppath = btrfs_alloc_path();
3174         if (!ppath) {
3175                 btrfs_free_path(path);
3176                 return -ENOMEM;
3177         }
3178
3179         /*
3180          * work on commit root. The related disk blocks are static as
3181          * long as COW is applied. This means, it is save to rewrite
3182          * them to repair disk errors without any race conditions
3183          */
3184         path->search_commit_root = 1;
3185         path->skip_locking = 1;
3186
3187         ppath->search_commit_root = 1;
3188         ppath->skip_locking = 1;
3189         /*
3190          * trigger the readahead for extent tree csum tree and wait for
3191          * completion. During readahead, the scrub is officially paused
3192          * to not hold off transaction commits
3193          */
3194         logical = base + offset;
3195         physical_end = physical + nstripes * map->stripe_len;
3196         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3197                 get_raid56_logic_offset(physical_end, num,
3198                                         map, &logic_end, NULL);
3199                 logic_end += base;
3200         } else {
3201                 logic_end = logical + increment * nstripes;
3202         }
3203         wait_event(sctx->list_wait,
3204                    atomic_read(&sctx->bios_in_flight) == 0);
3205         scrub_blocked_if_needed(fs_info);
3206
3207         /* FIXME it might be better to start readahead at commit root */
3208         key_start.objectid = logical;
3209         key_start.type = BTRFS_EXTENT_ITEM_KEY;
3210         key_start.offset = (u64)0;
3211         key_end.objectid = logic_end;
3212         key_end.type = BTRFS_METADATA_ITEM_KEY;
3213         key_end.offset = (u64)-1;
3214         reada1 = btrfs_reada_add(root, &key_start, &key_end);
3215
3216         key_start.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3217         key_start.type = BTRFS_EXTENT_CSUM_KEY;
3218         key_start.offset = logical;
3219         key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3220         key_end.type = BTRFS_EXTENT_CSUM_KEY;
3221         key_end.offset = logic_end;
3222         reada2 = btrfs_reada_add(csum_root, &key_start, &key_end);
3223
3224         if (!IS_ERR(reada1))
3225                 btrfs_reada_wait(reada1);
3226         if (!IS_ERR(reada2))
3227                 btrfs_reada_wait(reada2);
3228
3229
3230         /*
3231          * collect all data csums for the stripe to avoid seeking during
3232          * the scrub. This might currently (crc32) end up to be about 1MB
3233          */
3234         blk_start_plug(&plug);
3235
3236         /*
3237          * now find all extents for each stripe and scrub them
3238          */
3239         ret = 0;
3240         while (physical < physical_end) {
3241                 /*
3242                  * canceled?
3243                  */
3244                 if (atomic_read(&fs_info->scrub_cancel_req) ||
3245                     atomic_read(&sctx->cancel_req)) {
3246                         ret = -ECANCELED;
3247                         goto out;
3248                 }
3249                 /*
3250                  * check to see if we have to pause
3251                  */
3252                 if (atomic_read(&fs_info->scrub_pause_req)) {
3253                         /* push queued extents */
3254                         atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3255                         scrub_submit(sctx);
3256                         mutex_lock(&sctx->wr_ctx.wr_lock);
3257                         scrub_wr_submit(sctx);
3258                         mutex_unlock(&sctx->wr_ctx.wr_lock);
3259                         wait_event(sctx->list_wait,
3260                                    atomic_read(&sctx->bios_in_flight) == 0);
3261                         atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3262                         scrub_blocked_if_needed(fs_info);
3263                 }
3264
3265                 /* for raid56, we skip parity stripe */
3266                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3267                         ret = get_raid56_logic_offset(physical, num, map,
3268                                                       &logical,
3269                                                       &stripe_logical);
3270                         logical += base;
3271                         if (ret) {
3272                                 stripe_logical += base;
3273                                 stripe_end = stripe_logical + increment;
3274                                 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3275                                                           ppath, stripe_logical,
3276                                                           stripe_end);
3277                                 if (ret)
3278                                         goto out;
3279                                 goto skip;
3280                         }
3281                 }
3282
3283                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3284                         key.type = BTRFS_METADATA_ITEM_KEY;
3285                 else
3286                         key.type = BTRFS_EXTENT_ITEM_KEY;
3287                 key.objectid = logical;
3288                 key.offset = (u64)-1;
3289
3290                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3291                 if (ret < 0)
3292                         goto out;
3293
3294                 if (ret > 0) {
3295                         ret = btrfs_previous_extent_item(root, path, 0);
3296                         if (ret < 0)
3297                                 goto out;
3298                         if (ret > 0) {
3299                                 /* there's no smaller item, so stick with the
3300                                  * larger one */
3301                                 btrfs_release_path(path);
3302                                 ret = btrfs_search_slot(NULL, root, &key,
3303                                                         path, 0, 0);
3304                                 if (ret < 0)
3305                                         goto out;
3306                         }
3307                 }
3308
3309                 stop_loop = 0;
3310                 while (1) {
3311                         u64 bytes;
3312
3313                         l = path->nodes[0];
3314                         slot = path->slots[0];
3315                         if (slot >= btrfs_header_nritems(l)) {
3316                                 ret = btrfs_next_leaf(root, path);
3317                                 if (ret == 0)
3318                                         continue;
3319                                 if (ret < 0)
3320                                         goto out;
3321
3322                                 stop_loop = 1;
3323                                 break;
3324                         }
3325                         btrfs_item_key_to_cpu(l, &key, slot);
3326
3327                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3328                             key.type != BTRFS_METADATA_ITEM_KEY)
3329                                 goto next;
3330
3331                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3332                                 bytes = root->nodesize;
3333                         else
3334                                 bytes = key.offset;
3335
3336                         if (key.objectid + bytes <= logical)
3337                                 goto next;
3338
3339                         if (key.objectid >= logical + map->stripe_len) {
3340                                 /* out of this device extent */
3341                                 if (key.objectid >= logic_end)
3342                                         stop_loop = 1;
3343                                 break;
3344                         }
3345
3346                         extent = btrfs_item_ptr(l, slot,
3347                                                 struct btrfs_extent_item);
3348                         flags = btrfs_extent_flags(l, extent);
3349                         generation = btrfs_extent_generation(l, extent);
3350
3351                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3352                             (key.objectid < logical ||
3353                              key.objectid + bytes >
3354                              logical + map->stripe_len)) {
3355                                 btrfs_err(fs_info,
3356                                            "scrub: tree block %llu spanning "
3357                                            "stripes, ignored. logical=%llu",
3358                                        key.objectid, logical);
3359                                 goto next;
3360                         }
3361
3362 again:
3363                         extent_logical = key.objectid;
3364                         extent_len = bytes;
3365
3366                         /*
3367                          * trim extent to this stripe
3368                          */
3369                         if (extent_logical < logical) {
3370                                 extent_len -= logical - extent_logical;
3371                                 extent_logical = logical;
3372                         }
3373                         if (extent_logical + extent_len >
3374                             logical + map->stripe_len) {
3375                                 extent_len = logical + map->stripe_len -
3376                                              extent_logical;
3377                         }
3378
3379                         extent_physical = extent_logical - logical + physical;
3380                         extent_dev = scrub_dev;
3381                         extent_mirror_num = mirror_num;
3382                         if (is_dev_replace)
3383                                 scrub_remap_extent(fs_info, extent_logical,
3384                                                    extent_len, &extent_physical,
3385                                                    &extent_dev,
3386                                                    &extent_mirror_num);
3387
3388                         ret = btrfs_lookup_csums_range(csum_root,
3389                                                        extent_logical,
3390                                                        extent_logical +
3391                                                        extent_len - 1,
3392                                                        &sctx->csum_list, 1);
3393                         if (ret)
3394                                 goto out;
3395
3396                         ret = scrub_extent(sctx, extent_logical, extent_len,
3397                                            extent_physical, extent_dev, flags,
3398                                            generation, extent_mirror_num,
3399                                            extent_logical - logical + physical);
3400
3401                         scrub_free_csums(sctx);
3402
3403                         if (ret)
3404                                 goto out;
3405
3406                         if (extent_logical + extent_len <
3407                             key.objectid + bytes) {
3408                                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3409                                         /*
3410                                          * loop until we find next data stripe
3411                                          * or we have finished all stripes.
3412                                          */
3413 loop:
3414                                         physical += map->stripe_len;
3415                                         ret = get_raid56_logic_offset(physical,
3416                                                         num, map, &logical,
3417                                                         &stripe_logical);
3418                                         logical += base;
3419
3420                                         if (ret && physical < physical_end) {
3421                                                 stripe_logical += base;
3422                                                 stripe_end = stripe_logical +
3423                                                                 increment;
3424                                                 ret = scrub_raid56_parity(sctx,
3425                                                         map, scrub_dev, ppath,
3426                                                         stripe_logical,
3427                                                         stripe_end);
3428                                                 if (ret)
3429                                                         goto out;
3430                                                 goto loop;
3431                                         }
3432                                 } else {
3433                                         physical += map->stripe_len;
3434                                         logical += increment;
3435                                 }
3436                                 if (logical < key.objectid + bytes) {
3437                                         cond_resched();
3438                                         goto again;
3439                                 }
3440
3441                                 if (physical >= physical_end) {
3442                                         stop_loop = 1;
3443                                         break;
3444                                 }
3445                         }
3446 next:
3447                         path->slots[0]++;
3448                 }
3449                 btrfs_release_path(path);
3450 skip:
3451                 logical += increment;
3452                 physical += map->stripe_len;
3453                 spin_lock(&sctx->stat_lock);
3454                 if (stop_loop)
3455                         sctx->stat.last_physical = map->stripes[num].physical +
3456                                                    length;
3457                 else
3458                         sctx->stat.last_physical = physical;
3459                 spin_unlock(&sctx->stat_lock);
3460                 if (stop_loop)
3461                         break;
3462         }
3463 out:
3464         /* push queued extents */
3465         scrub_submit(sctx);
3466         mutex_lock(&sctx->wr_ctx.wr_lock);
3467         scrub_wr_submit(sctx);
3468         mutex_unlock(&sctx->wr_ctx.wr_lock);
3469
3470         blk_finish_plug(&plug);
3471         btrfs_free_path(path);
3472         btrfs_free_path(ppath);
3473         return ret < 0 ? ret : 0;
3474 }
3475
3476 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3477                                           struct btrfs_device *scrub_dev,
3478                                           u64 chunk_tree, u64 chunk_objectid,
3479                                           u64 chunk_offset, u64 length,
3480                                           u64 dev_offset, int is_dev_replace)
3481 {
3482         struct btrfs_mapping_tree *map_tree =
3483                 &sctx->dev_root->fs_info->mapping_tree;
3484         struct map_lookup *map;
3485         struct extent_map *em;
3486         int i;
3487         int ret = 0;
3488
3489         read_lock(&map_tree->map_tree.lock);
3490         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
3491         read_unlock(&map_tree->map_tree.lock);
3492
3493         if (!em)
3494                 return -EINVAL;
3495
3496         map = (struct map_lookup *)em->bdev;
3497         if (em->start != chunk_offset)
3498                 goto out;
3499
3500         if (em->len < length)
3501                 goto out;
3502
3503         for (i = 0; i < map->num_stripes; ++i) {
3504                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3505                     map->stripes[i].physical == dev_offset) {
3506                         ret = scrub_stripe(sctx, map, scrub_dev, i,
3507                                            chunk_offset, length,
3508                                            is_dev_replace);
3509                         if (ret)
3510                                 goto out;
3511                 }
3512         }
3513 out:
3514         free_extent_map(em);
3515
3516         return ret;
3517 }
3518
3519 static noinline_for_stack
3520 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3521                            struct btrfs_device *scrub_dev, u64 start, u64 end,
3522                            int is_dev_replace)
3523 {
3524         struct btrfs_dev_extent *dev_extent = NULL;
3525         struct btrfs_path *path;
3526         struct btrfs_root *root = sctx->dev_root;
3527         struct btrfs_fs_info *fs_info = root->fs_info;
3528         u64 length;
3529         u64 chunk_tree;
3530         u64 chunk_objectid;
3531         u64 chunk_offset;
3532         int ret = 0;
3533         int slot;
3534         struct extent_buffer *l;
3535         struct btrfs_key key;
3536         struct btrfs_key found_key;
3537         struct btrfs_block_group_cache *cache;
3538         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
3539
3540         path = btrfs_alloc_path();
3541         if (!path)
3542                 return -ENOMEM;
3543
3544         path->reada = 2;
3545         path->search_commit_root = 1;
3546         path->skip_locking = 1;
3547
3548         key.objectid = scrub_dev->devid;
3549         key.offset = 0ull;
3550         key.type = BTRFS_DEV_EXTENT_KEY;
3551
3552         while (1) {
3553                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3554                 if (ret < 0)
3555                         break;
3556                 if (ret > 0) {
3557                         if (path->slots[0] >=
3558                             btrfs_header_nritems(path->nodes[0])) {
3559                                 ret = btrfs_next_leaf(root, path);
3560                                 if (ret < 0)
3561                                         break;
3562                                 if (ret > 0) {
3563                                         ret = 0;
3564                                         break;
3565                                 }
3566                         } else {
3567                                 ret = 0;
3568                         }
3569                 }
3570
3571                 l = path->nodes[0];
3572                 slot = path->slots[0];
3573
3574                 btrfs_item_key_to_cpu(l, &found_key, slot);
3575
3576                 if (found_key.objectid != scrub_dev->devid)
3577                         break;
3578
3579                 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
3580                         break;
3581
3582                 if (found_key.offset >= end)
3583                         break;
3584
3585                 if (found_key.offset < key.offset)
3586                         break;
3587
3588                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3589                 length = btrfs_dev_extent_length(l, dev_extent);
3590
3591                 if (found_key.offset + length <= start)
3592                         goto skip;
3593
3594                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
3595                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
3596                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3597
3598                 /*
3599                  * get a reference on the corresponding block group to prevent
3600                  * the chunk from going away while we scrub it
3601                  */
3602                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3603
3604                 /* some chunks are removed but not committed to disk yet,
3605                  * continue scrubbing */
3606                 if (!cache)
3607                         goto skip;
3608
3609                 /*
3610                  * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3611                  * to avoid deadlock caused by:
3612                  * btrfs_inc_block_group_ro()
3613                  * -> btrfs_wait_for_commit()
3614                  * -> btrfs_commit_transaction()
3615                  * -> btrfs_scrub_pause()
3616                  */
3617                 scrub_pause_on(fs_info);
3618                 ret = btrfs_inc_block_group_ro(root, cache);
3619                 scrub_pause_off(fs_info);
3620                 if (ret) {
3621                         btrfs_put_block_group(cache);
3622                         break;
3623                 }
3624
3625                 dev_replace->cursor_right = found_key.offset + length;
3626                 dev_replace->cursor_left = found_key.offset;
3627                 dev_replace->item_needs_writeback = 1;
3628                 ret = scrub_chunk(sctx, scrub_dev, chunk_tree, chunk_objectid,
3629                                   chunk_offset, length, found_key.offset,
3630                                   is_dev_replace);
3631
3632                 /*
3633                  * flush, submit all pending read and write bios, afterwards
3634                  * wait for them.
3635                  * Note that in the dev replace case, a read request causes
3636                  * write requests that are submitted in the read completion
3637                  * worker. Therefore in the current situation, it is required
3638                  * that all write requests are flushed, so that all read and
3639                  * write requests are really completed when bios_in_flight
3640                  * changes to 0.
3641                  */
3642                 atomic_set(&sctx->wr_ctx.flush_all_writes, 1);
3643                 scrub_submit(sctx);
3644                 mutex_lock(&sctx->wr_ctx.wr_lock);
3645                 scrub_wr_submit(sctx);
3646                 mutex_unlock(&sctx->wr_ctx.wr_lock);
3647
3648                 wait_event(sctx->list_wait,
3649                            atomic_read(&sctx->bios_in_flight) == 0);
3650
3651                 scrub_pause_on(fs_info);
3652
3653                 /*
3654                  * must be called before we decrease @scrub_paused.
3655                  * make sure we don't block transaction commit while
3656                  * we are waiting pending workers finished.
3657                  */
3658                 wait_event(sctx->list_wait,
3659                            atomic_read(&sctx->workers_pending) == 0);
3660                 atomic_set(&sctx->wr_ctx.flush_all_writes, 0);
3661
3662                 scrub_pause_off(fs_info);
3663
3664                 btrfs_dec_block_group_ro(root, cache);
3665
3666                 btrfs_put_block_group(cache);
3667                 if (ret)
3668                         break;
3669                 if (is_dev_replace &&
3670                     atomic64_read(&dev_replace->num_write_errors) > 0) {
3671                         ret = -EIO;
3672                         break;
3673                 }
3674                 if (sctx->stat.malloc_errors > 0) {
3675                         ret = -ENOMEM;
3676                         break;
3677                 }
3678
3679                 dev_replace->cursor_left = dev_replace->cursor_right;
3680                 dev_replace->item_needs_writeback = 1;
3681 skip:
3682                 key.offset = found_key.offset + length;
3683                 btrfs_release_path(path);
3684         }
3685
3686         btrfs_free_path(path);
3687
3688         return ret;
3689 }
3690
3691 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3692                                            struct btrfs_device *scrub_dev)
3693 {
3694         int     i;
3695         u64     bytenr;
3696         u64     gen;
3697         int     ret;
3698         struct btrfs_root *root = sctx->dev_root;
3699
3700         if (test_bit(BTRFS_FS_STATE_ERROR, &root->fs_info->fs_state))
3701                 return -EIO;
3702
3703         /* Seed devices of a new filesystem has their own generation. */
3704         if (scrub_dev->fs_devices != root->fs_info->fs_devices)
3705                 gen = scrub_dev->generation;
3706         else
3707                 gen = root->fs_info->last_trans_committed;
3708
3709         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3710                 bytenr = btrfs_sb_offset(i);
3711                 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3712                     scrub_dev->commit_total_bytes)
3713                         break;
3714
3715                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
3716                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
3717                                   NULL, 1, bytenr);
3718                 if (ret)
3719                         return ret;
3720         }
3721         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3722
3723         return 0;
3724 }
3725
3726 /*
3727  * get a reference count on fs_info->scrub_workers. start worker if necessary
3728  */
3729 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3730                                                 int is_dev_replace)
3731 {
3732         unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3733         int max_active = fs_info->thread_pool_size;
3734
3735         if (fs_info->scrub_workers_refcnt == 0) {
3736                 if (is_dev_replace)
3737                         fs_info->scrub_workers =
3738                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
3739                                                       1, 4);
3740                 else
3741                         fs_info->scrub_workers =
3742                                 btrfs_alloc_workqueue("btrfs-scrub", flags,
3743                                                       max_active, 4);
3744                 if (!fs_info->scrub_workers)
3745                         goto fail_scrub_workers;
3746
3747                 fs_info->scrub_wr_completion_workers =
3748                         btrfs_alloc_workqueue("btrfs-scrubwrc", flags,
3749                                               max_active, 2);
3750                 if (!fs_info->scrub_wr_completion_workers)
3751                         goto fail_scrub_wr_completion_workers;
3752
3753                 fs_info->scrub_nocow_workers =
3754                         btrfs_alloc_workqueue("btrfs-scrubnc", flags, 1, 0);
3755                 if (!fs_info->scrub_nocow_workers)
3756                         goto fail_scrub_nocow_workers;
3757                 fs_info->scrub_parity_workers =
3758                         btrfs_alloc_workqueue("btrfs-scrubparity", flags,
3759                                               max_active, 2);
3760                 if (!fs_info->scrub_parity_workers)
3761                         goto fail_scrub_parity_workers;
3762         }
3763         ++fs_info->scrub_workers_refcnt;
3764         return 0;
3765
3766 fail_scrub_parity_workers:
3767         btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3768 fail_scrub_nocow_workers:
3769         btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3770 fail_scrub_wr_completion_workers:
3771         btrfs_destroy_workqueue(fs_info->scrub_workers);
3772 fail_scrub_workers:
3773         return -ENOMEM;
3774 }
3775
3776 static noinline_for_stack void scrub_workers_put(struct btrfs_fs_info *fs_info)
3777 {
3778         if (--fs_info->scrub_workers_refcnt == 0) {
3779                 btrfs_destroy_workqueue(fs_info->scrub_workers);
3780                 btrfs_destroy_workqueue(fs_info->scrub_wr_completion_workers);
3781                 btrfs_destroy_workqueue(fs_info->scrub_nocow_workers);
3782                 btrfs_destroy_workqueue(fs_info->scrub_parity_workers);
3783         }
3784         WARN_ON(fs_info->scrub_workers_refcnt < 0);
3785 }
3786
3787 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3788                     u64 end, struct btrfs_scrub_progress *progress,
3789                     int readonly, int is_dev_replace)
3790 {
3791         struct scrub_ctx *sctx;
3792         int ret;
3793         struct btrfs_device *dev;
3794         struct rcu_string *name;
3795
3796         if (btrfs_fs_closing(fs_info))
3797                 return -EINVAL;
3798
3799         if (fs_info->chunk_root->nodesize > BTRFS_STRIPE_LEN) {
3800                 /*
3801                  * in this case scrub is unable to calculate the checksum
3802                  * the way scrub is implemented. Do not handle this
3803                  * situation at all because it won't ever happen.
3804                  */
3805                 btrfs_err(fs_info,
3806                            "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3807                        fs_info->chunk_root->nodesize, BTRFS_STRIPE_LEN);
3808                 return -EINVAL;
3809         }
3810
3811         if (fs_info->chunk_root->sectorsize != PAGE_SIZE) {
3812                 /* not supported for data w/o checksums */
3813                 btrfs_err(fs_info,
3814                            "scrub: size assumption sectorsize != PAGE_SIZE "
3815                            "(%d != %lu) fails",
3816                        fs_info->chunk_root->sectorsize, PAGE_SIZE);
3817                 return -EINVAL;
3818         }
3819
3820         if (fs_info->chunk_root->nodesize >
3821             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3822             fs_info->chunk_root->sectorsize >
3823             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3824                 /*
3825                  * would exhaust the array bounds of pagev member in
3826                  * struct scrub_block
3827                  */
3828                 btrfs_err(fs_info, "scrub: size assumption nodesize and sectorsize "
3829                            "<= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3830                        fs_info->chunk_root->nodesize,
3831                        SCRUB_MAX_PAGES_PER_BLOCK,
3832                        fs_info->chunk_root->sectorsize,
3833                        SCRUB_MAX_PAGES_PER_BLOCK);
3834                 return -EINVAL;
3835         }
3836
3837
3838         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3839         dev = btrfs_find_device(fs_info, devid, NULL, NULL);
3840         if (!dev || (dev->missing && !is_dev_replace)) {
3841                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3842                 return -ENODEV;
3843         }
3844
3845         if (!is_dev_replace && !readonly && !dev->writeable) {
3846                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3847                 rcu_read_lock();
3848                 name = rcu_dereference(dev->name);
3849                 btrfs_err(fs_info, "scrub: device %s is not writable",
3850                           name->str);
3851                 rcu_read_unlock();
3852                 return -EROFS;
3853         }
3854
3855         mutex_lock(&fs_info->scrub_lock);
3856         if (!dev->in_fs_metadata || dev->is_tgtdev_for_dev_replace) {
3857                 mutex_unlock(&fs_info->scrub_lock);
3858                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3859                 return -EIO;
3860         }
3861
3862         btrfs_dev_replace_lock(&fs_info->dev_replace);
3863         if (dev->scrub_device ||
3864             (!is_dev_replace &&
3865              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3866                 btrfs_dev_replace_unlock(&fs_info->dev_replace);
3867                 mutex_unlock(&fs_info->scrub_lock);
3868                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3869                 return -EINPROGRESS;
3870         }
3871         btrfs_dev_replace_unlock(&fs_info->dev_replace);
3872
3873         ret = scrub_workers_get(fs_info, is_dev_replace);
3874         if (ret) {
3875                 mutex_unlock(&fs_info->scrub_lock);
3876                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3877                 return ret;
3878         }
3879
3880         sctx = scrub_setup_ctx(dev, is_dev_replace);
3881         if (IS_ERR(sctx)) {
3882                 mutex_unlock(&fs_info->scrub_lock);
3883                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3884                 scrub_workers_put(fs_info);
3885                 return PTR_ERR(sctx);
3886         }
3887         sctx->readonly = readonly;
3888         dev->scrub_device = sctx;
3889         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3890
3891         /*
3892          * checking @scrub_pause_req here, we can avoid
3893          * race between committing transaction and scrubbing.
3894          */
3895         __scrub_blocked_if_needed(fs_info);
3896         atomic_inc(&fs_info->scrubs_running);
3897         mutex_unlock(&fs_info->scrub_lock);
3898
3899         if (!is_dev_replace) {
3900                 /*
3901                  * by holding device list mutex, we can
3902                  * kick off writing super in log tree sync.
3903                  */
3904                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3905                 ret = scrub_supers(sctx, dev);
3906                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3907         }
3908
3909         if (!ret)
3910                 ret = scrub_enumerate_chunks(sctx, dev, start, end,
3911                                              is_dev_replace);
3912
3913         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3914         atomic_dec(&fs_info->scrubs_running);
3915         wake_up(&fs_info->scrub_pause_wait);
3916
3917         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3918
3919         if (progress)
3920                 memcpy(progress, &sctx->stat, sizeof(*progress));
3921
3922         mutex_lock(&fs_info->scrub_lock);
3923         dev->scrub_device = NULL;
3924         scrub_workers_put(fs_info);
3925         mutex_unlock(&fs_info->scrub_lock);
3926
3927         scrub_put_ctx(sctx);
3928
3929         return ret;
3930 }
3931
3932 void btrfs_scrub_pause(struct btrfs_root *root)
3933 {
3934         struct btrfs_fs_info *fs_info = root->fs_info;
3935
3936         mutex_lock(&fs_info->scrub_lock);
3937         atomic_inc(&fs_info->scrub_pause_req);
3938         while (atomic_read(&fs_info->scrubs_paused) !=
3939                atomic_read(&fs_info->scrubs_running)) {
3940                 mutex_unlock(&fs_info->scrub_lock);
3941                 wait_event(fs_info->scrub_pause_wait,
3942                            atomic_read(&fs_info->scrubs_paused) ==
3943                            atomic_read(&fs_info->scrubs_running));
3944                 mutex_lock(&fs_info->scrub_lock);
3945         }
3946         mutex_unlock(&fs_info->scrub_lock);
3947 }
3948
3949 void btrfs_scrub_continue(struct btrfs_root *root)
3950 {
3951         struct btrfs_fs_info *fs_info = root->fs_info;
3952
3953         atomic_dec(&fs_info->scrub_pause_req);
3954         wake_up(&fs_info->scrub_pause_wait);
3955 }
3956
3957 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
3958 {
3959         mutex_lock(&fs_info->scrub_lock);
3960         if (!atomic_read(&fs_info->scrubs_running)) {
3961                 mutex_unlock(&fs_info->scrub_lock);
3962                 return -ENOTCONN;
3963         }
3964
3965         atomic_inc(&fs_info->scrub_cancel_req);
3966         while (atomic_read(&fs_info->scrubs_running)) {
3967                 mutex_unlock(&fs_info->scrub_lock);
3968                 wait_event(fs_info->scrub_pause_wait,
3969                            atomic_read(&fs_info->scrubs_running) == 0);
3970                 mutex_lock(&fs_info->scrub_lock);
3971         }
3972         atomic_dec(&fs_info->scrub_cancel_req);
3973         mutex_unlock(&fs_info->scrub_lock);
3974
3975         return 0;
3976 }
3977
3978 int btrfs_scrub_cancel_dev(struct btrfs_fs_info *fs_info,
3979                            struct btrfs_device *dev)
3980 {
3981         struct scrub_ctx *sctx;
3982
3983         mutex_lock(&fs_info->scrub_lock);
3984         sctx = dev->scrub_device;
3985         if (!sctx) {
3986                 mutex_unlock(&fs_info->scrub_lock);
3987                 return -ENOTCONN;
3988         }
3989         atomic_inc(&sctx->cancel_req);
3990         while (dev->scrub_device) {
3991                 mutex_unlock(&fs_info->scrub_lock);
3992                 wait_event(fs_info->scrub_pause_wait,
3993                            dev->scrub_device == NULL);
3994                 mutex_lock(&fs_info->scrub_lock);
3995         }
3996         mutex_unlock(&fs_info->scrub_lock);
3997
3998         return 0;
3999 }
4000
4001 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
4002                          struct btrfs_scrub_progress *progress)
4003 {
4004         struct btrfs_device *dev;
4005         struct scrub_ctx *sctx = NULL;
4006
4007         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
4008         dev = btrfs_find_device(root->fs_info, devid, NULL, NULL);
4009         if (dev)
4010                 sctx = dev->scrub_device;
4011         if (sctx)
4012                 memcpy(progress, &sctx->stat, sizeof(*progress));
4013         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
4014
4015         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
4016 }
4017
4018 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
4019                                u64 extent_logical, u64 extent_len,
4020                                u64 *extent_physical,
4021                                struct btrfs_device **extent_dev,
4022                                int *extent_mirror_num)
4023 {
4024         u64 mapped_length;
4025         struct btrfs_bio *bbio = NULL;
4026         int ret;
4027
4028         mapped_length = extent_len;
4029         ret = btrfs_map_block(fs_info, READ, extent_logical,
4030                               &mapped_length, &bbio, 0);
4031         if (ret || !bbio || mapped_length < extent_len ||
4032             !bbio->stripes[0].dev->bdev) {
4033                 btrfs_put_bbio(bbio);
4034                 return;
4035         }
4036
4037         *extent_physical = bbio->stripes[0].physical;
4038         *extent_mirror_num = bbio->mirror_num;
4039         *extent_dev = bbio->stripes[0].dev;
4040         btrfs_put_bbio(bbio);
4041 }
4042
4043 static int scrub_setup_wr_ctx(struct scrub_ctx *sctx,
4044                               struct scrub_wr_ctx *wr_ctx,
4045                               struct btrfs_fs_info *fs_info,
4046                               struct btrfs_device *dev,
4047                               int is_dev_replace)
4048 {
4049         WARN_ON(wr_ctx->wr_curr_bio != NULL);
4050
4051         mutex_init(&wr_ctx->wr_lock);
4052         wr_ctx->wr_curr_bio = NULL;
4053         if (!is_dev_replace)
4054                 return 0;
4055
4056         WARN_ON(!dev->bdev);
4057         wr_ctx->pages_per_wr_bio = min_t(int, SCRUB_PAGES_PER_WR_BIO,
4058                                          bio_get_nr_vecs(dev->bdev));
4059         wr_ctx->tgtdev = dev;
4060         atomic_set(&wr_ctx->flush_all_writes, 0);
4061         return 0;
4062 }
4063
4064 static void scrub_free_wr_ctx(struct scrub_wr_ctx *wr_ctx)
4065 {
4066         mutex_lock(&wr_ctx->wr_lock);
4067         kfree(wr_ctx->wr_curr_bio);
4068         wr_ctx->wr_curr_bio = NULL;
4069         mutex_unlock(&wr_ctx->wr_lock);
4070 }
4071
4072 static int copy_nocow_pages(struct scrub_ctx *sctx, u64 logical, u64 len,
4073                             int mirror_num, u64 physical_for_dev_replace)
4074 {
4075         struct scrub_copy_nocow_ctx *nocow_ctx;
4076         struct btrfs_fs_info *fs_info = sctx->dev_root->fs_info;
4077
4078         nocow_ctx = kzalloc(sizeof(*nocow_ctx), GFP_NOFS);
4079         if (!nocow_ctx) {
4080                 spin_lock(&sctx->stat_lock);
4081                 sctx->stat.malloc_errors++;
4082                 spin_unlock(&sctx->stat_lock);
4083                 return -ENOMEM;
4084         }
4085
4086         scrub_pending_trans_workers_inc(sctx);
4087
4088         nocow_ctx->sctx = sctx;
4089         nocow_ctx->logical = logical;
4090         nocow_ctx->len = len;
4091         nocow_ctx->mirror_num = mirror_num;
4092         nocow_ctx->physical_for_dev_replace = physical_for_dev_replace;
4093         btrfs_init_work(&nocow_ctx->work, btrfs_scrubnc_helper,
4094                         copy_nocow_pages_worker, NULL, NULL);
4095         INIT_LIST_HEAD(&nocow_ctx->inodes);
4096         btrfs_queue_work(fs_info->scrub_nocow_workers,
4097                          &nocow_ctx->work);
4098
4099         return 0;
4100 }
4101
4102 static int record_inode_for_nocow(u64 inum, u64 offset, u64 root, void *ctx)
4103 {
4104         struct scrub_copy_nocow_ctx *nocow_ctx = ctx;
4105         struct scrub_nocow_inode *nocow_inode;
4106
4107         nocow_inode = kzalloc(sizeof(*nocow_inode), GFP_NOFS);
4108         if (!nocow_inode)
4109                 return -ENOMEM;
4110         nocow_inode->inum = inum;
4111         nocow_inode->offset = offset;
4112         nocow_inode->root = root;
4113         list_add_tail(&nocow_inode->list, &nocow_ctx->inodes);
4114         return 0;
4115 }
4116
4117 #define COPY_COMPLETE 1
4118
4119 static void copy_nocow_pages_worker(struct btrfs_work *work)
4120 {
4121         struct scrub_copy_nocow_ctx *nocow_ctx =
4122                 container_of(work, struct scrub_copy_nocow_ctx, work);
4123         struct scrub_ctx *sctx = nocow_ctx->sctx;
4124         u64 logical = nocow_ctx->logical;
4125         u64 len = nocow_ctx->len;
4126         int mirror_num = nocow_ctx->mirror_num;
4127         u64 physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4128         int ret;
4129         struct btrfs_trans_handle *trans = NULL;
4130         struct btrfs_fs_info *fs_info;
4131         struct btrfs_path *path;
4132         struct btrfs_root *root;
4133         int not_written = 0;
4134
4135         fs_info = sctx->dev_root->fs_info;
4136         root = fs_info->extent_root;
4137
4138         path = btrfs_alloc_path();
4139         if (!path) {
4140                 spin_lock(&sctx->stat_lock);
4141                 sctx->stat.malloc_errors++;
4142                 spin_unlock(&sctx->stat_lock);
4143                 not_written = 1;
4144                 goto out;
4145         }
4146
4147         trans = btrfs_join_transaction(root);
4148         if (IS_ERR(trans)) {
4149                 not_written = 1;
4150                 goto out;
4151         }
4152
4153         ret = iterate_inodes_from_logical(logical, fs_info, path,
4154                                           record_inode_for_nocow, nocow_ctx);
4155         if (ret != 0 && ret != -ENOENT) {
4156                 btrfs_warn(fs_info, "iterate_inodes_from_logical() failed: log %llu, "
4157                         "phys %llu, len %llu, mir %u, ret %d",
4158                         logical, physical_for_dev_replace, len, mirror_num,
4159                         ret);
4160                 not_written = 1;
4161                 goto out;
4162         }
4163
4164         btrfs_end_transaction(trans, root);
4165         trans = NULL;
4166         while (!list_empty(&nocow_ctx->inodes)) {
4167                 struct scrub_nocow_inode *entry;
4168                 entry = list_first_entry(&nocow_ctx->inodes,
4169                                          struct scrub_nocow_inode,
4170                                          list);
4171                 list_del_init(&entry->list);
4172                 ret = copy_nocow_pages_for_inode(entry->inum, entry->offset,
4173                                                  entry->root, nocow_ctx);
4174                 kfree(entry);
4175                 if (ret == COPY_COMPLETE) {
4176                         ret = 0;
4177                         break;
4178                 } else if (ret) {
4179                         break;
4180                 }
4181         }
4182 out:
4183         while (!list_empty(&nocow_ctx->inodes)) {
4184                 struct scrub_nocow_inode *entry;
4185                 entry = list_first_entry(&nocow_ctx->inodes,
4186                                          struct scrub_nocow_inode,
4187                                          list);
4188                 list_del_init(&entry->list);
4189                 kfree(entry);
4190         }
4191         if (trans && !IS_ERR(trans))
4192                 btrfs_end_transaction(trans, root);
4193         if (not_written)
4194                 btrfs_dev_replace_stats_inc(&fs_info->dev_replace.
4195                                             num_uncorrectable_read_errors);
4196
4197         btrfs_free_path(path);
4198         kfree(nocow_ctx);
4199
4200         scrub_pending_trans_workers_dec(sctx);
4201 }
4202
4203 static int check_extent_to_block(struct inode *inode, u64 start, u64 len,
4204                                  u64 logical)
4205 {
4206         struct extent_state *cached_state = NULL;
4207         struct btrfs_ordered_extent *ordered;
4208         struct extent_io_tree *io_tree;
4209         struct extent_map *em;
4210         u64 lockstart = start, lockend = start + len - 1;
4211         int ret = 0;
4212
4213         io_tree = &BTRFS_I(inode)->io_tree;
4214
4215         lock_extent_bits(io_tree, lockstart, lockend, 0, &cached_state);
4216         ordered = btrfs_lookup_ordered_range(inode, lockstart, len);
4217         if (ordered) {
4218                 btrfs_put_ordered_extent(ordered);
4219                 ret = 1;
4220                 goto out_unlock;
4221         }
4222
4223         em = btrfs_get_extent(inode, NULL, 0, start, len, 0);
4224         if (IS_ERR(em)) {
4225                 ret = PTR_ERR(em);
4226                 goto out_unlock;
4227         }
4228
4229         /*
4230          * This extent does not actually cover the logical extent anymore,
4231          * move on to the next inode.
4232          */
4233         if (em->block_start > logical ||
4234             em->block_start + em->block_len < logical + len) {
4235                 free_extent_map(em);
4236                 ret = 1;
4237                 goto out_unlock;
4238         }
4239         free_extent_map(em);
4240
4241 out_unlock:
4242         unlock_extent_cached(io_tree, lockstart, lockend, &cached_state,
4243                              GFP_NOFS);
4244         return ret;
4245 }
4246
4247 static int copy_nocow_pages_for_inode(u64 inum, u64 offset, u64 root,
4248                                       struct scrub_copy_nocow_ctx *nocow_ctx)
4249 {
4250         struct btrfs_fs_info *fs_info = nocow_ctx->sctx->dev_root->fs_info;
4251         struct btrfs_key key;
4252         struct inode *inode;
4253         struct page *page;
4254         struct btrfs_root *local_root;
4255         struct extent_io_tree *io_tree;
4256         u64 physical_for_dev_replace;
4257         u64 nocow_ctx_logical;
4258         u64 len = nocow_ctx->len;
4259         unsigned long index;
4260         int srcu_index;
4261         int ret = 0;
4262         int err = 0;
4263
4264         key.objectid = root;
4265         key.type = BTRFS_ROOT_ITEM_KEY;
4266         key.offset = (u64)-1;
4267
4268         srcu_index = srcu_read_lock(&fs_info->subvol_srcu);
4269
4270         local_root = btrfs_read_fs_root_no_name(fs_info, &key);
4271         if (IS_ERR(local_root)) {
4272                 srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4273                 return PTR_ERR(local_root);
4274         }
4275
4276         key.type = BTRFS_INODE_ITEM_KEY;
4277         key.objectid = inum;
4278         key.offset = 0;
4279         inode = btrfs_iget(fs_info->sb, &key, local_root, NULL);
4280         srcu_read_unlock(&fs_info->subvol_srcu, srcu_index);
4281         if (IS_ERR(inode))
4282                 return PTR_ERR(inode);
4283
4284         /* Avoid truncate/dio/punch hole.. */
4285         mutex_lock(&inode->i_mutex);
4286         inode_dio_wait(inode);
4287
4288         physical_for_dev_replace = nocow_ctx->physical_for_dev_replace;
4289         io_tree = &BTRFS_I(inode)->io_tree;
4290         nocow_ctx_logical = nocow_ctx->logical;
4291
4292         ret = check_extent_to_block(inode, offset, len, nocow_ctx_logical);
4293         if (ret) {
4294                 ret = ret > 0 ? 0 : ret;
4295                 goto out;
4296         }
4297
4298         while (len >= PAGE_CACHE_SIZE) {
4299                 index = offset >> PAGE_CACHE_SHIFT;
4300 again:
4301                 page = find_or_create_page(inode->i_mapping, index, GFP_NOFS);
4302                 if (!page) {
4303                         btrfs_err(fs_info, "find_or_create_page() failed");
4304                         ret = -ENOMEM;
4305                         goto out;
4306                 }
4307
4308                 if (PageUptodate(page)) {
4309                         if (PageDirty(page))
4310                                 goto next_page;
4311                 } else {
4312                         ClearPageError(page);
4313                         err = extent_read_full_page(io_tree, page,
4314                                                            btrfs_get_extent,
4315                                                            nocow_ctx->mirror_num);
4316                         if (err) {
4317                                 ret = err;
4318                                 goto next_page;
4319                         }
4320
4321                         lock_page(page);
4322                         /*
4323                          * If the page has been remove from the page cache,
4324                          * the data on it is meaningless, because it may be
4325                          * old one, the new data may be written into the new
4326                          * page in the page cache.
4327                          */
4328                         if (page->mapping != inode->i_mapping) {
4329                                 unlock_page(page);
4330                                 page_cache_release(page);
4331                                 goto again;
4332                         }
4333                         if (!PageUptodate(page)) {
4334                                 ret = -EIO;
4335                                 goto next_page;
4336                         }
4337                 }
4338
4339                 ret = check_extent_to_block(inode, offset, len,
4340                                             nocow_ctx_logical);
4341                 if (ret) {
4342                         ret = ret > 0 ? 0 : ret;
4343                         goto next_page;
4344                 }
4345
4346                 err = write_page_nocow(nocow_ctx->sctx,
4347                                        physical_for_dev_replace, page);
4348                 if (err)
4349                         ret = err;
4350 next_page:
4351                 unlock_page(page);
4352                 page_cache_release(page);
4353
4354                 if (ret)
4355                         break;
4356
4357                 offset += PAGE_CACHE_SIZE;
4358                 physical_for_dev_replace += PAGE_CACHE_SIZE;
4359                 nocow_ctx_logical += PAGE_CACHE_SIZE;
4360                 len -= PAGE_CACHE_SIZE;
4361         }
4362         ret = COPY_COMPLETE;
4363 out:
4364         mutex_unlock(&inode->i_mutex);
4365         iput(inode);
4366         return ret;
4367 }
4368
4369 static int write_page_nocow(struct scrub_ctx *sctx,
4370                             u64 physical_for_dev_replace, struct page *page)
4371 {
4372         struct bio *bio;
4373         struct btrfs_device *dev;
4374         int ret;
4375
4376         dev = sctx->wr_ctx.tgtdev;
4377         if (!dev)
4378                 return -EIO;
4379         if (!dev->bdev) {
4380                 printk_ratelimited(KERN_WARNING
4381                         "BTRFS: scrub write_page_nocow(bdev == NULL) is unexpected!\n");
4382                 return -EIO;
4383         }
4384         bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
4385         if (!bio) {
4386                 spin_lock(&sctx->stat_lock);
4387                 sctx->stat.malloc_errors++;
4388                 spin_unlock(&sctx->stat_lock);
4389                 return -ENOMEM;
4390         }
4391         bio->bi_iter.bi_size = 0;
4392         bio->bi_iter.bi_sector = physical_for_dev_replace >> 9;
4393         bio->bi_bdev = dev->bdev;
4394         ret = bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
4395         if (ret != PAGE_CACHE_SIZE) {
4396 leave_with_eio:
4397                 bio_put(bio);
4398                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
4399                 return -EIO;
4400         }
4401
4402         if (btrfsic_submit_bio_wait(WRITE_SYNC, bio))
4403                 goto leave_with_eio;
4404
4405         bio_put(bio);
4406         return 0;
4407 }