fs/btrfs/extent-tree.c

   1 /*
   2  * Copyright (C) 2007 Oracle.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18 #include <linux/sched.h>
  19 #include <linux/pagemap.h>
  20 #include <linux/writeback.h>
  21 #include <linux/blkdev.h>
  22 #include <linux/sort.h>
  23 #include <linux/rcupdate.h>
  24 #include <linux/kthread.h>
  25 #include <linux/slab.h>
  26 #include <linux/ratelimit.h>
  27 #include "compat.h"
  28 #include "hash.h"
  29 #include "ctree.h"
  30 #include "disk-io.h"
  31 #include "print-tree.h"
  32 #include "transaction.h"
  33 #include "volumes.h"
  34 #include "raid56.h"
  35 #include "locking.h"
  36 #include "free-space-cache.h"
  37 #include "math.h"
  38
  39 #undef SCRAMBLE_DELAYED_REFS
  40
  41 /*
  42  * control flags for do_chunk_alloc's force field
  43  * CHUNK_ALLOC_NO_FORCE means to only allocate a chunk
  44  * if we really need one.
  45  *
  46  * CHUNK_ALLOC_LIMITED means to only try and allocate one
  47  * if we have very few chunks already allocated.  This is
  48  * used as part of the clustering code to help make sure
  49  * we have a good pool of storage to cluster in, without
  50  * filling the FS with empty chunks
  51  *
  52  * CHUNK_ALLOC_FORCE means it must try to allocate one
  53  *
  54  */
  55 enum {
  56         CHUNK_ALLOC_NO_FORCE = 0,
  57         CHUNK_ALLOC_LIMITED = 1,
  58         CHUNK_ALLOC_FORCE = 2,
  59 };
  60
  61 /*
  62  * Control how reservations are dealt with.
  63  *
  64  * RESERVE_FREE - freeing a reservation.
  65  * RESERVE_ALLOC - allocating space and we need to update bytes_may_use for
  66  *   ENOSPC accounting
  67  * RESERVE_ALLOC_NO_ACCOUNT - allocating space and we should not update
  68  *   bytes_may_use as the ENOSPC accounting is done elsewhere
  69  */
  70 enum {
  71         RESERVE_FREE = 0,
  72         RESERVE_ALLOC = 1,
  73         RESERVE_ALLOC_NO_ACCOUNT = 2,
  74 };
  75
  76 static int update_block_group(struct btrfs_trans_handle *trans,
  77                               struct btrfs_root *root,
  78                               u64 bytenr, u64 num_bytes, int alloc);
  79 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
  80                                 struct btrfs_root *root,
  81                                 u64 bytenr, u64 num_bytes, u64 parent,
  82                                 u64 root_objectid, u64 owner_objectid,
  83                                 u64 owner_offset, int refs_to_drop,
  84                                 struct btrfs_delayed_extent_op *extra_op);
  85 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
  86                                     struct extent_buffer *leaf,
  87                                     struct btrfs_extent_item *ei);
  88 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
  89                                       struct btrfs_root *root,
  90                                       u64 parent, u64 root_objectid,
  91                                       u64 flags, u64 owner, u64 offset,
  92                                       struct btrfs_key *ins, int ref_mod);
  93 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
  94                                      struct btrfs_root *root,
  95                                      u64 parent, u64 root_objectid,
  96                                      u64 flags, struct btrfs_disk_key *key,
  97                                      int level, struct btrfs_key *ins);
  98 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
  99                           struct btrfs_root *extent_root, u64 flags,
 100                           int force);
 101 static int find_next_key(struct btrfs_path *path, int level,
 102                          struct btrfs_key *key);
 103 static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
 104                             int dump_block_groups);
 105 static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
 106                                        u64 num_bytes, int reserve);
 107
 108 static noinline int
 109 block_group_cache_done(struct btrfs_block_group_cache *cache)
 110 {
 111         smp_mb();
 112         return cache->cached == BTRFS_CACHE_FINISHED;
 113 }
 114
 115 static int block_group_bits(struct btrfs_block_group_cache *cache, u64 bits)
 116 {
 117         return (cache->flags & bits) == bits;
 118 }
 119
 120 static void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
 121 {
 122         atomic_inc(&cache->count);
 123 }
 124
 125 void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
 126 {
 127         if (atomic_dec_and_test(&cache->count)) {
 128                 WARN_ON(cache->pinned > 0);
 129                 WARN_ON(cache->reserved > 0);
 130                 kfree(cache->free_space_ctl);
 131                 kfree(cache);
 132         }
 133 }
 134
 135 /*
 136  * this adds the block group to the fs_info rb tree for the block group
 137  * cache
 138  */
 139 static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
 140                                 struct btrfs_block_group_cache *block_group)
 141 {
 142         struct rb_node **p;
 143         struct rb_node *parent = NULL;
 144         struct btrfs_block_group_cache *cache;
 145
 146         spin_lock(&info->block_group_cache_lock);
 147         p = &info->block_group_cache_tree.rb_node;
 148
 149         while (*p) {
 150                 parent = *p;
 151                 cache = rb_entry(parent, struct btrfs_block_group_cache,
 152                                  cache_node);
 153                 if (block_group->key.objectid < cache->key.objectid) {
 154                         p = &(*p)->rb_left;
 155                 } else if (block_group->key.objectid > cache->key.objectid) {
 156                         p = &(*p)->rb_right;
 157                 } else {
 158                         spin_unlock(&info->block_group_cache_lock);
 159                         return -EEXIST;
 160                 }
 161         }
 162
 163         rb_link_node(&block_group->cache_node, parent, p);
 164         rb_insert_color(&block_group->cache_node,
 165                         &info->block_group_cache_tree);
 166         spin_unlock(&info->block_group_cache_lock);
 167
 168         return 0;
 169 }
 170
 171 /*
 172  * This will return the block group at or after bytenr if contains is 0, else
 173  * it will return the block group that contains the bytenr
 174  */
 175 static struct btrfs_block_group_cache *
 176 block_group_cache_tree_search(struct btrfs_fs_info *info, u64 bytenr,
 177                               int contains)
 178 {
 179         struct btrfs_block_group_cache *cache, *ret = NULL;
 180         struct rb_node *n;
 181         u64 end, start;
 182
 183         spin_lock(&info->block_group_cache_lock);
 184         n = info->block_group_cache_tree.rb_node;
 185
 186         while (n) {
 187                 cache = rb_entry(n, struct btrfs_block_group_cache,
 188                                  cache_node);
 189                 end = cache->key.objectid + cache->key.offset - 1;
 190                 start = cache->key.objectid;
 191
 192                 if (bytenr < start) {
 193                         if (!contains && (!ret || start < ret->key.objectid))
 194                                 ret = cache;
 195                         n = n->rb_left;
 196                 } else if (bytenr > start) {
 197                         if (contains && bytenr <= end) {
 198                                 ret = cache;
 199                                 break;
 200                         }
 201                         n = n->rb_right;
 202                 } else {
 203                         ret = cache;
 204                         break;
 205                 }
 206         }
 207         if (ret)
 208                 btrfs_get_block_group(ret);
 209         spin_unlock(&info->block_group_cache_lock);
 210
 211         return ret;
 212 }
 213
 214 static int add_excluded_extent(struct btrfs_root *root,
 215                                u64 start, u64 num_bytes)
 216 {
 217         u64 end = start + num_bytes - 1;
 218         set_extent_bits(&root->fs_info->freed_extents[0],
 219                         start, end, EXTENT_UPTODATE, GFP_NOFS);
 220         set_extent_bits(&root->fs_info->freed_extents[1],
 221                         start, end, EXTENT_UPTODATE, GFP_NOFS);
 222         return 0;
 223 }
 224
 225 static void free_excluded_extents(struct btrfs_root *root,
 226                                   struct btrfs_block_group_cache *cache)
 227 {
 228         u64 start, end;
 229
 230         start = cache->key.objectid;
 231         end = start + cache->key.offset - 1;
 232
 233         clear_extent_bits(&root->fs_info->freed_extents[0],
 234                           start, end, EXTENT_UPTODATE, GFP_NOFS);
 235         clear_extent_bits(&root->fs_info->freed_extents[1],
 236                           start, end, EXTENT_UPTODATE, GFP_NOFS);
 237 }
 238
 239 static int exclude_super_stripes(struct btrfs_root *root,
 240                                  struct btrfs_block_group_cache *cache)
 241 {
 242         u64 bytenr;
 243         u64 *logical;
 244         int stripe_len;
 245         int i, nr, ret;
 246
 247         if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
 248                 stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
 249                 cache->bytes_super += stripe_len;
 250                 ret = add_excluded_extent(root, cache->key.objectid,
 251                                           stripe_len);
 252                 BUG_ON(ret); /* -ENOMEM */
 253         }
 254
 255         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
 256                 bytenr = btrfs_sb_offset(i);
 257                 ret = btrfs_rmap_block(&root->fs_info->mapping_tree,
 258                                        cache->key.objectid, bytenr,
 259                                        0, &logical, &nr, &stripe_len);
 260                 BUG_ON(ret); /* -ENOMEM */
 261
 262                 while (nr--) {
 263                         cache->bytes_super += stripe_len;
 264                         ret = add_excluded_extent(root, logical[nr],
 265                                                   stripe_len);
 266                         BUG_ON(ret); /* -ENOMEM */
 267                 }
 268
 269                 kfree(logical);
 270         }
 271         return 0;
 272 }
 273
 274 static struct btrfs_caching_control *
 275 get_caching_control(struct btrfs_block_group_cache *cache)
 276 {
 277         struct btrfs_caching_control *ctl;
 278
 279         spin_lock(&cache->lock);
 280         if (cache->cached != BTRFS_CACHE_STARTED) {
 281                 spin_unlock(&cache->lock);
 282                 return NULL;
 283         }
 284
 285         /* We're loading it the fast way, so we don't have a caching_ctl. */
 286         if (!cache->caching_ctl) {
 287                 spin_unlock(&cache->lock);
 288                 return NULL;
 289         }
 290
 291         ctl = cache->caching_ctl;
 292         atomic_inc(&ctl->count);
 293         spin_unlock(&cache->lock);
 294         return ctl;
 295 }
 296
 297 static void put_caching_control(struct btrfs_caching_control *ctl)
 298 {
 299         if (atomic_dec_and_test(&ctl->count))
 300                 kfree(ctl);
 301 }
 302
 303 /*
 304  * this is only called by cache_block_group, since we could have freed extents
 305  * we need to check the pinned_extents for any extents that can't be used yet
 306  * since their free space will be released as soon as the transaction commits.
 307  */
 308 static u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
 309                               struct btrfs_fs_info *info, u64 start, u64 end)
 310 {
 311         u64 extent_start, extent_end, size, total_added = 0;
 312         int ret;
 313
 314         while (start < end) {
 315                 ret = find_first_extent_bit(info->pinned_extents, start,
 316                                             &extent_start, &extent_end,
 317                                             EXTENT_DIRTY | EXTENT_UPTODATE,
 318                                             NULL);
 319                 if (ret)
 320                         break;
 321
 322                 if (extent_start <= start) {
 323                         start = extent_end + 1;
 324                 } else if (extent_start > start && extent_start < end) {
 325                         size = extent_start - start;
 326                         total_added += size;
 327                         ret = btrfs_add_free_space(block_group, start,
 328                                                    size);
 329                         BUG_ON(ret); /* -ENOMEM or logic error */
 330                         start = extent_end + 1;
 331                 } else {
 332                         break;
 333                 }
 334         }
 335
 336         if (start < end) {
 337                 size = end - start;
 338                 total_added += size;
 339                 ret = btrfs_add_free_space(block_group, start, size);
 340                 BUG_ON(ret); /* -ENOMEM or logic error */
 341         }
 342
 343         return total_added;
 344 }
 345
 346 static noinline void caching_thread(struct btrfs_work *work)
 347 {
 348         struct btrfs_block_group_cache *block_group;
 349         struct btrfs_fs_info *fs_info;
 350         struct btrfs_caching_control *caching_ctl;
 351         struct btrfs_root *extent_root;
 352         struct btrfs_path *path;
 353         struct extent_buffer *leaf;
 354         struct btrfs_key key;
 355         u64 total_found = 0;
 356         u64 last = 0;
 357         u32 nritems;
 358         int ret = 0;
 359
 360         caching_ctl = container_of(work, struct btrfs_caching_control, work);
 361         block_group = caching_ctl->block_group;
 362         fs_info = block_group->fs_info;
 363         extent_root = fs_info->extent_root;
 364
 365         path = btrfs_alloc_path();
 366         if (!path)
 367                 goto out;
 368
 369         last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
 370
 371         /*
 372          * We don't want to deadlock with somebody trying to allocate a new
 373          * extent for the extent root while also trying to search the extent
 374          * root to add free space.  So we skip locking and search the commit
 375          * root, since its read-only
 376          */
 377         path->skip_locking = 1;
 378         path->search_commit_root = 1;
 379         path->reada = 1;
 380
 381         key.objectid = last;
 382         key.offset = 0;
 383         key.type = BTRFS_EXTENT_ITEM_KEY;
 384 again:
 385         mutex_lock(&caching_ctl->mutex);
 386         /* need to make sure the commit_root doesn't disappear */
 387         down_read(&fs_info->extent_commit_sem);
 388
 389         ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
 390         if (ret < 0)
 391                 goto err;
 392
 393         leaf = path->nodes[0];
 394         nritems = btrfs_header_nritems(leaf);
 395
 396         while (1) {
 397                 if (btrfs_fs_closing(fs_info) > 1) {
 398                         last = (u64)-1;
 399                         break;
 400                 }
 401
 402                 if (path->slots[0] < nritems) {
 403                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 404                 } else {
 405                         ret = find_next_key(path, 0, &key);
 406                         if (ret)
 407                                 break;
 408
 409                         if (need_resched() ||
 410                             btrfs_next_leaf(extent_root, path)) {
 411                                 caching_ctl->progress = last;
 412                                 btrfs_release_path(path);
 413                                 up_read(&fs_info->extent_commit_sem);
 414                                 mutex_unlock(&caching_ctl->mutex);
 415                                 cond_resched();
 416                                 goto again;
 417                         }
 418                         leaf = path->nodes[0];
 419                         nritems = btrfs_header_nritems(leaf);
 420                         continue;
 421                 }
 422
 423                 if (key.objectid < block_group->key.objectid) {
 424                         path->slots[0]++;
 425                         continue;
 426                 }
 427
 428                 if (key.objectid >= block_group->key.objectid +
 429                     block_group->key.offset)
 430                         break;
 431
 432                 if (key.type == BTRFS_EXTENT_ITEM_KEY) {
 433                         total_found += add_new_free_space(block_group,
 434                                                           fs_info, last,
 435                                                           key.objectid);
 436                         last = key.objectid + key.offset;
 437
 438                         if (total_found > (1024 * 1024 * 2)) {
 439                                 total_found = 0;
 440                                 wake_up(&caching_ctl->wait);
 441                         }
 442                 }
 443                 path->slots[0]++;
 444         }
 445         ret = 0;
 446
 447         total_found += add_new_free_space(block_group, fs_info, last,
 448                                           block_group->key.objectid +
 449                                           block_group->key.offset);
 450         caching_ctl->progress = (u64)-1;
 451
 452         spin_lock(&block_group->lock);
 453         block_group->caching_ctl = NULL;
 454         block_group->cached = BTRFS_CACHE_FINISHED;
 455         spin_unlock(&block_group->lock);
 456
 457 err:
 458         btrfs_free_path(path);
 459         up_read(&fs_info->extent_commit_sem);
 460
 461         free_excluded_extents(extent_root, block_group);
 462
 463         mutex_unlock(&caching_ctl->mutex);
 464 out:
 465         wake_up(&caching_ctl->wait);
 466
 467         put_caching_control(caching_ctl);
 468         btrfs_put_block_group(block_group);
 469 }
 470
 471 static int cache_block_group(struct btrfs_block_group_cache *cache,
 472                              struct btrfs_trans_handle *trans,
 473                              struct btrfs_root *root,
 474                              int load_cache_only)
 475 {
 476         DEFINE_WAIT(wait);
 477         struct btrfs_fs_info *fs_info = cache->fs_info;
 478         struct btrfs_caching_control *caching_ctl;
 479         int ret = 0;
 480
 481         caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
 482         if (!caching_ctl)
 483                 return -ENOMEM;
 484
 485         INIT_LIST_HEAD(&caching_ctl->list);
 486         mutex_init(&caching_ctl->mutex);
 487         init_waitqueue_head(&caching_ctl->wait);
 488         caching_ctl->block_group = cache;
 489         caching_ctl->progress = cache->key.objectid;
 490         atomic_set(&caching_ctl->count, 1);
 491         caching_ctl->work.func = caching_thread;
 492
 493         spin_lock(&cache->lock);
 494         /*
 495          * This should be a rare occasion, but this could happen I think in the
 496          * case where one thread starts to load the space cache info, and then
 497          * some other thread starts a transaction commit which tries to do an
 498          * allocation while the other thread is still loading the space cache
 499          * info.  The previous loop should have kept us from choosing this block
 500          * group, but if we've moved to the state where we will wait on caching
 501          * block groups we need to first check if we're doing a fast load here,
 502          * so we can wait for it to finish, otherwise we could end up allocating
 503          * from a block group who's cache gets evicted for one reason or
 504          * another.
 505          */
 506         while (cache->cached == BTRFS_CACHE_FAST) {
 507                 struct btrfs_caching_control *ctl;
 508
 509                 ctl = cache->caching_ctl;
 510                 atomic_inc(&ctl->count);
 511                 prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
 512                 spin_unlock(&cache->lock);
 513
 514                 schedule();
 515
 516                 finish_wait(&ctl->wait, &wait);
 517                 put_caching_control(ctl);
 518                 spin_lock(&cache->lock);
 519         }
 520
 521         if (cache->cached != BTRFS_CACHE_NO) {
 522                 spin_unlock(&cache->lock);
 523                 kfree(caching_ctl);
 524                 return 0;
 525         }
 526         WARN_ON(cache->caching_ctl);
 527         cache->caching_ctl = caching_ctl;
 528         cache->cached = BTRFS_CACHE_FAST;
 529         spin_unlock(&cache->lock);
 530
 531         /*
 532          * We can't do the read from on-disk cache during a commit since we need
 533          * to have the normal tree locking.  Also if we are currently trying to
 534          * allocate blocks for the tree root we can't do the fast caching since
 535          * we likely hold important locks.
 536          */
 537         if (fs_info->mount_opt & BTRFS_MOUNT_SPACE_CACHE) {
 538                 ret = load_free_space_cache(fs_info, cache);
 539
 540                 spin_lock(&cache->lock);
 541                 if (ret == 1) {
 542                         cache->caching_ctl = NULL;
 543                         cache->cached = BTRFS_CACHE_FINISHED;
 544                         cache->last_byte_to_unpin = (u64)-1;
 545                 } else {
 546                         if (load_cache_only) {
 547                                 cache->caching_ctl = NULL;
 548                                 cache->cached = BTRFS_CACHE_NO;
 549                         } else {
 550                                 cache->cached = BTRFS_CACHE_STARTED;
 551                         }
 552                 }
 553                 spin_unlock(&cache->lock);
 554                 wake_up(&caching_ctl->wait);
 555                 if (ret == 1) {
 556                         put_caching_control(caching_ctl);
 557                         free_excluded_extents(fs_info->extent_root, cache);
 558                         return 0;
 559                 }
 560         } else {
 561                 /*
 562                  * We are not going to do the fast caching, set cached to the
 563                  * appropriate value and wakeup any waiters.
 564                  */
 565                 spin_lock(&cache->lock);
 566                 if (load_cache_only) {
 567                         cache->caching_ctl = NULL;
 568                         cache->cached = BTRFS_CACHE_NO;
 569                 } else {
 570                         cache->cached = BTRFS_CACHE_STARTED;
 571                 }
 572                 spin_unlock(&cache->lock);
 573                 wake_up(&caching_ctl->wait);
 574         }
 575
 576         if (load_cache_only) {
 577                 put_caching_control(caching_ctl);
 578                 return 0;
 579         }
 580
 581         down_write(&fs_info->extent_commit_sem);
 582         atomic_inc(&caching_ctl->count);
 583         list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
 584         up_write(&fs_info->extent_commit_sem);
 585
 586         btrfs_get_block_group(cache);
 587
 588         btrfs_queue_worker(&fs_info->caching_workers, &caching_ctl->work);
 589
 590         return ret;
 591 }
 592
 593 /*
 594  * return the block group that starts at or after bytenr
 595  */
 596 static struct btrfs_block_group_cache *
 597 btrfs_lookup_first_block_group(struct btrfs_fs_info *info, u64 bytenr)
 598 {
 599         struct btrfs_block_group_cache *cache;
 600
 601         cache = block_group_cache_tree_search(info, bytenr, 0);
 602
 603         return cache;
 604 }
 605
 606 /*
 607  * return the block group that contains the given bytenr
 608  */
 609 struct btrfs_block_group_cache *btrfs_lookup_block_group(
 610                                                  struct btrfs_fs_info *info,
 611                                                  u64 bytenr)
 612 {
 613         struct btrfs_block_group_cache *cache;
 614
 615         cache = block_group_cache_tree_search(info, bytenr, 1);
 616
 617         return cache;
 618 }
 619
 620 static struct btrfs_space_info *__find_space_info(struct btrfs_fs_info *info,
 621                                                   u64 flags)
 622 {
 623         struct list_head *head = &info->space_info;
 624         struct btrfs_space_info *found;
 625
 626         flags &= BTRFS_BLOCK_GROUP_TYPE_MASK;
 627
 628         rcu_read_lock();
 629         list_for_each_entry_rcu(found, head, list) {
 630                 if (found->flags & flags) {
 631                         rcu_read_unlock();
 632                         return found;
 633                 }
 634         }
 635         rcu_read_unlock();
 636         return NULL;
 637 }
 638
 639 /*
 640  * after adding space to the filesystem, we need to clear the full flags
 641  * on all the space infos.
 642  */
 643 void btrfs_clear_space_info_full(struct btrfs_fs_info *info)
 644 {
 645         struct list_head *head = &info->space_info;
 646         struct btrfs_space_info *found;
 647
 648         rcu_read_lock();
 649         list_for_each_entry_rcu(found, head, list)
 650                 found->full = 0;
 651         rcu_read_unlock();
 652 }
 653
 654 u64 btrfs_find_block_group(struct btrfs_root *root,
 655                            u64 search_start, u64 search_hint, int owner)
 656 {
 657         struct btrfs_block_group_cache *cache;
 658         u64 used;
 659         u64 last = max(search_hint, search_start);
 660         u64 group_start = 0;
 661         int full_search = 0;
 662         int factor = 9;
 663         int wrapped = 0;
 664 again:
 665         while (1) {
 666                 cache = btrfs_lookup_first_block_group(root->fs_info, last);
 667                 if (!cache)
 668                         break;
 669
 670                 spin_lock(&cache->lock);
 671                 last = cache->key.objectid + cache->key.offset;
 672                 used = btrfs_block_group_used(&cache->item);
 673
 674                 if ((full_search || !cache->ro) &&
 675                     block_group_bits(cache, BTRFS_BLOCK_GROUP_METADATA)) {
 676                         if (used + cache->pinned + cache->reserved <
 677                             div_factor(cache->key.offset, factor)) {
 678                                 group_start = cache->key.objectid;
 679                                 spin_unlock(&cache->lock);
 680                                 btrfs_put_block_group(cache);
 681                                 goto found;
 682                         }
 683                 }
 684                 spin_unlock(&cache->lock);
 685                 btrfs_put_block_group(cache);
 686                 cond_resched();
 687         }
 688         if (!wrapped) {
 689                 last = search_start;
 690                 wrapped = 1;
 691                 goto again;
 692         }
 693         if (!full_search && factor < 10) {
 694                 last = search_start;
 695                 full_search = 1;
 696                 factor = 10;
 697                 goto again;
 698         }
 699 found:
 700         return group_start;
 701 }
 702
 703 /* simple helper to search for an existing extent at a given offset */
 704 int btrfs_lookup_extent(struct btrfs_root *root, u64 start, u64 len)
 705 {
 706         int ret;
 707         struct btrfs_key key;
 708         struct btrfs_path *path;
 709
 710         path = btrfs_alloc_path();
 711         if (!path)
 712                 return -ENOMEM;
 713
 714         key.objectid = start;
 715         key.offset = len;
 716         btrfs_set_key_type(&key, BTRFS_EXTENT_ITEM_KEY);
 717         ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &key, path,
 718                                 0, 0);
 719         btrfs_free_path(path);
 720         return ret;
 721 }
 722
 723 /*
 724  * helper function to lookup reference count and flags of extent.
 725  *
 726  * the head node for delayed ref is used to store the sum of all the
 727  * reference count modifications queued up in the rbtree. the head
 728  * node may also store the extent flags to set. This way you can check
 729  * to see what the reference count and extent flags would be if all of
 730  * the delayed refs are not processed.
 731  */
 732 int btrfs_lookup_extent_info(struct btrfs_trans_handle *trans,
 733                              struct btrfs_root *root, u64 bytenr,
 734                              u64 num_bytes, u64 *refs, u64 *flags)
 735 {
 736         struct btrfs_delayed_ref_head *head;
 737         struct btrfs_delayed_ref_root *delayed_refs;
 738         struct btrfs_path *path;
 739         struct btrfs_extent_item *ei;
 740         struct extent_buffer *leaf;
 741         struct btrfs_key key;
 742         u32 item_size;
 743         u64 num_refs;
 744         u64 extent_flags;
 745         int ret;
 746
 747         path = btrfs_alloc_path();
 748         if (!path)
 749                 return -ENOMEM;
 750
 751         key.objectid = bytenr;
 752         key.type = BTRFS_EXTENT_ITEM_KEY;
 753         key.offset = num_bytes;
 754         if (!trans) {
 755                 path->skip_locking = 1;
 756                 path->search_commit_root = 1;
 757         }
 758 again:
 759         ret = btrfs_search_slot(trans, root->fs_info->extent_root,
 760                                 &key, path, 0, 0);
 761         if (ret < 0)
 762                 goto out_free;
 763
 764         if (ret == 0) {
 765                 leaf = path->nodes[0];
 766                 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
 767                 if (item_size >= sizeof(*ei)) {
 768                         ei = btrfs_item_ptr(leaf, path->slots[0],
 769                                             struct btrfs_extent_item);
 770                         num_refs = btrfs_extent_refs(leaf, ei);
 771                         extent_flags = btrfs_extent_flags(leaf, ei);
 772                 } else {
 773 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
 774                         struct btrfs_extent_item_v0 *ei0;
 775                         BUG_ON(item_size != sizeof(*ei0));
 776                         ei0 = btrfs_item_ptr(leaf, path->slots[0],
 777                                              struct btrfs_extent_item_v0);
 778                         num_refs = btrfs_extent_refs_v0(leaf, ei0);
 779                         /* FIXME: this isn't correct for data */
 780                         extent_flags = BTRFS_BLOCK_FLAG_FULL_BACKREF;
 781 #else
 782                         BUG();
 783 #endif
 784                 }
 785                 BUG_ON(num_refs == 0);
 786         } else {
 787                 num_refs = 0;
 788                 extent_flags = 0;
 789                 ret = 0;
 790         }
 791
 792         if (!trans)
 793                 goto out;
 794
 795         delayed_refs = &trans->transaction->delayed_refs;
 796         spin_lock(&delayed_refs->lock);
 797         head = btrfs_find_delayed_ref_head(trans, bytenr);
 798         if (head) {
 799                 if (!mutex_trylock(&head->mutex)) {
 800                         atomic_inc(&head->node.refs);
 801                         spin_unlock(&delayed_refs->lock);
 802
 803                         btrfs_release_path(path);
 804
 805                         /*
 806                          * Mutex was contended, block until it's released and try
 807                          * again
 808                          */
 809                         mutex_lock(&head->mutex);
 810                         mutex_unlock(&head->mutex);
 811                         btrfs_put_delayed_ref(&head->node);
 812                         goto again;
 813                 }
 814                 if (head->extent_op && head->extent_op->update_flags)
 815                         extent_flags |= head->extent_op->flags_to_set;
 816                 else
 817                         BUG_ON(num_refs == 0);
 818
 819                 num_refs += head->node.ref_mod;
 820                 mutex_unlock(&head->mutex);
 821         }
 822         spin_unlock(&delayed_refs->lock);
 823 out:
 824         WARN_ON(num_refs == 0);
 825         if (refs)
 826                 *refs = num_refs;
 827         if (flags)
 828                 *flags = extent_flags;
 829 out_free:
 830         btrfs_free_path(path);
 831         return ret;
 832 }
 833
 834 /*
 835  * Back reference rules.  Back refs have three main goals:
 836  *
 837  * 1) differentiate between all holders of references to an extent so that
 838  *    when a reference is dropped we can make sure it was a valid reference
 839  *    before freeing the extent.
 840  *
 841  * 2) Provide enough information to quickly find the holders of an extent
 842  *    if we notice a given block is corrupted or bad.
 843  *
 844  * 3) Make it easy to migrate blocks for FS shrinking or storage pool
 845  *    maintenance.  This is actually the same as #2, but with a slightly
 846  *    different use case.
 847  *
 848  * There are two kinds of back refs. The implicit back refs is optimized
 849  * for pointers in non-shared tree blocks. For a given pointer in a block,
 850  * back refs of this kind provide information about the block's owner tree
 851  * and the pointer's key. These information allow us to find the block by
 852  * b-tree searching. The full back refs is for pointers in tree blocks not
 853  * referenced by their owner trees. The location of tree block is recorded
 854  * in the back refs. Actually the full back refs is generic, and can be
 855  * used in all cases the implicit back refs is used. The major shortcoming
 856  * of the full back refs is its overhead. Every time a tree block gets
 857  * COWed, we have to update back refs entry for all pointers in it.
 858  *
 859  * For a newly allocated tree block, we use implicit back refs for
 860  * pointers in it. This means most tree related operations only involve
 861  * implicit back refs. For a tree block created in old transaction, the
 862  * only way to drop a reference to it is COW it. So we can detect the
 863  * event that tree block loses its owner tree's reference and do the
 864  * back refs conversion.
 865  *
 866  * When a tree block is COW'd through a tree, there are four cases:
 867  *
 868  * The reference count of the block is one and the tree is the block's
 869  * owner tree. Nothing to do in this case.
 870  *
 871  * The reference count of the block is one and the tree is not the
 872  * block's owner tree. In this case, full back refs is used for pointers
 873  * in the block. Remove these full back refs, add implicit back refs for
 874  * every pointers in the new block.
 875  *
 876  * The reference count of the block is greater than one and the tree is
 877  * the block's owner tree. In this case, implicit back refs is used for
 878  * pointers in the block. Add full back refs for every pointers in the
 879  * block, increase lower level extents' reference counts. The original
 880  * implicit back refs are entailed to the new block.
 881  *
 882  * The reference count of the block is greater than one and the tree is
 883  * not the block's owner tree. Add implicit back refs for every pointer in
 884  * the new block, increase lower level extents' reference count.
 885  *
 886  * Back Reference Key composing:
 887  *
 888  * The key objectid corresponds to the first byte in the extent,
 889  * The key type is used to differentiate between types of back refs.
 890  * There are different meanings of the key offset for different types
 891  * of back refs.
 892  *
 893  * File extents can be referenced by:
 894  *
 895  * - multiple snapshots, subvolumes, or different generations in one subvol
 896  * - different files inside a single subvolume
 897  * - different offsets inside a file (bookend extents in file.c)
 898  *
 899  * The extent ref structure for the implicit back refs has fields for:
 900  *
 901  * - Objectid of the subvolume root
 902  * - objectid of the file holding the reference
 903  * - original offset in the file
 904  * - how many bookend extents
 905  *
 906  * The key offset for the implicit back refs is hash of the first
 907  * three fields.
 908  *
 909  * The extent ref structure for the full back refs has field for:
 910  *
 911  * - number of pointers in the tree leaf
 912  *
 913  * The key offset for the implicit back refs is the first byte of
 914  * the tree leaf
 915  *
 916  * When a file extent is allocated, The implicit back refs is used.
 917  * the fields are filled in:
 918  *
 919  *     (root_key.objectid, inode objectid, offset in file, 1)
 920  *
 921  * When a file extent is removed file truncation, we find the
 922  * corresponding implicit back refs and check the following fields:
 923  *
 924  *     (btrfs_header_owner(leaf), inode objectid, offset in file)
 925  *
 926  * Btree extents can be referenced by:
 927  *
 928  * - Different subvolumes
 929  *
 930  * Both the implicit back refs and the full back refs for tree blocks
 931  * only consist of key. The key offset for the implicit back refs is
 932  * objectid of block's owner tree. The key offset for the full back refs
 933  * is the first byte of parent block.
 934  *
 935  * When implicit back refs is used, information about the lowest key and
 936  * level of the tree block are required. These information are stored in
 937  * tree block info structure.
 938  */
 939
 940 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
 941 static int convert_extent_item_v0(struct btrfs_trans_handle *trans,
 942                                   struct btrfs_root *root,
 943                                   struct btrfs_path *path,
 944                                   u64 owner, u32 extra_size)
 945 {
 946         struct btrfs_extent_item *item;
 947         struct btrfs_extent_item_v0 *ei0;
 948         struct btrfs_extent_ref_v0 *ref0;
 949         struct btrfs_tree_block_info *bi;
 950         struct extent_buffer *leaf;
 951         struct btrfs_key key;
 952         struct btrfs_key found_key;
 953         u32 new_size = sizeof(*item);
 954         u64 refs;
 955         int ret;
 956
 957         leaf = path->nodes[0];
 958         BUG_ON(btrfs_item_size_nr(leaf, path->slots[0]) != sizeof(*ei0));
 959
 960         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 961         ei0 = btrfs_item_ptr(leaf, path->slots[0],
 962                              struct btrfs_extent_item_v0);
 963         refs = btrfs_extent_refs_v0(leaf, ei0);
 964
 965         if (owner == (u64)-1) {
 966                 while (1) {
 967                         if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 968                                 ret = btrfs_next_leaf(root, path);
 969                                 if (ret < 0)
 970                                         return ret;
 971                                 BUG_ON(ret > 0); /* Corruption */
 972                                 leaf = path->nodes[0];
 973                         }
 974                         btrfs_item_key_to_cpu(leaf, &found_key,
 975                                               path->slots[0]);
 976                         BUG_ON(key.objectid != found_key.objectid);
 977                         if (found_key.type != BTRFS_EXTENT_REF_V0_KEY) {
 978                                 path->slots[0]++;
 979                                 continue;
 980                         }
 981                         ref0 = btrfs_item_ptr(leaf, path->slots[0],
 982                                               struct btrfs_extent_ref_v0);
 983                         owner = btrfs_ref_objectid_v0(leaf, ref0);
 984                         break;
 985                 }
 986         }
 987         btrfs_release_path(path);
 988
 989         if (owner < BTRFS_FIRST_FREE_OBJECTID)
 990                 new_size += sizeof(*bi);
 991
 992         new_size -= sizeof(*ei0);
 993         ret = btrfs_search_slot(trans, root, &key, path,
 994                                 new_size + extra_size, 1);
 995         if (ret < 0)
 996                 return ret;
 997         BUG_ON(ret); /* Corruption */
 998
 999         btrfs_extend_item(trans, root, path, new_size);
1000
1001         leaf = path->nodes[0];
1002         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1003         btrfs_set_extent_refs(leaf, item, refs);
1004         /* FIXME: get real generation */
1005         btrfs_set_extent_generation(leaf, item, 0);
1006         if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1007                 btrfs_set_extent_flags(leaf, item,
1008                                        BTRFS_EXTENT_FLAG_TREE_BLOCK |
1009                                        BTRFS_BLOCK_FLAG_FULL_BACKREF);
1010                 bi = (struct btrfs_tree_block_info *)(item + 1);
1011                 /* FIXME: get first key of the block */
1012                 memset_extent_buffer(leaf, 0, (unsigned long)bi, sizeof(*bi));
1013                 btrfs_set_tree_block_level(leaf, bi, (int)owner);
1014         } else {
1015                 btrfs_set_extent_flags(leaf, item, BTRFS_EXTENT_FLAG_DATA);
1016         }
1017         btrfs_mark_buffer_dirty(leaf);
1018         return 0;
1019 }
1020 #endif
1021
1022 static u64 hash_extent_data_ref(u64 root_objectid, u64 owner, u64 offset)
1023 {
1024         u32 high_crc = ~(u32)0;
1025         u32 low_crc = ~(u32)0;
1026         __le64 lenum;
1027
1028         lenum = cpu_to_le64(root_objectid);
1029         high_crc = crc32c(high_crc, &lenum, sizeof(lenum));
1030         lenum = cpu_to_le64(owner);
1031         low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1032         lenum = cpu_to_le64(offset);
1033         low_crc = crc32c(low_crc, &lenum, sizeof(lenum));
1034
1035         return ((u64)high_crc << 31) ^ (u64)low_crc;
1036 }
1037
1038 static u64 hash_extent_data_ref_item(struct extent_buffer *leaf,
1039                                      struct btrfs_extent_data_ref *ref)
1040 {
1041         return hash_extent_data_ref(btrfs_extent_data_ref_root(leaf, ref),
1042                                     btrfs_extent_data_ref_objectid(leaf, ref),
1043                                     btrfs_extent_data_ref_offset(leaf, ref));
1044 }
1045
1046 static int match_extent_data_ref(struct extent_buffer *leaf,
1047                                  struct btrfs_extent_data_ref *ref,
1048                                  u64 root_objectid, u64 owner, u64 offset)
1049 {
1050         if (btrfs_extent_data_ref_root(leaf, ref) != root_objectid ||
1051             btrfs_extent_data_ref_objectid(leaf, ref) != owner ||
1052             btrfs_extent_data_ref_offset(leaf, ref) != offset)
1053                 return 0;
1054         return 1;
1055 }
1056
1057 static noinline int lookup_extent_data_ref(struct btrfs_trans_handle *trans,
1058                                            struct btrfs_root *root,
1059                                            struct btrfs_path *path,
1060                                            u64 bytenr, u64 parent,
1061                                            u64 root_objectid,
1062                                            u64 owner, u64 offset)
1063 {
1064         struct btrfs_key key;
1065         struct btrfs_extent_data_ref *ref;
1066         struct extent_buffer *leaf;
1067         u32 nritems;
1068         int ret;
1069         int recow;
1070         int err = -ENOENT;
1071
1072         key.objectid = bytenr;
1073         if (parent) {
1074                 key.type = BTRFS_SHARED_DATA_REF_KEY;
1075                 key.offset = parent;
1076         } else {
1077                 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1078                 key.offset = hash_extent_data_ref(root_objectid,
1079                                                   owner, offset);
1080         }
1081 again:
1082         recow = 0;
1083         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1084         if (ret < 0) {
1085                 err = ret;
1086                 goto fail;
1087         }
1088
1089         if (parent) {
1090                 if (!ret)
1091                         return 0;
1092 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1093                 key.type = BTRFS_EXTENT_REF_V0_KEY;
1094                 btrfs_release_path(path);
1095                 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1096                 if (ret < 0) {
1097                         err = ret;
1098                         goto fail;
1099                 }
1100                 if (!ret)
1101                         return 0;
1102 #endif
1103                 goto fail;
1104         }
1105
1106         leaf = path->nodes[0];
1107         nritems = btrfs_header_nritems(leaf);
1108         while (1) {
1109                 if (path->slots[0] >= nritems) {
1110                         ret = btrfs_next_leaf(root, path);
1111                         if (ret < 0)
1112                                 err = ret;
1113                         if (ret)
1114                                 goto fail;
1115
1116                         leaf = path->nodes[0];
1117                         nritems = btrfs_header_nritems(leaf);
1118                         recow = 1;
1119                 }
1120
1121                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1122                 if (key.objectid != bytenr ||
1123                     key.type != BTRFS_EXTENT_DATA_REF_KEY)
1124                         goto fail;
1125
1126                 ref = btrfs_item_ptr(leaf, path->slots[0],
1127                                      struct btrfs_extent_data_ref);
1128
1129                 if (match_extent_data_ref(leaf, ref, root_objectid,
1130                                           owner, offset)) {
1131                         if (recow) {
1132                                 btrfs_release_path(path);
1133                                 goto again;
1134                         }
1135                         err = 0;
1136                         break;
1137                 }
1138                 path->slots[0]++;
1139         }
1140 fail:
1141         return err;
1142 }
1143
1144 static noinline int insert_extent_data_ref(struct btrfs_trans_handle *trans,
1145                                            struct btrfs_root *root,
1146                                            struct btrfs_path *path,
1147                                            u64 bytenr, u64 parent,
1148                                            u64 root_objectid, u64 owner,
1149                                            u64 offset, int refs_to_add)
1150 {
1151         struct btrfs_key key;
1152         struct extent_buffer *leaf;
1153         u32 size;
1154         u32 num_refs;
1155         int ret;
1156
1157         key.objectid = bytenr;
1158         if (parent) {
1159                 key.type = BTRFS_SHARED_DATA_REF_KEY;
1160                 key.offset = parent;
1161                 size = sizeof(struct btrfs_shared_data_ref);
1162         } else {
1163                 key.type = BTRFS_EXTENT_DATA_REF_KEY;
1164                 key.offset = hash_extent_data_ref(root_objectid,
1165                                                   owner, offset);
1166                 size = sizeof(struct btrfs_extent_data_ref);
1167         }
1168
1169         ret = btrfs_insert_empty_item(trans, root, path, &key, size);
1170         if (ret && ret != -EEXIST)
1171                 goto fail;
1172
1173         leaf = path->nodes[0];
1174         if (parent) {
1175                 struct btrfs_shared_data_ref *ref;
1176                 ref = btrfs_item_ptr(leaf, path->slots[0],
1177                                      struct btrfs_shared_data_ref);
1178                 if (ret == 0) {
1179                         btrfs_set_shared_data_ref_count(leaf, ref, refs_to_add);
1180                 } else {
1181                         num_refs = btrfs_shared_data_ref_count(leaf, ref);
1182                         num_refs += refs_to_add;
1183                         btrfs_set_shared_data_ref_count(leaf, ref, num_refs);
1184                 }
1185         } else {
1186                 struct btrfs_extent_data_ref *ref;
1187                 while (ret == -EEXIST) {
1188                         ref = btrfs_item_ptr(leaf, path->slots[0],
1189                                              struct btrfs_extent_data_ref);
1190                         if (match_extent_data_ref(leaf, ref, root_objectid,
1191                                                   owner, offset))
1192                                 break;
1193                         btrfs_release_path(path);
1194                         key.offset++;
1195                         ret = btrfs_insert_empty_item(trans, root, path, &key,
1196                                                       size);
1197                         if (ret && ret != -EEXIST)
1198                                 goto fail;
1199
1200                         leaf = path->nodes[0];
1201                 }
1202                 ref = btrfs_item_ptr(leaf, path->slots[0],
1203                                      struct btrfs_extent_data_ref);
1204                 if (ret == 0) {
1205                         btrfs_set_extent_data_ref_root(leaf, ref,
1206                                                        root_objectid);
1207                         btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
1208                         btrfs_set_extent_data_ref_offset(leaf, ref, offset);
1209                         btrfs_set_extent_data_ref_count(leaf, ref, refs_to_add);
1210                 } else {
1211                         num_refs = btrfs_extent_data_ref_count(leaf, ref);
1212                         num_refs += refs_to_add;
1213                         btrfs_set_extent_data_ref_count(leaf, ref, num_refs);
1214                 }
1215         }
1216         btrfs_mark_buffer_dirty(leaf);
1217         ret = 0;
1218 fail:
1219         btrfs_release_path(path);
1220         return ret;
1221 }
1222
1223 static noinline int remove_extent_data_ref(struct btrfs_trans_handle *trans,
1224                                            struct btrfs_root *root,
1225                                            struct btrfs_path *path,
1226                                            int refs_to_drop)
1227 {
1228         struct btrfs_key key;
1229         struct btrfs_extent_data_ref *ref1 = NULL;
1230         struct btrfs_shared_data_ref *ref2 = NULL;
1231         struct extent_buffer *leaf;
1232         u32 num_refs = 0;
1233         int ret = 0;
1234
1235         leaf = path->nodes[0];
1236         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1237
1238         if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1239                 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1240                                       struct btrfs_extent_data_ref);
1241                 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1242         } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1243                 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1244                                       struct btrfs_shared_data_ref);
1245                 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1246 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1247         } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1248                 struct btrfs_extent_ref_v0 *ref0;
1249                 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1250                                       struct btrfs_extent_ref_v0);
1251                 num_refs = btrfs_ref_count_v0(leaf, ref0);
1252 #endif
1253         } else {
1254                 BUG();
1255         }
1256
1257         BUG_ON(num_refs < refs_to_drop);
1258         num_refs -= refs_to_drop;
1259
1260         if (num_refs == 0) {
1261                 ret = btrfs_del_item(trans, root, path);
1262         } else {
1263                 if (key.type == BTRFS_EXTENT_DATA_REF_KEY)
1264                         btrfs_set_extent_data_ref_count(leaf, ref1, num_refs);
1265                 else if (key.type == BTRFS_SHARED_DATA_REF_KEY)
1266                         btrfs_set_shared_data_ref_count(leaf, ref2, num_refs);
1267 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1268                 else {
1269                         struct btrfs_extent_ref_v0 *ref0;
1270                         ref0 = btrfs_item_ptr(leaf, path->slots[0],
1271                                         struct btrfs_extent_ref_v0);
1272                         btrfs_set_ref_count_v0(leaf, ref0, num_refs);
1273                 }
1274 #endif
1275                 btrfs_mark_buffer_dirty(leaf);
1276         }
1277         return ret;
1278 }
1279
1280 static noinline u32 extent_data_ref_count(struct btrfs_root *root,
1281                                           struct btrfs_path *path,
1282                                           struct btrfs_extent_inline_ref *iref)
1283 {
1284         struct btrfs_key key;
1285         struct extent_buffer *leaf;
1286         struct btrfs_extent_data_ref *ref1;
1287         struct btrfs_shared_data_ref *ref2;
1288         u32 num_refs = 0;
1289
1290         leaf = path->nodes[0];
1291         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1292         if (iref) {
1293                 if (btrfs_extent_inline_ref_type(leaf, iref) ==
1294                     BTRFS_EXTENT_DATA_REF_KEY) {
1295                         ref1 = (struct btrfs_extent_data_ref *)(&iref->offset);
1296                         num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1297                 } else {
1298                         ref2 = (struct btrfs_shared_data_ref *)(iref + 1);
1299                         num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1300                 }
1301         } else if (key.type == BTRFS_EXTENT_DATA_REF_KEY) {
1302                 ref1 = btrfs_item_ptr(leaf, path->slots[0],
1303                                       struct btrfs_extent_data_ref);
1304                 num_refs = btrfs_extent_data_ref_count(leaf, ref1);
1305         } else if (key.type == BTRFS_SHARED_DATA_REF_KEY) {
1306                 ref2 = btrfs_item_ptr(leaf, path->slots[0],
1307                                       struct btrfs_shared_data_ref);
1308                 num_refs = btrfs_shared_data_ref_count(leaf, ref2);
1309 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1310         } else if (key.type == BTRFS_EXTENT_REF_V0_KEY) {
1311                 struct btrfs_extent_ref_v0 *ref0;
1312                 ref0 = btrfs_item_ptr(leaf, path->slots[0],
1313                                       struct btrfs_extent_ref_v0);
1314                 num_refs = btrfs_ref_count_v0(leaf, ref0);
1315 #endif
1316         } else {
1317                 WARN_ON(1);
1318         }
1319         return num_refs;
1320 }
1321
1322 static noinline int lookup_tree_block_ref(struct btrfs_trans_handle *trans,
1323                                           struct btrfs_root *root,
1324                                           struct btrfs_path *path,
1325                                           u64 bytenr, u64 parent,
1326                                           u64 root_objectid)
1327 {
1328         struct btrfs_key key;
1329         int ret;
1330
1331         key.objectid = bytenr;
1332         if (parent) {
1333                 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1334                 key.offset = parent;
1335         } else {
1336                 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1337                 key.offset = root_objectid;
1338         }
1339
1340         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1341         if (ret > 0)
1342                 ret = -ENOENT;
1343 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1344         if (ret == -ENOENT && parent) {
1345                 btrfs_release_path(path);
1346                 key.type = BTRFS_EXTENT_REF_V0_KEY;
1347                 ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1348                 if (ret > 0)
1349                         ret = -ENOENT;
1350         }
1351 #endif
1352         return ret;
1353 }
1354
1355 static noinline int insert_tree_block_ref(struct btrfs_trans_handle *trans,
1356                                           struct btrfs_root *root,
1357                                           struct btrfs_path *path,
1358                                           u64 bytenr, u64 parent,
1359                                           u64 root_objectid)
1360 {
1361         struct btrfs_key key;
1362         int ret;
1363
1364         key.objectid = bytenr;
1365         if (parent) {
1366                 key.type = BTRFS_SHARED_BLOCK_REF_KEY;
1367                 key.offset = parent;
1368         } else {
1369                 key.type = BTRFS_TREE_BLOCK_REF_KEY;
1370                 key.offset = root_objectid;
1371         }
1372
1373         ret = btrfs_insert_empty_item(trans, root, path, &key, 0);
1374         btrfs_release_path(path);
1375         return ret;
1376 }
1377
1378 static inline int extent_ref_type(u64 parent, u64 owner)
1379 {
1380         int type;
1381         if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1382                 if (parent > 0)
1383                         type = BTRFS_SHARED_BLOCK_REF_KEY;
1384                 else
1385                         type = BTRFS_TREE_BLOCK_REF_KEY;
1386         } else {
1387                 if (parent > 0)
1388                         type = BTRFS_SHARED_DATA_REF_KEY;
1389                 else
1390                         type = BTRFS_EXTENT_DATA_REF_KEY;
1391         }
1392         return type;
1393 }
1394
1395 static int find_next_key(struct btrfs_path *path, int level,
1396                          struct btrfs_key *key)
1397
1398 {
1399         for (; level < BTRFS_MAX_LEVEL; level++) {
1400                 if (!path->nodes[level])
1401                         break;
1402                 if (path->slots[level] + 1 >=
1403                     btrfs_header_nritems(path->nodes[level]))
1404                         continue;
1405                 if (level == 0)
1406                         btrfs_item_key_to_cpu(path->nodes[level], key,
1407                                               path->slots[level] + 1);
1408                 else
1409                         btrfs_node_key_to_cpu(path->nodes[level], key,
1410                                               path->slots[level] + 1);
1411                 return 0;
1412         }
1413         return 1;
1414 }
1415
1416 /*
1417  * look for inline back ref. if back ref is found, *ref_ret is set
1418  * to the address of inline back ref, and 0 is returned.
1419  *
1420  * if back ref isn't found, *ref_ret is set to the address where it
1421  * should be inserted, and -ENOENT is returned.
1422  *
1423  * if insert is true and there are too many inline back refs, the path
1424  * points to the extent item, and -EAGAIN is returned.
1425  *
1426  * NOTE: inline back refs are ordered in the same way that back ref
1427  *       items in the tree are ordered.
1428  */
1429 static noinline_for_stack
1430 int lookup_inline_extent_backref(struct btrfs_trans_handle *trans,
1431                                  struct btrfs_root *root,
1432                                  struct btrfs_path *path,
1433                                  struct btrfs_extent_inline_ref **ref_ret,
1434                                  u64 bytenr, u64 num_bytes,
1435                                  u64 parent, u64 root_objectid,
1436                                  u64 owner, u64 offset, int insert)
1437 {
1438         struct btrfs_key key;
1439         struct extent_buffer *leaf;
1440         struct btrfs_extent_item *ei;
1441         struct btrfs_extent_inline_ref *iref;
1442         u64 flags;
1443         u64 item_size;
1444         unsigned long ptr;
1445         unsigned long end;
1446         int extra_size;
1447         int type;
1448         int want;
1449         int ret;
1450         int err = 0;
1451
1452         key.objectid = bytenr;
1453         key.type = BTRFS_EXTENT_ITEM_KEY;
1454         key.offset = num_bytes;
1455
1456         want = extent_ref_type(parent, owner);
1457         if (insert) {
1458                 extra_size = btrfs_extent_inline_ref_size(want);
1459                 path->keep_locks = 1;
1460         } else
1461                 extra_size = -1;
1462         ret = btrfs_search_slot(trans, root, &key, path, extra_size, 1);
1463         if (ret < 0) {
1464                 err = ret;
1465                 goto out;
1466         }
1467         if (ret && !insert) {
1468                 err = -ENOENT;
1469                 goto out;
1470         }
1471         BUG_ON(ret); /* Corruption */
1472
1473         leaf = path->nodes[0];
1474         item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1475 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
1476         if (item_size < sizeof(*ei)) {
1477                 if (!insert) {
1478                         err = -ENOENT;
1479                         goto out;
1480                 }
1481                 ret = convert_extent_item_v0(trans, root, path, owner,
1482                                              extra_size);
1483                 if (ret < 0) {
1484                         err = ret;
1485                         goto out;
1486                 }
1487                 leaf = path->nodes[0];
1488                 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1489         }
1490 #endif
1491         BUG_ON(item_size < sizeof(*ei));
1492
1493         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1494         flags = btrfs_extent_flags(leaf, ei);
1495
1496         ptr = (unsigned long)(ei + 1);
1497         end = (unsigned long)ei + item_size;
1498
1499         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
1500                 ptr += sizeof(struct btrfs_tree_block_info);
1501                 BUG_ON(ptr > end);
1502         } else {
1503                 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_DATA));
1504         }
1505
1506         err = -ENOENT;
1507         while (1) {
1508                 if (ptr >= end) {
1509                         WARN_ON(ptr > end);
1510                         break;
1511                 }
1512                 iref = (struct btrfs_extent_inline_ref *)ptr;
1513                 type = btrfs_extent_inline_ref_type(leaf, iref);
1514                 if (want < type)
1515                         break;
1516                 if (want > type) {
1517                         ptr += btrfs_extent_inline_ref_size(type);
1518                         continue;
1519                 }
1520
1521                 if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1522                         struct btrfs_extent_data_ref *dref;
1523                         dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1524                         if (match_extent_data_ref(leaf, dref, root_objectid,
1525                                                   owner, offset)) {
1526                                 err = 0;
1527                                 break;
1528                         }
1529                         if (hash_extent_data_ref_item(leaf, dref) <
1530                             hash_extent_data_ref(root_objectid, owner, offset))
1531                                 break;
1532                 } else {
1533                         u64 ref_offset;
1534                         ref_offset = btrfs_extent_inline_ref_offset(leaf, iref);
1535                         if (parent > 0) {
1536                                 if (parent == ref_offset) {
1537                                         err = 0;
1538                                         break;
1539                                 }
1540                                 if (ref_offset < parent)
1541                                         break;
1542                         } else {
1543                                 if (root_objectid == ref_offset) {
1544                                         err = 0;
1545                                         break;
1546                                 }
1547                                 if (ref_offset < root_objectid)
1548                                         break;
1549                         }
1550                 }
1551                 ptr += btrfs_extent_inline_ref_size(type);
1552         }
1553         if (err == -ENOENT && insert) {
1554                 if (item_size + extra_size >=
1555                     BTRFS_MAX_EXTENT_ITEM_SIZE(root)) {
1556                         err = -EAGAIN;
1557                         goto out;
1558                 }
1559                 /*
1560                  * To add new inline back ref, we have to make sure
1561                  * there is no corresponding back ref item.
1562                  * For simplicity, we just do not add new inline back
1563                  * ref if there is any kind of item for this block
1564                  */
1565                 if (find_next_key(path, 0, &key) == 0 &&
1566                     key.objectid == bytenr &&
1567                     key.type < BTRFS_BLOCK_GROUP_ITEM_KEY) {
1568                         err = -EAGAIN;
1569                         goto out;
1570                 }
1571         }
1572         *ref_ret = (struct btrfs_extent_inline_ref *)ptr;
1573 out:
1574         if (insert) {
1575                 path->keep_locks = 0;
1576                 btrfs_unlock_up_safe(path, 1);
1577         }
1578         return err;
1579 }
1580
1581 /*
1582  * helper to add new inline back ref
1583  */
1584 static noinline_for_stack
1585 void setup_inline_extent_backref(struct btrfs_trans_handle *trans,
1586                                  struct btrfs_root *root,
1587                                  struct btrfs_path *path,
1588                                  struct btrfs_extent_inline_ref *iref,
1589                                  u64 parent, u64 root_objectid,
1590                                  u64 owner, u64 offset, int refs_to_add,
1591                                  struct btrfs_delayed_extent_op *extent_op)
1592 {
1593         struct extent_buffer *leaf;
1594         struct btrfs_extent_item *ei;
1595         unsigned long ptr;
1596         unsigned long end;
1597         unsigned long item_offset;
1598         u64 refs;
1599         int size;
1600         int type;
1601
1602         leaf = path->nodes[0];
1603         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1604         item_offset = (unsigned long)iref - (unsigned long)ei;
1605
1606         type = extent_ref_type(parent, owner);
1607         size = btrfs_extent_inline_ref_size(type);
1608
1609         btrfs_extend_item(trans, root, path, size);
1610
1611         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1612         refs = btrfs_extent_refs(leaf, ei);
1613         refs += refs_to_add;
1614         btrfs_set_extent_refs(leaf, ei, refs);
1615         if (extent_op)
1616                 __run_delayed_extent_op(extent_op, leaf, ei);
1617
1618         ptr = (unsigned long)ei + item_offset;
1619         end = (unsigned long)ei + btrfs_item_size_nr(leaf, path->slots[0]);
1620         if (ptr < end - size)
1621                 memmove_extent_buffer(leaf, ptr + size, ptr,
1622                                       end - size - ptr);
1623
1624         iref = (struct btrfs_extent_inline_ref *)ptr;
1625         btrfs_set_extent_inline_ref_type(leaf, iref, type);
1626         if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1627                 struct btrfs_extent_data_ref *dref;
1628                 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1629                 btrfs_set_extent_data_ref_root(leaf, dref, root_objectid);
1630                 btrfs_set_extent_data_ref_objectid(leaf, dref, owner);
1631                 btrfs_set_extent_data_ref_offset(leaf, dref, offset);
1632                 btrfs_set_extent_data_ref_count(leaf, dref, refs_to_add);
1633         } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1634                 struct btrfs_shared_data_ref *sref;
1635                 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1636                 btrfs_set_shared_data_ref_count(leaf, sref, refs_to_add);
1637                 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1638         } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) {
1639                 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
1640         } else {
1641                 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
1642         }
1643         btrfs_mark_buffer_dirty(leaf);
1644 }
1645
1646 static int lookup_extent_backref(struct btrfs_trans_handle *trans,
1647                                  struct btrfs_root *root,
1648                                  struct btrfs_path *path,
1649                                  struct btrfs_extent_inline_ref **ref_ret,
1650                                  u64 bytenr, u64 num_bytes, u64 parent,
1651                                  u64 root_objectid, u64 owner, u64 offset)
1652 {
1653         int ret;
1654
1655         ret = lookup_inline_extent_backref(trans, root, path, ref_ret,
1656                                            bytenr, num_bytes, parent,
1657                                            root_objectid, owner, offset, 0);
1658         if (ret != -ENOENT)
1659                 return ret;
1660
1661         btrfs_release_path(path);
1662         *ref_ret = NULL;
1663
1664         if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1665                 ret = lookup_tree_block_ref(trans, root, path, bytenr, parent,
1666                                             root_objectid);
1667         } else {
1668                 ret = lookup_extent_data_ref(trans, root, path, bytenr, parent,
1669                                              root_objectid, owner, offset);
1670         }
1671         return ret;
1672 }
1673
1674 /*
1675  * helper to update/remove inline back ref
1676  */
1677 static noinline_for_stack
1678 void update_inline_extent_backref(struct btrfs_trans_handle *trans,
1679                                   struct btrfs_root *root,
1680                                   struct btrfs_path *path,
1681                                   struct btrfs_extent_inline_ref *iref,
1682                                   int refs_to_mod,
1683                                   struct btrfs_delayed_extent_op *extent_op)
1684 {
1685         struct extent_buffer *leaf;
1686         struct btrfs_extent_item *ei;
1687         struct btrfs_extent_data_ref *dref = NULL;
1688         struct btrfs_shared_data_ref *sref = NULL;
1689         unsigned long ptr;
1690         unsigned long end;
1691         u32 item_size;
1692         int size;
1693         int type;
1694         u64 refs;
1695
1696         leaf = path->nodes[0];
1697         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1698         refs = btrfs_extent_refs(leaf, ei);
1699         WARN_ON(refs_to_mod < 0 && refs + refs_to_mod <= 0);
1700         refs += refs_to_mod;
1701         btrfs_set_extent_refs(leaf, ei, refs);
1702         if (extent_op)
1703                 __run_delayed_extent_op(extent_op, leaf, ei);
1704
1705         type = btrfs_extent_inline_ref_type(leaf, iref);
1706
1707         if (type == BTRFS_EXTENT_DATA_REF_KEY) {
1708                 dref = (struct btrfs_extent_data_ref *)(&iref->offset);
1709                 refs = btrfs_extent_data_ref_count(leaf, dref);
1710         } else if (type == BTRFS_SHARED_DATA_REF_KEY) {
1711                 sref = (struct btrfs_shared_data_ref *)(iref + 1);
1712                 refs = btrfs_shared_data_ref_count(leaf, sref);
1713         } else {
1714                 refs = 1;
1715                 BUG_ON(refs_to_mod != -1);
1716         }
1717
1718         BUG_ON(refs_to_mod < 0 && refs < -refs_to_mod);
1719         refs += refs_to_mod;
1720
1721         if (refs > 0) {
1722                 if (type == BTRFS_EXTENT_DATA_REF_KEY)
1723                         btrfs_set_extent_data_ref_count(leaf, dref, refs);
1724                 else
1725                         btrfs_set_shared_data_ref_count(leaf, sref, refs);
1726         } else {
1727                 size =  btrfs_extent_inline_ref_size(type);
1728                 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
1729                 ptr = (unsigned long)iref;
1730                 end = (unsigned long)ei + item_size;
1731                 if (ptr + size < end)
1732                         memmove_extent_buffer(leaf, ptr, ptr + size,
1733                                               end - ptr - size);
1734                 item_size -= size;
1735                 btrfs_truncate_item(trans, root, path, item_size, 1);
1736         }
1737         btrfs_mark_buffer_dirty(leaf);
1738 }
1739
1740 static noinline_for_stack
1741 int insert_inline_extent_backref(struct btrfs_trans_handle *trans,
1742                                  struct btrfs_root *root,
1743                                  struct btrfs_path *path,
1744                                  u64 bytenr, u64 num_bytes, u64 parent,
1745                                  u64 root_objectid, u64 owner,
1746                                  u64 offset, int refs_to_add,
1747                                  struct btrfs_delayed_extent_op *extent_op)
1748 {
1749         struct btrfs_extent_inline_ref *iref;
1750         int ret;
1751
1752         ret = lookup_inline_extent_backref(trans, root, path, &iref,
1753                                            bytenr, num_bytes, parent,
1754                                            root_objectid, owner, offset, 1);
1755         if (ret == 0) {
1756                 BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID);
1757                 update_inline_extent_backref(trans, root, path, iref,
1758                                              refs_to_add, extent_op);
1759         } else if (ret == -ENOENT) {
1760                 setup_inline_extent_backref(trans, root, path, iref, parent,
1761                                             root_objectid, owner, offset,
1762                                             refs_to_add, extent_op);
1763                 ret = 0;
1764         }
1765         return ret;
1766 }
1767
1768 static int insert_extent_backref(struct btrfs_trans_handle *trans,
1769                                  struct btrfs_root *root,
1770                                  struct btrfs_path *path,
1771                                  u64 bytenr, u64 parent, u64 root_objectid,
1772                                  u64 owner, u64 offset, int refs_to_add)
1773 {
1774         int ret;
1775         if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1776                 BUG_ON(refs_to_add != 1);
1777                 ret = insert_tree_block_ref(trans, root, path, bytenr,
1778                                             parent, root_objectid);
1779         } else {
1780                 ret = insert_extent_data_ref(trans, root, path, bytenr,
1781                                              parent, root_objectid,
1782                                              owner, offset, refs_to_add);
1783         }
1784         return ret;
1785 }
1786
1787 static int remove_extent_backref(struct btrfs_trans_handle *trans,
1788                                  struct btrfs_root *root,
1789                                  struct btrfs_path *path,
1790                                  struct btrfs_extent_inline_ref *iref,
1791                                  int refs_to_drop, int is_data)
1792 {
1793         int ret = 0;
1794
1795         BUG_ON(!is_data && refs_to_drop != 1);
1796         if (iref) {
1797                 update_inline_extent_backref(trans, root, path, iref,
1798                                              -refs_to_drop, NULL);
1799         } else if (is_data) {
1800                 ret = remove_extent_data_ref(trans, root, path, refs_to_drop);
1801         } else {
1802                 ret = btrfs_del_item(trans, root, path);
1803         }
1804         return ret;
1805 }
1806
1807 static int btrfs_issue_discard(struct block_device *bdev,
1808                                 u64 start, u64 len)
1809 {
1810         return blkdev_issue_discard(bdev, start >> 9, len >> 9, GFP_NOFS, 0);
1811 }
1812
1813 static int btrfs_discard_extent(struct btrfs_root *root, u64 bytenr,
1814                                 u64 num_bytes, u64 *actual_bytes)
1815 {
1816         int ret;
1817         u64 discarded_bytes = 0;
1818         struct btrfs_bio *bbio = NULL;
1819
1820
1821         /* Tell the block device(s) that the sectors can be discarded */
1822         ret = btrfs_map_block(root->fs_info, REQ_DISCARD,
1823                               bytenr, &num_bytes, &bbio, 0);
1824         /* Error condition is -ENOMEM */
1825         if (!ret) {
1826                 struct btrfs_bio_stripe *stripe = bbio->stripes;
1827                 int i;
1828
1829
1830                 for (i = 0; i < bbio->num_stripes; i++, stripe++) {
1831                         if (!stripe->dev->can_discard)
1832                                 continue;
1833
1834                         ret = btrfs_issue_discard(stripe->dev->bdev,
1835                                                   stripe->physical,
1836                                                   stripe->length);
1837                         if (!ret)
1838                                 discarded_bytes += stripe->length;
1839                         else if (ret != -EOPNOTSUPP)
1840                                 break; /* Logic errors or -ENOMEM, or -EIO but I don't know how that could happen JDM */
1841
1842                         /*
1843                          * Just in case we get back EOPNOTSUPP for some reason,
1844                          * just ignore the return value so we don't screw up
1845                          * people calling discard_extent.
1846                          */
1847                         ret = 0;
1848                 }
1849                 kfree(bbio);
1850         }
1851
1852         if (actual_bytes)
1853                 *actual_bytes = discarded_bytes;
1854
1855
1856         if (ret == -EOPNOTSUPP)
1857                 ret = 0;
1858         return ret;
1859 }
1860
1861 /* Can return -ENOMEM */
1862 int btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1863                          struct btrfs_root *root,
1864                          u64 bytenr, u64 num_bytes, u64 parent,
1865                          u64 root_objectid, u64 owner, u64 offset, int for_cow)
1866 {
1867         int ret;
1868         struct btrfs_fs_info *fs_info = root->fs_info;
1869
1870         BUG_ON(owner < BTRFS_FIRST_FREE_OBJECTID &&
1871                root_objectid == BTRFS_TREE_LOG_OBJECTID);
1872
1873         if (owner < BTRFS_FIRST_FREE_OBJECTID) {
1874                 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
1875                                         num_bytes,
1876                                         parent, root_objectid, (int)owner,
1877                                         BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1878         } else {
1879                 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
1880                                         num_bytes,
1881                                         parent, root_objectid, owner, offset,
1882                                         BTRFS_ADD_DELAYED_REF, NULL, for_cow);
1883         }
1884         return ret;
1885 }
1886
1887 static int __btrfs_inc_extent_ref(struct btrfs_trans_handle *trans,
1888                                   struct btrfs_root *root,
1889                                   u64 bytenr, u64 num_bytes,
1890                                   u64 parent, u64 root_objectid,
1891                                   u64 owner, u64 offset, int refs_to_add,
1892                                   struct btrfs_delayed_extent_op *extent_op)
1893 {
1894         struct btrfs_path *path;
1895         struct extent_buffer *leaf;
1896         struct btrfs_extent_item *item;
1897         u64 refs;
1898         int ret;
1899         int err = 0;
1900
1901         path = btrfs_alloc_path();
1902         if (!path)
1903                 return -ENOMEM;
1904
1905         path->reada = 1;
1906         path->leave_spinning = 1;
1907         /* this will setup the path even if it fails to insert the back ref */
1908         ret = insert_inline_extent_backref(trans, root->fs_info->extent_root,
1909                                            path, bytenr, num_bytes, parent,
1910                                            root_objectid, owner, offset,
1911                                            refs_to_add, extent_op);
1912         if (ret == 0)
1913                 goto out;
1914
1915         if (ret != -EAGAIN) {
1916                 err = ret;
1917                 goto out;
1918         }
1919
1920         leaf = path->nodes[0];
1921         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
1922         refs = btrfs_extent_refs(leaf, item);
1923         btrfs_set_extent_refs(leaf, item, refs + refs_to_add);
1924         if (extent_op)
1925                 __run_delayed_extent_op(extent_op, leaf, item);
1926
1927         btrfs_mark_buffer_dirty(leaf);
1928         btrfs_release_path(path);
1929
1930         path->reada = 1;
1931         path->leave_spinning = 1;
1932
1933         /* now insert the actual backref */
1934         ret = insert_extent_backref(trans, root->fs_info->extent_root,
1935                                     path, bytenr, parent, root_objectid,
1936                                     owner, offset, refs_to_add);
1937         if (ret)
1938                 btrfs_abort_transaction(trans, root, ret);
1939 out:
1940         btrfs_free_path(path);
1941         return err;
1942 }
1943
1944 static int run_delayed_data_ref(struct btrfs_trans_handle *trans,
1945                                 struct btrfs_root *root,
1946                                 struct btrfs_delayed_ref_node *node,
1947                                 struct btrfs_delayed_extent_op *extent_op,
1948                                 int insert_reserved)
1949 {
1950         int ret = 0;
1951         struct btrfs_delayed_data_ref *ref;
1952         struct btrfs_key ins;
1953         u64 parent = 0;
1954         u64 ref_root = 0;
1955         u64 flags = 0;
1956
1957         ins.objectid = node->bytenr;
1958         ins.offset = node->num_bytes;
1959         ins.type = BTRFS_EXTENT_ITEM_KEY;
1960
1961         ref = btrfs_delayed_node_to_data_ref(node);
1962         if (node->type == BTRFS_SHARED_DATA_REF_KEY)
1963                 parent = ref->parent;
1964         else
1965                 ref_root = ref->root;
1966
1967         if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
1968                 if (extent_op) {
1969                         BUG_ON(extent_op->update_key);
1970                         flags |= extent_op->flags_to_set;
1971                 }
1972                 ret = alloc_reserved_file_extent(trans, root,
1973                                                  parent, ref_root, flags,
1974                                                  ref->objectid, ref->offset,
1975                                                  &ins, node->ref_mod);
1976         } else if (node->action == BTRFS_ADD_DELAYED_REF) {
1977                 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
1978                                              node->num_bytes, parent,
1979                                              ref_root, ref->objectid,
1980                                              ref->offset, node->ref_mod,
1981                                              extent_op);
1982         } else if (node->action == BTRFS_DROP_DELAYED_REF) {
1983                 ret = __btrfs_free_extent(trans, root, node->bytenr,
1984                                           node->num_bytes, parent,
1985                                           ref_root, ref->objectid,
1986                                           ref->offset, node->ref_mod,
1987                                           extent_op);
1988         } else {
1989                 BUG();
1990         }
1991         return ret;
1992 }
1993
1994 static void __run_delayed_extent_op(struct btrfs_delayed_extent_op *extent_op,
1995                                     struct extent_buffer *leaf,
1996                                     struct btrfs_extent_item *ei)
1997 {
1998         u64 flags = btrfs_extent_flags(leaf, ei);
1999         if (extent_op->update_flags) {
2000                 flags |= extent_op->flags_to_set;
2001                 btrfs_set_extent_flags(leaf, ei, flags);
2002         }
2003
2004         if (extent_op->update_key) {
2005                 struct btrfs_tree_block_info *bi;
2006                 BUG_ON(!(flags & BTRFS_EXTENT_FLAG_TREE_BLOCK));
2007                 bi = (struct btrfs_tree_block_info *)(ei + 1);
2008                 btrfs_set_tree_block_key(leaf, bi, &extent_op->key);
2009         }
2010 }
2011
2012 static int run_delayed_extent_op(struct btrfs_trans_handle *trans,
2013                                  struct btrfs_root *root,
2014                                  struct btrfs_delayed_ref_node *node,
2015                                  struct btrfs_delayed_extent_op *extent_op)
2016 {
2017         struct btrfs_key key;
2018         struct btrfs_path *path;
2019         struct btrfs_extent_item *ei;
2020         struct extent_buffer *leaf;
2021         u32 item_size;
2022         int ret;
2023         int err = 0;
2024
2025         if (trans->aborted)
2026                 return 0;
2027
2028         path = btrfs_alloc_path();
2029         if (!path)
2030                 return -ENOMEM;
2031
2032         key.objectid = node->bytenr;
2033         key.type = BTRFS_EXTENT_ITEM_KEY;
2034         key.offset = node->num_bytes;
2035
2036         path->reada = 1;
2037         path->leave_spinning = 1;
2038         ret = btrfs_search_slot(trans, root->fs_info->extent_root, &key,
2039                                 path, 0, 1);
2040         if (ret < 0) {
2041                 err = ret;
2042                 goto out;
2043         }
2044         if (ret > 0) {
2045                 err = -EIO;
2046                 goto out;
2047         }
2048
2049         leaf = path->nodes[0];
2050         item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2051 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2052         if (item_size < sizeof(*ei)) {
2053                 ret = convert_extent_item_v0(trans, root->fs_info->extent_root,
2054                                              path, (u64)-1, 0);
2055                 if (ret < 0) {
2056                         err = ret;
2057                         goto out;
2058                 }
2059                 leaf = path->nodes[0];
2060                 item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2061         }
2062 #endif
2063         BUG_ON(item_size < sizeof(*ei));
2064         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2065         __run_delayed_extent_op(extent_op, leaf, ei);
2066
2067         btrfs_mark_buffer_dirty(leaf);
2068 out:
2069         btrfs_free_path(path);
2070         return err;
2071 }
2072
2073 static int run_delayed_tree_ref(struct btrfs_trans_handle *trans,
2074                                 struct btrfs_root *root,
2075                                 struct btrfs_delayed_ref_node *node,
2076                                 struct btrfs_delayed_extent_op *extent_op,
2077                                 int insert_reserved)
2078 {
2079         int ret = 0;
2080         struct btrfs_delayed_tree_ref *ref;
2081         struct btrfs_key ins;
2082         u64 parent = 0;
2083         u64 ref_root = 0;
2084
2085         ins.objectid = node->bytenr;
2086         ins.offset = node->num_bytes;
2087         ins.type = BTRFS_EXTENT_ITEM_KEY;
2088
2089         ref = btrfs_delayed_node_to_tree_ref(node);
2090         if (node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2091                 parent = ref->parent;
2092         else
2093                 ref_root = ref->root;
2094
2095         BUG_ON(node->ref_mod != 1);
2096         if (node->action == BTRFS_ADD_DELAYED_REF && insert_reserved) {
2097                 BUG_ON(!extent_op || !extent_op->update_flags ||
2098                        !extent_op->update_key);
2099                 ret = alloc_reserved_tree_block(trans, root,
2100                                                 parent, ref_root,
2101                                                 extent_op->flags_to_set,
2102                                                 &extent_op->key,
2103                                                 ref->level, &ins);
2104         } else if (node->action == BTRFS_ADD_DELAYED_REF) {
2105                 ret = __btrfs_inc_extent_ref(trans, root, node->bytenr,
2106                                              node->num_bytes, parent, ref_root,
2107                                              ref->level, 0, 1, extent_op);
2108         } else if (node->action == BTRFS_DROP_DELAYED_REF) {
2109                 ret = __btrfs_free_extent(trans, root, node->bytenr,
2110                                           node->num_bytes, parent, ref_root,
2111                                           ref->level, 0, 1, extent_op);
2112         } else {
2113                 BUG();
2114         }
2115         return ret;
2116 }
2117
2118 /* helper function to actually process a single delayed ref entry */
2119 static int run_one_delayed_ref(struct btrfs_trans_handle *trans,
2120                                struct btrfs_root *root,
2121                                struct btrfs_delayed_ref_node *node,
2122                                struct btrfs_delayed_extent_op *extent_op,
2123                                int insert_reserved)
2124 {
2125         int ret = 0;
2126
2127         if (trans->aborted)
2128                 return 0;
2129
2130         if (btrfs_delayed_ref_is_head(node)) {
2131                 struct btrfs_delayed_ref_head *head;
2132                 /*
2133                  * we've hit the end of the chain and we were supposed
2134                  * to insert this extent into the tree.  But, it got
2135                  * deleted before we ever needed to insert it, so all
2136                  * we have to do is clean up the accounting
2137                  */
2138                 BUG_ON(extent_op);
2139                 head = btrfs_delayed_node_to_head(node);
2140                 if (insert_reserved) {
2141                         btrfs_pin_extent(root, node->bytenr,
2142                                          node->num_bytes, 1);
2143                         if (head->is_data) {
2144                                 ret = btrfs_del_csums(trans, root,
2145                                                       node->bytenr,
2146                                                       node->num_bytes);
2147                         }
2148                 }
2149                 mutex_unlock(&head->mutex);
2150                 return ret;
2151         }
2152
2153         if (node->type == BTRFS_TREE_BLOCK_REF_KEY ||
2154             node->type == BTRFS_SHARED_BLOCK_REF_KEY)
2155                 ret = run_delayed_tree_ref(trans, root, node, extent_op,
2156                                            insert_reserved);
2157         else if (node->type == BTRFS_EXTENT_DATA_REF_KEY ||
2158                  node->type == BTRFS_SHARED_DATA_REF_KEY)
2159                 ret = run_delayed_data_ref(trans, root, node, extent_op,
2160                                            insert_reserved);
2161         else
2162                 BUG();
2163         return ret;
2164 }
2165
2166 static noinline struct btrfs_delayed_ref_node *
2167 select_delayed_ref(struct btrfs_delayed_ref_head *head)
2168 {
2169         struct rb_node *node;
2170         struct btrfs_delayed_ref_node *ref;
2171         int action = BTRFS_ADD_DELAYED_REF;
2172 again:
2173         /*
2174          * select delayed ref of type BTRFS_ADD_DELAYED_REF first.
2175          * this prevents ref count from going down to zero when
2176          * there still are pending delayed ref.
2177          */
2178         node = rb_prev(&head->node.rb_node);
2179         while (1) {
2180                 if (!node)
2181                         break;
2182                 ref = rb_entry(node, struct btrfs_delayed_ref_node,
2183                                 rb_node);
2184                 if (ref->bytenr != head->node.bytenr)
2185                         break;
2186                 if (ref->action == action)
2187                         return ref;
2188                 node = rb_prev(node);
2189         }
2190         if (action == BTRFS_ADD_DELAYED_REF) {
2191                 action = BTRFS_DROP_DELAYED_REF;
2192                 goto again;
2193         }
2194         return NULL;
2195 }
2196
2197 /*
2198  * Returns 0 on success or if called with an already aborted transaction.
2199  * Returns -ENOMEM or -EIO on failure and will abort the transaction.
2200  */
2201 static noinline int run_clustered_refs(struct btrfs_trans_handle *trans,
2202                                        struct btrfs_root *root,
2203                                        struct list_head *cluster)
2204 {
2205         struct btrfs_delayed_ref_root *delayed_refs;
2206         struct btrfs_delayed_ref_node *ref;
2207         struct btrfs_delayed_ref_head *locked_ref = NULL;
2208         struct btrfs_delayed_extent_op *extent_op;
2209         struct btrfs_fs_info *fs_info = root->fs_info;
2210         int ret;
2211         int count = 0;
2212         int must_insert_reserved = 0;
2213
2214         delayed_refs = &trans->transaction->delayed_refs;
2215         while (1) {
2216                 if (!locked_ref) {
2217                         /* pick a new head ref from the cluster list */
2218                         if (list_empty(cluster))
2219                                 break;
2220
2221                         locked_ref = list_entry(cluster->next,
2222                                      struct btrfs_delayed_ref_head, cluster);
2223
2224                         /* grab the lock that says we are going to process
2225                          * all the refs for this head */
2226                         ret = btrfs_delayed_ref_lock(trans, locked_ref);
2227
2228                         /*
2229                          * we may have dropped the spin lock to get the head
2230                          * mutex lock, and that might have given someone else
2231                          * time to free the head.  If that's true, it has been
2232                          * removed from our list and we can move on.
2233                          */
2234                         if (ret == -EAGAIN) {
2235                                 locked_ref = NULL;
2236                                 count++;
2237                                 continue;
2238                         }
2239                 }
2240
2241                 /*
2242                  * We need to try and merge add/drops of the same ref since we
2243                  * can run into issues with relocate dropping the implicit ref
2244                  * and then it being added back again before the drop can
2245                  * finish.  If we merged anything we need to re-loop so we can
2246                  * get a good ref.
2247                  */
2248                 btrfs_merge_delayed_refs(trans, fs_info, delayed_refs,
2249                                          locked_ref);
2250
2251                 /*
2252                  * locked_ref is the head node, so we have to go one
2253                  * node back for any delayed ref updates
2254                  */
2255                 ref = select_delayed_ref(locked_ref);
2256
2257                 if (ref && ref->seq &&
2258                     btrfs_check_delayed_seq(fs_info, delayed_refs, ref->seq)) {
2259                         /*
2260                          * there are still refs with lower seq numbers in the
2261                          * process of being added. Don't run this ref yet.
2262                          */
2263                         list_del_init(&locked_ref->cluster);
2264                         mutex_unlock(&locked_ref->mutex);
2265                         locked_ref = NULL;
2266                         delayed_refs->num_heads_ready++;
2267                         spin_unlock(&delayed_refs->lock);
2268                         cond_resched();
2269                         spin_lock(&delayed_refs->lock);
2270                         continue;
2271                 }
2272
2273                 /*
2274                  * record the must insert reserved flag before we
2275                  * drop the spin lock.
2276                  */
2277                 must_insert_reserved = locked_ref->must_insert_reserved;
2278                 locked_ref->must_insert_reserved = 0;
2279
2280                 extent_op = locked_ref->extent_op;
2281                 locked_ref->extent_op = NULL;
2282
2283                 if (!ref) {
2284                         /* All delayed refs have been processed, Go ahead
2285                          * and send the head node to run_one_delayed_ref,
2286                          * so that any accounting fixes can happen
2287                          */
2288                         ref = &locked_ref->node;
2289
2290                         if (extent_op && must_insert_reserved) {
2291                                 kfree(extent_op);
2292                                 extent_op = NULL;
2293                         }
2294
2295                         if (extent_op) {
2296                                 spin_unlock(&delayed_refs->lock);
2297
2298                                 ret = run_delayed_extent_op(trans, root,
2299                                                             ref, extent_op);
2300                                 kfree(extent_op);
2301
2302                                 if (ret) {
2303                                         list_del_init(&locked_ref->cluster);
2304                                         mutex_unlock(&locked_ref->mutex);
2305
2306                                         printk(KERN_DEBUG "btrfs: run_delayed_extent_op returned %d\n", ret);
2307                                         spin_lock(&delayed_refs->lock);
2308                                         return ret;
2309                                 }
2310
2311                                 goto next;
2312                         }
2313
2314                         list_del_init(&locked_ref->cluster);
2315                         locked_ref = NULL;
2316                 }
2317
2318                 ref->in_tree = 0;
2319                 rb_erase(&ref->rb_node, &delayed_refs->root);
2320                 delayed_refs->num_entries--;
2321                 if (locked_ref) {
2322                         /*
2323                          * when we play the delayed ref, also correct the
2324                          * ref_mod on head
2325                          */
2326                         switch (ref->action) {
2327                         case BTRFS_ADD_DELAYED_REF:
2328                         case BTRFS_ADD_DELAYED_EXTENT:
2329                                 locked_ref->node.ref_mod -= ref->ref_mod;
2330                                 break;
2331                         case BTRFS_DROP_DELAYED_REF:
2332                                 locked_ref->node.ref_mod += ref->ref_mod;
2333                                 break;
2334                         default:
2335                                 WARN_ON(1);
2336                         }
2337                 }
2338                 spin_unlock(&delayed_refs->lock);
2339
2340                 ret = run_one_delayed_ref(trans, root, ref, extent_op,
2341                                           must_insert_reserved);
2342
2343                 btrfs_put_delayed_ref(ref);
2344                 kfree(extent_op);
2345                 count++;
2346
2347                 if (ret) {
2348                         if (locked_ref) {
2349                                 list_del_init(&locked_ref->cluster);
2350                                 mutex_unlock(&locked_ref->mutex);
2351                         }
2352                         printk(KERN_DEBUG "btrfs: run_one_delayed_ref returned %d\n", ret);
2353                         spin_lock(&delayed_refs->lock);
2354                         return ret;
2355                 }
2356
2357 next:
2358                 cond_resched();
2359                 spin_lock(&delayed_refs->lock);
2360         }
2361         return count;
2362 }
2363
2364 #ifdef SCRAMBLE_DELAYED_REFS
2365 /*
2366  * Normally delayed refs get processed in ascending bytenr order. This
2367  * correlates in most cases to the order added. To expose dependencies on this
2368  * order, we start to process the tree in the middle instead of the beginning
2369  */
2370 static u64 find_middle(struct rb_root *root)
2371 {
2372         struct rb_node *n = root->rb_node;
2373         struct btrfs_delayed_ref_node *entry;
2374         int alt = 1;
2375         u64 middle;
2376         u64 first = 0, last = 0;
2377
2378         n = rb_first(root);
2379         if (n) {
2380                 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2381                 first = entry->bytenr;
2382         }
2383         n = rb_last(root);
2384         if (n) {
2385                 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2386                 last = entry->bytenr;
2387         }
2388         n = root->rb_node;
2389
2390         while (n) {
2391                 entry = rb_entry(n, struct btrfs_delayed_ref_node, rb_node);
2392                 WARN_ON(!entry->in_tree);
2393
2394                 middle = entry->bytenr;
2395
2396                 if (alt)
2397                         n = n->rb_left;
2398                 else
2399                         n = n->rb_right;
2400
2401                 alt = 1 - alt;
2402         }
2403         return middle;
2404 }
2405 #endif
2406
2407 int btrfs_delayed_refs_qgroup_accounting(struct btrfs_trans_handle *trans,
2408                                          struct btrfs_fs_info *fs_info)
2409 {
2410         struct qgroup_update *qgroup_update;
2411         int ret = 0;
2412
2413         if (list_empty(&trans->qgroup_ref_list) !=
2414             !trans->delayed_ref_elem.seq) {
2415                 /* list without seq or seq without list */
2416                 printk(KERN_ERR "btrfs: qgroup accounting update error, list is%s empty, seq is %llu\n",
2417                         list_empty(&trans->qgroup_ref_list) ? "" : " not",
2418                         trans->delayed_ref_elem.seq);
2419                 BUG();
2420         }
2421
2422         if (!trans->delayed_ref_elem.seq)
2423                 return 0;
2424
2425         while (!list_empty(&trans->qgroup_ref_list)) {
2426                 qgroup_update = list_first_entry(&trans->qgroup_ref_list,
2427                                                  struct qgroup_update, list);
2428                 list_del(&qgroup_update->list);
2429                 if (!ret)
2430                         ret = btrfs_qgroup_account_ref(
2431                                         trans, fs_info, qgroup_update->node,
2432                                         qgroup_update->extent_op);
2433                 kfree(qgroup_update);
2434         }
2435
2436         btrfs_put_tree_mod_seq(fs_info, &trans->delayed_ref_elem);
2437
2438         return ret;
2439 }
2440
2441 static int refs_newer(struct btrfs_delayed_ref_root *delayed_refs, int seq,
2442                       int count)
2443 {
2444         int val = atomic_read(&delayed_refs->ref_seq);
2445
2446         if (val < seq || val >= seq + count)
2447                 return 1;
2448         return 0;
2449 }
2450
2451 /*
2452  * this starts processing the delayed reference count updates and
2453  * extent insertions we have queued up so far.  count can be
2454  * 0, which means to process everything in the tree at the start
2455  * of the run (but not newly added entries), or it can be some target
2456  * number you'd like to process.
2457  *
2458  * Returns 0 on success or if called with an aborted transaction
2459  * Returns <0 on error and aborts the transaction
2460  */
2461 int btrfs_run_delayed_refs(struct btrfs_trans_handle *trans,
2462                            struct btrfs_root *root, unsigned long count)
2463 {
2464         struct rb_node *node;
2465         struct btrfs_delayed_ref_root *delayed_refs;
2466         struct btrfs_delayed_ref_node *ref;
2467         struct list_head cluster;
2468         int ret;
2469         u64 delayed_start;
2470         int run_all = count == (unsigned long)-1;
2471         int run_most = 0;
2472         int loops;
2473
2474         /* We'll clean this up in btrfs_cleanup_transaction */
2475         if (trans->aborted)
2476                 return 0;
2477
2478         if (root == root->fs_info->extent_root)
2479                 root = root->fs_info->tree_root;
2480
2481         btrfs_delayed_refs_qgroup_accounting(trans, root->fs_info);
2482
2483         delayed_refs = &trans->transaction->delayed_refs;
2484         INIT_LIST_HEAD(&cluster);
2485         if (count == 0) {
2486                 count = delayed_refs->num_entries * 2;
2487                 run_most = 1;
2488         }
2489
2490         if (!run_all && !run_most) {
2491                 int old;
2492                 int seq = atomic_read(&delayed_refs->ref_seq);
2493
2494 progress:
2495                 old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
2496                 if (old) {
2497                         DEFINE_WAIT(__wait);
2498                         if (delayed_refs->num_entries < 16348)
2499                                 return 0;
2500
2501                         prepare_to_wait(&delayed_refs->wait, &__wait,
2502                                         TASK_UNINTERRUPTIBLE);
2503
2504                         old = atomic_cmpxchg(&delayed_refs->procs_running_refs, 0, 1);
2505                         if (old) {
2506                                 schedule();
2507                                 finish_wait(&delayed_refs->wait, &__wait);
2508
2509                                 if (!refs_newer(delayed_refs, seq, 256))
2510                                         goto progress;
2511                                 else
2512                                         return 0;
2513                         } else {
2514                                 finish_wait(&delayed_refs->wait, &__wait);
2515                                 goto again;
2516                         }
2517                 }
2518
2519         } else {
2520                 atomic_inc(&delayed_refs->procs_running_refs);
2521         }
2522
2523 again:
2524         loops = 0;
2525         spin_lock(&delayed_refs->lock);
2526
2527 #ifdef SCRAMBLE_DELAYED_REFS
2528         delayed_refs->run_delayed_start = find_middle(&delayed_refs->root);
2529 #endif
2530
2531         while (1) {
2532                 if (!(run_all || run_most) &&
2533                     delayed_refs->num_heads_ready < 64)
2534                         break;
2535
2536                 /*
2537                  * go find something we can process in the rbtree.  We start at
2538                  * the beginning of the tree, and then build a cluster
2539                  * of refs to process starting at the first one we are able to
2540                  * lock
2541                  */
2542                 delayed_start = delayed_refs->run_delayed_start;
2543                 ret = btrfs_find_ref_cluster(trans, &cluster,
2544                                              delayed_refs->run_delayed_start);
2545                 if (ret)
2546                         break;
2547
2548                 ret = run_clustered_refs(trans, root, &cluster);
2549                 if (ret < 0) {
2550                         spin_unlock(&delayed_refs->lock);
2551                         btrfs_abort_transaction(trans, root, ret);
2552                         atomic_dec(&delayed_refs->procs_running_refs);
2553                         return ret;
2554                 }
2555
2556                 atomic_add(ret, &delayed_refs->ref_seq);
2557
2558                 count -= min_t(unsigned long, ret, count);
2559
2560                 if (count == 0)
2561                         break;
2562
2563                 if (delayed_start >= delayed_refs->run_delayed_start) {
2564                         if (loops == 0) {
2565                                 /*
2566                                  * btrfs_find_ref_cluster looped. let's do one
2567                                  * more cycle. if we don't run any delayed ref
2568                                  * during that cycle (because we can't because
2569                                  * all of them are blocked), bail out.
2570                                  */
2571                                 loops = 1;
2572                         } else {
2573                                 /*
2574                                  * no runnable refs left, stop trying
2575                                  */
2576                                 BUG_ON(run_all);
2577                                 break;
2578                         }
2579                 }
2580                 if (ret) {
2581                         /* refs were run, let's reset staleness detection */
2582                         loops = 0;
2583                 }
2584         }
2585
2586         if (run_all) {
2587                 if (!list_empty(&trans->new_bgs)) {
2588                         spin_unlock(&delayed_refs->lock);
2589                         btrfs_create_pending_block_groups(trans, root);
2590                         spin_lock(&delayed_refs->lock);
2591                 }
2592
2593                 node = rb_first(&delayed_refs->root);
2594                 if (!node)
2595                         goto out;
2596                 count = (unsigned long)-1;
2597
2598                 while (node) {
2599                         ref = rb_entry(node, struct btrfs_delayed_ref_node,
2600                                        rb_node);
2601                         if (btrfs_delayed_ref_is_head(ref)) {
2602                                 struct btrfs_delayed_ref_head *head;
2603
2604                                 head = btrfs_delayed_node_to_head(ref);
2605                                 atomic_inc(&ref->refs);
2606
2607                                 spin_unlock(&delayed_refs->lock);
2608                                 /*
2609                                  * Mutex was contended, block until it's
2610                                  * released and try again
2611                                  */
2612                                 mutex_lock(&head->mutex);
2613                                 mutex_unlock(&head->mutex);
2614
2615                                 btrfs_put_delayed_ref(ref);
2616                                 cond_resched();
2617                                 goto again;
2618                         }
2619                         node = rb_next(node);
2620                 }
2621                 spin_unlock(&delayed_refs->lock);
2622                 schedule_timeout(1);
2623                 goto again;
2624         }
2625 out:
2626         atomic_dec(&delayed_refs->procs_running_refs);
2627         smp_mb();
2628         if (waitqueue_active(&delayed_refs->wait))
2629                 wake_up(&delayed_refs->wait);
2630
2631         spin_unlock(&delayed_refs->lock);
2632         assert_qgroups_uptodate(trans);
2633         return 0;
2634 }
2635
2636 int btrfs_set_disk_extent_flags(struct btrfs_trans_handle *trans,
2637                                 struct btrfs_root *root,
2638                                 u64 bytenr, u64 num_bytes, u64 flags,
2639                                 int is_data)
2640 {
2641         struct btrfs_delayed_extent_op *extent_op;
2642         int ret;
2643
2644         extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
2645         if (!extent_op)
2646                 return -ENOMEM;
2647
2648         extent_op->flags_to_set = flags;
2649         extent_op->update_flags = 1;
2650         extent_op->update_key = 0;
2651         extent_op->is_data = is_data ? 1 : 0;
2652
2653         ret = btrfs_add_delayed_extent_op(root->fs_info, trans, bytenr,
2654                                           num_bytes, extent_op);
2655         if (ret)
2656                 kfree(extent_op);
2657         return ret;
2658 }
2659
2660 static noinline int check_delayed_ref(struct btrfs_trans_handle *trans,
2661                                       struct btrfs_root *root,
2662                                       struct btrfs_path *path,
2663                                       u64 objectid, u64 offset, u64 bytenr)
2664 {
2665         struct btrfs_delayed_ref_head *head;
2666         struct btrfs_delayed_ref_node *ref;
2667         struct btrfs_delayed_data_ref *data_ref;
2668         struct btrfs_delayed_ref_root *delayed_refs;
2669         struct rb_node *node;
2670         int ret = 0;
2671
2672         ret = -ENOENT;
2673         delayed_refs = &trans->transaction->delayed_refs;
2674         spin_lock(&delayed_refs->lock);
2675         head = btrfs_find_delayed_ref_head(trans, bytenr);
2676         if (!head)
2677                 goto out;
2678
2679         if (!mutex_trylock(&head->mutex)) {
2680                 atomic_inc(&head->node.refs);
2681                 spin_unlock(&delayed_refs->lock);
2682
2683                 btrfs_release_path(path);
2684
2685                 /*
2686                  * Mutex was contended, block until it's released and let
2687                  * caller try again
2688                  */
2689                 mutex_lock(&head->mutex);
2690                 mutex_unlock(&head->mutex);
2691                 btrfs_put_delayed_ref(&head->node);
2692                 return -EAGAIN;
2693         }
2694
2695         node = rb_prev(&head->node.rb_node);
2696         if (!node)
2697                 goto out_unlock;
2698
2699         ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2700
2701         if (ref->bytenr != bytenr)
2702                 goto out_unlock;
2703
2704         ret = 1;
2705         if (ref->type != BTRFS_EXTENT_DATA_REF_KEY)
2706                 goto out_unlock;
2707
2708         data_ref = btrfs_delayed_node_to_data_ref(ref);
2709
2710         node = rb_prev(node);
2711         if (node) {
2712                 int seq = ref->seq;
2713
2714                 ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
2715                 if (ref->bytenr == bytenr && ref->seq == seq)
2716                         goto out_unlock;
2717         }
2718
2719         if (data_ref->root != root->root_key.objectid ||
2720             data_ref->objectid != objectid || data_ref->offset != offset)
2721                 goto out_unlock;
2722
2723         ret = 0;
2724 out_unlock:
2725         mutex_unlock(&head->mutex);
2726 out:
2727         spin_unlock(&delayed_refs->lock);
2728         return ret;
2729 }
2730
2731 static noinline int check_committed_ref(struct btrfs_trans_handle *trans,
2732                                         struct btrfs_root *root,
2733                                         struct btrfs_path *path,
2734                                         u64 objectid, u64 offset, u64 bytenr)
2735 {
2736         struct btrfs_root *extent_root = root->fs_info->extent_root;
2737         struct extent_buffer *leaf;
2738         struct btrfs_extent_data_ref *ref;
2739         struct btrfs_extent_inline_ref *iref;
2740         struct btrfs_extent_item *ei;
2741         struct btrfs_key key;
2742         u32 item_size;
2743         int ret;
2744
2745         key.objectid = bytenr;
2746         key.offset = (u64)-1;
2747         key.type = BTRFS_EXTENT_ITEM_KEY;
2748
2749         ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
2750         if (ret < 0)
2751                 goto out;
2752         BUG_ON(ret == 0); /* Corruption */
2753
2754         ret = -ENOENT;
2755         if (path->slots[0] == 0)
2756                 goto out;
2757
2758         path->slots[0]--;
2759         leaf = path->nodes[0];
2760         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2761
2762         if (key.objectid != bytenr || key.type != BTRFS_EXTENT_ITEM_KEY)
2763                 goto out;
2764
2765         ret = 1;
2766         item_size = btrfs_item_size_nr(leaf, path->slots[0]);
2767 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
2768         if (item_size < sizeof(*ei)) {
2769                 WARN_ON(item_size != sizeof(struct btrfs_extent_item_v0));
2770                 goto out;
2771         }
2772 #endif
2773         ei = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_extent_item);
2774
2775         if (item_size != sizeof(*ei) +
2776             btrfs_extent_inline_ref_size(BTRFS_EXTENT_DATA_REF_KEY))
2777                 goto out;
2778
2779         if (btrfs_extent_generation(leaf, ei) <=
2780             btrfs_root_last_snapshot(&root->root_item))
2781                 goto out;
2782
2783         iref = (struct btrfs_extent_inline_ref *)(ei + 1);
2784         if (btrfs_extent_inline_ref_type(leaf, iref) !=
2785             BTRFS_EXTENT_DATA_REF_KEY)
2786                 goto out;
2787
2788         ref = (struct btrfs_extent_data_ref *)(&iref->offset);
2789         if (btrfs_extent_refs(leaf, ei) !=
2790             btrfs_extent_data_ref_count(leaf, ref) ||
2791             btrfs_extent_data_ref_root(leaf, ref) !=
2792             root->root_key.objectid ||
2793             btrfs_extent_data_ref_objectid(leaf, ref) != objectid ||
2794             btrfs_extent_data_ref_offset(leaf, ref) != offset)
2795                 goto out;
2796
2797         ret = 0;
2798 out:
2799         return ret;
2800 }
2801
2802 int btrfs_cross_ref_exist(struct btrfs_trans_handle *trans,
2803                           struct btrfs_root *root,
2804                           u64 objectid, u64 offset, u64 bytenr)
2805 {
2806         struct btrfs_path *path;
2807         int ret;
2808         int ret2;
2809
2810         path = btrfs_alloc_path();
2811         if (!path)
2812                 return -ENOENT;
2813
2814         do {
2815                 ret = check_committed_ref(trans, root, path, objectid,
2816                                           offset, bytenr);
2817                 if (ret && ret != -ENOENT)
2818                         goto out;
2819
2820                 ret2 = check_delayed_ref(trans, root, path, objectid,
2821                                          offset, bytenr);
2822         } while (ret2 == -EAGAIN);
2823
2824         if (ret2 && ret2 != -ENOENT) {
2825                 ret = ret2;
2826                 goto out;
2827         }
2828
2829         if (ret != -ENOENT || ret2 != -ENOENT)
2830                 ret = 0;
2831 out:
2832         btrfs_free_path(path);
2833         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
2834                 WARN_ON(ret > 0);
2835         return ret;
2836 }
2837
2838 static int __btrfs_mod_ref(struct btrfs_trans_handle *trans,
2839                            struct btrfs_root *root,
2840                            struct extent_buffer *buf,
2841                            int full_backref, int inc, int for_cow)
2842 {
2843         u64 bytenr;
2844         u64 num_bytes;
2845         u64 parent;
2846         u64 ref_root;
2847         u32 nritems;
2848         struct btrfs_key key;
2849         struct btrfs_file_extent_item *fi;
2850         int i;
2851         int level;
2852         int ret = 0;
2853         int (*process_func)(struct btrfs_trans_handle *, struct btrfs_root *,
2854                             u64, u64, u64, u64, u64, u64, int);
2855
2856         ref_root = btrfs_header_owner(buf);
2857         nritems = btrfs_header_nritems(buf);
2858         level = btrfs_header_level(buf);
2859
2860         if (!root->ref_cows && level == 0)
2861                 return 0;
2862
2863         if (inc)
2864                 process_func = btrfs_inc_extent_ref;
2865         else
2866                 process_func = btrfs_free_extent;
2867
2868         if (full_backref)
2869                 parent = buf->start;
2870         else
2871                 parent = 0;
2872
2873         for (i = 0; i < nritems; i++) {
2874                 if (level == 0) {
2875                         btrfs_item_key_to_cpu(buf, &key, i);
2876                         if (btrfs_key_type(&key) != BTRFS_EXTENT_DATA_KEY)
2877                                 continue;
2878                         fi = btrfs_item_ptr(buf, i,
2879                                             struct btrfs_file_extent_item);
2880                         if (btrfs_file_extent_type(buf, fi) ==
2881                             BTRFS_FILE_EXTENT_INLINE)
2882                                 continue;
2883                         bytenr = btrfs_file_extent_disk_bytenr(buf, fi);
2884                         if (bytenr == 0)
2885                                 continue;
2886
2887                         num_bytes = btrfs_file_extent_disk_num_bytes(buf, fi);
2888                         key.offset -= btrfs_file_extent_offset(buf, fi);
2889                         ret = process_func(trans, root, bytenr, num_bytes,
2890                                            parent, ref_root, key.objectid,
2891                                            key.offset, for_cow);
2892                         if (ret)
2893                                 goto fail;
2894                 } else {
2895                         bytenr = btrfs_node_blockptr(buf, i);
2896                         num_bytes = btrfs_level_size(root, level - 1);
2897                         ret = process_func(trans, root, bytenr, num_bytes,
2898                                            parent, ref_root, level - 1, 0,
2899                                            for_cow);
2900                         if (ret)
2901                                 goto fail;
2902                 }
2903         }
2904         return 0;
2905 fail:
2906         return ret;
2907 }
2908
2909 int btrfs_inc_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2910                   struct extent_buffer *buf, int full_backref, int for_cow)
2911 {
2912         return __btrfs_mod_ref(trans, root, buf, full_backref, 1, for_cow);
2913 }
2914
2915 int btrfs_dec_ref(struct btrfs_trans_handle *trans, struct btrfs_root *root,
2916                   struct extent_buffer *buf, int full_backref, int for_cow)
2917 {
2918         return __btrfs_mod_ref(trans, root, buf, full_backref, 0, for_cow);
2919 }
2920
2921 static int write_one_cache_group(struct btrfs_trans_handle *trans,
2922                                  struct btrfs_root *root,
2923                                  struct btrfs_path *path,
2924                                  struct btrfs_block_group_cache *cache)
2925 {
2926         int ret;
2927         struct btrfs_root *extent_root = root->fs_info->extent_root;
2928         unsigned long bi;
2929         struct extent_buffer *leaf;
2930
2931         ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2932         if (ret < 0)
2933                 goto fail;
2934         BUG_ON(ret); /* Corruption */
2935
2936         leaf = path->nodes[0];
2937         bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
2938         write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
2939         btrfs_mark_buffer_dirty(leaf);
2940         btrfs_release_path(path);
2941 fail:
2942         if (ret) {
2943                 btrfs_abort_transaction(trans, root, ret);
2944                 return ret;
2945         }
2946         return 0;
2947
2948 }
2949
2950 static struct btrfs_block_group_cache *
2951 next_block_group(struct btrfs_root *root,
2952                  struct btrfs_block_group_cache *cache)
2953 {
2954         struct rb_node *node;
2955         spin_lock(&root->fs_info->block_group_cache_lock);
2956         node = rb_next(&cache->cache_node);
2957         btrfs_put_block_group(cache);
2958         if (node) {
2959                 cache = rb_entry(node, struct btrfs_block_group_cache,
2960                                  cache_node);
2961                 btrfs_get_block_group(cache);
2962         } else
2963                 cache = NULL;
2964         spin_unlock(&root->fs_info->block_group_cache_lock);
2965         return cache;
2966 }
2967
2968 static int cache_save_setup(struct btrfs_block_group_cache *block_group,
2969                             struct btrfs_trans_handle *trans,
2970                             struct btrfs_path *path)
2971 {
2972         struct btrfs_root *root = block_group->fs_info->tree_root;
2973         struct inode *inode = NULL;
2974         u64 alloc_hint = 0;
2975         int dcs = BTRFS_DC_ERROR;
2976         int num_pages = 0;
2977         int retries = 0;
2978         int ret = 0;
2979
2980         /*
2981          * If this block group is smaller than 100 megs don't bother caching the
2982          * block group.
2983          */
2984         if (block_group->key.offset < (100 * 1024 * 1024)) {
2985                 spin_lock(&block_group->lock);
2986                 block_group->disk_cache_state = BTRFS_DC_WRITTEN;
2987                 spin_unlock(&block_group->lock);
2988                 return 0;
2989         }
2990
2991 again:
2992         inode = lookup_free_space_inode(root, block_group, path);
2993         if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
2994                 ret = PTR_ERR(inode);
2995                 btrfs_release_path(path);
2996                 goto out;
2997         }
2998
2999         if (IS_ERR(inode)) {
3000                 BUG_ON(retries);
3001                 retries++;
3002
3003                 if (block_group->ro)
3004                         goto out_free;
3005
3006                 ret = create_free_space_inode(root, trans, block_group, path);
3007                 if (ret)
3008                         goto out_free;
3009                 goto again;
3010         }
3011
3012         /* We've already setup this transaction, go ahead and exit */
3013         if (block_group->cache_generation == trans->transid &&
3014             i_size_read(inode)) {
3015                 dcs = BTRFS_DC_SETUP;
3016                 goto out_put;
3017         }
3018
3019         /*
3020          * We want to set the generation to 0, that way if anything goes wrong
3021          * from here on out we know not to trust this cache when we load up next
3022          * time.
3023          */
3024         BTRFS_I(inode)->generation = 0;
3025         ret = btrfs_update_inode(trans, root, inode);
3026         WARN_ON(ret);
3027
3028         if (i_size_read(inode) > 0) {
3029                 ret = btrfs_truncate_free_space_cache(root, trans, path,
3030                                                       inode);
3031                 if (ret)
3032                         goto out_put;
3033         }
3034
3035         spin_lock(&block_group->lock);
3036         if (block_group->cached != BTRFS_CACHE_FINISHED ||
3037             !btrfs_test_opt(root, SPACE_CACHE)) {
3038                 /*
3039                  * don't bother trying to write stuff out _if_
3040                  * a) we're not cached,
3041                  * b) we're with nospace_cache mount option.
3042                  */
3043                 dcs = BTRFS_DC_WRITTEN;
3044                 spin_unlock(&block_group->lock);
3045                 goto out_put;
3046         }
3047         spin_unlock(&block_group->lock);
3048
3049         /*
3050          * Try to preallocate enough space based on how big the block group is.
3051          * Keep in mind this has to include any pinned space which could end up
3052          * taking up quite a bit since it's not folded into the other space
3053          * cache.
3054          */
3055         num_pages = (int)div64_u64(block_group->key.offset, 256 * 1024 * 1024);
3056         if (!num_pages)
3057                 num_pages = 1;
3058
3059         num_pages *= 16;
3060         num_pages *= PAGE_CACHE_SIZE;
3061
3062         ret = btrfs_check_data_free_space(inode, num_pages);
3063         if (ret)
3064                 goto out_put;
3065
3066         ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
3067                                               num_pages, num_pages,
3068                                               &alloc_hint);
3069         if (!ret)
3070                 dcs = BTRFS_DC_SETUP;
3071         btrfs_free_reserved_data_space(inode, num_pages);
3072
3073 out_put:
3074         iput(inode);
3075 out_free:
3076         btrfs_release_path(path);
3077 out:
3078         spin_lock(&block_group->lock);
3079         if (!ret && dcs == BTRFS_DC_SETUP)
3080                 block_group->cache_generation = trans->transid;
3081         block_group->disk_cache_state = dcs;
3082         spin_unlock(&block_group->lock);
3083
3084         return ret;
3085 }
3086
3087 int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans,
3088                                    struct btrfs_root *root)
3089 {
3090         struct btrfs_block_group_cache *cache;
3091         int err = 0;
3092         struct btrfs_path *path;
3093         u64 last = 0;
3094
3095         path = btrfs_alloc_path();
3096         if (!path)
3097                 return -ENOMEM;
3098
3099 again:
3100         while (1) {
3101                 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3102                 while (cache) {
3103                         if (cache->disk_cache_state == BTRFS_DC_CLEAR)
3104                                 break;
3105                         cache = next_block_group(root, cache);
3106                 }
3107                 if (!cache) {
3108                         if (last == 0)
3109                                 break;
3110                         last = 0;
3111                         continue;
3112                 }
3113                 err = cache_save_setup(cache, trans, path);
3114                 last = cache->key.objectid + cache->key.offset;
3115                 btrfs_put_block_group(cache);
3116         }
3117
3118         while (1) {
3119                 if (last == 0) {
3120                         err = btrfs_run_delayed_refs(trans, root,
3121                                                      (unsigned long)-1);
3122                         if (err) /* File system offline */
3123                                 goto out;
3124                 }
3125
3126                 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3127                 while (cache) {
3128                         if (cache->disk_cache_state == BTRFS_DC_CLEAR) {
3129                                 btrfs_put_block_group(cache);
3130                                 goto again;
3131                         }
3132
3133                         if (cache->dirty)
3134                                 break;
3135                         cache = next_block_group(root, cache);
3136                 }
3137                 if (!cache) {
3138                         if (last == 0)
3139                                 break;
3140                         last = 0;
3141                         continue;
3142                 }
3143
3144                 if (cache->disk_cache_state == BTRFS_DC_SETUP)
3145                         cache->disk_cache_state = BTRFS_DC_NEED_WRITE;
3146                 cache->dirty = 0;
3147                 last = cache->key.objectid + cache->key.offset;
3148
3149                 err = write_one_cache_group(trans, root, path, cache);
3150                 if (err) /* File system offline */
3151                         goto out;
3152
3153                 btrfs_put_block_group(cache);
3154         }
3155
3156         while (1) {
3157                 /*
3158                  * I don't think this is needed since we're just marking our
3159                  * preallocated extent as written, but just in case it can't
3160                  * hurt.
3161                  */
3162                 if (last == 0) {
3163                         err = btrfs_run_delayed_refs(trans, root,
3164                                                      (unsigned long)-1);
3165                         if (err) /* File system offline */
3166                                 goto out;
3167                 }
3168
3169                 cache = btrfs_lookup_first_block_group(root->fs_info, last);
3170                 while (cache) {
3171                         /*
3172                          * Really this shouldn't happen, but it could if we
3173                          * couldn't write the entire preallocated extent and
3174                          * splitting the extent resulted in a new block.
3175                          */
3176                         if (cache->dirty) {
3177                                 btrfs_put_block_group(cache);
3178                                 goto again;
3179                         }
3180                         if (cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
3181                                 break;
3182                         cache = next_block_group(root, cache);
3183                 }
3184                 if (!cache) {
3185                         if (last == 0)
3186                                 break;
3187                         last = 0;
3188                         continue;
3189                 }
3190
3191                 err = btrfs_write_out_cache(root, trans, cache, path);
3192
3193                 /*
3194                  * If we didn't have an error then the cache state is still
3195                  * NEED_WRITE, so we can set it to WRITTEN.
3196                  */
3197                 if (!err && cache->disk_cache_state == BTRFS_DC_NEED_WRITE)
3198                         cache->disk_cache_state = BTRFS_DC_WRITTEN;
3199                 last = cache->key.objectid + cache->key.offset;
3200                 btrfs_put_block_group(cache);
3201         }
3202 out:
3203
3204         btrfs_free_path(path);
3205         return err;
3206 }
3207
3208 int btrfs_extent_readonly(struct btrfs_root *root, u64 bytenr)
3209 {
3210         struct btrfs_block_group_cache *block_group;
3211         int readonly = 0;
3212
3213         block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
3214         if (!block_group || block_group->ro)
3215                 readonly = 1;
3216         if (block_group)
3217                 btrfs_put_block_group(block_group);
3218         return readonly;
3219 }
3220
3221 static int update_space_info(struct btrfs_fs_info *info, u64 flags,
3222                              u64 total_bytes, u64 bytes_used,
3223                              struct btrfs_space_info **space_info)
3224 {
3225         struct btrfs_space_info *found;
3226         int i;
3227         int factor;
3228
3229         if (flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1 |
3230                      BTRFS_BLOCK_GROUP_RAID10))
3231                 factor = 2;
3232         else
3233                 factor = 1;
3234
3235         found = __find_space_info(info, flags);
3236         if (found) {
3237                 spin_lock(&found->lock);
3238                 found->total_bytes += total_bytes;
3239                 found->disk_total += total_bytes * factor;
3240                 found->bytes_used += bytes_used;
3241                 found->disk_used += bytes_used * factor;
3242                 found->full = 0;
3243                 spin_unlock(&found->lock);
3244                 *space_info = found;
3245                 return 0;
3246         }
3247         found = kzalloc(sizeof(*found), GFP_NOFS);
3248         if (!found)
3249                 return -ENOMEM;
3250
3251         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
3252                 INIT_LIST_HEAD(&found->block_groups[i]);
3253         init_rwsem(&found->groups_sem);
3254         spin_lock_init(&found->lock);
3255         found->flags = flags & BTRFS_BLOCK_GROUP_TYPE_MASK;
3256         found->total_bytes = total_bytes;
3257         found->disk_total = total_bytes * factor;
3258         found->bytes_used = bytes_used;
3259         found->disk_used = bytes_used * factor;
3260         found->bytes_pinned = 0;
3261         found->bytes_reserved = 0;
3262         found->bytes_readonly = 0;
3263         found->bytes_may_use = 0;
3264         found->full = 0;
3265         found->force_alloc = CHUNK_ALLOC_NO_FORCE;
3266         found->chunk_alloc = 0;
3267         found->flush = 0;
3268         init_waitqueue_head(&found->wait);
3269         *space_info = found;
3270         list_add_rcu(&found->list, &info->space_info);
3271         if (flags & BTRFS_BLOCK_GROUP_DATA)
3272                 info->data_sinfo = found;
3273         return 0;
3274 }
3275
3276 static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
3277 {
3278         u64 extra_flags = chunk_to_extended(flags) &
3279                                 BTRFS_EXTENDED_PROFILE_MASK;
3280
3281         if (flags & BTRFS_BLOCK_GROUP_DATA)
3282                 fs_info->avail_data_alloc_bits |= extra_flags;
3283         if (flags & BTRFS_BLOCK_GROUP_METADATA)
3284                 fs_info->avail_metadata_alloc_bits |= extra_flags;
3285         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3286                 fs_info->avail_system_alloc_bits |= extra_flags;
3287 }
3288
3289 /*
3290  * returns target flags in extended format or 0 if restripe for this
3291  * chunk_type is not in progress
3292  *
3293  * should be called with either volume_mutex or balance_lock held
3294  */
3295 static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
3296 {
3297         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3298         u64 target = 0;
3299
3300         if (!bctl)
3301                 return 0;
3302
3303         if (flags & BTRFS_BLOCK_GROUP_DATA &&
3304             bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3305                 target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
3306         } else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
3307                    bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3308                 target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
3309         } else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
3310                    bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
3311                 target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
3312         }
3313
3314         return target;
3315 }
3316
3317 /*
3318  * @flags: available profiles in extended format (see ctree.h)
3319  *
3320  * Returns reduced profile in chunk format.  If profile changing is in
3321  * progress (either running or paused) picks the target profile (if it's
3322  * already available), otherwise falls back to plain reducing.
3323  */
3324 u64 btrfs_reduce_alloc_profile(struct btrfs_root *root, u64 flags)
3325 {
3326         /*
3327          * we add in the count of missing devices because we want
3328          * to make sure that any RAID levels on a degraded FS
3329          * continue to be honored.
3330          */
3331         u64 num_devices = root->fs_info->fs_devices->rw_devices +
3332                 root->fs_info->fs_devices->missing_devices;
3333         u64 target;
3334         u64 tmp;
3335
3336         /*
3337          * see if restripe for this chunk_type is in progress, if so
3338          * try to reduce to the target profile
3339          */
3340         spin_lock(&root->fs_info->balance_lock);
3341         target = get_restripe_target(root->fs_info, flags);
3342         if (target) {
3343                 /* pick target profile only if it's already available */
3344                 if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
3345                         spin_unlock(&root->fs_info->balance_lock);
3346                         return extended_to_chunk(target);
3347                 }
3348         }
3349         spin_unlock(&root->fs_info->balance_lock);
3350
3351         /* First, mask out the RAID levels which aren't possible */
3352         if (num_devices == 1)
3353                 flags &= ~(BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID0 |
3354                            BTRFS_BLOCK_GROUP_RAID5);
3355         if (num_devices < 3)
3356                 flags &= ~BTRFS_BLOCK_GROUP_RAID6;
3357         if (num_devices < 4)
3358                 flags &= ~BTRFS_BLOCK_GROUP_RAID10;
3359
3360         tmp = flags & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID0 |
3361                        BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID5 |
3362                        BTRFS_BLOCK_GROUP_RAID6 | BTRFS_BLOCK_GROUP_RAID10);
3363         flags &= ~tmp;
3364
3365         if (tmp & BTRFS_BLOCK_GROUP_RAID6)
3366                 tmp = BTRFS_BLOCK_GROUP_RAID6;
3367         else if (tmp & BTRFS_BLOCK_GROUP_RAID5)
3368                 tmp = BTRFS_BLOCK_GROUP_RAID5;
3369         else if (tmp & BTRFS_BLOCK_GROUP_RAID10)
3370                 tmp = BTRFS_BLOCK_GROUP_RAID10;
3371         else if (tmp & BTRFS_BLOCK_GROUP_RAID1)
3372                 tmp = BTRFS_BLOCK_GROUP_RAID1;
3373         else if (tmp & BTRFS_BLOCK_GROUP_RAID0)
3374                 tmp = BTRFS_BLOCK_GROUP_RAID0;
3375
3376         return extended_to_chunk(flags | tmp);
3377 }
3378
3379 static u64 get_alloc_profile(struct btrfs_root *root, u64 flags)
3380 {
3381         if (flags & BTRFS_BLOCK_GROUP_DATA)
3382                 flags |= root->fs_info->avail_data_alloc_bits;
3383         else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
3384                 flags |= root->fs_info->avail_system_alloc_bits;
3385         else if (flags & BTRFS_BLOCK_GROUP_METADATA)
3386                 flags |= root->fs_info->avail_metadata_alloc_bits;
3387
3388         return btrfs_reduce_alloc_profile(root, flags);
3389 }
3390
3391 u64 btrfs_get_alloc_profile(struct btrfs_root *root, int data)
3392 {
3393         u64 flags;
3394         u64 ret;
3395
3396         if (data)
3397                 flags = BTRFS_BLOCK_GROUP_DATA;
3398         else if (root == root->fs_info->chunk_root)
3399                 flags = BTRFS_BLOCK_GROUP_SYSTEM;
3400         else
3401                 flags = BTRFS_BLOCK_GROUP_METADATA;
3402
3403         ret = get_alloc_profile(root, flags);
3404         return ret;
3405 }
3406
3407 /*
3408  * This will check the space that the inode allocates from to make sure we have
3409  * enough space for bytes.
3410  */
3411 int btrfs_check_data_free_space(struct inode *inode, u64 bytes)
3412 {
3413         struct btrfs_space_info *data_sinfo;
3414         struct btrfs_root *root = BTRFS_I(inode)->root;
3415         struct btrfs_fs_info *fs_info = root->fs_info;
3416         u64 used;
3417         int ret = 0, committed = 0, alloc_chunk = 1;
3418
3419         /* make sure bytes are sectorsize aligned */
3420         bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3421
3422         if (root == root->fs_info->tree_root ||
3423             BTRFS_I(inode)->location.objectid == BTRFS_FREE_INO_OBJECTID) {
3424                 alloc_chunk = 0;
3425                 committed = 1;
3426         }
3427
3428         data_sinfo = fs_info->data_sinfo;
3429         if (!data_sinfo)
3430                 goto alloc;
3431
3432 again:
3433         /* make sure we have enough space to handle the data first */
3434         spin_lock(&data_sinfo->lock);
3435         used = data_sinfo->bytes_used + data_sinfo->bytes_reserved +
3436                 data_sinfo->bytes_pinned + data_sinfo->bytes_readonly +
3437                 data_sinfo->bytes_may_use;
3438
3439         if (used + bytes > data_sinfo->total_bytes) {
3440                 struct btrfs_trans_handle *trans;
3441
3442                 /*
3443                  * if we don't have enough free bytes in this space then we need
3444                  * to alloc a new chunk.
3445                  */
3446                 if (!data_sinfo->full && alloc_chunk) {
3447                         u64 alloc_target;
3448
3449                         data_sinfo->force_alloc = CHUNK_ALLOC_FORCE;
3450                         spin_unlock(&data_sinfo->lock);
3451 alloc:
3452                         alloc_target = btrfs_get_alloc_profile(root, 1);
3453                         trans = btrfs_join_transaction(root);
3454                         if (IS_ERR(trans))
3455                                 return PTR_ERR(trans);
3456
3457                         ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3458                                              alloc_target,
3459                                              CHUNK_ALLOC_NO_FORCE);
3460                         btrfs_end_transaction(trans, root);
3461                         if (ret < 0) {
3462                                 if (ret != -ENOSPC)
3463                                         return ret;
3464                                 else
3465                                         goto commit_trans;
3466                         }
3467
3468                         if (!data_sinfo)
3469                                 data_sinfo = fs_info->data_sinfo;
3470
3471                         goto again;
3472                 }
3473
3474                 /*
3475                  * If we have less pinned bytes than we want to allocate then
3476                  * don't bother committing the transaction, it won't help us.
3477                  */
3478                 if (data_sinfo->bytes_pinned < bytes)
3479                         committed = 1;
3480                 spin_unlock(&data_sinfo->lock);
3481
3482                 /* commit the current transaction and try again */
3483 commit_trans:
3484                 if (!committed &&
3485                     !atomic_read(&root->fs_info->open_ioctl_trans)) {
3486                         committed = 1;
3487                         trans = btrfs_join_transaction(root);
3488                         if (IS_ERR(trans))
3489                                 return PTR_ERR(trans);
3490                         ret = btrfs_commit_transaction(trans, root);
3491                         if (ret)
3492                                 return ret;
3493                         goto again;
3494                 }
3495
3496                 return -ENOSPC;
3497         }
3498         data_sinfo->bytes_may_use += bytes;
3499         trace_btrfs_space_reservation(root->fs_info, "space_info",
3500                                       data_sinfo->flags, bytes, 1);
3501         spin_unlock(&data_sinfo->lock);
3502
3503         return 0;
3504 }
3505
3506 /*
3507  * Called if we need to clear a data reservation for this inode.
3508  */
3509 void btrfs_free_reserved_data_space(struct inode *inode, u64 bytes)
3510 {
3511         struct btrfs_root *root = BTRFS_I(inode)->root;
3512         struct btrfs_space_info *data_sinfo;
3513
3514         /* make sure bytes are sectorsize aligned */
3515         bytes = (bytes + root->sectorsize - 1) & ~((u64)root->sectorsize - 1);
3516
3517         data_sinfo = root->fs_info->data_sinfo;
3518         spin_lock(&data_sinfo->lock);
3519         data_sinfo->bytes_may_use -= bytes;
3520         trace_btrfs_space_reservation(root->fs_info, "space_info",
3521                                       data_sinfo->flags, bytes, 0);
3522         spin_unlock(&data_sinfo->lock);
3523 }
3524
3525 static void force_metadata_allocation(struct btrfs_fs_info *info)
3526 {
3527         struct list_head *head = &info->space_info;
3528         struct btrfs_space_info *found;
3529
3530         rcu_read_lock();
3531         list_for_each_entry_rcu(found, head, list) {
3532                 if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
3533                         found->force_alloc = CHUNK_ALLOC_FORCE;
3534         }
3535         rcu_read_unlock();
3536 }
3537
3538 static int should_alloc_chunk(struct btrfs_root *root,
3539                               struct btrfs_space_info *sinfo, int force)
3540 {
3541         struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
3542         u64 num_bytes = sinfo->total_bytes - sinfo->bytes_readonly;
3543         u64 num_allocated = sinfo->bytes_used + sinfo->bytes_reserved;
3544         u64 thresh;
3545
3546         if (force == CHUNK_ALLOC_FORCE)
3547                 return 1;
3548
3549         /*
3550          * We need to take into account the global rsv because for all intents
3551          * and purposes it's used space.  Don't worry about locking the
3552          * global_rsv, it doesn't change except when the transaction commits.
3553          */
3554         if (sinfo->flags & BTRFS_BLOCK_GROUP_METADATA)
3555                 num_allocated += global_rsv->size;
3556
3557         /*
3558          * in limited mode, we want to have some free space up to
3559          * about 1% of the FS size.
3560          */
3561         if (force == CHUNK_ALLOC_LIMITED) {
3562                 thresh = btrfs_super_total_bytes(root->fs_info->super_copy);
3563                 thresh = max_t(u64, 64 * 1024 * 1024,
3564                                div_factor_fine(thresh, 1));
3565
3566                 if (num_bytes - num_allocated < thresh)
3567                         return 1;
3568         }
3569
3570         if (num_allocated + 2 * 1024 * 1024 < div_factor(num_bytes, 8))
3571                 return 0;
3572         return 1;
3573 }
3574
3575 static u64 get_system_chunk_thresh(struct btrfs_root *root, u64 type)
3576 {
3577         u64 num_dev;
3578
3579         if (type & (BTRFS_BLOCK_GROUP_RAID10 |
3580                     BTRFS_BLOCK_GROUP_RAID0 |
3581                     BTRFS_BLOCK_GROUP_RAID5 |
3582                     BTRFS_BLOCK_GROUP_RAID6))
3583                 num_dev = root->fs_info->fs_devices->rw_devices;
3584         else if (type & BTRFS_BLOCK_GROUP_RAID1)
3585                 num_dev = 2;
3586         else
3587                 num_dev = 1;    /* DUP or single */
3588
3589         /* metadata for updaing devices and chunk tree */
3590         return btrfs_calc_trans_metadata_size(root, num_dev + 1);
3591 }
3592
3593 static void check_system_chunk(struct btrfs_trans_handle *trans,
3594                                struct btrfs_root *root, u64 type)
3595 {
3596         struct btrfs_space_info *info;
3597         u64 left;
3598         u64 thresh;
3599
3600         info = __find_space_info(root->fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3601         spin_lock(&info->lock);
3602         left = info->total_bytes - info->bytes_used - info->bytes_pinned -
3603                 info->bytes_reserved - info->bytes_readonly;
3604         spin_unlock(&info->lock);
3605
3606         thresh = get_system_chunk_thresh(root, type);
3607         if (left < thresh && btrfs_test_opt(root, ENOSPC_DEBUG)) {
3608                 printk(KERN_INFO "left=%llu, need=%llu, flags=%llu\n",
3609                        left, thresh, type);
3610                 dump_space_info(info, 0, 0);
3611         }
3612
3613         if (left < thresh) {
3614                 u64 flags;
3615
3616                 flags = btrfs_get_alloc_profile(root->fs_info->chunk_root, 0);
3617                 btrfs_alloc_chunk(trans, root, flags);
3618         }
3619 }
3620
3621 static int do_chunk_alloc(struct btrfs_trans_handle *trans,
3622                           struct btrfs_root *extent_root, u64 flags, int force)
3623 {
3624         struct btrfs_space_info *space_info;
3625         struct btrfs_fs_info *fs_info = extent_root->fs_info;
3626         int wait_for_alloc = 0;
3627         int ret = 0;
3628
3629         space_info = __find_space_info(extent_root->fs_info, flags);
3630         if (!space_info) {
3631                 ret = update_space_info(extent_root->fs_info, flags,
3632                                         0, 0, &space_info);
3633                 BUG_ON(ret); /* -ENOMEM */
3634         }
3635         BUG_ON(!space_info); /* Logic error */
3636
3637 again:
3638         spin_lock(&space_info->lock);
3639         if (force < space_info->force_alloc)
3640                 force = space_info->force_alloc;
3641         if (space_info->full) {
3642                 spin_unlock(&space_info->lock);
3643                 return 0;
3644         }
3645
3646         if (!should_alloc_chunk(extent_root, space_info, force)) {
3647                 spin_unlock(&space_info->lock);
3648                 return 0;
3649         } else if (space_info->chunk_alloc) {
3650                 wait_for_alloc = 1;
3651         } else {
3652                 space_info->chunk_alloc = 1;
3653         }
3654
3655         spin_unlock(&space_info->lock);
3656
3657         mutex_lock(&fs_info->chunk_mutex);
3658
3659         /*
3660          * The chunk_mutex is held throughout the entirety of a chunk
3661          * allocation, so once we've acquired the chunk_mutex we know that the
3662          * other guy is done and we need to recheck and see if we should
3663          * allocate.
3664          */
3665         if (wait_for_alloc) {
3666                 mutex_unlock(&fs_info->chunk_mutex);
3667                 wait_for_alloc = 0;
3668                 goto again;
3669         }
3670
3671         /*
3672          * If we have mixed data/metadata chunks we want to make sure we keep
3673          * allocating mixed chunks instead of individual chunks.
3674          */
3675         if (btrfs_mixed_space_info(space_info))
3676                 flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
3677
3678         /*
3679          * if we're doing a data chunk, go ahead and make sure that
3680          * we keep a reasonable number of metadata chunks allocated in the
3681          * FS as well.
3682          */
3683         if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
3684                 fs_info->data_chunk_allocations++;
3685                 if (!(fs_info->data_chunk_allocations %
3686                       fs_info->metadata_ratio))
3687                         force_metadata_allocation(fs_info);
3688         }
3689
3690         /*
3691          * Check if we have enough space in SYSTEM chunk because we may need
3692          * to update devices.
3693          */
3694         check_system_chunk(trans, extent_root, flags);
3695
3696         ret = btrfs_alloc_chunk(trans, extent_root, flags);
3697         if (ret < 0 && ret != -ENOSPC)
3698                 goto out;
3699
3700         spin_lock(&space_info->lock);
3701         if (ret)
3702                 space_info->full = 1;
3703         else
3704                 ret = 1;
3705
3706         space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
3707         space_info->chunk_alloc = 0;
3708         spin_unlock(&space_info->lock);
3709 out:
3710         mutex_unlock(&fs_info->chunk_mutex);
3711         return ret;
3712 }
3713
3714 static int can_overcommit(struct btrfs_root *root,
3715                           struct btrfs_space_info *space_info, u64 bytes,
3716                           enum btrfs_reserve_flush_enum flush)
3717 {
3718         u64 profile = btrfs_get_alloc_profile(root, 0);
3719         u64 avail;
3720         u64 used;
3721
3722         used = space_info->bytes_used + space_info->bytes_reserved +
3723                 space_info->bytes_pinned + space_info->bytes_readonly +
3724                 space_info->bytes_may_use;
3725
3726         spin_lock(&root->fs_info->free_chunk_lock);
3727         avail = root->fs_info->free_chunk_space;
3728         spin_unlock(&root->fs_info->free_chunk_lock);
3729
3730         /*
3731          * If we have dup, raid1 or raid10 then only half of the free
3732          * space is actually useable.  For raid56, the space info used
3733          * doesn't include the parity drive, so we don't have to
3734          * change the math
3735          */
3736         if (profile & (BTRFS_BLOCK_GROUP_DUP |
3737                        BTRFS_BLOCK_GROUP_RAID1 |
3738                        BTRFS_BLOCK_GROUP_RAID10))
3739                 avail >>= 1;
3740
3741         /*
3742          * If we aren't flushing all things, let us overcommit up to
3743          * 1/2th of the space. If we can flush, don't let us overcommit
3744          * too much, let it overcommit up to 1/8 of the space.
3745          */
3746         if (flush == BTRFS_RESERVE_FLUSH_ALL)
3747                 avail >>= 3;
3748         else
3749                 avail >>= 1;
3750
3751         if (used + bytes < space_info->total_bytes + avail)
3752                 return 1;
3753         return 0;
3754 }
3755
3756 static int writeback_inodes_sb_nr_if_idle_safe(struct super_block *sb,
3757                                                unsigned long nr_pages,
3758                                                enum wb_reason reason)
3759 {
3760         if (!writeback_in_progress(sb->s_bdi) &&
3761             down_read_trylock(&sb->s_umount)) {
3762                 writeback_inodes_sb_nr(sb, nr_pages, reason);
3763                 up_read(&sb->s_umount);
3764                 return 1;
3765         }
3766
3767         return 0;
3768 }
3769
3770 /*
3771  * shrink metadata reservation for delalloc
3772  */
3773 static void shrink_delalloc(struct btrfs_root *root, u64 to_reclaim, u64 orig,
3774                             bool wait_ordered)
3775 {
3776         struct btrfs_block_rsv *block_rsv;
3777         struct btrfs_space_info *space_info;
3778         struct btrfs_trans_handle *trans;
3779         u64 delalloc_bytes;
3780         u64 max_reclaim;
3781         long time_left;
3782         unsigned long nr_pages = (2 * 1024 * 1024) >> PAGE_CACHE_SHIFT;
3783         int loops = 0;
3784         enum btrfs_reserve_flush_enum flush;
3785
3786         trans = (struct btrfs_trans_handle *)current->journal_info;
3787         block_rsv = &root->fs_info->delalloc_block_rsv;
3788         space_info = block_rsv->space_info;
3789
3790         smp_mb();
3791         delalloc_bytes = root->fs_info->delalloc_bytes;
3792         if (delalloc_bytes == 0) {
3793                 if (trans)
3794                         return;
3795                 btrfs_wait_ordered_extents(root, 0);
3796                 return;
3797         }
3798
3799         while (delalloc_bytes && loops < 3) {
3800                 max_reclaim = min(delalloc_bytes, to_reclaim);
3801                 nr_pages = max_reclaim >> PAGE_CACHE_SHIFT;
3802                 writeback_inodes_sb_nr_if_idle_safe(root->fs_info->sb,
3803                                                     nr_pages,
3804                                                     WB_REASON_FS_FREE_SPACE);
3805
3806                 /*
3807                  * We need to wait for the async pages to actually start before
3808                  * we do anything.
3809                  */
3810                 wait_event(root->fs_info->async_submit_wait,
3811                            !atomic_read(&root->fs_info->async_delalloc_pages));
3812
3813                 if (!trans)
3814                         flush = BTRFS_RESERVE_FLUSH_ALL;
3815                 else
3816                         flush = BTRFS_RESERVE_NO_FLUSH;
3817                 spin_lock(&space_info->lock);
3818                 if (can_overcommit(root, space_info, orig, flush)) {
3819                         spin_unlock(&space_info->lock);
3820                         break;
3821                 }
3822                 spin_unlock(&space_info->lock);
3823
3824                 loops++;
3825                 if (wait_ordered && !trans) {
3826                         btrfs_wait_ordered_extents(root, 0);
3827                 } else {
3828                         time_left = schedule_timeout_killable(1);
3829                         if (time_left)
3830                                 break;
3831                 }
3832                 smp_mb();
3833                 delalloc_bytes = root->fs_info->delalloc_bytes;
3834         }
3835 }
3836
3837 /**
3838  * maybe_commit_transaction - possibly commit the transaction if its ok to
3839  * @root - the root we're allocating for
3840  * @bytes - the number of bytes we want to reserve
3841  * @force - force the commit
3842  *
3843  * This will check to make sure that committing the transaction will actually
3844  * get us somewhere and then commit the transaction if it does.  Otherwise it
3845  * will return -ENOSPC.
3846  */
3847 static int may_commit_transaction(struct btrfs_root *root,
3848                                   struct btrfs_space_info *space_info,
3849                                   u64 bytes, int force)
3850 {
3851         struct btrfs_block_rsv *delayed_rsv = &root->fs_info->delayed_block_rsv;
3852         struct btrfs_trans_handle *trans;
3853
3854         trans = (struct btrfs_trans_handle *)current->journal_info;
3855         if (trans)
3856                 return -EAGAIN;
3857
3858         if (force)
3859                 goto commit;
3860
3861         /* See if there is enough pinned space to make this reservation */
3862         spin_lock(&space_info->lock);
3863         if (space_info->bytes_pinned >= bytes) {
3864                 spin_unlock(&space_info->lock);
3865                 goto commit;
3866         }
3867         spin_unlock(&space_info->lock);
3868
3869         /*
3870          * See if there is some space in the delayed insertion reservation for
3871          * this reservation.
3872          */
3873         if (space_info != delayed_rsv->space_info)
3874                 return -ENOSPC;
3875
3876         spin_lock(&space_info->lock);
3877         spin_lock(&delayed_rsv->lock);
3878         if (space_info->bytes_pinned + delayed_rsv->size < bytes) {
3879                 spin_unlock(&delayed_rsv->lock);
3880                 spin_unlock(&space_info->lock);
3881                 return -ENOSPC;
3882         }
3883         spin_unlock(&delayed_rsv->lock);
3884         spin_unlock(&space_info->lock);
3885
3886 commit:
3887         trans = btrfs_join_transaction(root);
3888         if (IS_ERR(trans))
3889                 return -ENOSPC;
3890
3891         return btrfs_commit_transaction(trans, root);
3892 }
3893
3894 enum flush_state {
3895         FLUSH_DELAYED_ITEMS_NR  =       1,
3896         FLUSH_DELAYED_ITEMS     =       2,
3897         FLUSH_DELALLOC          =       3,
3898         FLUSH_DELALLOC_WAIT     =       4,
3899         ALLOC_CHUNK             =       5,
3900         COMMIT_TRANS            =       6,
3901 };
3902
3903 static int flush_space(struct btrfs_root *root,
3904                        struct btrfs_space_info *space_info, u64 num_bytes,
3905                        u64 orig_bytes, int state)
3906 {
3907         struct btrfs_trans_handle *trans;
3908         int nr;
3909         int ret = 0;
3910
3911         switch (state) {
3912         case FLUSH_DELAYED_ITEMS_NR:
3913         case FLUSH_DELAYED_ITEMS:
3914                 if (state == FLUSH_DELAYED_ITEMS_NR) {
3915                         u64 bytes = btrfs_calc_trans_metadata_size(root, 1);
3916
3917                         nr = (int)div64_u64(num_bytes, bytes);
3918                         if (!nr)
3919                                 nr = 1;
3920                         nr *= 2;
3921                 } else {
3922                         nr = -1;
3923                 }
3924                 trans = btrfs_join_transaction(root);
3925                 if (IS_ERR(trans)) {
3926                         ret = PTR_ERR(trans);
3927                         break;
3928                 }
3929                 ret = btrfs_run_delayed_items_nr(trans, root, nr);
3930                 btrfs_end_transaction(trans, root);
3931                 break;
3932         case FLUSH_DELALLOC:
3933         case FLUSH_DELALLOC_WAIT:
3934                 shrink_delalloc(root, num_bytes, orig_bytes,
3935                                 state == FLUSH_DELALLOC_WAIT);
3936                 break;
3937         case ALLOC_CHUNK:
3938                 trans = btrfs_join_transaction(root);
3939                 if (IS_ERR(trans)) {
3940                         ret = PTR_ERR(trans);
3941                         break;
3942                 }
3943                 ret = do_chunk_alloc(trans, root->fs_info->extent_root,
3944                                      btrfs_get_alloc_profile(root, 0),
3945                                      CHUNK_ALLOC_NO_FORCE);
3946                 btrfs_end_transaction(trans, root);
3947                 if (ret == -ENOSPC)
3948                         ret = 0;
3949                 break;
3950         case COMMIT_TRANS:
3951                 ret = may_commit_transaction(root, space_info, orig_bytes, 0);
3952                 break;
3953         default:
3954                 ret = -ENOSPC;
3955                 break;
3956         }
3957
3958         return ret;
3959 }
3960 /**
3961  * reserve_metadata_bytes - try to reserve bytes from the block_rsv's space
3962  * @root - the root we're allocating for
3963  * @block_rsv - the block_rsv we're allocating for
3964  * @orig_bytes - the number of bytes we want
3965  * @flush - wether or not we can flush to make our reservation
3966  *
3967  * This will reserve orgi_bytes number of bytes from the space info associated
3968  * with the block_rsv.  If there is not enough space it will make an attempt to
3969  * flush out space to make room.  It will do this by flushing delalloc if
3970  * possible or committing the transaction.  If flush is 0 then no attempts to
3971  * regain reservations will be made and this will fail if there is not enough
3972  * space already.
3973  */
3974 static int reserve_metadata_bytes(struct btrfs_root *root,
3975                                   struct btrfs_block_rsv *block_rsv,
3976                                   u64 orig_bytes,
3977                                   enum btrfs_reserve_flush_enum flush)
3978 {
3979         struct btrfs_space_info *space_info = block_rsv->space_info;
3980         u64 used;
3981         u64 num_bytes = orig_bytes;
3982         int flush_state = FLUSH_DELAYED_ITEMS_NR;
3983         int ret = 0;
3984         bool flushing = false;
3985
3986 again:
3987         ret = 0;
3988         spin_lock(&space_info->lock);
3989         /*
3990          * We only want to wait if somebody other than us is flushing and we
3991          * are actually allowed to flush all things.
3992          */
3993         while (flush == BTRFS_RESERVE_FLUSH_ALL && !flushing &&
3994                space_info->flush) {
3995                 spin_unlock(&space_info->lock);
3996                 /*
3997                  * If we have a trans handle we can't wait because the flusher
3998                  * may have to commit the transaction, which would mean we would
3999                  * deadlock since we are waiting for the flusher to finish, but
4000                  * hold the current transaction open.
4001                  */
4002                 if (current->journal_info)
4003                         return -EAGAIN;
4004                 ret = wait_event_killable(space_info->wait, !space_info->flush);
4005                 /* Must have been killed, return */
4006                 if (ret)
4007                         return -EINTR;
4008
4009                 spin_lock(&space_info->lock);
4010         }
4011
4012         ret = -ENOSPC;
4013         used = space_info->bytes_used + space_info->bytes_reserved +
4014                 space_info->bytes_pinned + space_info->bytes_readonly +
4015                 space_info->bytes_may_use;
4016
4017         /*
4018          * The idea here is that we've not already over-reserved the block group
4019          * then we can go ahead and save our reservation first and then start
4020          * flushing if we need to.  Otherwise if we've already overcommitted
4021          * lets start flushing stuff first and then come back and try to make
4022          * our reservation.
4023          */
4024         if (used <= space_info->total_bytes) {
4025                 if (used + orig_bytes <= space_info->total_bytes) {
4026                         space_info->bytes_may_use += orig_bytes;
4027                         trace_btrfs_space_reservation(root->fs_info,
4028                                 "space_info", space_info->flags, orig_bytes, 1);
4029                         ret = 0;
4030                 } else {
4031                         /*
4032                          * Ok set num_bytes to orig_bytes since we aren't
4033                          * overocmmitted, this way we only try and reclaim what
4034                          * we need.
4035                          */
4036                         num_bytes = orig_bytes;
4037                 }
4038         } else {
4039                 /*
4040                  * Ok we're over committed, set num_bytes to the overcommitted
4041                  * amount plus the amount of bytes that we need for this
4042                  * reservation.
4043                  */
4044                 num_bytes = used - space_info->total_bytes +
4045                         (orig_bytes * 2);
4046         }
4047
4048         if (ret && can_overcommit(root, space_info, orig_bytes, flush)) {
4049                 space_info->bytes_may_use += orig_bytes;
4050                 trace_btrfs_space_reservation(root->fs_info, "space_info",
4051                                               space_info->flags, orig_bytes,
4052                                               1);
4053                 ret = 0;
4054         }
4055
4056         /*
4057          * Couldn't make our reservation, save our place so while we're trying
4058          * to reclaim space we can actually use it instead of somebody else
4059          * stealing it from us.
4060          *
4061          * We make the other tasks wait for the flush only when we can flush
4062          * all things.
4063          */
4064         if (ret && flush != BTRFS_RESERVE_NO_FLUSH) {
4065                 flushing = true;
4066                 space_info->flush = 1;
4067         }
4068
4069         spin_unlock(&space_info->lock);
4070
4071         if (!ret || flush == BTRFS_RESERVE_NO_FLUSH)
4072                 goto out;
4073
4074         ret = flush_space(root, space_info, num_bytes, orig_bytes,
4075                           flush_state);
4076         flush_state++;
4077
4078         /*
4079          * If we are FLUSH_LIMIT, we can not flush delalloc, or the deadlock
4080          * would happen. So skip delalloc flush.
4081          */
4082         if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
4083             (flush_state == FLUSH_DELALLOC ||
4084              flush_state == FLUSH_DELALLOC_WAIT))
4085                 flush_state = ALLOC_CHUNK;
4086
4087         if (!ret)
4088                 goto again;
4089         else if (flush == BTRFS_RESERVE_FLUSH_LIMIT &&
4090                  flush_state < COMMIT_TRANS)
4091                 goto again;
4092         else if (flush == BTRFS_RESERVE_FLUSH_ALL &&
4093                  flush_state <= COMMIT_TRANS)
4094                 goto again;
4095
4096 out:
4097         if (flushing) {
4098                 spin_lock(&space_info->lock);
4099                 space_info->flush = 0;
4100                 wake_up_all(&space_info->wait);
4101                 spin_unlock(&space_info->lock);
4102         }
4103         return ret;
4104 }
4105
4106 static struct btrfs_block_rsv *get_block_rsv(
4107                                         const struct btrfs_trans_handle *trans,
4108                                         const struct btrfs_root *root)
4109 {
4110         struct btrfs_block_rsv *block_rsv = NULL;
4111
4112         if (root->ref_cows)
4113                 block_rsv = trans->block_rsv;
4114
4115         if (root == root->fs_info->csum_root && trans->adding_csums)
4116                 block_rsv = trans->block_rsv;
4117
4118         if (!block_rsv)
4119                 block_rsv = root->block_rsv;
4120
4121         if (!block_rsv)
4122                 block_rsv = &root->fs_info->empty_block_rsv;
4123
4124         return block_rsv;
4125 }
4126
4127 static int block_rsv_use_bytes(struct btrfs_block_rsv *block_rsv,
4128                                u64 num_bytes)
4129 {
4130         int ret = -ENOSPC;
4131         spin_lock(&block_rsv->lock);
4132         if (block_rsv->reserved >= num_bytes) {
4133                 block_rsv->reserved -= num_bytes;
4134                 if (block_rsv->reserved < block_rsv->size)
4135                         block_rsv->full = 0;
4136                 ret = 0;
4137         }
4138         spin_unlock(&block_rsv->lock);
4139         return ret;
4140 }
4141
4142 static void block_rsv_add_bytes(struct btrfs_block_rsv *block_rsv,
4143                                 u64 num_bytes, int update_size)
4144 {
4145         spin_lock(&block_rsv->lock);
4146         block_rsv->reserved += num_bytes;
4147         if (update_size)
4148                 block_rsv->size += num_bytes;
4149         else if (block_rsv->reserved >= block_rsv->size)
4150                 block_rsv->full = 1;
4151         spin_unlock(&block_rsv->lock);
4152 }
4153
4154 static void block_rsv_release_bytes(struct btrfs_fs_info *fs_info,
4155                                     struct btrfs_block_rsv *block_rsv,
4156                                     struct btrfs_block_rsv *dest, u64 num_bytes)
4157 {
4158         struct btrfs_space_info *space_info = block_rsv->space_info;
4159
4160         spin_lock(&block_rsv->lock);
4161         if (num_bytes == (u64)-1)
4162                 num_bytes = block_rsv->size;
4163         block_rsv->size -= num_bytes;
4164         if (block_rsv->reserved >= block_rsv->size) {
4165                 num_bytes = block_rsv->reserved - block_rsv->size;
4166                 block_rsv->reserved = block_rsv->size;
4167                 block_rsv->full = 1;
4168         } else {
4169                 num_bytes = 0;
4170         }
4171         spin_unlock(&block_rsv->lock);
4172
4173         if (num_bytes > 0) {
4174                 if (dest) {
4175                         spin_lock(&dest->lock);
4176                         if (!dest->full) {
4177                                 u64 bytes_to_add;
4178
4179                                 bytes_to_add = dest->size - dest->reserved;
4180                                 bytes_to_add = min(num_bytes, bytes_to_add);
4181                                 dest->reserved += bytes_to_add;
4182                                 if (dest->reserved >= dest->size)
4183                                         dest->full = 1;
4184                                 num_bytes -= bytes_to_add;
4185                         }
4186                         spin_unlock(&dest->lock);
4187                 }
4188                 if (num_bytes) {
4189                         spin_lock(&space_info->lock);
4190                         space_info->bytes_may_use -= num_bytes;
4191                         trace_btrfs_space_reservation(fs_info, "space_info",
4192                                         space_info->flags, num_bytes, 0);
4193                         space_info->reservation_progress++;
4194                         spin_unlock(&space_info->lock);
4195                 }
4196         }
4197 }
4198
4199 static int block_rsv_migrate_bytes(struct btrfs_block_rsv *src,
4200                                    struct btrfs_block_rsv *dst, u64 num_bytes)
4201 {
4202         int ret;
4203
4204         ret = block_rsv_use_bytes(src, num_bytes);
4205         if (ret)
4206                 return ret;
4207
4208         block_rsv_add_bytes(dst, num_bytes, 1);
4209         return 0;
4210 }
4211
4212 void btrfs_init_block_rsv(struct btrfs_block_rsv *rsv, unsigned short type)
4213 {
4214         memset(rsv, 0, sizeof(*rsv));
4215         spin_lock_init(&rsv->lock);
4216         rsv->type = type;
4217 }
4218
4219 struct btrfs_block_rsv *btrfs_alloc_block_rsv(struct btrfs_root *root,
4220                                               unsigned short type)
4221 {
4222         struct btrfs_block_rsv *block_rsv;
4223         struct btrfs_fs_info *fs_info = root->fs_info;
4224
4225         block_rsv = kmalloc(sizeof(*block_rsv), GFP_NOFS);
4226         if (!block_rsv)
4227                 return NULL;
4228
4229         btrfs_init_block_rsv(block_rsv, type);
4230         block_rsv->space_info = __find_space_info(fs_info,
4231                                                   BTRFS_BLOCK_GROUP_METADATA);
4232         return block_rsv;
4233 }
4234
4235 void btrfs_free_block_rsv(struct btrfs_root *root,
4236                           struct btrfs_block_rsv *rsv)
4237 {
4238         if (!rsv)
4239                 return;
4240         btrfs_block_rsv_release(root, rsv, (u64)-1);
4241         kfree(rsv);
4242 }
4243
4244 int btrfs_block_rsv_add(struct btrfs_root *root,
4245                         struct btrfs_block_rsv *block_rsv, u64 num_bytes,
4246                         enum btrfs_reserve_flush_enum flush)
4247 {
4248         int ret;
4249
4250         if (num_bytes == 0)
4251                 return 0;
4252
4253         ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4254         if (!ret) {
4255                 block_rsv_add_bytes(block_rsv, num_bytes, 1);
4256                 return 0;
4257         }
4258
4259         return ret;
4260 }
4261
4262 int btrfs_block_rsv_check(struct btrfs_root *root,
4263                           struct btrfs_block_rsv *block_rsv, int min_factor)
4264 {
4265         u64 num_bytes = 0;
4266         int ret = -ENOSPC;
4267
4268         if (!block_rsv)
4269                 return 0;
4270
4271         spin_lock(&block_rsv->lock);
4272         num_bytes = div_factor(block_rsv->size, min_factor);
4273         if (block_rsv->reserved >= num_bytes)
4274                 ret = 0;
4275         spin_unlock(&block_rsv->lock);
4276
4277         return ret;
4278 }
4279
4280 int btrfs_block_rsv_refill(struct btrfs_root *root,
4281                            struct btrfs_block_rsv *block_rsv, u64 min_reserved,
4282                            enum btrfs_reserve_flush_enum flush)
4283 {
4284         u64 num_bytes = 0;
4285         int ret = -ENOSPC;
4286
4287         if (!block_rsv)
4288                 return 0;
4289
4290         spin_lock(&block_rsv->lock);
4291         num_bytes = min_reserved;
4292         if (block_rsv->reserved >= num_bytes)
4293                 ret = 0;
4294         else
4295                 num_bytes -= block_rsv->reserved;
4296         spin_unlock(&block_rsv->lock);
4297
4298         if (!ret)
4299                 return 0;
4300
4301         ret = reserve_metadata_bytes(root, block_rsv, num_bytes, flush);
4302         if (!ret) {
4303                 block_rsv_add_bytes(block_rsv, num_bytes, 0);
4304                 return 0;
4305         }
4306
4307         return ret;
4308 }
4309
4310 int btrfs_block_rsv_migrate(struct btrfs_block_rsv *src_rsv,
4311                             struct btrfs_block_rsv *dst_rsv,
4312                             u64 num_bytes)
4313 {
4314         return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4315 }
4316
4317 void btrfs_block_rsv_release(struct btrfs_root *root,
4318                              struct btrfs_block_rsv *block_rsv,
4319                              u64 num_bytes)
4320 {
4321         struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
4322         if (global_rsv->full || global_rsv == block_rsv ||
4323             block_rsv->space_info != global_rsv->space_info)
4324                 global_rsv = NULL;
4325         block_rsv_release_bytes(root->fs_info, block_rsv, global_rsv,
4326                                 num_bytes);
4327 }
4328
4329 /*
4330  * helper to calculate size of global block reservation.
4331  * the desired value is sum of space used by extent tree,
4332  * checksum tree and root tree
4333  */
4334 static u64 calc_global_metadata_size(struct btrfs_fs_info *fs_info)
4335 {
4336         struct btrfs_space_info *sinfo;
4337         u64 num_bytes;
4338         u64 meta_used;
4339         u64 data_used;
4340         int csum_size = btrfs_super_csum_size(fs_info->super_copy);
4341
4342         sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_DATA);
4343         spin_lock(&sinfo->lock);
4344         data_used = sinfo->bytes_used;
4345         spin_unlock(&sinfo->lock);
4346
4347         sinfo = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4348         spin_lock(&sinfo->lock);
4349         if (sinfo->flags & BTRFS_BLOCK_GROUP_DATA)
4350                 data_used = 0;
4351         meta_used = sinfo->bytes_used;
4352         spin_unlock(&sinfo->lock);
4353
4354         num_bytes = (data_used >> fs_info->sb->s_blocksize_bits) *
4355                     csum_size * 2;
4356         num_bytes += div64_u64(data_used + meta_used, 50);
4357
4358         if (num_bytes * 3 > meta_used)
4359                 num_bytes = div64_u64(meta_used, 3);
4360
4361         return ALIGN(num_bytes, fs_info->extent_root->leafsize << 10);
4362 }
4363
4364 static void update_global_block_rsv(struct btrfs_fs_info *fs_info)
4365 {
4366         struct btrfs_block_rsv *block_rsv = &fs_info->global_block_rsv;
4367         struct btrfs_space_info *sinfo = block_rsv->space_info;
4368         u64 num_bytes;
4369
4370         num_bytes = calc_global_metadata_size(fs_info);
4371
4372         spin_lock(&sinfo->lock);
4373         spin_lock(&block_rsv->lock);
4374
4375         block_rsv->size = num_bytes;
4376
4377         num_bytes = sinfo->bytes_used + sinfo->bytes_pinned +
4378                     sinfo->bytes_reserved + sinfo->bytes_readonly +
4379                     sinfo->bytes_may_use;
4380
4381         if (sinfo->total_bytes > num_bytes) {
4382                 num_bytes = sinfo->total_bytes - num_bytes;
4383                 block_rsv->reserved += num_bytes;
4384                 sinfo->bytes_may_use += num_bytes;
4385                 trace_btrfs_space_reservation(fs_info, "space_info",
4386                                       sinfo->flags, num_bytes, 1);
4387         }
4388
4389         if (block_rsv->reserved >= block_rsv->size) {
4390                 num_bytes = block_rsv->reserved - block_rsv->size;
4391                 sinfo->bytes_may_use -= num_bytes;
4392                 trace_btrfs_space_reservation(fs_info, "space_info",
4393                                       sinfo->flags, num_bytes, 0);
4394                 sinfo->reservation_progress++;
4395                 block_rsv->reserved = block_rsv->size;
4396                 block_rsv->full = 1;
4397         }
4398
4399         spin_unlock(&block_rsv->lock);
4400         spin_unlock(&sinfo->lock);
4401 }
4402
4403 static void init_global_block_rsv(struct btrfs_fs_info *fs_info)
4404 {
4405         struct btrfs_space_info *space_info;
4406
4407         space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
4408         fs_info->chunk_block_rsv.space_info = space_info;
4409
4410         space_info = __find_space_info(fs_info, BTRFS_BLOCK_GROUP_METADATA);
4411         fs_info->global_block_rsv.space_info = space_info;
4412         fs_info->delalloc_block_rsv.space_info = space_info;
4413         fs_info->trans_block_rsv.space_info = space_info;
4414         fs_info->empty_block_rsv.space_info = space_info;
4415         fs_info->delayed_block_rsv.space_info = space_info;
4416
4417         fs_info->extent_root->block_rsv = &fs_info->global_block_rsv;
4418         fs_info->csum_root->block_rsv = &fs_info->global_block_rsv;
4419         fs_info->dev_root->block_rsv = &fs_info->global_block_rsv;
4420         fs_info->tree_root->block_rsv = &fs_info->global_block_rsv;
4421         fs_info->chunk_root->block_rsv = &fs_info->chunk_block_rsv;
4422
4423         update_global_block_rsv(fs_info);
4424 }
4425
4426 static void release_global_block_rsv(struct btrfs_fs_info *fs_info)
4427 {
4428         block_rsv_release_bytes(fs_info, &fs_info->global_block_rsv, NULL,
4429                                 (u64)-1);
4430         WARN_ON(fs_info->delalloc_block_rsv.size > 0);
4431         WARN_ON(fs_info->delalloc_block_rsv.reserved > 0);
4432         WARN_ON(fs_info->trans_block_rsv.size > 0);
4433         WARN_ON(fs_info->trans_block_rsv.reserved > 0);
4434         WARN_ON(fs_info->chunk_block_rsv.size > 0);
4435         WARN_ON(fs_info->chunk_block_rsv.reserved > 0);
4436         WARN_ON(fs_info->delayed_block_rsv.size > 0);
4437         WARN_ON(fs_info->delayed_block_rsv.reserved > 0);
4438 }
4439
4440 void btrfs_trans_release_metadata(struct btrfs_trans_handle *trans,
4441                                   struct btrfs_root *root)
4442 {
4443         if (!trans->block_rsv)
4444                 return;
4445
4446         if (!trans->bytes_reserved)
4447                 return;
4448
4449         trace_btrfs_space_reservation(root->fs_info, "transaction",
4450                                       trans->transid, trans->bytes_reserved, 0);
4451         btrfs_block_rsv_release(root, trans->block_rsv, trans->bytes_reserved);
4452         trans->bytes_reserved = 0;
4453 }
4454
4455 /* Can only return 0 or -ENOSPC */
4456 int btrfs_orphan_reserve_metadata(struct btrfs_trans_handle *trans,
4457                                   struct inode *inode)
4458 {
4459         struct btrfs_root *root = BTRFS_I(inode)->root;
4460         struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
4461         struct btrfs_block_rsv *dst_rsv = root->orphan_block_rsv;
4462
4463         /*
4464          * We need to hold space in order to delete our orphan item once we've
4465          * added it, so this takes the reservation so we can release it later
4466          * when we are truly done with the orphan item.
4467          */
4468         u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
4469         trace_btrfs_space_reservation(root->fs_info, "orphan",
4470                                       btrfs_ino(inode), num_bytes, 1);
4471         return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4472 }
4473
4474 void btrfs_orphan_release_metadata(struct inode *inode)
4475 {
4476         struct btrfs_root *root = BTRFS_I(inode)->root;
4477         u64 num_bytes = btrfs_calc_trans_metadata_size(root, 1);
4478         trace_btrfs_space_reservation(root->fs_info, "orphan",
4479                                       btrfs_ino(inode), num_bytes, 0);
4480         btrfs_block_rsv_release(root, root->orphan_block_rsv, num_bytes);
4481 }
4482
4483 int btrfs_snap_reserve_metadata(struct btrfs_trans_handle *trans,
4484                                 struct btrfs_pending_snapshot *pending)
4485 {
4486         struct btrfs_root *root = pending->root;
4487         struct btrfs_block_rsv *src_rsv = get_block_rsv(trans, root);
4488         struct btrfs_block_rsv *dst_rsv = &pending->block_rsv;
4489         /*
4490          * two for root back/forward refs, two for directory entries,
4491          * one for root of the snapshot and one for parent inode.
4492          */
4493         u64 num_bytes = btrfs_calc_trans_metadata_size(root, 6);
4494         dst_rsv->space_info = src_rsv->space_info;
4495         return block_rsv_migrate_bytes(src_rsv, dst_rsv, num_bytes);
4496 }
4497
4498 /**
4499  * drop_outstanding_extent - drop an outstanding extent
4500  * @inode: the inode we're dropping the extent for
4501  *
4502  * This is called when we are freeing up an outstanding extent, either called
4503  * after an error or after an extent is written.  This will return the number of
4504  * reserved extents that need to be freed.  This must be called with
4505  * BTRFS_I(inode)->lock held.
4506  */
4507 static unsigned drop_outstanding_extent(struct inode *inode)
4508 {
4509         unsigned drop_inode_space = 0;
4510         unsigned dropped_extents = 0;
4511
4512         BUG_ON(!BTRFS_I(inode)->outstanding_extents);
4513         BTRFS_I(inode)->outstanding_extents--;
4514
4515         if (BTRFS_I(inode)->outstanding_extents == 0 &&
4516             test_and_clear_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4517                                &BTRFS_I(inode)->runtime_flags))
4518                 drop_inode_space = 1;
4519
4520         /*
4521          * If we have more or the same amount of outsanding extents than we have
4522          * reserved then we need to leave the reserved extents count alone.
4523          */
4524         if (BTRFS_I(inode)->outstanding_extents >=
4525             BTRFS_I(inode)->reserved_extents)
4526                 return drop_inode_space;
4527
4528         dropped_extents = BTRFS_I(inode)->reserved_extents -
4529                 BTRFS_I(inode)->outstanding_extents;
4530         BTRFS_I(inode)->reserved_extents -= dropped_extents;
4531         return dropped_extents + drop_inode_space;
4532 }
4533
4534 /**
4535  * calc_csum_metadata_size - return the amount of metada space that must be
4536  *      reserved/free'd for the given bytes.
4537  * @inode: the inode we're manipulating
4538  * @num_bytes: the number of bytes in question
4539  * @reserve: 1 if we are reserving space, 0 if we are freeing space
4540  *
4541  * This adjusts the number of csum_bytes in the inode and then returns the
4542  * correct amount of metadata that must either be reserved or freed.  We
4543  * calculate how many checksums we can fit into one leaf and then divide the
4544  * number of bytes that will need to be checksumed by this value to figure out
4545  * how many checksums will be required.  If we are adding bytes then the number
4546  * may go up and we will return the number of additional bytes that must be
4547  * reserved.  If it is going down we will return the number of bytes that must
4548  * be freed.
4549  *
4550  * This must be called with BTRFS_I(inode)->lock held.
4551  */
4552 static u64 calc_csum_metadata_size(struct inode *inode, u64 num_bytes,
4553                                    int reserve)
4554 {
4555         struct btrfs_root *root = BTRFS_I(inode)->root;
4556         u64 csum_size;
4557         int num_csums_per_leaf;
4558         int num_csums;
4559         int old_csums;
4560
4561         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM &&
4562             BTRFS_I(inode)->csum_bytes == 0)
4563                 return 0;
4564
4565         old_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
4566         if (reserve)
4567                 BTRFS_I(inode)->csum_bytes += num_bytes;
4568         else
4569                 BTRFS_I(inode)->csum_bytes -= num_bytes;
4570         csum_size = BTRFS_LEAF_DATA_SIZE(root) - sizeof(struct btrfs_item);
4571         num_csums_per_leaf = (int)div64_u64(csum_size,
4572                                             sizeof(struct btrfs_csum_item) +
4573                                             sizeof(struct btrfs_disk_key));
4574         num_csums = (int)div64_u64(BTRFS_I(inode)->csum_bytes, root->sectorsize);
4575         num_csums = num_csums + num_csums_per_leaf - 1;
4576         num_csums = num_csums / num_csums_per_leaf;
4577
4578         old_csums = old_csums + num_csums_per_leaf - 1;
4579         old_csums = old_csums / num_csums_per_leaf;
4580
4581         /* No change, no need to reserve more */
4582         if (old_csums == num_csums)
4583                 return 0;
4584
4585         if (reserve)
4586                 return btrfs_calc_trans_metadata_size(root,
4587                                                       num_csums - old_csums);
4588
4589         return btrfs_calc_trans_metadata_size(root, old_csums - num_csums);
4590 }
4591
4592 int btrfs_delalloc_reserve_metadata(struct inode *inode, u64 num_bytes)
4593 {
4594         struct btrfs_root *root = BTRFS_I(inode)->root;
4595         struct btrfs_block_rsv *block_rsv = &root->fs_info->delalloc_block_rsv;
4596         u64 to_reserve = 0;
4597         u64 csum_bytes;
4598         unsigned nr_extents = 0;
4599         int extra_reserve = 0;
4600         enum btrfs_reserve_flush_enum flush = BTRFS_RESERVE_FLUSH_ALL;
4601         int ret;
4602         bool delalloc_lock = true;
4603
4604         /* If we are a free space inode we need to not flush since we will be in
4605          * the middle of a transaction commit.  We also don't need the delalloc
4606          * mutex since we won't race with anybody.  We need this mostly to make
4607          * lockdep shut its filthy mouth.
4608          */
4609         if (btrfs_is_free_space_inode(inode)) {
4610                 flush = BTRFS_RESERVE_NO_FLUSH;
4611                 delalloc_lock = false;
4612         }
4613
4614         if (flush != BTRFS_RESERVE_NO_FLUSH &&
4615             btrfs_transaction_in_commit(root->fs_info))
4616                 schedule_timeout(1);
4617
4618         if (delalloc_lock)
4619                 mutex_lock(&BTRFS_I(inode)->delalloc_mutex);
4620
4621         num_bytes = ALIGN(num_bytes, root->sectorsize);
4622
4623         spin_lock(&BTRFS_I(inode)->lock);
4624         BTRFS_I(inode)->outstanding_extents++;
4625
4626         if (BTRFS_I(inode)->outstanding_extents >
4627             BTRFS_I(inode)->reserved_extents)
4628                 nr_extents = BTRFS_I(inode)->outstanding_extents -
4629                         BTRFS_I(inode)->reserved_extents;
4630
4631         /*
4632          * Add an item to reserve for updating the inode when we complete the
4633          * delalloc io.
4634          */
4635         if (!test_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4636                       &BTRFS_I(inode)->runtime_flags)) {
4637                 nr_extents++;
4638                 extra_reserve = 1;
4639         }
4640
4641         to_reserve = btrfs_calc_trans_metadata_size(root, nr_extents);
4642         to_reserve += calc_csum_metadata_size(inode, num_bytes, 1);
4643         csum_bytes = BTRFS_I(inode)->csum_bytes;
4644         spin_unlock(&BTRFS_I(inode)->lock);
4645
4646         if (root->fs_info->quota_enabled) {
4647                 ret = btrfs_qgroup_reserve(root, num_bytes +
4648                                            nr_extents * root->leafsize);
4649                 if (ret) {
4650                         spin_lock(&BTRFS_I(inode)->lock);
4651                         calc_csum_metadata_size(inode, num_bytes, 0);
4652                         spin_unlock(&BTRFS_I(inode)->lock);
4653                         if (delalloc_lock)
4654                                 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4655                         return ret;
4656                 }
4657         }
4658
4659         ret = reserve_metadata_bytes(root, block_rsv, to_reserve, flush);
4660         if (ret) {
4661                 u64 to_free = 0;
4662                 unsigned dropped;
4663
4664                 spin_lock(&BTRFS_I(inode)->lock);
4665                 dropped = drop_outstanding_extent(inode);
4666                 /*
4667                  * If the inodes csum_bytes is the same as the original
4668                  * csum_bytes then we know we haven't raced with any free()ers
4669                  * so we can just reduce our inodes csum bytes and carry on.
4670                  * Otherwise we have to do the normal free thing to account for
4671                  * the case that the free side didn't free up its reserve
4672                  * because of this outstanding reservation.
4673                  */
4674                 if (BTRFS_I(inode)->csum_bytes == csum_bytes)
4675                         calc_csum_metadata_size(inode, num_bytes, 0);
4676                 else
4677                         to_free = calc_csum_metadata_size(inode, num_bytes, 0);
4678                 spin_unlock(&BTRFS_I(inode)->lock);
4679                 if (dropped)
4680                         to_free += btrfs_calc_trans_metadata_size(root, dropped);
4681
4682                 if (to_free) {
4683                         btrfs_block_rsv_release(root, block_rsv, to_free);
4684                         trace_btrfs_space_reservation(root->fs_info,
4685                                                       "delalloc",
4686                                                       btrfs_ino(inode),
4687                                                       to_free, 0);
4688                 }
4689                 if (root->fs_info->quota_enabled) {
4690                         btrfs_qgroup_free(root, num_bytes +
4691                                                 nr_extents * root->leafsize);
4692                 }
4693                 if (delalloc_lock)
4694                         mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4695                 return ret;
4696         }
4697
4698         spin_lock(&BTRFS_I(inode)->lock);
4699         if (extra_reserve) {
4700                 set_bit(BTRFS_INODE_DELALLOC_META_RESERVED,
4701                         &BTRFS_I(inode)->runtime_flags);
4702                 nr_extents--;
4703         }
4704         BTRFS_I(inode)->reserved_extents += nr_extents;
4705         spin_unlock(&BTRFS_I(inode)->lock);
4706
4707         if (delalloc_lock)
4708                 mutex_unlock(&BTRFS_I(inode)->delalloc_mutex);
4709
4710         if (to_reserve)
4711                 trace_btrfs_space_reservation(root->fs_info,"delalloc",
4712                                               btrfs_ino(inode), to_reserve, 1);
4713         block_rsv_add_bytes(block_rsv, to_reserve, 1);
4714
4715         return 0;
4716 }
4717
4718 /**
4719  * btrfs_delalloc_release_metadata - release a metadata reservation for an inode
4720  * @inode: the inode to release the reservation for
4721  * @num_bytes: the number of bytes we're releasing
4722  *
4723  * This will release the metadata reservation for an inode.  This can be called
4724  * once we complete IO for a given set of bytes to release their metadata
4725  * reservations.
4726  */
4727 void btrfs_delalloc_release_metadata(struct inode *inode, u64 num_bytes)
4728 {
4729         struct btrfs_root *root = BTRFS_I(inode)->root;
4730         u64 to_free = 0;
4731         unsigned dropped;
4732
4733         num_bytes = ALIGN(num_bytes, root->sectorsize);
4734         spin_lock(&BTRFS_I(inode)->lock);
4735         dropped = drop_outstanding_extent(inode);
4736
4737         to_free = calc_csum_metadata_size(inode, num_bytes, 0);
4738         spin_unlock(&BTRFS_I(inode)->lock);
4739         if (dropped > 0)
4740                 to_free += btrfs_calc_trans_metadata_size(root, dropped);
4741
4742         trace_btrfs_space_reservation(root->fs_info, "delalloc",
4743                                       btrfs_ino(inode), to_free, 0);
4744         if (root->fs_info->quota_enabled) {
4745                 btrfs_qgroup_free(root, num_bytes +
4746                                         dropped * root->leafsize);
4747         }
4748
4749         btrfs_block_rsv_release(root, &root->fs_info->delalloc_block_rsv,
4750                                 to_free);
4751 }
4752
4753 /**
4754  * btrfs_delalloc_reserve_space - reserve data and metadata space for delalloc
4755  * @inode: inode we're writing to
4756  * @num_bytes: the number of bytes we want to allocate
4757  *
4758  * This will do the following things
4759  *
4760  * o reserve space in the data space info for num_bytes
4761  * o reserve space in the metadata space info based on number of outstanding
4762  *   extents and how much csums will be needed
4763  * o add to the inodes ->delalloc_bytes
4764  * o add it to the fs_info's delalloc inodes list.
4765  *
4766  * This will return 0 for success and -ENOSPC if there is no space left.
4767  */
4768 int btrfs_delalloc_reserve_space(struct inode *inode, u64 num_bytes)
4769 {
4770         int ret;
4771
4772         ret = btrfs_check_data_free_space(inode, num_bytes);
4773         if (ret)
4774                 return ret;
4775
4776         ret = btrfs_delalloc_reserve_metadata(inode, num_bytes);
4777         if (ret) {
4778                 btrfs_free_reserved_data_space(inode, num_bytes);
4779                 return ret;
4780         }
4781
4782         return 0;
4783 }
4784
4785 /**
4786  * btrfs_delalloc_release_space - release data and metadata space for delalloc
4787  * @inode: inode we're releasing space for
4788  * @num_bytes: the number of bytes we want to free up
4789  *
4790  * This must be matched with a call to btrfs_delalloc_reserve_space.  This is
4791  * called in the case that we don't need the metadata AND data reservations
4792  * anymore.  So if there is an error or we insert an inline extent.
4793  *
4794  * This function will release the metadata space that was not used and will
4795  * decrement ->delalloc_bytes and remove it from the fs_info delalloc_inodes
4796  * list if there are no delalloc bytes left.
4797  */
4798 void btrfs_delalloc_release_space(struct inode *inode, u64 num_bytes)
4799 {
4800         btrfs_delalloc_release_metadata(inode, num_bytes);
4801         btrfs_free_reserved_data_space(inode, num_bytes);
4802 }
4803
4804 static int update_block_group(struct btrfs_trans_handle *trans,
4805                               struct btrfs_root *root,
4806                               u64 bytenr, u64 num_bytes, int alloc)
4807 {
4808         struct btrfs_block_group_cache *cache = NULL;
4809         struct btrfs_fs_info *info = root->fs_info;
4810         u64 total = num_bytes;
4811         u64 old_val;
4812         u64 byte_in_group;
4813         int factor;
4814
4815         /* block accounting for super block */
4816         spin_lock(&info->delalloc_lock);
4817         old_val = btrfs_super_bytes_used(info->super_copy);
4818         if (alloc)
4819                 old_val += num_bytes;
4820         else
4821                 old_val -= num_bytes;
4822         btrfs_set_super_bytes_used(info->super_copy, old_val);
4823         spin_unlock(&info->delalloc_lock);
4824
4825         while (total) {
4826                 cache = btrfs_lookup_block_group(info, bytenr);
4827                 if (!cache)
4828                         return -ENOENT;
4829                 if (cache->flags & (BTRFS_BLOCK_GROUP_DUP |
4830                                     BTRFS_BLOCK_GROUP_RAID1 |
4831                                     BTRFS_BLOCK_GROUP_RAID10))
4832                         factor = 2;
4833                 else
4834                         factor = 1;
4835                 /*
4836                  * If this block group has free space cache written out, we
4837                  * need to make sure to load it if we are removing space.  This
4838                  * is because we need the unpinning stage to actually add the
4839                  * space back to the block group, otherwise we will leak space.
4840                  */
4841                 if (!alloc && cache->cached == BTRFS_CACHE_NO)
4842                         cache_block_group(cache, trans, NULL, 1);
4843
4844                 byte_in_group = bytenr - cache->key.objectid;
4845                 WARN_ON(byte_in_group > cache->key.offset);
4846
4847                 spin_lock(&cache->space_info->lock);
4848                 spin_lock(&cache->lock);
4849
4850                 if (btrfs_test_opt(root, SPACE_CACHE) &&
4851                     cache->disk_cache_state < BTRFS_DC_CLEAR)
4852                         cache->disk_cache_state = BTRFS_DC_CLEAR;
4853
4854                 cache->dirty = 1;
4855                 old_val = btrfs_block_group_used(&cache->item);
4856                 num_bytes = min(total, cache->key.offset - byte_in_group);
4857                 if (alloc) {
4858                         old_val += num_bytes;
4859                         btrfs_set_block_group_used(&cache->item, old_val);
4860                         cache->reserved -= num_bytes;
4861                         cache->space_info->bytes_reserved -= num_bytes;
4862                         cache->space_info->bytes_used += num_bytes;
4863                         cache->space_info->disk_used += num_bytes * factor;
4864                         spin_unlock(&cache->lock);
4865                         spin_unlock(&cache->space_info->lock);
4866                 } else {
4867                         old_val -= num_bytes;
4868                         btrfs_set_block_group_used(&cache->item, old_val);
4869                         cache->pinned += num_bytes;
4870                         cache->space_info->bytes_pinned += num_bytes;
4871                         cache->space_info->bytes_used -= num_bytes;
4872                         cache->space_info->disk_used -= num_bytes * factor;
4873                         spin_unlock(&cache->lock);
4874                         spin_unlock(&cache->space_info->lock);
4875
4876                         set_extent_dirty(info->pinned_extents,
4877                                          bytenr, bytenr + num_bytes - 1,
4878                                          GFP_NOFS | __GFP_NOFAIL);
4879                 }
4880                 btrfs_put_block_group(cache);
4881                 total -= num_bytes;
4882                 bytenr += num_bytes;
4883         }
4884         return 0;
4885 }
4886
4887 static u64 first_logical_byte(struct btrfs_root *root, u64 search_start)
4888 {
4889         struct btrfs_block_group_cache *cache;
4890         u64 bytenr;
4891
4892         cache = btrfs_lookup_first_block_group(root->fs_info, search_start);
4893         if (!cache)
4894                 return 0;
4895
4896         bytenr = cache->key.objectid;
4897         btrfs_put_block_group(cache);
4898
4899         return bytenr;
4900 }
4901
4902 static int pin_down_extent(struct btrfs_root *root,
4903                            struct btrfs_block_group_cache *cache,
4904                            u64 bytenr, u64 num_bytes, int reserved)
4905 {
4906         spin_lock(&cache->space_info->lock);
4907         spin_lock(&cache->lock);
4908         cache->pinned += num_bytes;
4909         cache->space_info->bytes_pinned += num_bytes;
4910         if (reserved) {
4911                 cache->reserved -= num_bytes;
4912                 cache->space_info->bytes_reserved -= num_bytes;
4913         }
4914         spin_unlock(&cache->lock);
4915         spin_unlock(&cache->space_info->lock);
4916
4917         set_extent_dirty(root->fs_info->pinned_extents, bytenr,
4918                          bytenr + num_bytes - 1, GFP_NOFS | __GFP_NOFAIL);
4919         return 0;
4920 }
4921
4922 /*
4923  * this function must be called within transaction
4924  */
4925 int btrfs_pin_extent(struct btrfs_root *root,
4926                      u64 bytenr, u64 num_bytes, int reserved)
4927 {
4928         struct btrfs_block_group_cache *cache;
4929
4930         cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4931         BUG_ON(!cache); /* Logic error */
4932
4933         pin_down_extent(root, cache, bytenr, num_bytes, reserved);
4934
4935         btrfs_put_block_group(cache);
4936         return 0;
4937 }
4938
4939 /*
4940  * this function must be called within transaction
4941  */
4942 int btrfs_pin_extent_for_log_replay(struct btrfs_trans_handle *trans,
4943                                     struct btrfs_root *root,
4944                                     u64 bytenr, u64 num_bytes)
4945 {
4946         struct btrfs_block_group_cache *cache;
4947
4948         cache = btrfs_lookup_block_group(root->fs_info, bytenr);
4949         BUG_ON(!cache); /* Logic error */
4950
4951         /*
4952          * pull in the free space cache (if any) so that our pin
4953          * removes the free space from the cache.  We have load_only set
4954          * to one because the slow code to read in the free extents does check
4955          * the pinned extents.
4956          */
4957         cache_block_group(cache, trans, root, 1);
4958
4959         pin_down_extent(root, cache, bytenr, num_bytes, 0);
4960
4961         /* remove us from the free space cache (if we're there at all) */
4962         btrfs_remove_free_space(cache, bytenr, num_bytes);
4963         btrfs_put_block_group(cache);
4964         return 0;
4965 }
4966
4967 /**
4968  * btrfs_update_reserved_bytes - update the block_group and space info counters
4969  * @cache:      The cache we are manipulating
4970  * @num_bytes:  The number of bytes in question
4971  * @reserve:    One of the reservation enums
4972  *
4973  * This is called by the allocator when it reserves space, or by somebody who is
4974  * freeing space that was never actually used on disk.  For example if you
4975  * reserve some space for a new leaf in transaction A and before transaction A
4976  * commits you free that leaf, you call this with reserve set to 0 in order to
4977  * clear the reservation.
4978  *
4979  * Metadata reservations should be called with RESERVE_ALLOC so we do the proper
4980  * ENOSPC accounting.  For data we handle the reservation through clearing the
4981  * delalloc bits in the io_tree.  We have to do this since we could end up
4982  * allocating less disk space for the amount of data we have reserved in the
4983  * case of compression.
4984  *
4985  * If this is a reservation and the block group has become read only we cannot
4986  * make the reservation and return -EAGAIN, otherwise this function always
4987  * succeeds.
4988  */
4989 static int btrfs_update_reserved_bytes(struct btrfs_block_group_cache *cache,
4990                                        u64 num_bytes, int reserve)
4991 {
4992         struct btrfs_space_info *space_info = cache->space_info;
4993         int ret = 0;
4994
4995         spin_lock(&space_info->lock);
4996         spin_lock(&cache->lock);
4997         if (reserve != RESERVE_FREE) {
4998                 if (cache->ro) {
4999                         ret = -EAGAIN;
5000                 } else {
5001                         cache->reserved += num_bytes;
5002                         space_info->bytes_reserved += num_bytes;
5003                         if (reserve == RESERVE_ALLOC) {
5004                                 trace_btrfs_space_reservation(cache->fs_info,
5005                                                 "space_info", space_info->flags,
5006                                                 num_bytes, 0);
5007                                 space_info->bytes_may_use -= num_bytes;
5008                         }
5009                 }
5010         } else {
5011                 if (cache->ro)
5012                         space_info->bytes_readonly += num_bytes;
5013                 cache->reserved -= num_bytes;
5014                 space_info->bytes_reserved -= num_bytes;
5015                 space_info->reservation_progress++;
5016         }
5017         spin_unlock(&cache->lock);
5018         spin_unlock(&space_info->lock);
5019         return ret;
5020 }
5021
5022 void btrfs_prepare_extent_commit(struct btrfs_trans_handle *trans,
5023                                 struct btrfs_root *root)
5024 {
5025         struct btrfs_fs_info *fs_info = root->fs_info;
5026         struct btrfs_caching_control *next;
5027         struct btrfs_caching_control *caching_ctl;
5028         struct btrfs_block_group_cache *cache;
5029
5030         down_write(&fs_info->extent_commit_sem);
5031
5032         list_for_each_entry_safe(caching_ctl, next,
5033                                  &fs_info->caching_block_groups, list) {
5034                 cache = caching_ctl->block_group;
5035                 if (block_group_cache_done(cache)) {
5036                         cache->last_byte_to_unpin = (u64)-1;
5037                         list_del_init(&caching_ctl->list);
5038                         put_caching_control(caching_ctl);
5039                 } else {
5040                         cache->last_byte_to_unpin = caching_ctl->progress;
5041                 }
5042         }
5043
5044         if (fs_info->pinned_extents == &fs_info->freed_extents[0])
5045                 fs_info->pinned_extents = &fs_info->freed_extents[1];
5046         else
5047                 fs_info->pinned_extents = &fs_info->freed_extents[0];
5048
5049         up_write(&fs_info->extent_commit_sem);
5050
5051         update_global_block_rsv(fs_info);
5052 }
5053
5054 static int unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
5055 {
5056         struct btrfs_fs_info *fs_info = root->fs_info;
5057         struct btrfs_block_group_cache *cache = NULL;
5058         struct btrfs_space_info *space_info;
5059         struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
5060         u64 len;
5061         bool readonly;
5062
5063         while (start <= end) {
5064                 readonly = false;
5065                 if (!cache ||
5066                     start >= cache->key.objectid + cache->key.offset) {
5067                         if (cache)
5068                                 btrfs_put_block_group(cache);
5069                         cache = btrfs_lookup_block_group(fs_info, start);
5070                         BUG_ON(!cache); /* Logic error */
5071                 }
5072
5073                 len = cache->key.objectid + cache->key.offset - start;
5074                 len = min(len, end + 1 - start);
5075
5076                 if (start < cache->last_byte_to_unpin) {
5077                         len = min(len, cache->last_byte_to_unpin - start);
5078                         btrfs_add_free_space(cache, start, len);
5079                 }
5080
5081                 start += len;
5082                 space_info = cache->space_info;
5083
5084                 spin_lock(&space_info->lock);
5085                 spin_lock(&cache->lock);
5086                 cache->pinned -= len;
5087                 space_info->bytes_pinned -= len;
5088                 if (cache->ro) {
5089                         space_info->bytes_readonly += len;
5090                         readonly = true;
5091                 }
5092                 spin_unlock(&cache->lock);
5093                 if (!readonly && global_rsv->space_info == space_info) {
5094                         spin_lock(&global_rsv->lock);
5095                         if (!global_rsv->full) {
5096                                 len = min(len, global_rsv->size -
5097                                           global_rsv->reserved);
5098                                 global_rsv->reserved += len;
5099                                 space_info->bytes_may_use += len;
5100                                 if (global_rsv->reserved >= global_rsv->size)
5101                                         global_rsv->full = 1;
5102                         }
5103                         spin_unlock(&global_rsv->lock);
5104                 }
5105                 spin_unlock(&space_info->lock);
5106         }
5107
5108         if (cache)
5109                 btrfs_put_block_group(cache);
5110         return 0;
5111 }
5112
5113 int btrfs_finish_extent_commit(struct btrfs_trans_handle *trans,
5114                                struct btrfs_root *root)
5115 {
5116         struct btrfs_fs_info *fs_info = root->fs_info;
5117         struct extent_io_tree *unpin;
5118         u64 start;
5119         u64 end;
5120         int ret;
5121
5122         if (trans->aborted)
5123                 return 0;
5124
5125         if (fs_info->pinned_extents == &fs_info->freed_extents[0])
5126                 unpin = &fs_info->freed_extents[1];
5127         else
5128                 unpin = &fs_info->freed_extents[0];
5129
5130         while (1) {
5131                 ret = find_first_extent_bit(unpin, 0, &start, &end,
5132                                             EXTENT_DIRTY, NULL);
5133                 if (ret)
5134                         break;
5135
5136                 if (btrfs_test_opt(root, DISCARD))
5137                         ret = btrfs_discard_extent(root, start,
5138                                                    end + 1 - start, NULL);
5139
5140                 clear_extent_dirty(unpin, start, end, GFP_NOFS);
5141                 unpin_extent_range(root, start, end);
5142                 cond_resched();
5143         }
5144
5145         return 0;
5146 }
5147
5148 static int __btrfs_free_extent(struct btrfs_trans_handle *trans,
5149                                 struct btrfs_root *root,
5150                                 u64 bytenr, u64 num_bytes, u64 parent,
5151                                 u64 root_objectid, u64 owner_objectid,
5152                                 u64 owner_offset, int refs_to_drop,
5153                                 struct btrfs_delayed_extent_op *extent_op)
5154 {
5155         struct btrfs_key key;
5156         struct btrfs_path *path;
5157         struct btrfs_fs_info *info = root->fs_info;
5158         struct btrfs_root *extent_root = info->extent_root;
5159         struct extent_buffer *leaf;
5160         struct btrfs_extent_item *ei;
5161         struct btrfs_extent_inline_ref *iref;
5162         int ret;
5163         int is_data;
5164         int extent_slot = 0;
5165         int found_extent = 0;
5166         int num_to_del = 1;
5167         u32 item_size;
5168         u64 refs;
5169
5170         path = btrfs_alloc_path();
5171         if (!path)
5172                 return -ENOMEM;
5173
5174         path->reada = 1;
5175         path->leave_spinning = 1;
5176
5177         is_data = owner_objectid >= BTRFS_FIRST_FREE_OBJECTID;
5178         BUG_ON(!is_data && refs_to_drop != 1);
5179
5180         ret = lookup_extent_backref(trans, extent_root, path, &iref,
5181                                     bytenr, num_bytes, parent,
5182                                     root_objectid, owner_objectid,
5183                                     owner_offset);
5184         if (ret == 0) {
5185                 extent_slot = path->slots[0];
5186                 while (extent_slot >= 0) {
5187                         btrfs_item_key_to_cpu(path->nodes[0], &key,
5188                                               extent_slot);
5189                         if (key.objectid != bytenr)
5190                                 break;
5191                         if (key.type == BTRFS_EXTENT_ITEM_KEY &&
5192                             key.offset == num_bytes) {
5193                                 found_extent = 1;
5194                                 break;
5195                         }
5196                         if (path->slots[0] - extent_slot > 5)
5197                                 break;
5198                         extent_slot--;
5199                 }
5200 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
5201                 item_size = btrfs_item_size_nr(path->nodes[0], extent_slot);
5202                 if (found_extent && item_size < sizeof(*ei))
5203                         found_extent = 0;
5204 #endif
5205                 if (!found_extent) {
5206                         BUG_ON(iref);
5207                         ret = remove_extent_backref(trans, extent_root, path,
5208                                                     NULL, refs_to_drop,
5209                                                     is_data);
5210                         if (ret) {
5211                                 btrfs_abort_transaction(trans, extent_root, ret);
5212                                 goto out;
5213                         }
5214                         btrfs_release_path(path);
5215                         path->leave_spinning = 1;
5216
5217                         key.objectid = bytenr;
5218                         key.type = BTRFS_EXTENT_ITEM_KEY;
5219                         key.offset = num_bytes;
5220
5221                         ret = btrfs_search_slot(trans, extent_root,
5222                                                 &key, path, -1, 1);
5223                         if (ret) {
5224                                 printk(KERN_ERR "umm, got %d back from search"
5225                                        ", was looking for %llu\n", ret,
5226                                        (unsigned long long)bytenr);
5227                                 if (ret > 0)
5228                                         btrfs_print_leaf(extent_root,
5229                                                          path->nodes[0]);
5230                         }
5231                         if (ret < 0) {
5232                                 btrfs_abort_transaction(trans, extent_root, ret);
5233                                 goto out;
5234                         }
5235                         extent_slot = path->slots[0];
5236                 }
5237         } else if (ret == -ENOENT) {
5238                 btrfs_print_leaf(extent_root, path->nodes[0]);
5239                 WARN_ON(1);
5240                 printk(KERN_ERR "btrfs unable to find ref byte nr %llu "
5241                        "parent %llu root %llu  owner %llu offset %llu\n",
5242                        (unsigned long long)bytenr,
5243                        (unsigned long long)parent,
5244                        (unsigned long long)root_objectid,
5245                        (unsigned long long)owner_objectid,
5246                        (unsigned long long)owner_offset);
5247         } else {
5248                 btrfs_abort_transaction(trans, extent_root, ret);
5249                 goto out;
5250         }
5251
5252         leaf = path->nodes[0];
5253         item_size = btrfs_item_size_nr(leaf, extent_slot);
5254 #ifdef BTRFS_COMPAT_EXTENT_TREE_V0
5255         if (item_size < sizeof(*ei)) {
5256                 BUG_ON(found_extent || extent_slot != path->slots[0]);
5257                 ret = convert_extent_item_v0(trans, extent_root, path,
5258                                              owner_objectid, 0);
5259                 if (ret < 0) {
5260                         btrfs_abort_transaction(trans, extent_root, ret);
5261                         goto out;
5262                 }
5263
5264                 btrfs_release_path(path);
5265                 path->leave_spinning = 1;
5266
5267                 key.objectid = bytenr;
5268                 key.type = BTRFS_EXTENT_ITEM_KEY;
5269                 key.offset = num_bytes;
5270
5271                 ret = btrfs_search_slot(trans, extent_root, &key, path,
5272                                         -1, 1);
5273                 if (ret) {
5274                         printk(KERN_ERR "umm, got %d back from search"
5275                                ", was looking for %llu\n", ret,
5276                                (unsigned long long)bytenr);
5277                         btrfs_print_leaf(extent_root, path->nodes[0]);
5278                 }
5279                 if (ret < 0) {
5280                         btrfs_abort_transaction(trans, extent_root, ret);
5281                         goto out;
5282                 }
5283
5284                 extent_slot = path->slots[0];
5285                 leaf = path->nodes[0];
5286                 item_size = btrfs_item_size_nr(leaf, extent_slot);
5287         }
5288 #endif
5289         BUG_ON(item_size < sizeof(*ei));
5290         ei = btrfs_item_ptr(leaf, extent_slot,
5291                             struct btrfs_extent_item);
5292         if (owner_objectid < BTRFS_FIRST_FREE_OBJECTID) {
5293                 struct btrfs_tree_block_info *bi;
5294                 BUG_ON(item_size < sizeof(*ei) + sizeof(*bi));
5295                 bi = (struct btrfs_tree_block_info *)(ei + 1);
5296                 WARN_ON(owner_objectid != btrfs_tree_block_level(leaf, bi));
5297         }
5298
5299         refs = btrfs_extent_refs(leaf, ei);
5300         BUG_ON(refs < refs_to_drop);
5301         refs -= refs_to_drop;
5302
5303         if (refs > 0) {
5304                 if (extent_op)
5305                         __run_delayed_extent_op(extent_op, leaf, ei);
5306                 /*
5307                  * In the case of inline back ref, reference count will
5308                  * be updated by remove_extent_backref
5309                  */
5310                 if (iref) {
5311                         BUG_ON(!found_extent);
5312                 } else {
5313                         btrfs_set_extent_refs(leaf, ei, refs);
5314                         btrfs_mark_buffer_dirty(leaf);
5315                 }
5316                 if (found_extent) {
5317                         ret = remove_extent_backref(trans, extent_root, path,
5318                                                     iref, refs_to_drop,
5319                                                     is_data);
5320                         if (ret) {
5321                                 btrfs_abort_transaction(trans, extent_root, ret);
5322                                 goto out;
5323                         }
5324                 }
5325         } else {
5326                 if (found_extent) {
5327                         BUG_ON(is_data && refs_to_drop !=
5328                                extent_data_ref_count(root, path, iref));
5329                         if (iref) {
5330                                 BUG_ON(path->slots[0] != extent_slot);
5331                         } else {
5332                                 BUG_ON(path->slots[0] != extent_slot + 1);
5333                                 path->slots[0] = extent_slot;
5334                                 num_to_del = 2;
5335                         }
5336                 }
5337
5338                 ret = btrfs_del_items(trans, extent_root, path, path->slots[0],
5339                                       num_to_del);
5340                 if (ret) {
5341                         btrfs_abort_transaction(trans, extent_root, ret);
5342                         goto out;
5343                 }
5344                 btrfs_release_path(path);
5345
5346                 if (is_data) {
5347                         ret = btrfs_del_csums(trans, root, bytenr, num_bytes);
5348                         if (ret) {
5349                                 btrfs_abort_transaction(trans, extent_root, ret);
5350                                 goto out;
5351                         }
5352                 }
5353
5354                 ret = update_block_group(trans, root, bytenr, num_bytes, 0);
5355                 if (ret) {
5356                         btrfs_abort_transaction(trans, extent_root, ret);
5357                         goto out;
5358                 }
5359         }
5360 out:
5361         btrfs_free_path(path);
5362         return ret;
5363 }
5364
5365 /*
5366  * when we free an block, it is possible (and likely) that we free the last
5367  * delayed ref for that extent as well.  This searches the delayed ref tree for
5368  * a given extent, and if there are no other delayed refs to be processed, it
5369  * removes it from the tree.
5370  */
5371 static noinline int check_ref_cleanup(struct btrfs_trans_handle *trans,
5372                                       struct btrfs_root *root, u64 bytenr)
5373 {
5374         struct btrfs_delayed_ref_head *head;
5375         struct btrfs_delayed_ref_root *delayed_refs;
5376         struct btrfs_delayed_ref_node *ref;
5377         struct rb_node *node;
5378         int ret = 0;
5379
5380         delayed_refs = &trans->transaction->delayed_refs;
5381         spin_lock(&delayed_refs->lock);
5382         head = btrfs_find_delayed_ref_head(trans, bytenr);
5383         if (!head)
5384                 goto out;
5385
5386         node = rb_prev(&head->node.rb_node);
5387         if (!node)
5388                 goto out;
5389
5390         ref = rb_entry(node, struct btrfs_delayed_ref_node, rb_node);
5391
5392         /* there are still entries for this ref, we can't drop it */
5393         if (ref->bytenr == bytenr)
5394                 goto out;
5395
5396         if (head->extent_op) {
5397                 if (!head->must_insert_reserved)
5398                         goto out;
5399                 kfree(head->extent_op);
5400                 head->extent_op = NULL;
5401         }
5402
5403         /*
5404          * waiting for the lock here would deadlock.  If someone else has it
5405          * locked they are already in the process of dropping it anyway
5406          */
5407         if (!mutex_trylock(&head->mutex))
5408                 goto out;
5409
5410         /*
5411          * at this point we have a head with no other entries.  Go
5412          * ahead and process it.
5413          */
5414         head->node.in_tree = 0;
5415         rb_erase(&head->node.rb_node, &delayed_refs->root);
5416
5417         delayed_refs->num_entries--;
5418
5419         /*
5420          * we don't take a ref on the node because we're removing it from the
5421          * tree, so we just steal the ref the tree was holding.
5422          */
5423         delayed_refs->num_heads--;
5424         if (list_empty(&head->cluster))
5425                 delayed_refs->num_heads_ready--;
5426
5427         list_del_init(&head->cluster);
5428         spin_unlock(&delayed_refs->lock);
5429
5430         BUG_ON(head->extent_op);
5431         if (head->must_insert_reserved)
5432                 ret = 1;
5433
5434         mutex_unlock(&head->mutex);
5435         btrfs_put_delayed_ref(&head->node);
5436         return ret;
5437 out:
5438         spin_unlock(&delayed_refs->lock);
5439         return 0;
5440 }
5441
5442 void btrfs_free_tree_block(struct btrfs_trans_handle *trans,
5443                            struct btrfs_root *root,
5444                            struct extent_buffer *buf,
5445                            u64 parent, int last_ref)
5446 {
5447         struct btrfs_block_group_cache *cache = NULL;
5448         int ret;
5449
5450         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5451                 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
5452                                         buf->start, buf->len,
5453                                         parent, root->root_key.objectid,
5454                                         btrfs_header_level(buf),
5455                                         BTRFS_DROP_DELAYED_REF, NULL, 0);
5456                 BUG_ON(ret); /* -ENOMEM */
5457         }
5458
5459         if (!last_ref)
5460                 return;
5461
5462         cache = btrfs_lookup_block_group(root->fs_info, buf->start);
5463
5464         if (btrfs_header_generation(buf) == trans->transid) {
5465                 if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
5466                         ret = check_ref_cleanup(trans, root, buf->start);
5467                         if (!ret)
5468                                 goto out;
5469                 }
5470
5471                 if (btrfs_header_flag(buf, BTRFS_HEADER_FLAG_WRITTEN)) {
5472                         pin_down_extent(root, cache, buf->start, buf->len, 1);
5473                         goto out;
5474                 }
5475
5476                 WARN_ON(test_bit(EXTENT_BUFFER_DIRTY, &buf->bflags));
5477
5478                 btrfs_add_free_space(cache, buf->start, buf->len);
5479                 btrfs_update_reserved_bytes(cache, buf->len, RESERVE_FREE);
5480         }
5481 out:
5482         /*
5483          * Deleting the buffer, clear the corrupt flag since it doesn't matter
5484          * anymore.
5485          */
5486         clear_bit(EXTENT_BUFFER_CORRUPT, &buf->bflags);
5487         btrfs_put_block_group(cache);
5488 }
5489
5490 /* Can return -ENOMEM */
5491 int btrfs_free_extent(struct btrfs_trans_handle *trans, struct btrfs_root *root,
5492                       u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid,
5493                       u64 owner, u64 offset, int for_cow)
5494 {
5495         int ret;
5496         struct btrfs_fs_info *fs_info = root->fs_info;
5497
5498         /*
5499          * tree log blocks never actually go into the extent allocation
5500          * tree, just update pinning info and exit early.
5501          */
5502         if (root_objectid == BTRFS_TREE_LOG_OBJECTID) {
5503                 WARN_ON(owner >= BTRFS_FIRST_FREE_OBJECTID);
5504                 /* unlocks the pinned mutex */
5505                 btrfs_pin_extent(root, bytenr, num_bytes, 1);
5506                 ret = 0;
5507         } else if (owner < BTRFS_FIRST_FREE_OBJECTID) {
5508                 ret = btrfs_add_delayed_tree_ref(fs_info, trans, bytenr,
5509                                         num_bytes,
5510                                         parent, root_objectid, (int)owner,
5511                                         BTRFS_DROP_DELAYED_REF, NULL, for_cow);
5512         } else {
5513                 ret = btrfs_add_delayed_data_ref(fs_info, trans, bytenr,
5514                                                 num_bytes,
5515                                                 parent, root_objectid, owner,
5516                                                 offset, BTRFS_DROP_DELAYED_REF,
5517                                                 NULL, for_cow);
5518         }
5519         return ret;
5520 }
5521
5522 static u64 stripe_align(struct btrfs_root *root,
5523                         struct btrfs_block_group_cache *cache,
5524                         u64 val, u64 num_bytes)
5525 {
5526         u64 mask;
5527         u64 ret;
5528         mask = ((u64)root->stripesize - 1);
5529         ret = (val + mask) & ~mask;
5530         return ret;
5531 }
5532
5533 /*
5534  * when we wait for progress in the block group caching, its because
5535  * our allocation attempt failed at least once.  So, we must sleep
5536  * and let some progress happen before we try again.
5537  *
5538  * This function will sleep at least once waiting for new free space to
5539  * show up, and then it will check the block group free space numbers
5540  * for our min num_bytes.  Another option is to have it go ahead
5541  * and look in the rbtree for a free extent of a given size, but this
5542  * is a good start.
5543  */
5544 static noinline int
5545 wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
5546                                 u64 num_bytes)
5547 {
5548         struct btrfs_caching_control *caching_ctl;
5549         DEFINE_WAIT(wait);
5550
5551         caching_ctl = get_caching_control(cache);
5552         if (!caching_ctl)
5553                 return 0;
5554
5555         wait_event(caching_ctl->wait, block_group_cache_done(cache) ||
5556                    (cache->free_space_ctl->free_space >= num_bytes));
5557
5558         put_caching_control(caching_ctl);
5559         return 0;
5560 }
5561
5562 static noinline int
5563 wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
5564 {
5565         struct btrfs_caching_control *caching_ctl;
5566         DEFINE_WAIT(wait);
5567
5568         caching_ctl = get_caching_control(cache);
5569         if (!caching_ctl)
5570                 return 0;
5571
5572         wait_event(caching_ctl->wait, block_group_cache_done(cache));
5573
5574         put_caching_control(caching_ctl);
5575         return 0;
5576 }
5577
5578 int __get_raid_index(u64 flags)
5579 {
5580         int index;
5581
5582         if (flags & BTRFS_BLOCK_GROUP_RAID10)
5583                 index = 0;
5584         else if (flags & BTRFS_BLOCK_GROUP_RAID1)
5585                 index = 1;
5586         else if (flags & BTRFS_BLOCK_GROUP_DUP)
5587                 index = 2;
5588         else if (flags & BTRFS_BLOCK_GROUP_RAID0)
5589                 index = 3;
5590         else if (flags & BTRFS_BLOCK_GROUP_RAID5)
5591                 index = 5;
5592         else if (flags & BTRFS_BLOCK_GROUP_RAID6)
5593                 index = 6;
5594         else
5595                 index = 4; /* BTRFS_BLOCK_GROUP_SINGLE */
5596         return index;
5597 }
5598
5599 static int get_block_group_index(struct btrfs_block_group_cache *cache)
5600 {
5601         return __get_raid_index(cache->flags);
5602 }
5603
5604 enum btrfs_loop_type {
5605         LOOP_CACHING_NOWAIT = 0,
5606         LOOP_CACHING_WAIT = 1,
5607         LOOP_ALLOC_CHUNK = 2,
5608         LOOP_NO_EMPTY_SIZE = 3,
5609 };
5610
5611 /*
5612  * walks the btree of allocated extents and find a hole of a given size.
5613  * The key ins is changed to record the hole:
5614  * ins->objectid == block start
5615  * ins->flags = BTRFS_EXTENT_ITEM_KEY
5616  * ins->offset == number of blocks
5617  * Any available blocks before search_start are skipped.
5618  */
5619 static noinline int find_free_extent(struct btrfs_trans_handle *trans,
5620                                      struct btrfs_root *orig_root,
5621                                      u64 num_bytes, u64 empty_size,
5622                                      u64 hint_byte, struct btrfs_key *ins,
5623                                      u64 data)
5624 {
5625         int ret = 0;
5626         struct btrfs_root *root = orig_root->fs_info->extent_root;
5627         struct btrfs_free_cluster *last_ptr = NULL;
5628         struct btrfs_block_group_cache *block_group = NULL;
5629         struct btrfs_block_group_cache *used_block_group;
5630         u64 search_start = 0;
5631         int empty_cluster = 2 * 1024 * 1024;
5632         struct btrfs_space_info *space_info;
5633         int loop = 0;
5634         int index = __get_raid_index(data);
5635         int alloc_type = (data & BTRFS_BLOCK_GROUP_DATA) ?
5636                 RESERVE_ALLOC_NO_ACCOUNT : RESERVE_ALLOC;
5637         bool found_uncached_bg = false;
5638         bool failed_cluster_refill = false;
5639         bool failed_alloc = false;
5640         bool use_cluster = true;
5641         bool have_caching_bg = false;
5642
5643         WARN_ON(num_bytes < root->sectorsize);
5644         btrfs_set_key_type(ins, BTRFS_EXTENT_ITEM_KEY);
5645         ins->objectid = 0;
5646         ins->offset = 0;
5647
5648         trace_find_free_extent(orig_root, num_bytes, empty_size, data);
5649
5650         space_info = __find_space_info(root->fs_info, data);
5651         if (!space_info) {
5652                 printk(KERN_ERR "No space info for %llu\n", data);
5653                 return -ENOSPC;
5654         }
5655
5656         /*
5657          * If the space info is for both data and metadata it means we have a
5658          * small filesystem and we can't use the clustering stuff.
5659          */
5660         if (btrfs_mixed_space_info(space_info))
5661                 use_cluster = false;
5662
5663         if (data & BTRFS_BLOCK_GROUP_METADATA && use_cluster) {
5664                 last_ptr = &root->fs_info->meta_alloc_cluster;
5665                 if (!btrfs_test_opt(root, SSD))
5666                         empty_cluster = 64 * 1024;
5667         }
5668
5669         if ((data & BTRFS_BLOCK_GROUP_DATA) && use_cluster &&
5670             btrfs_test_opt(root, SSD)) {
5671                 last_ptr = &root->fs_info->data_alloc_cluster;
5672         }
5673
5674         if (last_ptr) {
5675                 spin_lock(&last_ptr->lock);
5676                 if (last_ptr->block_group)
5677                         hint_byte = last_ptr->window_start;
5678                 spin_unlock(&last_ptr->lock);
5679         }
5680
5681         search_start = max(search_start, first_logical_byte(root, 0));
5682         search_start = max(search_start, hint_byte);
5683
5684         if (!last_ptr)
5685                 empty_cluster = 0;
5686
5687         if (search_start == hint_byte) {
5688                 block_group = btrfs_lookup_block_group(root->fs_info,
5689                                                        search_start);
5690                 used_block_group = block_group;
5691                 /*
5692                  * we don't want to use the block group if it doesn't match our
5693                  * allocation bits, or if its not cached.
5694                  *
5695                  * However if we are re-searching with an ideal block group
5696                  * picked out then we don't care that the block group is cached.
5697                  */
5698                 if (block_group && block_group_bits(block_group, data) &&
5699                     block_group->cached != BTRFS_CACHE_NO) {
5700                         down_read(&space_info->groups_sem);
5701                         if (list_empty(&block_group->list) ||
5702                             block_group->ro) {
5703                                 /*
5704                                  * someone is removing this block group,
5705                                  * we can't jump into the have_block_group
5706                                  * target because our list pointers are not
5707                                  * valid
5708                                  */
5709                                 btrfs_put_block_group(block_group);
5710                                 up_read(&space_info->groups_sem);
5711                         } else {
5712                                 index = get_block_group_index(block_group);
5713                                 goto have_block_group;
5714                         }
5715                 } else if (block_group) {
5716                         btrfs_put_block_group(block_group);
5717                 }
5718         }
5719 search:
5720         have_caching_bg = false;
5721         down_read(&space_info->groups_sem);
5722         list_for_each_entry(block_group, &space_info->block_groups[index],
5723                             list) {
5724                 u64 offset;
5725                 int cached;
5726
5727                 used_block_group = block_group;
5728                 btrfs_get_block_group(block_group);
5729                 search_start = block_group->key.objectid;
5730
5731                 /*
5732                  * this can happen if we end up cycling through all the
5733                  * raid types, but we want to make sure we only allocate
5734                  * for the proper type.
5735                  */
5736                 if (!block_group_bits(block_group, data)) {
5737                     u64 extra = BTRFS_BLOCK_GROUP_DUP |
5738                                 BTRFS_BLOCK_GROUP_RAID1 |
5739                                 BTRFS_BLOCK_GROUP_RAID5 |
5740                                 BTRFS_BLOCK_GROUP_RAID6 |
5741                                 BTRFS_BLOCK_GROUP_RAID10;
5742
5743                         /*
5744                          * if they asked for extra copies and this block group
5745                          * doesn't provide them, bail.  This does allow us to
5746                          * fill raid0 from raid1.
5747                          */
5748                         if ((data & extra) && !(block_group->flags & extra))
5749                                 goto loop;
5750                 }
5751
5752 have_block_group:
5753                 cached = block_group_cache_done(block_group);
5754                 if (unlikely(!cached)) {
5755                         found_uncached_bg = true;
5756                         ret = cache_block_group(block_group, trans,
5757                                                 orig_root, 0);
5758                         BUG_ON(ret < 0);
5759                         ret = 0;
5760                 }
5761
5762                 if (unlikely(block_group->ro))
5763                         goto loop;
5764
5765                 /*
5766                  * Ok we want to try and use the cluster allocator, so
5767                  * lets look there
5768                  */
5769                 if (last_ptr) {
5770                         unsigned long aligned_cluster;
5771                         /*
5772                          * the refill lock keeps out other
5773                          * people trying to start a new cluster
5774                          */
5775                         spin_lock(&last_ptr->refill_lock);
5776                         used_block_group = last_ptr->block_group;
5777                         if (used_block_group != block_group &&
5778                             (!used_block_group ||
5779                              used_block_group->ro ||
5780                              !block_group_bits(used_block_group, data))) {
5781                                 used_block_group = block_group;
5782                                 goto refill_cluster;
5783                         }
5784
5785                         if (used_block_group != block_group)
5786                                 btrfs_get_block_group(used_block_group);
5787
5788                         offset = btrfs_alloc_from_cluster(used_block_group,
5789                           last_ptr, num_bytes, used_block_group->key.objectid);
5790                         if (offset) {
5791                                 /* we have a block, we're done */
5792                                 spin_unlock(&last_ptr->refill_lock);
5793                                 trace_btrfs_reserve_extent_cluster(root,
5794                                         block_group, search_start, num_bytes);
5795                                 goto checks;
5796                         }
5797
5798                         WARN_ON(last_ptr->block_group != used_block_group);
5799                         if (used_block_group != block_group) {
5800                                 btrfs_put_block_group(used_block_group);
5801                                 used_block_group = block_group;
5802                         }
5803 refill_cluster:
5804                         BUG_ON(used_block_group != block_group);
5805                         /* If we are on LOOP_NO_EMPTY_SIZE, we can't
5806                          * set up a new clusters, so lets just skip it
5807                          * and let the allocator find whatever block
5808                          * it can find.  If we reach this point, we
5809                          * will have tried the cluster allocator
5810                          * plenty of times and not have found
5811                          * anything, so we are likely way too
5812                          * fragmented for the clustering stuff to find
5813                          * anything.
5814                          *
5815                          * However, if the cluster is taken from the
5816                          * current block group, release the cluster
5817                          * first, so that we stand a better chance of
5818                          * succeeding in the unclustered
5819                          * allocation.  */
5820                         if (loop >= LOOP_NO_EMPTY_SIZE &&
5821                             last_ptr->block_group != block_group) {
5822                                 spin_unlock(&last_ptr->refill_lock);
5823                                 goto unclustered_alloc;
5824                         }
5825
5826                         /*
5827                          * this cluster didn't work out, free it and
5828                          * start over
5829                          */
5830                         btrfs_return_cluster_to_free_space(NULL, last_ptr);
5831
5832                         if (loop >= LOOP_NO_EMPTY_SIZE) {
5833                                 spin_unlock(&last_ptr->refill_lock);
5834                                 goto unclustered_alloc;
5835                         }
5836
5837                         aligned_cluster = max_t(unsigned long,
5838                                                 empty_cluster + empty_size,
5839                                               block_group->full_stripe_len);
5840
5841                         /* allocate a cluster in this block group */
5842                         ret = btrfs_find_space_cluster(trans, root,
5843                                                block_group, last_ptr,
5844                                                search_start, num_bytes,
5845                                                aligned_cluster);
5846                         if (ret == 0) {
5847                                 /*
5848                                  * now pull our allocation out of this
5849                                  * cluster
5850                                  */
5851                                 offset = btrfs_alloc_from_cluster(block_group,
5852                                                   last_ptr, num_bytes,
5853                                                   search_start);
5854                                 if (offset) {
5855                                         /* we found one, proceed */
5856                                         spin_unlock(&last_ptr->refill_lock);
5857                                         trace_btrfs_reserve_extent_cluster(root,
5858                                                 block_group, search_start,
5859                                                 num_bytes);
5860                                         goto checks;
5861                                 }
5862                         } else if (!cached && loop > LOOP_CACHING_NOWAIT
5863                                    && !failed_cluster_refill) {
5864                                 spin_unlock(&last_ptr->refill_lock);
5865
5866                                 failed_cluster_refill = true;
5867                                 wait_block_group_cache_progress(block_group,
5868                                        num_bytes + empty_cluster + empty_size);
5869                                 goto have_block_group;
5870                         }
5871
5872                         /*
5873                          * at this point we either didn't find a cluster
5874                          * or we weren't able to allocate a block from our
5875                          * cluster.  Free the cluster we've been trying
5876                          * to use, and go to the next block group
5877                          */
5878                         btrfs_return_cluster_to_free_space(NULL, last_ptr);
5879                         spin_unlock(&last_ptr->refill_lock);
5880                         goto loop;
5881                 }
5882
5883 unclustered_alloc:
5884                 spin_lock(&block_group->free_space_ctl->tree_lock);
5885                 if (cached &&
5886                     block_group->free_space_ctl->free_space <
5887                     num_bytes + empty_cluster + empty_size) {
5888                         spin_unlock(&block_group->free_space_ctl->tree_lock);
5889                         goto loop;
5890                 }
5891                 spin_unlock(&block_group->free_space_ctl->tree_lock);
5892
5893                 offset = btrfs_find_space_for_alloc(block_group, search_start,
5894                                                     num_bytes, empty_size);
5895                 /*
5896                  * If we didn't find a chunk, and we haven't failed on this
5897                  * block group before, and this block group is in the middle of
5898                  * caching and we are ok with waiting, then go ahead and wait
5899                  * for progress to be made, and set failed_alloc to true.
5900                  *
5901                  * If failed_alloc is true then we've already waited on this
5902                  * block group once and should move on to the next block group.
5903                  */
5904                 if (!offset && !failed_alloc && !cached &&
5905                     loop > LOOP_CACHING_NOWAIT) {
5906                         wait_block_group_cache_progress(block_group,
5907                                                 num_bytes + empty_size);
5908                         failed_alloc = true;
5909                         goto have_block_group;
5910                 } else if (!offset) {
5911                         if (!cached)
5912                                 have_caching_bg = true;
5913                         goto loop;
5914                 }
5915 checks:
5916                 search_start = stripe_align(root, used_block_group,
5917                                             offset, num_bytes);
5918
5919                 /* move on to the next group */
5920                 if (search_start + num_bytes >
5921                     used_block_group->key.objectid + used_block_group->key.offset) {
5922                         btrfs_add_free_space(used_block_group, offset, num_bytes);
5923                         goto loop;
5924                 }
5925
5926                 if (offset < search_start)
5927                         btrfs_add_free_space(used_block_group, offset,
5928                                              search_start - offset);
5929                 BUG_ON(offset > search_start);
5930
5931                 ret = btrfs_update_reserved_bytes(used_block_group, num_bytes,
5932                                                   alloc_type);
5933                 if (ret == -EAGAIN) {
5934                         btrfs_add_free_space(used_block_group, offset, num_bytes);
5935                         goto loop;
5936                 }
5937
5938                 /* we are all good, lets return */
5939                 ins->objectid = search_start;
5940                 ins->offset = num_bytes;
5941
5942                 trace_btrfs_reserve_extent(orig_root, block_group,
5943                                            search_start, num_bytes);
5944                 if (used_block_group != block_group)
5945                         btrfs_put_block_group(used_block_group);
5946                 btrfs_put_block_group(block_group);
5947                 break;
5948 loop:
5949                 failed_cluster_refill = false;
5950                 failed_alloc = false;
5951                 BUG_ON(index != get_block_group_index(block_group));
5952                 if (used_block_group != block_group)
5953                         btrfs_put_block_group(used_block_group);
5954                 btrfs_put_block_group(block_group);
5955         }
5956         up_read(&space_info->groups_sem);
5957
5958         if (!ins->objectid && loop >= LOOP_CACHING_WAIT && have_caching_bg)
5959                 goto search;
5960
5961         if (!ins->objectid && ++index < BTRFS_NR_RAID_TYPES)
5962                 goto search;
5963
5964         /*
5965          * LOOP_CACHING_NOWAIT, search partially cached block groups, kicking
5966          *                      caching kthreads as we move along
5967          * LOOP_CACHING_WAIT, search everything, and wait if our bg is caching
5968          * LOOP_ALLOC_CHUNK, force a chunk allocation and try again
5969          * LOOP_NO_EMPTY_SIZE, set empty_size and empty_cluster to 0 and try
5970          *                      again
5971          */
5972         if (!ins->objectid && loop < LOOP_NO_EMPTY_SIZE) {
5973                 index = 0;
5974                 loop++;
5975                 if (loop == LOOP_ALLOC_CHUNK) {
5976                         ret = do_chunk_alloc(trans, root, data,
5977                                              CHUNK_ALLOC_FORCE);
5978                         /*
5979                          * Do not bail out on ENOSPC since we
5980                          * can do more things.
5981                          */
5982                         if (ret < 0 && ret != -ENOSPC) {
5983                                 btrfs_abort_transaction(trans,
5984                                                         root, ret);
5985                                 goto out;
5986                         }
5987                 }
5988
5989                 if (loop == LOOP_NO_EMPTY_SIZE) {
5990                         empty_size = 0;
5991                         empty_cluster = 0;
5992                 }
5993
5994                 goto search;
5995         } else if (!ins->objectid) {
5996                 ret = -ENOSPC;
5997         } else if (ins->objectid) {
5998                 ret = 0;
5999         }
6000 out:
6001
6002         return ret;
6003 }
6004
6005 static void dump_space_info(struct btrfs_space_info *info, u64 bytes,
6006                             int dump_block_groups)
6007 {
6008         struct btrfs_block_group_cache *cache;
6009         int index = 0;
6010
6011         spin_lock(&info->lock);
6012         printk(KERN_INFO "space_info %llu has %llu free, is %sfull\n",
6013                (unsigned long long)info->flags,
6014                (unsigned long long)(info->total_bytes - info->bytes_used -
6015                                     info->bytes_pinned - info->bytes_reserved -
6016                                     info->bytes_readonly),
6017                (info->full) ? "" : "not ");
6018         printk(KERN_INFO "space_info total=%llu, used=%llu, pinned=%llu, "
6019                "reserved=%llu, may_use=%llu, readonly=%llu\n",
6020                (unsigned long long)info->total_bytes,
6021                (unsigned long long)info->bytes_used,
6022                (unsigned long long)info->bytes_pinned,
6023                (unsigned long long)info->bytes_reserved,
6024                (unsigned long long)info->bytes_may_use,
6025                (unsigned long long)info->bytes_readonly);
6026         spin_unlock(&info->lock);
6027
6028         if (!dump_block_groups)
6029                 return;
6030
6031         down_read(&info->groups_sem);
6032 again:
6033         list_for_each_entry(cache, &info->block_groups[index], list) {
6034                 spin_lock(&cache->lock);
6035                 printk(KERN_INFO "block group %llu has %llu bytes, %llu used %llu pinned %llu reserved %s\n",
6036                        (unsigned long long)cache->key.objectid,
6037                        (unsigned long long)cache->key.offset,
6038                        (unsigned long long)btrfs_block_group_used(&cache->item),
6039                        (unsigned long long)cache->pinned,
6040                        (unsigned long long)cache->reserved,
6041                        cache->ro ? "[readonly]" : "");
6042                 btrfs_dump_free_space(cache, bytes);
6043                 spin_unlock(&cache->lock);
6044         }
6045         if (++index < BTRFS_NR_RAID_TYPES)
6046                 goto again;
6047         up_read(&info->groups_sem);
6048 }
6049
6050 int btrfs_reserve_extent(struct btrfs_trans_handle *trans,
6051                          struct btrfs_root *root,
6052                          u64 num_bytes, u64 min_alloc_size,
6053                          u64 empty_size, u64 hint_byte,
6054                          struct btrfs_key *ins, u64 data)
6055 {
6056         bool final_tried = false;
6057         int ret;
6058
6059         data = btrfs_get_alloc_profile(root, data);
6060 again:
6061         WARN_ON(num_bytes < root->sectorsize);
6062         ret = find_free_extent(trans, root, num_bytes, empty_size,
6063                                hint_byte, ins, data);
6064
6065         if (ret == -ENOSPC) {
6066                 if (!final_tried) {
6067                         num_bytes = num_bytes >> 1;
6068                         num_bytes = num_bytes & ~(root->sectorsize - 1);
6069                         num_bytes = max(num_bytes, min_alloc_size);
6070                         if (num_bytes == min_alloc_size)
6071                                 final_tried = true;
6072                         goto again;
6073                 } else if (btrfs_test_opt(root, ENOSPC_DEBUG)) {
6074                         struct btrfs_space_info *sinfo;
6075
6076                         sinfo = __find_space_info(root->fs_info, data);
6077                         printk(KERN_ERR "btrfs allocation failed flags %llu, "
6078                                "wanted %llu\n", (unsigned long long)data,
6079                                (unsigned long long)num_bytes);
6080                         if (sinfo)
6081                                 dump_space_info(sinfo, num_bytes, 1);
6082                 }
6083         }
6084
6085         trace_btrfs_reserved_extent_alloc(root, ins->objectid, ins->offset);
6086
6087         return ret;
6088 }
6089
6090 static int __btrfs_free_reserved_extent(struct btrfs_root *root,
6091                                         u64 start, u64 len, int pin)
6092 {
6093         struct btrfs_block_group_cache *cache;
6094         int ret = 0;
6095
6096         cache = btrfs_lookup_block_group(root->fs_info, start);
6097         if (!cache) {
6098                 printk(KERN_ERR "Unable to find block group for %llu\n",
6099                        (unsigned long long)start);
6100                 return -ENOSPC;
6101         }
6102
6103         if (btrfs_test_opt(root, DISCARD))
6104                 ret = btrfs_discard_extent(root, start, len, NULL);
6105
6106         if (pin)
6107                 pin_down_extent(root, cache, start, len, 1);
6108         else {
6109                 btrfs_add_free_space(cache, start, len);
6110                 btrfs_update_reserved_bytes(cache, len, RESERVE_FREE);
6111         }
6112         btrfs_put_block_group(cache);
6113
6114         trace_btrfs_reserved_extent_free(root, start, len);
6115
6116         return ret;
6117 }
6118
6119 int btrfs_free_reserved_extent(struct btrfs_root *root,
6120                                         u64 start, u64 len)
6121 {
6122         return __btrfs_free_reserved_extent(root, start, len, 0);
6123 }
6124
6125 int btrfs_free_and_pin_reserved_extent(struct btrfs_root *root,
6126                                        u64 start, u64 len)
6127 {
6128         return __btrfs_free_reserved_extent(root, start, len, 1);
6129 }
6130
6131 static int alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
6132                                       struct btrfs_root *root,
6133                                       u64 parent, u64 root_objectid,
6134                                       u64 flags, u64 owner, u64 offset,
6135                                       struct btrfs_key *ins, int ref_mod)
6136 {
6137         int ret;
6138         struct btrfs_fs_info *fs_info = root->fs_info;
6139         struct btrfs_extent_item *extent_item;
6140         struct btrfs_extent_inline_ref *iref;
6141         struct btrfs_path *path;
6142         struct extent_buffer *leaf;
6143         int type;
6144         u32 size;
6145
6146         if (parent > 0)
6147                 type = BTRFS_SHARED_DATA_REF_KEY;
6148         else
6149                 type = BTRFS_EXTENT_DATA_REF_KEY;
6150
6151         size = sizeof(*extent_item) + btrfs_extent_inline_ref_size(type);
6152
6153         path = btrfs_alloc_path();
6154         if (!path)
6155                 return -ENOMEM;
6156
6157         path->leave_spinning = 1;
6158         ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
6159                                       ins, size);
6160         if (ret) {
6161                 btrfs_free_path(path);
6162                 return ret;
6163         }
6164
6165         leaf = path->nodes[0];
6166         extent_item = btrfs_item_ptr(leaf, path->slots[0],
6167                                      struct btrfs_extent_item);
6168         btrfs_set_extent_refs(leaf, extent_item, ref_mod);
6169         btrfs_set_extent_generation(leaf, extent_item, trans->transid);
6170         btrfs_set_extent_flags(leaf, extent_item,
6171                                flags | BTRFS_EXTENT_FLAG_DATA);
6172
6173         iref = (struct btrfs_extent_inline_ref *)(extent_item + 1);
6174         btrfs_set_extent_inline_ref_type(leaf, iref, type);
6175         if (parent > 0) {
6176                 struct btrfs_shared_data_ref *ref;
6177                 ref = (struct btrfs_shared_data_ref *)(iref + 1);
6178                 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
6179                 btrfs_set_shared_data_ref_count(leaf, ref, ref_mod);
6180         } else {
6181                 struct btrfs_extent_data_ref *ref;
6182                 ref = (struct btrfs_extent_data_ref *)(&iref->offset);
6183                 btrfs_set_extent_data_ref_root(leaf, ref, root_objectid);
6184                 btrfs_set_extent_data_ref_objectid(leaf, ref, owner);
6185                 btrfs_set_extent_data_ref_offset(leaf, ref, offset);
6186                 btrfs_set_extent_data_ref_count(leaf, ref, ref_mod);
6187         }
6188
6189         btrfs_mark_buffer_dirty(path->nodes[0]);
6190         btrfs_free_path(path);
6191
6192         ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6193         if (ret) { /* -ENOENT, logic error */
6194                 printk(KERN_ERR "btrfs update block group failed for %llu "
6195                        "%llu\n", (unsigned long long)ins->objectid,
6196                        (unsigned long long)ins->offset);
6197                 BUG();
6198         }
6199         return ret;
6200 }
6201
6202 static int alloc_reserved_tree_block(struct btrfs_trans_handle *trans,
6203                                      struct btrfs_root *root,
6204                                      u64 parent, u64 root_objectid,
6205                                      u64 flags, struct btrfs_disk_key *key,
6206                                      int level, struct btrfs_key *ins)
6207 {
6208         int ret;
6209         struct btrfs_fs_info *fs_info = root->fs_info;
6210         struct btrfs_extent_item *extent_item;
6211         struct btrfs_tree_block_info *block_info;
6212         struct btrfs_extent_inline_ref *iref;
6213         struct btrfs_path *path;
6214         struct extent_buffer *leaf;
6215         u32 size = sizeof(*extent_item) + sizeof(*block_info) + sizeof(*iref);
6216
6217         path = btrfs_alloc_path();
6218         if (!path)
6219                 return -ENOMEM;
6220
6221         path->leave_spinning = 1;
6222         ret = btrfs_insert_empty_item(trans, fs_info->extent_root, path,
6223                                       ins, size);
6224         if (ret) {
6225                 btrfs_free_path(path);
6226                 return ret;
6227         }
6228
6229         leaf = path->nodes[0];
6230         extent_item = btrfs_item_ptr(leaf, path->slots[0],
6231                                      struct btrfs_extent_item);
6232         btrfs_set_extent_refs(leaf, extent_item, 1);
6233         btrfs_set_extent_generation(leaf, extent_item, trans->transid);
6234         btrfs_set_extent_flags(leaf, extent_item,
6235                                flags | BTRFS_EXTENT_FLAG_TREE_BLOCK);
6236         block_info = (struct btrfs_tree_block_info *)(extent_item + 1);
6237
6238         btrfs_set_tree_block_key(leaf, block_info, key);
6239         btrfs_set_tree_block_level(leaf, block_info, level);
6240
6241         iref = (struct btrfs_extent_inline_ref *)(block_info + 1);
6242         if (parent > 0) {
6243                 BUG_ON(!(flags & BTRFS_BLOCK_FLAG_FULL_BACKREF));
6244                 btrfs_set_extent_inline_ref_type(leaf, iref,
6245                                                  BTRFS_SHARED_BLOCK_REF_KEY);
6246                 btrfs_set_extent_inline_ref_offset(leaf, iref, parent);
6247         } else {
6248                 btrfs_set_extent_inline_ref_type(leaf, iref,
6249                                                  BTRFS_TREE_BLOCK_REF_KEY);
6250                 btrfs_set_extent_inline_ref_offset(leaf, iref, root_objectid);
6251         }
6252
6253         btrfs_mark_buffer_dirty(leaf);
6254         btrfs_free_path(path);
6255
6256         ret = update_block_group(trans, root, ins->objectid, ins->offset, 1);
6257         if (ret) { /* -ENOENT, logic error */
6258                 printk(KERN_ERR "btrfs update block group failed for %llu "
6259                        "%llu\n", (unsigned long long)ins->objectid,
6260                        (unsigned long long)ins->offset);
6261                 BUG();
6262         }
6263         return ret;
6264 }
6265
6266 int btrfs_alloc_reserved_file_extent(struct btrfs_trans_handle *trans,
6267                                      struct btrfs_root *root,
6268                                      u64 root_objectid, u64 owner,
6269                                      u64 offset, struct btrfs_key *ins)
6270 {
6271         int ret;
6272
6273         BUG_ON(root_objectid == BTRFS_TREE_LOG_OBJECTID);
6274
6275         ret = btrfs_add_delayed_data_ref(root->fs_info, trans, ins->objectid,
6276                                          ins->offset, 0,
6277                                          root_objectid, owner, offset,
6278                                          BTRFS_ADD_DELAYED_EXTENT, NULL, 0);
6279         return ret;
6280 }
6281
6282 /*
6283  * this is used by the tree logging recovery code.  It records that
6284  * an extent has been allocated and makes sure to clear the free
6285  * space cache bits as well
6286  */
6287 int btrfs_alloc_logged_file_extent(struct btrfs_trans_handle *trans,
6288                                    struct btrfs_root *root,
6289                                    u64 root_objectid, u64 owner, u64 offset,
6290                                    struct btrfs_key *ins)
6291 {
6292         int ret;
6293         struct btrfs_block_group_cache *block_group;
6294         struct btrfs_caching_control *caching_ctl;
6295         u64 start = ins->objectid;
6296         u64 num_bytes = ins->offset;
6297
6298         block_group = btrfs_lookup_block_group(root->fs_info, ins->objectid);
6299         cache_block_group(block_group, trans, NULL, 0);
6300         caching_ctl = get_caching_control(block_group);
6301
6302         if (!caching_ctl) {
6303                 BUG_ON(!block_group_cache_done(block_group));
6304                 ret = btrfs_remove_free_space(block_group, start, num_bytes);
6305                 BUG_ON(ret); /* -ENOMEM */
6306         } else {
6307                 mutex_lock(&caching_ctl->mutex);
6308
6309                 if (start >= caching_ctl->progress) {
6310                         ret = add_excluded_extent(root, start, num_bytes);
6311                         BUG_ON(ret); /* -ENOMEM */
6312                 } else if (start + num_bytes <= caching_ctl->progress) {
6313                         ret = btrfs_remove_free_space(block_group,
6314                                                       start, num_bytes);
6315                         BUG_ON(ret); /* -ENOMEM */
6316                 } else {
6317                         num_bytes = caching_ctl->progress - start;
6318                         ret = btrfs_remove_free_space(block_group,
6319                                                       start, num_bytes);
6320                         BUG_ON(ret); /* -ENOMEM */
6321
6322                         start = caching_ctl->progress;
6323                         num_bytes = ins->objectid + ins->offset -
6324                                     caching_ctl->progress;
6325                         ret = add_excluded_extent(root, start, num_bytes);
6326                         BUG_ON(ret); /* -ENOMEM */
6327                 }
6328
6329                 mutex_unlock(&caching_ctl->mutex);
6330                 put_caching_control(caching_ctl);
6331         }
6332
6333         ret = btrfs_update_reserved_bytes(block_group, ins->offset,
6334                                           RESERVE_ALLOC_NO_ACCOUNT);
6335         BUG_ON(ret); /* logic error */
6336         btrfs_put_block_group(block_group);
6337         ret = alloc_reserved_file_extent(trans, root, 0, root_objectid,
6338                                          0, owner, offset, ins, 1);
6339         return ret;
6340 }
6341
6342 struct extent_buffer *btrfs_init_new_buffer(struct btrfs_trans_handle *trans,
6343                                             struct btrfs_root *root,
6344                                             u64 bytenr, u32 blocksize,
6345                                             int level)
6346 {
6347         struct extent_buffer *buf;
6348
6349         buf = btrfs_find_create_tree_block(root, bytenr, blocksize);
6350         if (!buf)
6351                 return ERR_PTR(-ENOMEM);
6352         btrfs_set_header_generation(buf, trans->transid);
6353         btrfs_set_buffer_lockdep_class(root->root_key.objectid, buf, level);
6354         btrfs_tree_lock(buf);
6355         clean_tree_block(trans, root, buf);
6356         clear_bit(EXTENT_BUFFER_STALE, &buf->bflags);
6357
6358         btrfs_set_lock_blocking(buf);
6359         btrfs_set_buffer_uptodate(buf);
6360
6361         if (root->root_key.objectid == BTRFS_TREE_LOG_OBJECTID) {
6362                 /*
6363                  * we allow two log transactions at a time, use different
6364                  * EXENT bit to differentiate dirty pages.
6365                  */
6366                 if (root->log_transid % 2 == 0)
6367                         set_extent_dirty(&root->dirty_log_pages, buf->start,
6368                                         buf->start + buf->len - 1, GFP_NOFS);
6369                 else
6370                         set_extent_new(&root->dirty_log_pages, buf->start,
6371                                         buf->start + buf->len - 1, GFP_NOFS);
6372         } else {
6373                 set_extent_dirty(&trans->transaction->dirty_pages, buf->start,
6374                          buf->start + buf->len - 1, GFP_NOFS);
6375         }
6376         trans->blocks_used++;
6377         /* this returns a buffer locked for blocking */
6378         return buf;
6379 }
6380
6381 static struct btrfs_block_rsv *
6382 use_block_rsv(struct btrfs_trans_handle *trans,
6383               struct btrfs_root *root, u32 blocksize)
6384 {
6385         struct btrfs_block_rsv *block_rsv;
6386         struct btrfs_block_rsv *global_rsv = &root->fs_info->global_block_rsv;
6387         int ret;
6388
6389         block_rsv = get_block_rsv(trans, root);
6390
6391         if (block_rsv->size == 0) {
6392                 ret = reserve_metadata_bytes(root, block_rsv, blocksize,
6393                                              BTRFS_RESERVE_NO_FLUSH);
6394                 /*
6395                  * If we couldn't reserve metadata bytes try and use some from
6396                  * the global reserve.
6397                  */
6398                 if (ret && block_rsv != global_rsv) {
6399                         ret = block_rsv_use_bytes(global_rsv, blocksize);
6400                         if (!ret)
6401                                 return global_rsv;
6402                         return ERR_PTR(ret);
6403                 } else if (ret) {
6404                         return ERR_PTR(ret);
6405                 }
6406                 return block_rsv;
6407         }
6408
6409         ret = block_rsv_use_bytes(block_rsv, blocksize);
6410         if (!ret)
6411                 return block_rsv;
6412         if (ret && !block_rsv->failfast) {
6413                 static DEFINE_RATELIMIT_STATE(_rs,
6414                                 DEFAULT_RATELIMIT_INTERVAL,
6415                                 /*DEFAULT_RATELIMIT_BURST*/ 2);
6416                 if (__ratelimit(&_rs))
6417                         WARN(1, KERN_DEBUG "btrfs: block rsv returned %d\n",
6418                              ret);
6419                 ret = reserve_metadata_bytes(root, block_rsv, blocksize,
6420                                              BTRFS_RESERVE_NO_FLUSH);
6421                 if (!ret) {
6422                         return block_rsv;
6423                 } else if (ret && block_rsv != global_rsv) {
6424                         ret = block_rsv_use_bytes(global_rsv, blocksize);
6425                         if (!ret)
6426                                 return global_rsv;
6427                 }
6428         }
6429
6430         return ERR_PTR(-ENOSPC);
6431 }
6432
6433 static void unuse_block_rsv(struct btrfs_fs_info *fs_info,
6434                             struct btrfs_block_rsv *block_rsv, u32 blocksize)
6435 {
6436         block_rsv_add_bytes(block_rsv, blocksize, 0);
6437         block_rsv_release_bytes(fs_info, block_rsv, NULL, 0);
6438 }
6439
6440 /*
6441  * finds a free extent and does all the dirty work required for allocation
6442  * returns the key for the extent through ins, and a tree buffer for
6443  * the first block of the extent through buf.
6444  *
6445  * returns the tree buffer or NULL.
6446  */
6447 struct extent_buffer *btrfs_alloc_free_block(struct btrfs_trans_handle *trans,
6448                                         struct btrfs_root *root, u32 blocksize,
6449                                         u64 parent, u64 root_objectid,
6450                                         struct btrfs_disk_key *key, int level,
6451                                         u64 hint, u64 empty_size)
6452 {
6453         struct btrfs_key ins;
6454         struct btrfs_block_rsv *block_rsv;
6455         struct extent_buffer *buf;
6456         u64 flags = 0;
6457         int ret;
6458
6459
6460         block_rsv = use_block_rsv(trans, root, blocksize);
6461         if (IS_ERR(block_rsv))
6462                 return ERR_CAST(block_rsv);
6463
6464         ret = btrfs_reserve_extent(trans, root, blocksize, blocksize,
6465                                    empty_size, hint, &ins, 0);
6466         if (ret) {
6467                 unuse_block_rsv(root->fs_info, block_rsv, blocksize);
6468                 return ERR_PTR(ret);
6469         }
6470
6471         buf = btrfs_init_new_buffer(trans, root, ins.objectid,
6472                                     blocksize, level);
6473         BUG_ON(IS_ERR(buf)); /* -ENOMEM */
6474
6475         if (root_objectid == BTRFS_TREE_RELOC_OBJECTID) {
6476                 if (parent == 0)
6477                         parent = ins.objectid;
6478                 flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF;
6479         } else
6480                 BUG_ON(parent > 0);
6481
6482         if (root_objectid != BTRFS_TREE_LOG_OBJECTID) {
6483                 struct btrfs_delayed_extent_op *extent_op;
6484                 extent_op = kmalloc(sizeof(*extent_op), GFP_NOFS);
6485                 BUG_ON(!extent_op); /* -ENOMEM */
6486                 if (key)
6487                         memcpy(&extent_op->key, key, sizeof(extent_op->key));
6488                 else
6489                         memset(&extent_op->key, 0, sizeof(extent_op->key));
6490                 extent_op->flags_to_set = flags;
6491                 extent_op->update_key = 1;
6492                 extent_op->update_flags = 1;
6493                 extent_op->is_data = 0;
6494
6495                 ret = btrfs_add_delayed_tree_ref(root->fs_info, trans,
6496                                         ins.objectid,
6497                                         ins.offset, parent, root_objectid,
6498                                         level, BTRFS_ADD_DELAYED_EXTENT,
6499                                         extent_op, 0);
6500                 BUG_ON(ret); /* -ENOMEM */
6501         }
6502         return buf;
6503 }
6504
6505 struct walk_control {
6506         u64 refs[BTRFS_MAX_LEVEL];
6507         u64 flags[BTRFS_MAX_LEVEL];
6508         struct btrfs_key update_progress;
6509         int stage;
6510         int level;
6511         int shared_level;
6512         int update_ref;
6513         int keep_locks;
6514         int reada_slot;
6515         int reada_count;
6516         int for_reloc;
6517 };
6518
6519 #define DROP_REFERENCE  1
6520 #define UPDATE_BACKREF  2
6521
6522 static noinline void reada_walk_down(struct btrfs_trans_handle *trans,
6523                                      struct btrfs_root *root,
6524                                      struct walk_control *wc,
6525                                      struct btrfs_path *path)
6526 {
6527         u64 bytenr;
6528         u64 generation;
6529         u64 refs;
6530         u64 flags;
6531         u32 nritems;
6532         u32 blocksize;
6533         struct btrfs_key key;
6534         struct extent_buffer *eb;
6535         int ret;
6536         int slot;
6537         int nread = 0;
6538
6539         if (path->slots[wc->level] < wc->reada_slot) {
6540                 wc->reada_count = wc->reada_count * 2 / 3;
6541                 wc->reada_count = max(wc->reada_count, 2);
6542         } else {
6543                 wc->reada_count = wc->reada_count * 3 / 2;
6544                 wc->reada_count = min_t(int, wc->reada_count,
6545                                         BTRFS_NODEPTRS_PER_BLOCK(root));
6546         }
6547
6548         eb = path->nodes[wc->level];
6549         nritems = btrfs_header_nritems(eb);
6550         blocksize = btrfs_level_size(root, wc->level - 1);
6551
6552         for (slot = path->slots[wc->level]; slot < nritems; slot++) {
6553                 if (nread >= wc->reada_count)
6554                         break;
6555
6556                 cond_resched();
6557                 bytenr = btrfs_node_blockptr(eb, slot);
6558                 generation = btrfs_node_ptr_generation(eb, slot);
6559
6560                 if (slot == path->slots[wc->level])
6561                         goto reada;
6562
6563                 if (wc->stage == UPDATE_BACKREF &&
6564                     generation <= root->root_key.offset)
6565                         continue;
6566
6567                 /* We don't lock the tree block, it's OK to be racy here */
6568                 ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
6569                                                &refs, &flags);
6570                 /* We don't care about errors in readahead. */
6571                 if (ret < 0)
6572                         continue;
6573                 BUG_ON(refs == 0);
6574
6575                 if (wc->stage == DROP_REFERENCE) {
6576                         if (refs == 1)
6577                                 goto reada;
6578
6579                         if (wc->level == 1 &&
6580                             (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
6581                                 continue;
6582                         if (!wc->update_ref ||
6583                             generation <= root->root_key.offset)
6584                                 continue;
6585                         btrfs_node_key_to_cpu(eb, &key, slot);
6586                         ret = btrfs_comp_cpu_keys(&key,
6587                                                   &wc->update_progress);
6588                         if (ret < 0)
6589                                 continue;
6590                 } else {
6591                         if (wc->level == 1 &&
6592                             (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF))
6593                                 continue;
6594                 }
6595 reada:
6596                 ret = readahead_tree_block(root, bytenr, blocksize,
6597                                            generation);
6598                 if (ret)
6599                         break;
6600                 nread++;
6601         }
6602         wc->reada_slot = slot;
6603 }
6604
6605 /*
6606  * hepler to process tree block while walking down the tree.
6607  *
6608  * when wc->stage == UPDATE_BACKREF, this function updates
6609  * back refs for pointers in the block.
6610  *
6611  * NOTE: return value 1 means we should stop walking down.
6612  */
6613 static noinline int walk_down_proc(struct btrfs_trans_handle *trans,
6614                                    struct btrfs_root *root,
6615                                    struct btrfs_path *path,
6616                                    struct walk_control *wc, int lookup_info)
6617 {
6618         int level = wc->level;
6619         struct extent_buffer *eb = path->nodes[level];
6620         u64 flag = BTRFS_BLOCK_FLAG_FULL_BACKREF;
6621         int ret;
6622
6623         if (wc->stage == UPDATE_BACKREF &&
6624             btrfs_header_owner(eb) != root->root_key.objectid)
6625                 return 1;
6626
6627         /*
6628          * when reference count of tree block is 1, it won't increase
6629          * again. once full backref flag is set, we never clear it.
6630          */
6631         if (lookup_info &&
6632             ((wc->stage == DROP_REFERENCE && wc->refs[level] != 1) ||
6633              (wc->stage == UPDATE_BACKREF && !(wc->flags[level] & flag)))) {
6634                 BUG_ON(!path->locks[level]);
6635                 ret = btrfs_lookup_extent_info(trans, root,
6636                                                eb->start, eb->len,
6637                                                &wc->refs[level],
6638                                                &wc->flags[level]);
6639                 BUG_ON(ret == -ENOMEM);
6640                 if (ret)
6641                         return ret;
6642                 BUG_ON(wc->refs[level] == 0);
6643         }
6644
6645         if (wc->stage == DROP_REFERENCE) {
6646                 if (wc->refs[level] > 1)
6647                         return 1;
6648
6649                 if (path->locks[level] && !wc->keep_locks) {
6650                         btrfs_tree_unlock_rw(eb, path->locks[level]);
6651                         path->locks[level] = 0;
6652                 }
6653                 return 0;
6654         }
6655
6656         /* wc->stage == UPDATE_BACKREF */
6657         if (!(wc->flags[level] & flag)) {
6658                 BUG_ON(!path->locks[level]);
6659                 ret = btrfs_inc_ref(trans, root, eb, 1, wc->for_reloc);
6660                 BUG_ON(ret); /* -ENOMEM */
6661                 ret = btrfs_dec_ref(trans, root, eb, 0, wc->for_reloc);
6662                 BUG_ON(ret); /* -ENOMEM */
6663                 ret = btrfs_set_disk_extent_flags(trans, root, eb->start,
6664                                                   eb->len, flag, 0);
6665                 BUG_ON(ret); /* -ENOMEM */
6666                 wc->flags[level] |= flag;
6667         }
6668
6669         /*
6670          * the block is shared by multiple trees, so it's not good to
6671          * keep the tree lock
6672          */
6673         if (path->locks[level] && level > 0) {
6674                 btrfs_tree_unlock_rw(eb, path->locks[level]);
6675                 path->locks[level] = 0;
6676         }
6677         return 0;
6678 }
6679
6680 /*
6681  * hepler to process tree block pointer.
6682  *
6683  * when wc->stage == DROP_REFERENCE, this function checks
6684  * reference count of the block pointed to. if the block
6685  * is shared and we need update back refs for the subtree
6686  * rooted at the block, this function changes wc->stage to
6687  * UPDATE_BACKREF. if the block is shared and there is no
6688  * need to update back, this function drops the reference
6689  * to the block.
6690  *
6691  * NOTE: return value 1 means we should stop walking down.
6692  */
6693 static noinline int do_walk_down(struct btrfs_trans_handle *trans,
6694                                  struct btrfs_root *root,
6695                                  struct btrfs_path *path,
6696                                  struct walk_control *wc, int *lookup_info)
6697 {
6698         u64 bytenr;
6699         u64 generation;
6700         u64 parent;
6701         u32 blocksize;
6702         struct btrfs_key key;
6703         struct extent_buffer *next;
6704         int level = wc->level;
6705         int reada = 0;
6706         int ret = 0;
6707
6708         generation = btrfs_node_ptr_generation(path->nodes[level],
6709                                                path->slots[level]);
6710         /*
6711          * if the lower level block was created before the snapshot
6712          * was created, we know there is no need to update back refs
6713          * for the subtree
6714          */
6715         if (wc->stage == UPDATE_BACKREF &&
6716             generation <= root->root_key.offset) {
6717                 *lookup_info = 1;
6718                 return 1;
6719         }
6720
6721         bytenr = btrfs_node_blockptr(path->nodes[level], path->slots[level]);
6722         blocksize = btrfs_level_size(root, level - 1);
6723
6724         next = btrfs_find_tree_block(root, bytenr, blocksize);
6725         if (!next) {
6726                 next = btrfs_find_create_tree_block(root, bytenr, blocksize);
6727                 if (!next)
6728                         return -ENOMEM;
6729                 reada = 1;
6730         }
6731         btrfs_tree_lock(next);
6732         btrfs_set_lock_blocking(next);
6733
6734         ret = btrfs_lookup_extent_info(trans, root, bytenr, blocksize,
6735                                        &wc->refs[level - 1],
6736                                        &wc->flags[level - 1]);
6737         if (ret < 0) {
6738                 btrfs_tree_unlock(next);
6739                 return ret;
6740         }
6741
6742         BUG_ON(wc->refs[level - 1] == 0);
6743         *lookup_info = 0;
6744
6745         if (wc->stage == DROP_REFERENCE) {
6746                 if (wc->refs[level - 1] > 1) {
6747                         if (level == 1 &&
6748                             (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
6749                                 goto skip;
6750
6751                         if (!wc->update_ref ||
6752                             generation <= root->root_key.offset)
6753                                 goto skip;
6754
6755                         btrfs_node_key_to_cpu(path->nodes[level], &key,
6756                                               path->slots[level]);
6757                         ret = btrfs_comp_cpu_keys(&key, &wc->update_progress);
6758                         if (ret < 0)
6759                                 goto skip;
6760
6761                         wc->stage = UPDATE_BACKREF;
6762                         wc->shared_level = level - 1;
6763                 }
6764         } else {
6765                 if (level == 1 &&
6766                     (wc->flags[0] & BTRFS_BLOCK_FLAG_FULL_BACKREF))
6767                         goto skip;
6768         }
6769
6770         if (!btrfs_buffer_uptodate(next, generation, 0)) {
6771                 btrfs_tree_unlock(next);
6772                 free_extent_buffer(next);
6773                 next = NULL;
6774                 *lookup_info = 1;
6775         }
6776
6777         if (!next) {
6778                 if (reada && level == 1)
6779                         reada_walk_down(trans, root, wc, path);
6780                 next = read_tree_block(root, bytenr, blocksize, generation);
6781                 if (!next)
6782                         return -EIO;
6783                 btrfs_tree_lock(next);
6784                 btrfs_set_lock_blocking(next);
6785         }
6786
6787         level--;
6788         BUG_ON(level != btrfs_header_level(next));
6789         path->nodes[level] = next;
6790         path->slots[level] = 0;
6791         path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6792         wc->level = level;
6793         if (wc->level == 1)
6794                 wc->reada_slot = 0;
6795         return 0;
6796 skip:
6797         wc->refs[level - 1] = 0;
6798         wc->flags[level - 1] = 0;
6799         if (wc->stage == DROP_REFERENCE) {
6800                 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF) {
6801                         parent = path->nodes[level]->start;
6802                 } else {
6803                         BUG_ON(root->root_key.objectid !=
6804                                btrfs_header_owner(path->nodes[level]));
6805                         parent = 0;
6806                 }
6807
6808                 ret = btrfs_free_extent(trans, root, bytenr, blocksize, parent,
6809                                 root->root_key.objectid, level - 1, 0, 0);
6810                 BUG_ON(ret); /* -ENOMEM */
6811         }
6812         btrfs_tree_unlock(next);
6813         free_extent_buffer(next);
6814         *lookup_info = 1;
6815         return 1;
6816 }
6817
6818 /*
6819  * hepler to process tree block while walking up the tree.
6820  *
6821  * when wc->stage == DROP_REFERENCE, this function drops
6822  * reference count on the block.
6823  *
6824  * when wc->stage == UPDATE_BACKREF, this function changes
6825  * wc->stage back to DROP_REFERENCE if we changed wc->stage
6826  * to UPDATE_BACKREF previously while processing the block.
6827  *
6828  * NOTE: return value 1 means we should stop walking up.
6829  */
6830 static noinline int walk_up_proc(struct btrfs_trans_handle *trans,
6831                                  struct btrfs_root *root,
6832                                  struct btrfs_path *path,
6833                                  struct walk_control *wc)
6834 {
6835         int ret;
6836         int level = wc->level;
6837         struct extent_buffer *eb = path->nodes[level];
6838         u64 parent = 0;
6839
6840         if (wc->stage == UPDATE_BACKREF) {
6841                 BUG_ON(wc->shared_level < level);
6842                 if (level < wc->shared_level)
6843                         goto out;
6844
6845                 ret = find_next_key(path, level + 1, &wc->update_progress);
6846                 if (ret > 0)
6847                         wc->update_ref = 0;
6848
6849                 wc->stage = DROP_REFERENCE;
6850                 wc->shared_level = -1;
6851                 path->slots[level] = 0;
6852
6853                 /*
6854                  * check reference count again if the block isn't locked.
6855                  * we should start walking down the tree again if reference
6856                  * count is one.
6857                  */
6858                 if (!path->locks[level]) {
6859                         BUG_ON(level == 0);
6860                         btrfs_tree_lock(eb);
6861                         btrfs_set_lock_blocking(eb);
6862                         path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6863
6864                         ret = btrfs_lookup_extent_info(trans, root,
6865                                                        eb->start, eb->len,
6866                                                        &wc->refs[level],
6867                                                        &wc->flags[level]);
6868                         if (ret < 0) {
6869                                 btrfs_tree_unlock_rw(eb, path->locks[level]);
6870                                 path->locks[level] = 0;
6871                                 return ret;
6872                         }
6873                         BUG_ON(wc->refs[level] == 0);
6874                         if (wc->refs[level] == 1) {
6875                                 btrfs_tree_unlock_rw(eb, path->locks[level]);
6876                                 path->locks[level] = 0;
6877                                 return 1;
6878                         }
6879                 }
6880         }
6881
6882         /* wc->stage == DROP_REFERENCE */
6883         BUG_ON(wc->refs[level] > 1 && !path->locks[level]);
6884
6885         if (wc->refs[level] == 1) {
6886                 if (level == 0) {
6887                         if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6888                                 ret = btrfs_dec_ref(trans, root, eb, 1,
6889                                                     wc->for_reloc);
6890                         else
6891                                 ret = btrfs_dec_ref(trans, root, eb, 0,
6892                                                     wc->for_reloc);
6893                         BUG_ON(ret); /* -ENOMEM */
6894                 }
6895                 /* make block locked assertion in clean_tree_block happy */
6896                 if (!path->locks[level] &&
6897                     btrfs_header_generation(eb) == trans->transid) {
6898                         btrfs_tree_lock(eb);
6899                         btrfs_set_lock_blocking(eb);
6900                         path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
6901                 }
6902                 clean_tree_block(trans, root, eb);
6903         }
6904
6905         if (eb == root->node) {
6906                 if (wc->flags[level] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6907                         parent = eb->start;
6908                 else
6909                         BUG_ON(root->root_key.objectid !=
6910                                btrfs_header_owner(eb));
6911         } else {
6912                 if (wc->flags[level + 1] & BTRFS_BLOCK_FLAG_FULL_BACKREF)
6913                         parent = path->nodes[level + 1]->start;
6914                 else
6915                         BUG_ON(root->root_key.objectid !=
6916                                btrfs_header_owner(path->nodes[level + 1]));
6917         }
6918
6919         btrfs_free_tree_block(trans, root, eb, parent, wc->refs[level] == 1);
6920 out:
6921         wc->refs[level] = 0;
6922         wc->flags[level] = 0;
6923         return 0;
6924 }
6925
6926 static noinline int walk_down_tree(struct btrfs_trans_handle *trans,
6927                                    struct btrfs_root *root,
6928                                    struct btrfs_path *path,
6929                                    struct walk_control *wc)
6930 {
6931         int level = wc->level;
6932         int lookup_info = 1;
6933         int ret;
6934
6935         while (level >= 0) {
6936                 ret = walk_down_proc(trans, root, path, wc, lookup_info);
6937                 if (ret > 0)
6938                         break;
6939
6940                 if (level == 0)
6941                         break;
6942
6943                 if (path->slots[level] >=
6944                     btrfs_header_nritems(path->nodes[level]))
6945                         break;
6946
6947                 ret = do_walk_down(trans, root, path, wc, &lookup_info);
6948                 if (ret > 0) {
6949                         path->slots[level]++;
6950                         continue;
6951                 } else if (ret < 0)
6952                         return ret;
6953                 level = wc->level;
6954         }
6955         return 0;
6956 }
6957
6958 static noinline int walk_up_tree(struct btrfs_trans_handle *trans,
6959                                  struct btrfs_root *root,
6960                                  struct btrfs_path *path,
6961                                  struct walk_control *wc, int max_level)
6962 {
6963         int level = wc->level;
6964         int ret;
6965
6966         path->slots[level] = btrfs_header_nritems(path->nodes[level]);
6967         while (level < max_level && path->nodes[level]) {
6968                 wc->level = level;
6969                 if (path->slots[level] + 1 <
6970                     btrfs_header_nritems(path->nodes[level])) {
6971                         path->slots[level]++;
6972                         return 0;
6973                 } else {
6974                         ret = walk_up_proc(trans, root, path, wc);
6975                         if (ret > 0)
6976                                 return 0;
6977
6978                         if (path->locks[level]) {
6979                                 btrfs_tree_unlock_rw(path->nodes[level],
6980                                                      path->locks[level]);
6981                                 path->locks[level] = 0;
6982                         }
6983                         free_extent_buffer(path->nodes[level]);
6984                         path->nodes[level] = NULL;
6985                         level++;
6986                 }
6987         }
6988         return 1;
6989 }
6990
6991 /*
6992  * drop a subvolume tree.
6993  *
6994  * this function traverses the tree freeing any blocks that only
6995  * referenced by the tree.
6996  *
6997  * when a shared tree block is found. this function decreases its
6998  * reference count by one. if update_ref is true, this function
6999  * also make sure backrefs for the shared block and all lower level
7000  * blocks are properly updated.
7001  */
7002 int btrfs_drop_snapshot(struct btrfs_root *root,
7003                          struct btrfs_block_rsv *block_rsv, int update_ref,
7004                          int for_reloc)
7005 {
7006         struct btrfs_path *path;
7007         struct btrfs_trans_handle *trans;
7008         struct btrfs_root *tree_root = root->fs_info->tree_root;
7009         struct btrfs_root_item *root_item = &root->root_item;
7010         struct walk_control *wc;
7011         struct btrfs_key key;
7012         int err = 0;
7013         int ret;
7014         int level;
7015
7016         path = btrfs_alloc_path();
7017         if (!path) {
7018                 err = -ENOMEM;
7019                 goto out;
7020         }
7021
7022         wc = kzalloc(sizeof(*wc), GFP_NOFS);
7023         if (!wc) {
7024                 btrfs_free_path(path);
7025                 err = -ENOMEM;
7026                 goto out;
7027         }
7028
7029         trans = btrfs_start_transaction(tree_root, 0);
7030         if (IS_ERR(trans)) {
7031                 err = PTR_ERR(trans);
7032                 goto out_free;
7033         }
7034
7035         if (block_rsv)
7036                 trans->block_rsv = block_rsv;
7037
7038         if (btrfs_disk_key_objectid(&root_item->drop_progress) == 0) {
7039                 level = btrfs_header_level(root->node);
7040                 path->nodes[level] = btrfs_lock_root_node(root);
7041                 btrfs_set_lock_blocking(path->nodes[level]);
7042                 path->slots[level] = 0;
7043                 path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7044                 memset(&wc->update_progress, 0,
7045                        sizeof(wc->update_progress));
7046         } else {
7047                 btrfs_disk_key_to_cpu(&key, &root_item->drop_progress);
7048                 memcpy(&wc->update_progress, &key,
7049                        sizeof(wc->update_progress));
7050
7051                 level = root_item->drop_level;
7052                 BUG_ON(level == 0);
7053                 path->lowest_level = level;
7054                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7055                 path->lowest_level = 0;
7056                 if (ret < 0) {
7057                         err = ret;
7058                         goto out_end_trans;
7059                 }
7060                 WARN_ON(ret > 0);
7061
7062                 /*
7063                  * unlock our path, this is safe because only this
7064                  * function is allowed to delete this snapshot
7065                  */
7066                 btrfs_unlock_up_safe(path, 0);
7067
7068                 level = btrfs_header_level(root->node);
7069                 while (1) {
7070                         btrfs_tree_lock(path->nodes[level]);
7071                         btrfs_set_lock_blocking(path->nodes[level]);
7072
7073                         ret = btrfs_lookup_extent_info(trans, root,
7074                                                 path->nodes[level]->start,
7075                                                 path->nodes[level]->len,
7076                                                 &wc->refs[level],
7077                                                 &wc->flags[level]);
7078                         if (ret < 0) {
7079                                 err = ret;
7080                                 goto out_end_trans;
7081                         }
7082                         BUG_ON(wc->refs[level] == 0);
7083
7084                         if (level == root_item->drop_level)
7085                                 break;
7086
7087                         btrfs_tree_unlock(path->nodes[level]);
7088                         WARN_ON(wc->refs[level] != 1);
7089                         level--;
7090                 }
7091         }
7092
7093         wc->level = level;
7094         wc->shared_level = -1;
7095         wc->stage = DROP_REFERENCE;
7096         wc->update_ref = update_ref;
7097         wc->keep_locks = 0;
7098         wc->for_reloc = for_reloc;
7099         wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7100
7101         while (1) {
7102                 ret = walk_down_tree(trans, root, path, wc);
7103                 if (ret < 0) {
7104                         err = ret;
7105                         break;
7106                 }
7107
7108                 ret = walk_up_tree(trans, root, path, wc, BTRFS_MAX_LEVEL);
7109                 if (ret < 0) {
7110                         err = ret;
7111                         break;
7112                 }
7113
7114                 if (ret > 0) {
7115                         BUG_ON(wc->stage != DROP_REFERENCE);
7116                         break;
7117                 }
7118
7119                 if (wc->stage == DROP_REFERENCE) {
7120                         level = wc->level;
7121                         btrfs_node_key(path->nodes[level],
7122                                        &root_item->drop_progress,
7123                                        path->slots[level]);
7124                         root_item->drop_level = level;
7125                 }
7126
7127                 BUG_ON(wc->level == 0);
7128                 if (btrfs_should_end_transaction(trans, tree_root)) {
7129                         ret = btrfs_update_root(trans, tree_root,
7130                                                 &root->root_key,
7131                                                 root_item);
7132                         if (ret) {
7133                                 btrfs_abort_transaction(trans, tree_root, ret);
7134                                 err = ret;
7135                                 goto out_end_trans;
7136                         }
7137
7138                         btrfs_end_transaction_throttle(trans, tree_root);
7139                         trans = btrfs_start_transaction(tree_root, 0);
7140                         if (IS_ERR(trans)) {
7141                                 err = PTR_ERR(trans);
7142                                 goto out_free;
7143                         }
7144                         if (block_rsv)
7145                                 trans->block_rsv = block_rsv;
7146                 }
7147         }
7148         btrfs_release_path(path);
7149         if (err)
7150                 goto out_end_trans;
7151
7152         ret = btrfs_del_root(trans, tree_root, &root->root_key);
7153         if (ret) {
7154                 btrfs_abort_transaction(trans, tree_root, ret);
7155                 goto out_end_trans;
7156         }
7157
7158         if (root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID) {
7159                 ret = btrfs_find_last_root(tree_root, root->root_key.objectid,
7160                                            NULL, NULL);
7161                 if (ret < 0) {
7162                         btrfs_abort_transaction(trans, tree_root, ret);
7163                         err = ret;
7164                         goto out_end_trans;
7165                 } else if (ret > 0) {
7166                         /* if we fail to delete the orphan item this time
7167                          * around, it'll get picked up the next time.
7168                          *
7169                          * The most common failure here is just -ENOENT.
7170                          */
7171                         btrfs_del_orphan_item(trans, tree_root,
7172                                               root->root_key.objectid);
7173                 }
7174         }
7175
7176         if (root->in_radix) {
7177                 btrfs_free_fs_root(tree_root->fs_info, root);
7178         } else {
7179                 free_extent_buffer(root->node);
7180                 free_extent_buffer(root->commit_root);
7181                 kfree(root);
7182         }
7183 out_end_trans:
7184         btrfs_end_transaction_throttle(trans, tree_root);
7185 out_free:
7186         kfree(wc);
7187         btrfs_free_path(path);
7188 out:
7189         if (err)
7190                 btrfs_std_error(root->fs_info, err);
7191         return err;
7192 }
7193
7194 /*
7195  * drop subtree rooted at tree block 'node'.
7196  *
7197  * NOTE: this function will unlock and release tree block 'node'
7198  * only used by relocation code
7199  */
7200 int btrfs_drop_subtree(struct btrfs_trans_handle *trans,
7201                         struct btrfs_root *root,
7202                         struct extent_buffer *node,
7203                         struct extent_buffer *parent)
7204 {
7205         struct btrfs_path *path;
7206         struct walk_control *wc;
7207         int level;
7208         int parent_level;
7209         int ret = 0;
7210         int wret;
7211
7212         BUG_ON(root->root_key.objectid != BTRFS_TREE_RELOC_OBJECTID);
7213
7214         path = btrfs_alloc_path();
7215         if (!path)
7216                 return -ENOMEM;
7217
7218         wc = kzalloc(sizeof(*wc), GFP_NOFS);
7219         if (!wc) {
7220                 btrfs_free_path(path);
7221                 return -ENOMEM;
7222         }
7223
7224         btrfs_assert_tree_locked(parent);
7225         parent_level = btrfs_header_level(parent);
7226         extent_buffer_get(parent);
7227         path->nodes[parent_level] = parent;
7228         path->slots[parent_level] = btrfs_header_nritems(parent);
7229
7230         btrfs_assert_tree_locked(node);
7231         level = btrfs_header_level(node);
7232         path->nodes[level] = node;
7233         path->slots[level] = 0;
7234         path->locks[level] = BTRFS_WRITE_LOCK_BLOCKING;
7235
7236         wc->refs[parent_level] = 1;
7237         wc->flags[parent_level] = BTRFS_BLOCK_FLAG_FULL_BACKREF;
7238         wc->level = level;
7239         wc->shared_level = -1;
7240         wc->stage = DROP_REFERENCE;
7241         wc->update_ref = 0;
7242         wc->keep_locks = 1;
7243         wc->for_reloc = 1;
7244         wc->reada_count = BTRFS_NODEPTRS_PER_BLOCK(root);
7245
7246         while (1) {
7247                 wret = walk_down_tree(trans, root, path, wc);
7248                 if (wret < 0) {
7249                         ret = wret;
7250                         break;
7251                 }
7252
7253                 wret = walk_up_tree(trans, root, path, wc, parent_level);
7254                 if (wret < 0)
7255                         ret = wret;
7256                 if (wret != 0)
7257                         break;
7258         }
7259
7260         kfree(wc);
7261         btrfs_free_path(path);
7262         return ret;
7263 }
7264
7265 static u64 update_block_group_flags(struct btrfs_root *root, u64 flags)
7266 {
7267         u64 num_devices;
7268         u64 stripped;
7269
7270         /*
7271          * if restripe for this chunk_type is on pick target profile and
7272          * return, otherwise do the usual balance
7273          */
7274         stripped = get_restripe_target(root->fs_info, flags);
7275         if (stripped)
7276                 return extended_to_chunk(stripped);
7277
7278         /*
7279          * we add in the count of missing devices because we want
7280          * to make sure that any RAID levels on a degraded FS
7281          * continue to be honored.
7282          */
7283         num_devices = root->fs_info->fs_devices->rw_devices +
7284                 root->fs_info->fs_devices->missing_devices;
7285
7286         stripped = BTRFS_BLOCK_GROUP_RAID0 |
7287                 BTRFS_BLOCK_GROUP_RAID5 | BTRFS_BLOCK_GROUP_RAID6 |
7288                 BTRFS_BLOCK_GROUP_RAID1 | BTRFS_BLOCK_GROUP_RAID10;
7289
7290         if (num_devices == 1) {
7291                 stripped |= BTRFS_BLOCK_GROUP_DUP;
7292                 stripped = flags & ~stripped;
7293
7294                 /* turn raid0 into single device chunks */
7295                 if (flags & BTRFS_BLOCK_GROUP_RAID0)
7296                         return stripped;
7297
7298                 /* turn mirroring into duplication */
7299                 if (flags & (BTRFS_BLOCK_GROUP_RAID1 |
7300                              BTRFS_BLOCK_GROUP_RAID10))
7301                         return stripped | BTRFS_BLOCK_GROUP_DUP;
7302         } else {
7303                 /* they already had raid on here, just return */
7304                 if (flags & stripped)
7305                         return flags;
7306
7307                 stripped |= BTRFS_BLOCK_GROUP_DUP;
7308                 stripped = flags & ~stripped;
7309
7310                 /* switch duplicated blocks with raid1 */
7311                 if (flags & BTRFS_BLOCK_GROUP_DUP)
7312                         return stripped | BTRFS_BLOCK_GROUP_RAID1;
7313
7314                 /* this is drive concat, leave it alone */
7315         }
7316
7317         return flags;
7318 }
7319
7320 static int set_block_group_ro(struct btrfs_block_group_cache *cache, int force)
7321 {
7322         struct btrfs_space_info *sinfo = cache->space_info;
7323         u64 num_bytes;
7324         u64 min_allocable_bytes;
7325         int ret = -ENOSPC;
7326
7327
7328         /*
7329          * We need some metadata space and system metadata space for
7330          * allocating chunks in some corner cases until we force to set
7331          * it to be readonly.
7332          */
7333         if ((sinfo->flags &
7334              (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
7335             !force)
7336                 min_allocable_bytes = 1 * 1024 * 1024;
7337         else
7338                 min_allocable_bytes = 0;
7339
7340         spin_lock(&sinfo->lock);
7341         spin_lock(&cache->lock);
7342
7343         if (cache->ro) {
7344                 ret = 0;
7345                 goto out;
7346         }
7347
7348         num_bytes = cache->key.offset - cache->reserved - cache->pinned -
7349                     cache->bytes_super - btrfs_block_group_used(&cache->item);
7350
7351         if (sinfo->bytes_used + sinfo->bytes_reserved + sinfo->bytes_pinned +
7352             sinfo->bytes_may_use + sinfo->bytes_readonly + num_bytes +
7353             min_allocable_bytes <= sinfo->total_bytes) {
7354                 sinfo->bytes_readonly += num_bytes;
7355                 cache->ro = 1;
7356                 ret = 0;
7357         }
7358 out:
7359         spin_unlock(&cache->lock);
7360         spin_unlock(&sinfo->lock);
7361         return ret;
7362 }
7363
7364 int btrfs_set_block_group_ro(struct btrfs_root *root,
7365                              struct btrfs_block_group_cache *cache)
7366
7367 {
7368         struct btrfs_trans_handle *trans;
7369         u64 alloc_flags;
7370         int ret;
7371
7372         BUG_ON(cache->ro);
7373
7374         trans = btrfs_join_transaction(root);
7375         if (IS_ERR(trans))
7376                 return PTR_ERR(trans);
7377
7378         alloc_flags = update_block_group_flags(root, cache->flags);
7379         if (alloc_flags != cache->flags) {
7380                 ret = do_chunk_alloc(trans, root, alloc_flags,
7381                                      CHUNK_ALLOC_FORCE);
7382                 if (ret < 0)
7383                         goto out;
7384         }
7385
7386         ret = set_block_group_ro(cache, 0);
7387         if (!ret)
7388                 goto out;
7389         alloc_flags = get_alloc_profile(root, cache->space_info->flags);
7390         ret = do_chunk_alloc(trans, root, alloc_flags,
7391                              CHUNK_ALLOC_FORCE);
7392         if (ret < 0)
7393                 goto out;
7394         ret = set_block_group_ro(cache, 0);
7395 out:
7396         btrfs_end_transaction(trans, root);
7397         return ret;
7398 }
7399
7400 int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans,
7401                             struct btrfs_root *root, u64 type)
7402 {
7403         u64 alloc_flags = get_alloc_profile(root, type);
7404         return do_chunk_alloc(trans, root, alloc_flags,
7405                               CHUNK_ALLOC_FORCE);
7406 }
7407
7408 /*
7409  * helper to account the unused space of all the readonly block group in the
7410  * list. takes mirrors into account.
7411  */
7412 static u64 __btrfs_get_ro_block_group_free_space(struct list_head *groups_list)
7413 {
7414         struct btrfs_block_group_cache *block_group;
7415         u64 free_bytes = 0;
7416         int factor;
7417
7418         list_for_each_entry(block_group, groups_list, list) {
7419                 spin_lock(&block_group->lock);
7420
7421                 if (!block_group->ro) {
7422                         spin_unlock(&block_group->lock);
7423                         continue;
7424                 }
7425
7426                 if (block_group->flags & (BTRFS_BLOCK_GROUP_RAID1 |
7427                                           BTRFS_BLOCK_GROUP_RAID10 |
7428                                           BTRFS_BLOCK_GROUP_DUP))
7429                         factor = 2;
7430                 else
7431                         factor = 1;
7432
7433                 free_bytes += (block_group->key.offset -
7434                                btrfs_block_group_used(&block_group->item)) *
7435                                factor;
7436
7437                 spin_unlock(&block_group->lock);
7438         }
7439
7440         return free_bytes;
7441 }
7442
7443 /*
7444  * helper to account the unused space of all the readonly block group in the
7445  * space_info. takes mirrors into account.
7446  */
7447 u64 btrfs_account_ro_block_groups_free_space(struct btrfs_space_info *sinfo)
7448 {
7449         int i;
7450         u64 free_bytes = 0;
7451
7452         spin_lock(&sinfo->lock);
7453
7454         for(i = 0; i < BTRFS_NR_RAID_TYPES; i++)
7455                 if (!list_empty(&sinfo->block_groups[i]))
7456                         free_bytes += __btrfs_get_ro_block_group_free_space(
7457                                                 &sinfo->block_groups[i]);
7458
7459         spin_unlock(&sinfo->lock);
7460
7461         return free_bytes;
7462 }
7463
7464 void btrfs_set_block_group_rw(struct btrfs_root *root,
7465                               struct btrfs_block_group_cache *cache)
7466 {
7467         struct btrfs_space_info *sinfo = cache->space_info;
7468         u64 num_bytes;
7469
7470         BUG_ON(!cache->ro);
7471
7472         spin_lock(&sinfo->lock);
7473         spin_lock(&cache->lock);
7474         num_bytes = cache->key.offset - cache->reserved - cache->pinned -
7475                     cache->bytes_super - btrfs_block_group_used(&cache->item);
7476         sinfo->bytes_readonly -= num_bytes;
7477         cache->ro = 0;
7478         spin_unlock(&cache->lock);
7479         spin_unlock(&sinfo->lock);
7480 }
7481
7482 /*
7483  * checks to see if its even possible to relocate this block group.
7484  *
7485  * @return - -1 if it's not a good idea to relocate this block group, 0 if its
7486  * ok to go ahead and try.
7487  */
7488 int btrfs_can_relocate(struct btrfs_root *root, u64 bytenr)
7489 {
7490         struct btrfs_block_group_cache *block_group;
7491         struct btrfs_space_info *space_info;
7492         struct btrfs_fs_devices *fs_devices = root->fs_info->fs_devices;
7493         struct btrfs_device *device;
7494         u64 min_free;
7495         u64 dev_min = 1;
7496         u64 dev_nr = 0;
7497         u64 target;
7498         int index;
7499         int full = 0;
7500         int ret = 0;
7501
7502         block_group = btrfs_lookup_block_group(root->fs_info, bytenr);
7503
7504         /* odd, couldn't find the block group, leave it alone */
7505         if (!block_group)
7506                 return -1;
7507
7508         min_free = btrfs_block_group_used(&block_group->item);
7509
7510         /* no bytes used, we're good */
7511         if (!min_free)
7512                 goto out;
7513
7514         space_info = block_group->space_info;
7515         spin_lock(&space_info->lock);
7516
7517         full = space_info->full;
7518
7519         /*
7520          * if this is the last block group we have in this space, we can't
7521          * relocate it unless we're able to allocate a new chunk below.
7522          *
7523          * Otherwise, we need to make sure we have room in the space to handle
7524          * all of the extents from this block group.  If we can, we're good
7525          */
7526         if ((space_info->total_bytes != block_group->key.offset) &&
7527             (space_info->bytes_used + space_info->bytes_reserved +
7528              space_info->bytes_pinned + space_info->bytes_readonly +
7529              min_free < space_info->total_bytes)) {
7530                 spin_unlock(&space_info->lock);
7531                 goto out;
7532         }
7533         spin_unlock(&space_info->lock);
7534
7535         /*
7536          * ok we don't have enough space, but maybe we have free space on our
7537          * devices to allocate new chunks for relocation, so loop through our
7538          * alloc devices and guess if we have enough space.  if this block
7539          * group is going to be restriped, run checks against the target
7540          * profile instead of the current one.
7541          */
7542         ret = -1;
7543
7544         /*
7545          * index:
7546          *      0: raid10
7547          *      1: raid1
7548          *      2: dup
7549          *      3: raid0
7550          *      4: single
7551          */
7552         target = get_restripe_target(root->fs_info, block_group->flags);
7553         if (target) {
7554                 index = __get_raid_index(extended_to_chunk(target));
7555         } else {
7556                 /*
7557                  * this is just a balance, so if we were marked as full
7558                  * we know there is no space for a new chunk
7559                  */
7560                 if (full)
7561                         goto out;
7562
7563                 index = get_block_group_index(block_group);
7564         }
7565
7566         if (index == 0) {
7567                 dev_min = 4;
7568                 /* Divide by 2 */
7569                 min_free >>= 1;
7570         } else if (index == 1) {
7571                 dev_min = 2;
7572         } else if (index == 2) {
7573                 /* Multiply by 2 */
7574                 min_free <<= 1;
7575         } else if (index == 3) {
7576                 dev_min = fs_devices->rw_devices;
7577                 do_div(min_free, dev_min);
7578         }
7579
7580         mutex_lock(&root->fs_info->chunk_mutex);
7581         list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
7582                 u64 dev_offset;
7583
7584                 /*
7585                  * check to make sure we can actually find a chunk with enough
7586                  * space to fit our block group in.
7587                  */
7588                 if (device->total_bytes > device->bytes_used + min_free &&
7589                     !device->is_tgtdev_for_dev_replace) {
7590                         ret = find_free_dev_extent(device, min_free,
7591                                                    &dev_offset, NULL);
7592                         if (!ret)
7593                                 dev_nr++;
7594
7595                         if (dev_nr >= dev_min)
7596                                 break;
7597
7598                         ret = -1;
7599                 }
7600         }
7601         mutex_unlock(&root->fs_info->chunk_mutex);
7602 out:
7603         btrfs_put_block_group(block_group);
7604         return ret;
7605 }
7606
7607 static int find_first_block_group(struct btrfs_root *root,
7608                 struct btrfs_path *path, struct btrfs_key *key)
7609 {
7610         int ret = 0;
7611         struct btrfs_key found_key;
7612         struct extent_buffer *leaf;
7613         int slot;
7614
7615         ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
7616         if (ret < 0)
7617                 goto out;
7618
7619         while (1) {
7620                 slot = path->slots[0];
7621                 leaf = path->nodes[0];
7622                 if (slot >= btrfs_header_nritems(leaf)) {
7623                         ret = btrfs_next_leaf(root, path);
7624                         if (ret == 0)
7625                                 continue;
7626                         if (ret < 0)
7627                                 goto out;
7628                         break;
7629                 }
7630                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
7631
7632                 if (found_key.objectid >= key->objectid &&
7633                     found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
7634                         ret = 0;
7635                         goto out;
7636                 }
7637                 path->slots[0]++;
7638         }
7639 out:
7640         return ret;
7641 }
7642
7643 void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
7644 {
7645         struct btrfs_block_group_cache *block_group;
7646         u64 last = 0;
7647
7648         while (1) {
7649                 struct inode *inode;
7650
7651                 block_group = btrfs_lookup_first_block_group(info, last);
7652                 while (block_group) {
7653                         spin_lock(&block_group->lock);
7654                         if (block_group->iref)
7655                                 break;
7656                         spin_unlock(&block_group->lock);
7657                         block_group = next_block_group(info->tree_root,
7658                                                        block_group);
7659                 }
7660                 if (!block_group) {
7661                         if (last == 0)
7662                                 break;
7663                         last = 0;
7664                         continue;
7665                 }
7666
7667                 inode = block_group->inode;
7668                 block_group->iref = 0;
7669                 block_group->inode = NULL;
7670                 spin_unlock(&block_group->lock);
7671                 iput(inode);
7672                 last = block_group->key.objectid + block_group->key.offset;
7673                 btrfs_put_block_group(block_group);
7674         }
7675 }
7676
7677 int btrfs_free_block_groups(struct btrfs_fs_info *info)
7678 {
7679         struct btrfs_block_group_cache *block_group;
7680         struct btrfs_space_info *space_info;
7681         struct btrfs_caching_control *caching_ctl;
7682         struct rb_node *n;
7683
7684         down_write(&info->extent_commit_sem);
7685         while (!list_empty(&info->caching_block_groups)) {
7686                 caching_ctl = list_entry(info->caching_block_groups.next,
7687                                          struct btrfs_caching_control, list);
7688                 list_del(&caching_ctl->list);
7689                 put_caching_control(caching_ctl);
7690         }
7691         up_write(&info->extent_commit_sem);
7692
7693         spin_lock(&info->block_group_cache_lock);
7694         while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
7695                 block_group = rb_entry(n, struct btrfs_block_group_cache,
7696                                        cache_node);
7697                 rb_erase(&block_group->cache_node,
7698                          &info->block_group_cache_tree);
7699                 spin_unlock(&info->block_group_cache_lock);
7700
7701                 down_write(&block_group->space_info->groups_sem);
7702                 list_del(&block_group->list);
7703                 up_write(&block_group->space_info->groups_sem);
7704
7705                 if (block_group->cached == BTRFS_CACHE_STARTED)
7706                         wait_block_group_cache_done(block_group);
7707
7708                 /*
7709                  * We haven't cached this block group, which means we could
7710                  * possibly have excluded extents on this block group.
7711                  */
7712                 if (block_group->cached == BTRFS_CACHE_NO)
7713                         free_excluded_extents(info->extent_root, block_group);
7714
7715                 btrfs_remove_free_space_cache(block_group);
7716                 btrfs_put_block_group(block_group);
7717
7718                 spin_lock(&info->block_group_cache_lock);
7719         }
7720         spin_unlock(&info->block_group_cache_lock);
7721
7722         /* now that all the block groups are freed, go through and
7723          * free all the space_info structs.  This is only called during
7724          * the final stages of unmount, and so we know nobody is
7725          * using them.  We call synchronize_rcu() once before we start,
7726          * just to be on the safe side.
7727          */
7728         synchronize_rcu();
7729
7730         release_global_block_rsv(info);
7731
7732         while(!list_empty(&info->space_info)) {
7733                 space_info = list_entry(info->space_info.next,
7734                                         struct btrfs_space_info,
7735                                         list);
7736                 if (space_info->bytes_pinned > 0 ||
7737                     space_info->bytes_reserved > 0 ||
7738                     space_info->bytes_may_use > 0) {
7739                         WARN_ON(1);
7740                         dump_space_info(space_info, 0, 0);
7741                 }
7742                 list_del(&space_info->list);
7743                 kfree(space_info);
7744         }
7745         return 0;
7746 }
7747
7748 static void __link_block_group(struct btrfs_space_info *space_info,
7749                                struct btrfs_block_group_cache *cache)
7750 {
7751         int index = get_block_group_index(cache);
7752
7753         down_write(&space_info->groups_sem);
7754         list_add_tail(&cache->list, &space_info->block_groups[index]);
7755         up_write(&space_info->groups_sem);
7756 }
7757
7758 int btrfs_read_block_groups(struct btrfs_root *root)
7759 {
7760         struct btrfs_path *path;
7761         int ret;
7762         struct btrfs_block_group_cache *cache;
7763         struct btrfs_fs_info *info = root->fs_info;
7764         struct btrfs_space_info *space_info;
7765         struct btrfs_key key;
7766         struct btrfs_key found_key;
7767         struct extent_buffer *leaf;
7768         int need_clear = 0;
7769         u64 cache_gen;
7770
7771         root = info->extent_root;
7772         key.objectid = 0;
7773         key.offset = 0;
7774         btrfs_set_key_type(&key, BTRFS_BLOCK_GROUP_ITEM_KEY);
7775         path = btrfs_alloc_path();
7776         if (!path)
7777                 return -ENOMEM;
7778         path->reada = 1;
7779
7780         cache_gen = btrfs_super_cache_generation(root->fs_info->super_copy);
7781         if (btrfs_test_opt(root, SPACE_CACHE) &&
7782             btrfs_super_generation(root->fs_info->super_copy) != cache_gen)
7783                 need_clear = 1;
7784         if (btrfs_test_opt(root, CLEAR_CACHE))
7785                 need_clear = 1;
7786
7787         while (1) {
7788                 ret = find_first_block_group(root, path, &key);
7789                 if (ret > 0)
7790                         break;
7791                 if (ret != 0)
7792                         goto error;
7793                 leaf = path->nodes[0];
7794                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7795                 cache = kzalloc(sizeof(*cache), GFP_NOFS);
7796                 if (!cache) {
7797                         ret = -ENOMEM;
7798                         goto error;
7799                 }
7800                 cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
7801                                                 GFP_NOFS);
7802                 if (!cache->free_space_ctl) {
7803                         kfree(cache);
7804                         ret = -ENOMEM;
7805                         goto error;
7806                 }
7807
7808                 atomic_set(&cache->count, 1);
7809                 spin_lock_init(&cache->lock);
7810                 cache->fs_info = info;
7811                 INIT_LIST_HEAD(&cache->list);
7812                 INIT_LIST_HEAD(&cache->cluster_list);
7813
7814                 if (need_clear) {
7815                         /*
7816                          * When we mount with old space cache, we need to
7817                          * set BTRFS_DC_CLEAR and set dirty flag.
7818                          *
7819                          * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
7820                          *    truncate the old free space cache inode and
7821                          *    setup a new one.
7822                          * b) Setting 'dirty flag' makes sure that we flush
7823                          *    the new space cache info onto disk.
7824                          */
7825                         cache->disk_cache_state = BTRFS_DC_CLEAR;
7826                         if (btrfs_test_opt(root, SPACE_CACHE))
7827                                 cache->dirty = 1;
7828                 }
7829
7830                 read_extent_buffer(leaf, &cache->item,
7831                                    btrfs_item_ptr_offset(leaf, path->slots[0]),
7832                                    sizeof(cache->item));
7833                 memcpy(&cache->key, &found_key, sizeof(found_key));
7834
7835                 key.objectid = found_key.objectid + found_key.offset;
7836                 btrfs_release_path(path);
7837                 cache->flags = btrfs_block_group_flags(&cache->item);
7838                 cache->sectorsize = root->sectorsize;
7839                 cache->full_stripe_len = btrfs_full_stripe_len(root,
7840                                                &root->fs_info->mapping_tree,
7841                                                found_key.objectid);
7842                 btrfs_init_free_space_ctl(cache);
7843
7844                 /*
7845                  * We need to exclude the super stripes now so that the space
7846                  * info has super bytes accounted for, otherwise we'll think
7847                  * we have more space than we actually do.
7848                  */
7849                 exclude_super_stripes(root, cache);
7850
7851                 /*
7852                  * check for two cases, either we are full, and therefore
7853                  * don't need to bother with the caching work since we won't
7854                  * find any space, or we are empty, and we can just add all
7855                  * the space in and be done with it.  This saves us _alot_ of
7856                  * time, particularly in the full case.
7857                  */
7858                 if (found_key.offset == btrfs_block_group_used(&cache->item)) {
7859                         cache->last_byte_to_unpin = (u64)-1;
7860                         cache->cached = BTRFS_CACHE_FINISHED;
7861                         free_excluded_extents(root, cache);
7862                 } else if (btrfs_block_group_used(&cache->item) == 0) {
7863                         cache->last_byte_to_unpin = (u64)-1;
7864                         cache->cached = BTRFS_CACHE_FINISHED;
7865                         add_new_free_space(cache, root->fs_info,
7866                                            found_key.objectid,
7867                                            found_key.objectid +
7868                                            found_key.offset);
7869                         free_excluded_extents(root, cache);
7870                 }
7871
7872                 ret = update_space_info(info, cache->flags, found_key.offset,
7873                                         btrfs_block_group_used(&cache->item),
7874                                         &space_info);
7875                 BUG_ON(ret); /* -ENOMEM */
7876                 cache->space_info = space_info;
7877                 spin_lock(&cache->space_info->lock);
7878                 cache->space_info->bytes_readonly += cache->bytes_super;
7879                 spin_unlock(&cache->space_info->lock);
7880
7881                 __link_block_group(space_info, cache);
7882
7883                 ret = btrfs_add_block_group_cache(root->fs_info, cache);
7884                 BUG_ON(ret); /* Logic error */
7885
7886                 set_avail_alloc_bits(root->fs_info, cache->flags);
7887                 if (btrfs_chunk_readonly(root, cache->key.objectid))
7888                         set_block_group_ro(cache, 1);
7889         }
7890
7891         list_for_each_entry_rcu(space_info, &root->fs_info->space_info, list) {
7892                 if (!(get_alloc_profile(root, space_info->flags) &
7893                       (BTRFS_BLOCK_GROUP_RAID10 |
7894                        BTRFS_BLOCK_GROUP_RAID1 |
7895                        BTRFS_BLOCK_GROUP_RAID5 |
7896                        BTRFS_BLOCK_GROUP_RAID6 |
7897                        BTRFS_BLOCK_GROUP_DUP)))
7898                         continue;
7899                 /*
7900                  * avoid allocating from un-mirrored block group if there are
7901                  * mirrored block groups.
7902                  */
7903                 list_for_each_entry(cache, &space_info->block_groups[3], list)
7904                         set_block_group_ro(cache, 1);
7905                 list_for_each_entry(cache, &space_info->block_groups[4], list)
7906                         set_block_group_ro(cache, 1);
7907         }
7908
7909         init_global_block_rsv(info);
7910         ret = 0;
7911 error:
7912         btrfs_free_path(path);
7913         return ret;
7914 }
7915
7916 void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans,
7917                                        struct btrfs_root *root)
7918 {
7919         struct btrfs_block_group_cache *block_group, *tmp;
7920         struct btrfs_root *extent_root = root->fs_info->extent_root;
7921         struct btrfs_block_group_item item;
7922         struct btrfs_key key;
7923         int ret = 0;
7924
7925         list_for_each_entry_safe(block_group, tmp, &trans->new_bgs,
7926                                  new_bg_list) {
7927                 list_del_init(&block_group->new_bg_list);
7928
7929                 if (ret)
7930                         continue;
7931
7932                 spin_lock(&block_group->lock);
7933                 memcpy(&item, &block_group->item, sizeof(item));
7934                 memcpy(&key, &block_group->key, sizeof(key));
7935                 spin_unlock(&block_group->lock);
7936
7937                 ret = btrfs_insert_item(trans, extent_root, &key, &item,
7938                                         sizeof(item));
7939                 if (ret)
7940                         btrfs_abort_transaction(trans, extent_root, ret);
7941         }
7942 }
7943
7944 int btrfs_make_block_group(struct btrfs_trans_handle *trans,
7945                            struct btrfs_root *root, u64 bytes_used,
7946                            u64 type, u64 chunk_objectid, u64 chunk_offset,
7947                            u64 size)
7948 {
7949         int ret;
7950         struct btrfs_root *extent_root;
7951         struct btrfs_block_group_cache *cache;
7952
7953         extent_root = root->fs_info->extent_root;
7954
7955         root->fs_info->last_trans_log_full_commit = trans->transid;
7956
7957         cache = kzalloc(sizeof(*cache), GFP_NOFS);
7958         if (!cache)
7959                 return -ENOMEM;
7960         cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
7961                                         GFP_NOFS);
7962         if (!cache->free_space_ctl) {
7963                 kfree(cache);
7964                 return -ENOMEM;
7965         }
7966
7967         cache->key.objectid = chunk_offset;
7968         cache->key.offset = size;
7969         cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
7970         cache->sectorsize = root->sectorsize;
7971         cache->fs_info = root->fs_info;
7972         cache->full_stripe_len = btrfs_full_stripe_len(root,
7973                                                &root->fs_info->mapping_tree,
7974                                                chunk_offset);
7975
7976         atomic_set(&cache->count, 1);
7977         spin_lock_init(&cache->lock);
7978         INIT_LIST_HEAD(&cache->list);
7979         INIT_LIST_HEAD(&cache->cluster_list);
7980         INIT_LIST_HEAD(&cache->new_bg_list);
7981
7982         btrfs_init_free_space_ctl(cache);
7983
7984         btrfs_set_block_group_used(&cache->item, bytes_used);
7985         btrfs_set_block_group_chunk_objectid(&cache->item, chunk_objectid);
7986         cache->flags = type;
7987         btrfs_set_block_group_flags(&cache->item, type);
7988
7989         cache->last_byte_to_unpin = (u64)-1;
7990         cache->cached = BTRFS_CACHE_FINISHED;
7991         exclude_super_stripes(root, cache);
7992
7993         add_new_free_space(cache, root->fs_info, chunk_offset,
7994                            chunk_offset + size);
7995
7996         free_excluded_extents(root, cache);
7997
7998         ret = update_space_info(root->fs_info, cache->flags, size, bytes_used,
7999                                 &cache->space_info);
8000         BUG_ON(ret); /* -ENOMEM */
8001         update_global_block_rsv(root->fs_info);
8002
8003         spin_lock(&cache->space_info->lock);
8004         cache->space_info->bytes_readonly += cache->bytes_super;
8005         spin_unlock(&cache->space_info->lock);
8006
8007         __link_block_group(cache->space_info, cache);
8008
8009         ret = btrfs_add_block_group_cache(root->fs_info, cache);
8010         BUG_ON(ret); /* Logic error */
8011
8012         list_add_tail(&cache->new_bg_list, &trans->new_bgs);
8013
8014         set_avail_alloc_bits(extent_root->fs_info, type);
8015
8016         return 0;
8017 }
8018
8019 static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
8020 {
8021         u64 extra_flags = chunk_to_extended(flags) &
8022                                 BTRFS_EXTENDED_PROFILE_MASK;
8023
8024         if (flags & BTRFS_BLOCK_GROUP_DATA)
8025                 fs_info->avail_data_alloc_bits &= ~extra_flags;
8026         if (flags & BTRFS_BLOCK_GROUP_METADATA)
8027                 fs_info->avail_metadata_alloc_bits &= ~extra_flags;
8028         if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
8029                 fs_info->avail_system_alloc_bits &= ~extra_flags;
8030 }
8031
8032 int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
8033                              struct btrfs_root *root, u64 group_start)
8034 {
8035         struct btrfs_path *path;
8036         struct btrfs_block_group_cache *block_group;
8037         struct btrfs_free_cluster *cluster;
8038         struct btrfs_root *tree_root = root->fs_info->tree_root;
8039         struct btrfs_key key;
8040         struct inode *inode;
8041         int ret;
8042         int index;
8043         int factor;
8044
8045         root = root->fs_info->extent_root;
8046
8047         block_group = btrfs_lookup_block_group(root->fs_info, group_start);
8048         BUG_ON(!block_group);
8049         BUG_ON(!block_group->ro);
8050
8051         /*
8052          * Free the reserved super bytes from this block group before
8053          * remove it.
8054          */
8055         free_excluded_extents(root, block_group);
8056
8057         memcpy(&key, &block_group->key, sizeof(key));
8058         index = get_block_group_index(block_group);
8059         if (block_group->flags & (BTRFS_BLOCK_GROUP_DUP |
8060                                   BTRFS_BLOCK_GROUP_RAID1 |
8061                                   BTRFS_BLOCK_GROUP_RAID10))
8062                 factor = 2;
8063         else
8064                 factor = 1;
8065
8066         /* make sure this block group isn't part of an allocation cluster */
8067         cluster = &root->fs_info->data_alloc_cluster;
8068         spin_lock(&cluster->refill_lock);
8069         btrfs_return_cluster_to_free_space(block_group, cluster);
8070         spin_unlock(&cluster->refill_lock);
8071
8072         /*
8073          * make sure this block group isn't part of a metadata
8074          * allocation cluster
8075          */
8076         cluster = &root->fs_info->meta_alloc_cluster;
8077         spin_lock(&cluster->refill_lock);
8078         btrfs_return_cluster_to_free_space(block_group, cluster);
8079         spin_unlock(&cluster->refill_lock);
8080
8081         path = btrfs_alloc_path();
8082         if (!path) {
8083                 ret = -ENOMEM;
8084                 goto out;
8085         }
8086
8087         inode = lookup_free_space_inode(tree_root, block_group, path);
8088         if (!IS_ERR(inode)) {
8089                 ret = btrfs_orphan_add(trans, inode);
8090                 if (ret) {
8091                         btrfs_add_delayed_iput(inode);
8092                         goto out;
8093                 }
8094                 clear_nlink(inode);
8095                 /* One for the block groups ref */
8096                 spin_lock(&block_group->lock);
8097                 if (block_group->iref) {
8098                         block_group->iref = 0;
8099                         block_group->inode = NULL;
8100                         spin_unlock(&block_group->lock);
8101                         iput(inode);
8102                 } else {
8103                         spin_unlock(&block_group->lock);
8104                 }
8105                 /* One for our lookup ref */
8106                 btrfs_add_delayed_iput(inode);
8107         }
8108
8109         key.objectid = BTRFS_FREE_SPACE_OBJECTID;
8110         key.offset = block_group->key.objectid;
8111         key.type = 0;
8112
8113         ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
8114         if (ret < 0)
8115                 goto out;
8116         if (ret > 0)
8117                 btrfs_release_path(path);
8118         if (ret == 0) {
8119                 ret = btrfs_del_item(trans, tree_root, path);
8120                 if (ret)
8121                         goto out;
8122                 btrfs_release_path(path);
8123         }
8124
8125         spin_lock(&root->fs_info->block_group_cache_lock);
8126         rb_erase(&block_group->cache_node,
8127                  &root->fs_info->block_group_cache_tree);
8128         spin_unlock(&root->fs_info->block_group_cache_lock);
8129
8130         down_write(&block_group->space_info->groups_sem);
8131         /*
8132          * we must use list_del_init so people can check to see if they
8133          * are still on the list after taking the semaphore
8134          */
8135         list_del_init(&block_group->list);
8136         if (list_empty(&block_group->space_info->block_groups[index]))
8137                 clear_avail_alloc_bits(root->fs_info, block_group->flags);
8138         up_write(&block_group->space_info->groups_sem);
8139
8140         if (block_group->cached == BTRFS_CACHE_STARTED)
8141                 wait_block_group_cache_done(block_group);
8142
8143         btrfs_remove_free_space_cache(block_group);
8144
8145         spin_lock(&block_group->space_info->lock);
8146         block_group->space_info->total_bytes -= block_group->key.offset;
8147         block_group->space_info->bytes_readonly -= block_group->key.offset;
8148         block_group->space_info->disk_total -= block_group->key.offset * factor;
8149         spin_unlock(&block_group->space_info->lock);
8150
8151         memcpy(&key, &block_group->key, sizeof(key));
8152
8153         btrfs_clear_space_info_full(root->fs_info);
8154
8155         btrfs_put_block_group(block_group);
8156         btrfs_put_block_group(block_group);
8157
8158         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
8159         if (ret > 0)
8160                 ret = -EIO;
8161         if (ret < 0)
8162                 goto out;
8163
8164         ret = btrfs_del_item(trans, root, path);
8165 out:
8166         btrfs_free_path(path);
8167         return ret;
8168 }
8169
8170 int btrfs_init_space_info(struct btrfs_fs_info *fs_info)
8171 {
8172         struct btrfs_space_info *space_info;
8173         struct btrfs_super_block *disk_super;
8174         u64 features;
8175         u64 flags;
8176         int mixed = 0;
8177         int ret;
8178
8179         disk_super = fs_info->super_copy;
8180         if (!btrfs_super_root(disk_super))
8181                 return 1;
8182
8183         features = btrfs_super_incompat_flags(disk_super);
8184         if (features & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
8185                 mixed = 1;
8186
8187         flags = BTRFS_BLOCK_GROUP_SYSTEM;
8188         ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8189         if (ret)
8190                 goto out;
8191
8192         if (mixed) {
8193                 flags = BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_DATA;
8194                 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8195         } else {
8196                 flags = BTRFS_BLOCK_GROUP_METADATA;
8197                 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8198                 if (ret)
8199                         goto out;
8200
8201                 flags = BTRFS_BLOCK_GROUP_DATA;
8202                 ret = update_space_info(fs_info, flags, 0, 0, &space_info);
8203         }
8204 out:
8205         return ret;
8206 }
8207
8208 int btrfs_error_unpin_extent_range(struct btrfs_root *root, u64 start, u64 end)
8209 {
8210         return unpin_extent_range(root, start, end);
8211 }
8212
8213 int btrfs_error_discard_extent(struct btrfs_root *root, u64 bytenr,
8214                                u64 num_bytes, u64 *actual_bytes)
8215 {
8216         return btrfs_discard_extent(root, bytenr, num_bytes, actual_bytes);
8217 }
8218
8219 int btrfs_trim_fs(struct btrfs_root *root, struct fstrim_range *range)
8220 {
8221         struct btrfs_fs_info *fs_info = root->fs_info;
8222         struct btrfs_block_group_cache *cache = NULL;
8223         u64 group_trimmed;
8224         u64 start;
8225         u64 end;
8226         u64 trimmed = 0;
8227         u64 total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
8228         int ret = 0;
8229
8230         /*
8231          * try to trim all FS space, our block group may start from non-zero.
8232          */
8233         if (range->len == total_bytes)
8234                 cache = btrfs_lookup_first_block_group(fs_info, range->start);
8235         else
8236                 cache = btrfs_lookup_block_group(fs_info, range->start);
8237
8238         while (cache) {
8239                 if (cache->key.objectid >= (range->start + range->len)) {
8240                         btrfs_put_block_group(cache);
8241                         break;
8242                 }
8243
8244                 start = max(range->start, cache->key.objectid);
8245                 end = min(range->start + range->len,
8246                                 cache->key.objectid + cache->key.offset);
8247
8248                 if (end - start >= range->minlen) {
8249                         if (!block_group_cache_done(cache)) {
8250                                 ret = cache_block_group(cache, NULL, root, 0);
8251                                 if (!ret)
8252                                         wait_block_group_cache_done(cache);
8253                         }
8254                         ret = btrfs_trim_block_group(cache,
8255                                                      &group_trimmed,
8256                                                      start,
8257                                                      end,
8258                                                      range->minlen);
8259
8260                         trimmed += group_trimmed;
8261                         if (ret) {
8262                                 btrfs_put_block_group(cache);
8263                                 break;
8264                         }
8265                 }
8266
8267                 cache = next_block_group(fs_info->tree_root, cache);
8268         }
8269
8270         range->len = trimmed;
8271         return ret;
8272 }