xref: /linux/fs/btrfs/raid56.c (revision 53b381b3abeb86f12787a6c40fee9b2f71edc23b)

/*
 * Copyright (C) 2012 Fusion-io  All rights reserved.
 * Copyright (C) 2012 Intel Corp. All rights reserved.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public
 * License v2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 021110-1307, USA.
 */
#include <linux/sched.h>
#include <linux/wait.h>
#include <linux/bio.h>
#include <linux/slab.h>
#include <linux/buffer_head.h>
#include <linux/blkdev.h>
#include <linux/random.h>
#include <linux/iocontext.h>
#include <linux/capability.h>
#include <linux/ratelimit.h>
#include <linux/kthread.h>
#include <linux/raid/pq.h>
#include <linux/hash.h>
#include <linux/list_sort.h>
#include <linux/raid/xor.h>
#include <asm/div64.h>
#include "compat.h"
#include "ctree.h"
#include "extent_map.h"
#include "disk-io.h"
#include "transaction.h"
#include "print-tree.h"
#include "volumes.h"
#include "raid56.h"
#include "async-thread.h"
#include "check-integrity.h"
#include "rcu-string.h"

/* set when additional merges to this rbio are not allowed */
#define RBIO_RMW_LOCKED_BIT	1

struct btrfs_raid_bio {
	struct btrfs_fs_info *fs_info;
	struct btrfs_bio *bbio;

	/*
	 * logical block numbers for the start of each stripe
	 * The last one or two are p/q.  These are sorted,
	 * so raid_map[0] is the start of our full stripe
	 */
	u64 *raid_map;

	/* while we're doing rmw on a stripe
	 * we put it into a hash table so we can
	 * lock the stripe and merge more rbios
	 * into it.
	 */
	struct list_head hash_list;

	/*
	 * for scheduling work in the helper threads
	 */
	struct btrfs_work work;

	/*
	 * bio list and bio_list_lock are used
	 * to add more bios into the stripe
	 * in hopes of avoiding the full rmw
	 */
	struct bio_list bio_list;
	spinlock_t bio_list_lock;

	/*
	 * also protected by the bio_list_lock, the
	 * stripe locking code uses plug_list to hand off
	 * the stripe lock to the next pending IO
	 */
	struct list_head plug_list;

	/*
	 * flags that tell us if it is safe to
	 * merge with this bio
	 */
	unsigned long flags;

	/* size of each individual stripe on disk */
	int stripe_len;

	/* number of data stripes (no p/q) */
	int nr_data;

	/*
	 * set if we're doing a parity rebuild
	 * for a read from higher up, which is handled
	 * differently from a parity rebuild as part of
	 * rmw
	 */
	int read_rebuild;

	/* first bad stripe */
	int faila;

	/* second bad stripe (for raid6 use) */
	int failb;

	/*
	 * number of pages needed to represent the full
	 * stripe
	 */
	int nr_pages;

	/*
	 * size of all the bios in the bio_list.  This
	 * helps us decide if the rbio maps to a full
	 * stripe or not
	 */
	int bio_list_bytes;

	atomic_t refs;

	/*
	 * these are two arrays of pointers.  We allocate the
	 * rbio big enough to hold them both and setup their
	 * locations when the rbio is allocated
	 */

	/* pointers to pages that we allocated for
	 * reading/writing stripes directly from the disk (including P/Q)
	 */
	struct page **stripe_pages;

	/*
	 * pointers to the pages in the bio_list.  Stored
	 * here for faster lookup
	 */
	struct page **bio_pages;
};

static int __raid56_parity_recover(struct btrfs_raid_bio *rbio);
static noinline void finish_rmw(struct btrfs_raid_bio *rbio);
static void rmw_work(struct btrfs_work *work);
static void read_rebuild_work(struct btrfs_work *work);
static void async_rmw_stripe(struct btrfs_raid_bio *rbio);
static void async_read_rebuild(struct btrfs_raid_bio *rbio);
static int fail_bio_stripe(struct btrfs_raid_bio *rbio, struct bio *bio);
static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed);
static void __free_raid_bio(struct btrfs_raid_bio *rbio);
static void index_rbio_pages(struct btrfs_raid_bio *rbio);
static int alloc_rbio_pages(struct btrfs_raid_bio *rbio);

/*
 * the stripe hash table is used for locking, and to collect
 * bios in hopes of making a full stripe
 */
int btrfs_alloc_stripe_hash_table(struct btrfs_fs_info *info)
{
	struct btrfs_stripe_hash_table *table;
	struct btrfs_stripe_hash_table *x;
	struct btrfs_stripe_hash *cur;
	struct btrfs_stripe_hash *h;
	int num_entries = 1 << BTRFS_STRIPE_HASH_TABLE_BITS;
	int i;

	if (info->stripe_hash_table)
		return 0;

	table = kzalloc(sizeof(*table) + sizeof(*h) * num_entries, GFP_NOFS);
	if (!table)
		return -ENOMEM;

	table->table = (void *)(table + 1);
	h = table->table;

	for (i = 0; i < num_entries; i++) {
		cur = h + i;
		INIT_LIST_HEAD(&cur->hash_list);
		spin_lock_init(&cur->lock);
		init_waitqueue_head(&cur->wait);
	}

	x = cmpxchg(&info->stripe_hash_table, NULL, table);
	if (x)
		kfree(x);
	return 0;
}

/*
 * we hash on the first logical address of the stripe
 */
static int rbio_bucket(struct btrfs_raid_bio *rbio)
{
	u64 num = rbio->raid_map[0];

	/*
	 * we shift down quite a bit.  We're using byte
	 * addressing, and most of the lower bits are zeros.
	 * This tends to upset hash_64, and it consistently
	 * returns just one or two different values.
	 *
	 * shifting off the lower bits fixes things.
	 */
	return hash_64(num >> 16, BTRFS_STRIPE_HASH_TABLE_BITS);
}

/*
 * merging means we take the bio_list from the victim and
 * splice it into the destination.  The victim should
 * be discarded afterwards.
 *
 * must be called with dest->rbio_list_lock held
 */
static void merge_rbio(struct btrfs_raid_bio *dest,
		       struct btrfs_raid_bio *victim)
{
	bio_list_merge(&dest->bio_list, &victim->bio_list);
	dest->bio_list_bytes += victim->bio_list_bytes;
	bio_list_init(&victim->bio_list);
}

/*
 * free the hash table used by unmount
 */
void btrfs_free_stripe_hash_table(struct btrfs_fs_info *info)
{
	if (!info->stripe_hash_table)
		return;
	kfree(info->stripe_hash_table);
	info->stripe_hash_table = NULL;
}

/*
 * helper function to run the xor_blocks api.  It is only
 * able to do MAX_XOR_BLOCKS at a time, so we need to
 * loop through.
 */
static void run_xor(void **pages, int src_cnt, ssize_t len)
{
	int src_off = 0;
	int xor_src_cnt = 0;
	void *dest = pages[src_cnt];

	while(src_cnt > 0) {
		xor_src_cnt = min(src_cnt, MAX_XOR_BLOCKS);
		xor_blocks(xor_src_cnt, len, dest, pages + src_off);

		src_cnt -= xor_src_cnt;
		src_off += xor_src_cnt;
	}
}

/*
 * returns true if the bio list inside this rbio
 * covers an entire stripe (no rmw required).
 * Must be called with the bio list lock held, or
 * at a time when you know it is impossible to add
 * new bios into the list
 */
static int __rbio_is_full(struct btrfs_raid_bio *rbio)
{
	unsigned long size = rbio->bio_list_bytes;
	int ret = 1;

	if (size != rbio->nr_data * rbio->stripe_len)
		ret = 0;

	BUG_ON(size > rbio->nr_data * rbio->stripe_len);
	return ret;
}

static int rbio_is_full(struct btrfs_raid_bio *rbio)
{
	unsigned long flags;
	int ret;

	spin_lock_irqsave(&rbio->bio_list_lock, flags);
	ret = __rbio_is_full(rbio);
	spin_unlock_irqrestore(&rbio->bio_list_lock, flags);
	return ret;
}

/*
 * returns 1 if it is safe to merge two rbios together.
 * The merging is safe if the two rbios correspond to
 * the same stripe and if they are both going in the same
 * direction (read vs write), and if neither one is
 * locked for final IO
 *
 * The caller is responsible for locking such that
 * rmw_locked is safe to test
 */
static int rbio_can_merge(struct btrfs_raid_bio *last,
			  struct btrfs_raid_bio *cur)
{
	if (test_bit(RBIO_RMW_LOCKED_BIT, &last->flags) ||
	    test_bit(RBIO_RMW_LOCKED_BIT, &cur->flags))
		return 0;

	if (last->raid_map[0] !=
	    cur->raid_map[0])
		return 0;

	/* reads can't merge with writes */
	if (last->read_rebuild !=
	    cur->read_rebuild) {
		return 0;
	}

	return 1;
}

/*
 * helper to index into the pstripe
 */
static struct page *rbio_pstripe_page(struct btrfs_raid_bio *rbio, int index)
{
	index += (rbio->nr_data * rbio->stripe_len) >> PAGE_CACHE_SHIFT;
	return rbio->stripe_pages[index];
}

/*
 * helper to index into the qstripe, returns null
 * if there is no qstripe
 */
static struct page *rbio_qstripe_page(struct btrfs_raid_bio *rbio, int index)
{
	if (rbio->nr_data + 1 == rbio->bbio->num_stripes)
		return NULL;

	index += ((rbio->nr_data + 1) * rbio->stripe_len) >>
		PAGE_CACHE_SHIFT;
	return rbio->stripe_pages[index];
}

/*
 * The first stripe in the table for a logical address
 * has the lock.  rbios are added in one of three ways:
 *
 * 1) Nobody has the stripe locked yet.  The rbio is given
 * the lock and 0 is returned.  The caller must start the IO
 * themselves.
 *
 * 2) Someone has the stripe locked, but we're able to merge
 * with the lock owner.  The rbio is freed and the IO will
 * start automatically along with the existing rbio.  1 is returned.
 *
 * 3) Someone has the stripe locked, but we're not able to merge.
 * The rbio is added to the lock owner's plug list, or merged into
 * an rbio already on the plug list.  When the lock owner unlocks,
 * the next rbio on the list is run and the IO is started automatically.
 * 1 is returned
 *
 * If we return 0, the caller still owns the rbio and must continue with
 * IO submission.  If we return 1, the caller must assume the rbio has
 * already been freed.
 */
static noinline int lock_stripe_add(struct btrfs_raid_bio *rbio)
{
	int bucket = rbio_bucket(rbio);
	struct btrfs_stripe_hash *h = rbio->fs_info->stripe_hash_table->table + bucket;
	struct btrfs_raid_bio *cur;
	struct btrfs_raid_bio *pending;
	unsigned long flags;
	DEFINE_WAIT(wait);
	struct btrfs_raid_bio *freeit = NULL;
	int ret = 0;
	int walk = 0;

	spin_lock_irqsave(&h->lock, flags);
	list_for_each_entry(cur, &h->hash_list, hash_list) {
		walk++;
		if (cur->raid_map[0] == rbio->raid_map[0]) {
			spin_lock(&cur->bio_list_lock);

			/* can we merge into the lock owner? */
			if (rbio_can_merge(cur, rbio)) {
				merge_rbio(cur, rbio);
				spin_unlock(&cur->bio_list_lock);
				freeit = rbio;
				ret = 1;
				goto out;
			}

			/*
			 * we couldn't merge with the running
			 * rbio, see if we can merge with the
			 * pending ones.  We don't have to
			 * check for rmw_locked because there
			 * is no way they are inside finish_rmw
			 * right now
			 */
			list_for_each_entry(pending, &cur->plug_list,
					    plug_list) {
				if (rbio_can_merge(pending, rbio)) {
					merge_rbio(pending, rbio);
					spin_unlock(&cur->bio_list_lock);
					freeit = rbio;
					ret = 1;
					goto out;
				}
			}

			/* no merging, put us on the tail of the plug list,
			 * our rbio will be started with the currently
			 * running rbio unlocks
			 */
			list_add_tail(&rbio->plug_list, &cur->plug_list);
			spin_unlock(&cur->bio_list_lock);
			ret = 1;
			goto out;
		}
	}

	atomic_inc(&rbio->refs);
	list_add(&rbio->hash_list, &h->hash_list);
out:
	spin_unlock_irqrestore(&h->lock, flags);
	if (freeit)
		__free_raid_bio(freeit);
	return ret;
}

/*
 * called as rmw or parity rebuild is completed.  If the plug list has more
 * rbios waiting for this stripe, the next one on the list will be started
 */
static noinline void unlock_stripe(struct btrfs_raid_bio *rbio)
{
	int bucket;
	struct btrfs_stripe_hash *h;
	unsigned long flags;

	bucket = rbio_bucket(rbio);
	h = rbio->fs_info->stripe_hash_table->table + bucket;

	spin_lock_irqsave(&h->lock, flags);
	spin_lock(&rbio->bio_list_lock);

	if (!list_empty(&rbio->hash_list)) {

		list_del_init(&rbio->hash_list);
		atomic_dec(&rbio->refs);

		/*
		 * we use the plug list to hold all the rbios
		 * waiting for the chance to lock this stripe.
		 * hand the lock over to one of them.
		 */
		if (!list_empty(&rbio->plug_list)) {
			struct btrfs_raid_bio *next;
			struct list_head *head = rbio->plug_list.next;

			next = list_entry(head, struct btrfs_raid_bio,
					  plug_list);

			list_del_init(&rbio->plug_list);

			list_add(&next->hash_list, &h->hash_list);
			atomic_inc(&next->refs);
			spin_unlock(&rbio->bio_list_lock);
			spin_unlock_irqrestore(&h->lock, flags);

			if (next->read_rebuild)
				async_read_rebuild(next);
			else
				async_rmw_stripe(next);

			goto done_nolock;

		} else  if (waitqueue_active(&h->wait)) {
			spin_unlock(&rbio->bio_list_lock);
			spin_unlock_irqrestore(&h->lock, flags);
			wake_up(&h->wait);
			goto done_nolock;
		}
	}
	spin_unlock(&rbio->bio_list_lock);
	spin_unlock_irqrestore(&h->lock, flags);

done_nolock:
	return;
}

static void __free_raid_bio(struct btrfs_raid_bio *rbio)
{
	int i;

	WARN_ON(atomic_read(&rbio->refs) < 0);
	if (!atomic_dec_and_test(&rbio->refs))
		return;

	WARN_ON(!list_empty(&rbio->hash_list));
	WARN_ON(!bio_list_empty(&rbio->bio_list));

	for (i = 0; i < rbio->nr_pages; i++) {
		if (rbio->stripe_pages[i]) {
			__free_page(rbio->stripe_pages[i]);
			rbio->stripe_pages[i] = NULL;
		}
	}
	kfree(rbio->raid_map);
	kfree(rbio->bbio);
	kfree(rbio);
}

static void free_raid_bio(struct btrfs_raid_bio *rbio)
{
	unlock_stripe(rbio);
	__free_raid_bio(rbio);
}

/*
 * this frees the rbio and runs through all the bios in the
 * bio_list and calls end_io on them
 */
static void rbio_orig_end_io(struct btrfs_raid_bio *rbio, int err, int uptodate)
{
	struct bio *cur = bio_list_get(&rbio->bio_list);
	struct bio *next;
	free_raid_bio(rbio);

	while (cur) {
		next = cur->bi_next;
		cur->bi_next = NULL;
		if (uptodate)
			set_bit(BIO_UPTODATE, &cur->bi_flags);
		bio_endio(cur, err);
		cur = next;
	}
}

/*
 * end io function used by finish_rmw.  When we finally
 * get here, we've written a full stripe
 */
static void raid_write_end_io(struct bio *bio, int err)
{
	struct btrfs_raid_bio *rbio = bio->bi_private;

	if (err)
		fail_bio_stripe(rbio, bio);

	bio_put(bio);

	if (!atomic_dec_and_test(&rbio->bbio->stripes_pending))
		return;

	err = 0;

	/* OK, we have read all the stripes we need to. */
	if (atomic_read(&rbio->bbio->error) > rbio->bbio->max_errors)
		err = -EIO;

	rbio_orig_end_io(rbio, err, 0);
	return;
}

/*
 * the read/modify/write code wants to use the original bio for
 * any pages it included, and then use the rbio for everything
 * else.  This function decides if a given index (stripe number)
 * and page number in that stripe fall inside the original bio
 * or the rbio.
 *
 * if you set bio_list_only, you'll get a NULL back for any ranges
 * that are outside the bio_list
 *
 * This doesn't take any refs on anything, you get a bare page pointer
 * and the caller must bump refs as required.
 *
 * You must call index_rbio_pages once before you can trust
 * the answers from this function.
 */
static struct page *page_in_rbio(struct btrfs_raid_bio *rbio,
				 int index, int pagenr, int bio_list_only)
{
	int chunk_page;
	struct page *p = NULL;

	chunk_page = index * (rbio->stripe_len >> PAGE_SHIFT) + pagenr;

	spin_lock_irq(&rbio->bio_list_lock);
	p = rbio->bio_pages[chunk_page];
	spin_unlock_irq(&rbio->bio_list_lock);

	if (p || bio_list_only)
		return p;

	return rbio->stripe_pages[chunk_page];
}

/*
 * number of pages we need for the entire stripe across all the
 * drives
 */
static unsigned long rbio_nr_pages(unsigned long stripe_len, int nr_stripes)
{
	unsigned long nr = stripe_len * nr_stripes;
	return (nr + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
}

/*
 * allocation and initial setup for the btrfs_raid_bio.  Not
 * this does not allocate any pages for rbio->pages.
 */
static struct btrfs_raid_bio *alloc_rbio(struct btrfs_root *root,
			  struct btrfs_bio *bbio, u64 *raid_map,
			  u64 stripe_len)
{
	struct btrfs_raid_bio *rbio;
	int nr_data = 0;
	int num_pages = rbio_nr_pages(stripe_len, bbio->num_stripes);
	void *p;

	rbio = kzalloc(sizeof(*rbio) + num_pages * sizeof(struct page *) * 2,
			GFP_NOFS);
	if (!rbio) {
		kfree(raid_map);
		kfree(bbio);
		return ERR_PTR(-ENOMEM);
	}

	bio_list_init(&rbio->bio_list);
	INIT_LIST_HEAD(&rbio->plug_list);
	spin_lock_init(&rbio->bio_list_lock);
	INIT_LIST_HEAD(&rbio->hash_list);
	rbio->bbio = bbio;
	rbio->raid_map = raid_map;
	rbio->fs_info = root->fs_info;
	rbio->stripe_len = stripe_len;
	rbio->nr_pages = num_pages;
	rbio->faila = -1;
	rbio->failb = -1;
	atomic_set(&rbio->refs, 1);

	/*
	 * the stripe_pages and bio_pages array point to the extra
	 * memory we allocated past the end of the rbio
	 */
	p = rbio + 1;
	rbio->stripe_pages = p;
	rbio->bio_pages = p + sizeof(struct page *) * num_pages;

	if (raid_map[bbio->num_stripes - 1] == RAID6_Q_STRIPE)
		nr_data = bbio->num_stripes - 2;
	else
		nr_data = bbio->num_stripes - 1;

	rbio->nr_data = nr_data;
	return rbio;
}

/* allocate pages for all the stripes in the bio, including parity */
static int alloc_rbio_pages(struct btrfs_raid_bio *rbio)
{
	int i;
	struct page *page;

	for (i = 0; i < rbio->nr_pages; i++) {
		if (rbio->stripe_pages[i])
			continue;
		page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
		if (!page)
			return -ENOMEM;
		rbio->stripe_pages[i] = page;
		ClearPageUptodate(page);
	}
	return 0;
}

/* allocate pages for just the p/q stripes */
static int alloc_rbio_parity_pages(struct btrfs_raid_bio *rbio)
{
	int i;
	struct page *page;

	i = (rbio->nr_data * rbio->stripe_len) >> PAGE_CACHE_SHIFT;

	for (; i < rbio->nr_pages; i++) {
		if (rbio->stripe_pages[i])
			continue;
		page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
		if (!page)
			return -ENOMEM;
		rbio->stripe_pages[i] = page;
	}
	return 0;
}

/*
 * add a single page from a specific stripe into our list of bios for IO
 * this will try to merge into existing bios if possible, and returns
 * zero if all went well.
 */
int rbio_add_io_page(struct btrfs_raid_bio *rbio,
		     struct bio_list *bio_list,
		     struct page *page,
		     int stripe_nr,
		     unsigned long page_index,
		     unsigned long bio_max_len)
{
	struct bio *last = bio_list->tail;
	u64 last_end = 0;
	int ret;
	struct bio *bio;
	struct btrfs_bio_stripe *stripe;
	u64 disk_start;

	stripe = &rbio->bbio->stripes[stripe_nr];
	disk_start = stripe->physical + (page_index << PAGE_CACHE_SHIFT);

	/* if the device is missing, just fail this stripe */
	if (!stripe->dev->bdev)
		return fail_rbio_index(rbio, stripe_nr);

	/* see if we can add this page onto our existing bio */
	if (last) {
		last_end = (u64)last->bi_sector << 9;
		last_end += last->bi_size;

		/*
		 * we can't merge these if they are from different
		 * devices or if they are not contiguous
		 */
		if (last_end == disk_start && stripe->dev->bdev &&
		    test_bit(BIO_UPTODATE, &last->bi_flags) &&
		    last->bi_bdev == stripe->dev->bdev) {
			ret = bio_add_page(last, page, PAGE_CACHE_SIZE, 0);
			if (ret == PAGE_CACHE_SIZE)
				return 0;
		}
	}

	/* put a new bio on the list */
	bio = bio_alloc(GFP_NOFS, bio_max_len >> PAGE_SHIFT?:1);
	if (!bio)
		return -ENOMEM;

	bio->bi_size = 0;
	bio->bi_bdev = stripe->dev->bdev;
	bio->bi_sector = disk_start >> 9;
	set_bit(BIO_UPTODATE, &bio->bi_flags);

	bio_add_page(bio, page, PAGE_CACHE_SIZE, 0);
	bio_list_add(bio_list, bio);
	return 0;
}

/*
 * while we're doing the read/modify/write cycle, we could
 * have errors in reading pages off the disk.  This checks
 * for errors and if we're not able to read the page it'll
 * trigger parity reconstruction.  The rmw will be finished
 * after we've reconstructed the failed stripes
 */
static void validate_rbio_for_rmw(struct btrfs_raid_bio *rbio)
{
	if (rbio->faila >= 0 || rbio->failb >= 0) {
		BUG_ON(rbio->faila == rbio->bbio->num_stripes - 1);
		__raid56_parity_recover(rbio);
	} else {
		finish_rmw(rbio);
	}
}

/*
 * these are just the pages from the rbio array, not from anything
 * the FS sent down to us
 */
static struct page *rbio_stripe_page(struct btrfs_raid_bio *rbio, int stripe, int page)
{
	int index;
	index = stripe * (rbio->stripe_len >> PAGE_CACHE_SHIFT);
	index += page;
	return rbio->stripe_pages[index];
}

/*
 * helper function to walk our bio list and populate the bio_pages array with
 * the result.  This seems expensive, but it is faster than constantly
 * searching through the bio list as we setup the IO in finish_rmw or stripe
 * reconstruction.
 *
 * This must be called before you trust the answers from page_in_rbio
 */
static void index_rbio_pages(struct btrfs_raid_bio *rbio)
{
	struct bio *bio;
	u64 start;
	unsigned long stripe_offset;
	unsigned long page_index;
	struct page *p;
	int i;

	spin_lock_irq(&rbio->bio_list_lock);
	bio_list_for_each(bio, &rbio->bio_list) {
		start = (u64)bio->bi_sector << 9;
		stripe_offset = start - rbio->raid_map[0];
		page_index = stripe_offset >> PAGE_CACHE_SHIFT;

		for (i = 0; i < bio->bi_vcnt; i++) {
			p = bio->bi_io_vec[i].bv_page;
			rbio->bio_pages[page_index + i] = p;
		}
	}
	spin_unlock_irq(&rbio->bio_list_lock);
}

/*
 * this is called from one of two situations.  We either
 * have a full stripe from the higher layers, or we've read all
 * the missing bits off disk.
 *
 * This will calculate the parity and then send down any
 * changed blocks.
 */
static noinline void finish_rmw(struct btrfs_raid_bio *rbio)
{
	struct btrfs_bio *bbio = rbio->bbio;
	void *pointers[bbio->num_stripes];
	int stripe_len = rbio->stripe_len;
	int nr_data = rbio->nr_data;
	int stripe;
	int pagenr;
	int p_stripe = -1;
	int q_stripe = -1;
	struct bio_list bio_list;
	struct bio *bio;
	int pages_per_stripe = stripe_len >> PAGE_CACHE_SHIFT;
	int ret;

	bio_list_init(&bio_list);

	if (bbio->num_stripes - rbio->nr_data == 1) {
		p_stripe = bbio->num_stripes - 1;
	} else if (bbio->num_stripes - rbio->nr_data == 2) {
		p_stripe = bbio->num_stripes - 2;
		q_stripe = bbio->num_stripes - 1;
	} else {
		BUG();
	}

	/* at this point we either have a full stripe,
	 * or we've read the full stripe from the drive.
	 * recalculate the parity and write the new results.
	 *
	 * We're not allowed to add any new bios to the
	 * bio list here, anyone else that wants to
	 * change this stripe needs to do their own rmw.
	 */
	spin_lock_irq(&rbio->bio_list_lock);
	set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
	spin_unlock_irq(&rbio->bio_list_lock);

	atomic_set(&rbio->bbio->error, 0);

	/*
	 * now that we've set rmw_locked, run through the
	 * bio list one last time and map the page pointers
	 */
	index_rbio_pages(rbio);

	for (pagenr = 0; pagenr < pages_per_stripe; pagenr++) {
		struct page *p;
		/* first collect one page from each data stripe */
		for (stripe = 0; stripe < nr_data; stripe++) {
			p = page_in_rbio(rbio, stripe, pagenr, 0);
			pointers[stripe] = kmap(p);
		}

		/* then add the parity stripe */
		p = rbio_pstripe_page(rbio, pagenr);
		SetPageUptodate(p);
		pointers[stripe++] = kmap(p);

		if (q_stripe != -1) {

			/*
			 * raid6, add the qstripe and call the
			 * library function to fill in our p/q
			 */
			p = rbio_qstripe_page(rbio, pagenr);
			SetPageUptodate(p);
			pointers[stripe++] = kmap(p);

			raid6_call.gen_syndrome(bbio->num_stripes, PAGE_SIZE,
						pointers);
		} else {
			/* raid5 */
			memcpy(pointers[nr_data], pointers[0], PAGE_SIZE);
			run_xor(pointers + 1, nr_data - 1, PAGE_CACHE_SIZE);
		}


		for (stripe = 0; stripe < bbio->num_stripes; stripe++)
			kunmap(page_in_rbio(rbio, stripe, pagenr, 0));
	}

	/*
	 * time to start writing.  Make bios for everything from the
	 * higher layers (the bio_list in our rbio) and our p/q.  Ignore
	 * everything else.
	 */
	for (stripe = 0; stripe < bbio->num_stripes; stripe++) {
		for (pagenr = 0; pagenr < pages_per_stripe; pagenr++) {
			struct page *page;
			if (stripe < rbio->nr_data) {
				page = page_in_rbio(rbio, stripe, pagenr, 1);
				if (!page)
					continue;
			} else {
			       page = rbio_stripe_page(rbio, stripe, pagenr);
			}

			ret = rbio_add_io_page(rbio, &bio_list,
				       page, stripe, pagenr, rbio->stripe_len);
			if (ret)
				goto cleanup;
		}
	}

	atomic_set(&bbio->stripes_pending, bio_list_size(&bio_list));
	BUG_ON(atomic_read(&bbio->stripes_pending) == 0);

	while (1) {
		bio = bio_list_pop(&bio_list);
		if (!bio)
			break;

		bio->bi_private = rbio;
		bio->bi_end_io = raid_write_end_io;
		BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
		submit_bio(WRITE, bio);
	}
	return;

cleanup:
	rbio_orig_end_io(rbio, -EIO, 0);
}

/*
 * helper to find the stripe number for a given bio.  Used to figure out which
 * stripe has failed.  This expects the bio to correspond to a physical disk,
 * so it looks up based on physical sector numbers.
 */
static int find_bio_stripe(struct btrfs_raid_bio *rbio,
			   struct bio *bio)
{
	u64 physical = bio->bi_sector;
	u64 stripe_start;
	int i;
	struct btrfs_bio_stripe *stripe;

	physical <<= 9;

	for (i = 0; i < rbio->bbio->num_stripes; i++) {
		stripe = &rbio->bbio->stripes[i];
		stripe_start = stripe->physical;
		if (physical >= stripe_start &&
		    physical < stripe_start + rbio->stripe_len) {
			return i;
		}
	}
	return -1;
}

/*
 * helper to find the stripe number for a given
 * bio (before mapping).  Used to figure out which stripe has
 * failed.  This looks up based on logical block numbers.
 */
static int find_logical_bio_stripe(struct btrfs_raid_bio *rbio,
				   struct bio *bio)
{
	u64 logical = bio->bi_sector;
	u64 stripe_start;
	int i;

	logical <<= 9;

	for (i = 0; i < rbio->nr_data; i++) {
		stripe_start = rbio->raid_map[i];
		if (logical >= stripe_start &&
		    logical < stripe_start + rbio->stripe_len) {
			return i;
		}
	}
	return -1;
}

/*
 * returns -EIO if we had too many failures
 */
static int fail_rbio_index(struct btrfs_raid_bio *rbio, int failed)
{
	unsigned long flags;
	int ret = 0;

	spin_lock_irqsave(&rbio->bio_list_lock, flags);

	/* we already know this stripe is bad, move on */
	if (rbio->faila == failed || rbio->failb == failed)
		goto out;

	if (rbio->faila == -1) {
		/* first failure on this rbio */
		rbio->faila = failed;
		atomic_inc(&rbio->bbio->error);
	} else if (rbio->failb == -1) {
		/* second failure on this rbio */
		rbio->failb = failed;
		atomic_inc(&rbio->bbio->error);
	} else {
		ret = -EIO;
	}
out:
	spin_unlock_irqrestore(&rbio->bio_list_lock, flags);

	return ret;
}

/*
 * helper to fail a stripe based on a physical disk
 * bio.
 */
static int fail_bio_stripe(struct btrfs_raid_bio *rbio,
			   struct bio *bio)
{
	int failed = find_bio_stripe(rbio, bio);

	if (failed < 0)
		return -EIO;

	return fail_rbio_index(rbio, failed);
}

/*
 * this sets each page in the bio uptodate.  It should only be used on private
 * rbio pages, nothing that comes in from the higher layers
 */
static void set_bio_pages_uptodate(struct bio *bio)
{
	int i;
	struct page *p;

	for (i = 0; i < bio->bi_vcnt; i++) {
		p = bio->bi_io_vec[i].bv_page;
		SetPageUptodate(p);
	}
}

/*
 * end io for the read phase of the rmw cycle.  All the bios here are physical
 * stripe bios we've read from the disk so we can recalculate the parity of the
 * stripe.
 *
 * This will usually kick off finish_rmw once all the bios are read in, but it
 * may trigger parity reconstruction if we had any errors along the way
 */
static void raid_rmw_end_io(struct bio *bio, int err)
{
	struct btrfs_raid_bio *rbio = bio->bi_private;

	if (err)
		fail_bio_stripe(rbio, bio);
	else
		set_bio_pages_uptodate(bio);

	bio_put(bio);

	if (!atomic_dec_and_test(&rbio->bbio->stripes_pending))
		return;

	err = 0;
	if (atomic_read(&rbio->bbio->error) > rbio->bbio->max_errors)
		goto cleanup;

	/*
	 * this will normally call finish_rmw to start our write
	 * but if there are any failed stripes we'll reconstruct
	 * from parity first
	 */
	validate_rbio_for_rmw(rbio);
	return;

cleanup:

	rbio_orig_end_io(rbio, -EIO, 0);
}

static void async_rmw_stripe(struct btrfs_raid_bio *rbio)
{
	rbio->work.flags = 0;
	rbio->work.func = rmw_work;

	btrfs_queue_worker(&rbio->fs_info->rmw_workers,
			   &rbio->work);
}

static void async_read_rebuild(struct btrfs_raid_bio *rbio)
{
	rbio->work.flags = 0;
	rbio->work.func = read_rebuild_work;

	btrfs_queue_worker(&rbio->fs_info->rmw_workers,
			   &rbio->work);
}

/*
 * the stripe must be locked by the caller.  It will
 * unlock after all the writes are done
 */
static int raid56_rmw_stripe(struct btrfs_raid_bio *rbio)
{
	int bios_to_read = 0;
	struct btrfs_bio *bbio = rbio->bbio;
	struct bio_list bio_list;
	int ret;
	int nr_pages = (rbio->stripe_len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	int pagenr;
	int stripe;
	struct bio *bio;

	bio_list_init(&bio_list);

	ret = alloc_rbio_pages(rbio);
	if (ret)
		goto cleanup;

	index_rbio_pages(rbio);

	atomic_set(&rbio->bbio->error, 0);
	/*
	 * build a list of bios to read all the missing parts of this
	 * stripe
	 */
	for (stripe = 0; stripe < rbio->nr_data; stripe++) {
		for (pagenr = 0; pagenr < nr_pages; pagenr++) {
			struct page *page;
			/*
			 * we want to find all the pages missing from
			 * the rbio and read them from the disk.  If
			 * page_in_rbio finds a page in the bio list
			 * we don't need to read it off the stripe.
			 */
			page = page_in_rbio(rbio, stripe, pagenr, 1);
			if (page)
				continue;

			page = rbio_stripe_page(rbio, stripe, pagenr);
			ret = rbio_add_io_page(rbio, &bio_list, page,
				       stripe, pagenr, rbio->stripe_len);
			if (ret)
				goto cleanup;
		}
	}

	bios_to_read = bio_list_size(&bio_list);
	if (!bios_to_read) {
		/*
		 * this can happen if others have merged with
		 * us, it means there is nothing left to read.
		 * But if there are missing devices it may not be
		 * safe to do the full stripe write yet.
		 */
		goto finish;
	}

	/*
	 * the bbio may be freed once we submit the last bio.  Make sure
	 * not to touch it after that
	 */
	atomic_set(&bbio->stripes_pending, bios_to_read);
	while (1) {
		bio = bio_list_pop(&bio_list);
		if (!bio)
			break;

		bio->bi_private = rbio;
		bio->bi_end_io = raid_rmw_end_io;

		btrfs_bio_wq_end_io(rbio->fs_info, bio,
				    BTRFS_WQ_ENDIO_RAID56);

		BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
		submit_bio(READ, bio);
	}
	/* the actual write will happen once the reads are done */
	return 0;

cleanup:
	rbio_orig_end_io(rbio, -EIO, 0);
	return -EIO;

finish:
	validate_rbio_for_rmw(rbio);
	return 0;
}

/*
 * if the upper layers pass in a full stripe, we thank them by only allocating
 * enough pages to hold the parity, and sending it all down quickly.
 */
static int full_stripe_write(struct btrfs_raid_bio *rbio)
{
	int ret;

	ret = alloc_rbio_parity_pages(rbio);
	if (ret)
		return ret;

	ret = lock_stripe_add(rbio);
	if (ret == 0)
		finish_rmw(rbio);
	return 0;
}

/*
 * partial stripe writes get handed over to async helpers.
 * We're really hoping to merge a few more writes into this
 * rbio before calculating new parity
 */
static int partial_stripe_write(struct btrfs_raid_bio *rbio)
{
	int ret;

	ret = lock_stripe_add(rbio);
	if (ret == 0)
		async_rmw_stripe(rbio);
	return 0;
}

/*
 * sometimes while we were reading from the drive to
 * recalculate parity, enough new bios come into create
 * a full stripe.  So we do a check here to see if we can
 * go directly to finish_rmw
 */
static int __raid56_parity_write(struct btrfs_raid_bio *rbio)
{
	/* head off into rmw land if we don't have a full stripe */
	if (!rbio_is_full(rbio))
		return partial_stripe_write(rbio);
	return full_stripe_write(rbio);
}

/*
 * our main entry point for writes from the rest of the FS.
 */
int raid56_parity_write(struct btrfs_root *root, struct bio *bio,
			struct btrfs_bio *bbio, u64 *raid_map,
			u64 stripe_len)
{
	struct btrfs_raid_bio *rbio;

	rbio = alloc_rbio(root, bbio, raid_map, stripe_len);
	if (IS_ERR(rbio)) {
		kfree(raid_map);
		kfree(bbio);
		return PTR_ERR(rbio);
	}
	bio_list_add(&rbio->bio_list, bio);
	rbio->bio_list_bytes = bio->bi_size;
	return __raid56_parity_write(rbio);
}

/*
 * all parity reconstruction happens here.  We've read in everything
 * we can find from the drives and this does the heavy lifting of
 * sorting the good from the bad.
 */
static void __raid_recover_end_io(struct btrfs_raid_bio *rbio)
{
	int pagenr, stripe;
	void **pointers;
	int faila = -1, failb = -1;
	int nr_pages = (rbio->stripe_len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	struct page *page;
	int err;
	int i;

	pointers = kzalloc(rbio->bbio->num_stripes * sizeof(void *),
			   GFP_NOFS);
	if (!pointers) {
		err = -ENOMEM;
		goto cleanup_io;
	}

	faila = rbio->faila;
	failb = rbio->failb;

	if (rbio->read_rebuild) {
		spin_lock_irq(&rbio->bio_list_lock);
		set_bit(RBIO_RMW_LOCKED_BIT, &rbio->flags);
		spin_unlock_irq(&rbio->bio_list_lock);
	}

	index_rbio_pages(rbio);

	for (pagenr = 0; pagenr < nr_pages; pagenr++) {
		/* setup our array of pointers with pages
		 * from each stripe
		 */
		for (stripe = 0; stripe < rbio->bbio->num_stripes; stripe++) {
			/*
			 * if we're rebuilding a read, we have to use
			 * pages from the bio list
			 */
			if (rbio->read_rebuild &&
			    (stripe == faila || stripe == failb)) {
				page = page_in_rbio(rbio, stripe, pagenr, 0);
			} else {
				page = rbio_stripe_page(rbio, stripe, pagenr);
			}
			pointers[stripe] = kmap(page);
		}

		/* all raid6 handling here */
		if (rbio->raid_map[rbio->bbio->num_stripes - 1] ==
		    RAID6_Q_STRIPE) {

			/*
			 * single failure, rebuild from parity raid5
			 * style
			 */
			if (failb < 0) {
				if (faila == rbio->nr_data) {
					/*
					 * Just the P stripe has failed, without
					 * a bad data or Q stripe.
					 * TODO, we should redo the xor here.
					 */
					err = -EIO;
					goto cleanup;
				}
				/*
				 * a single failure in raid6 is rebuilt
				 * in the pstripe code below
				 */
				goto pstripe;
			}

			/* make sure our ps and qs are in order */
			if (faila > failb) {
				int tmp = failb;
				failb = faila;
				faila = tmp;
			}

			/* if the q stripe is failed, do a pstripe reconstruction
			 * from the xors.
			 * If both the q stripe and the P stripe are failed, we're
			 * here due to a crc mismatch and we can't give them the
			 * data they want
			 */
			if (rbio->raid_map[failb] == RAID6_Q_STRIPE) {
				if (rbio->raid_map[faila] == RAID5_P_STRIPE) {
					err = -EIO;
					goto cleanup;
				}
				/*
				 * otherwise we have one bad data stripe and
				 * a good P stripe.  raid5!
				 */
				goto pstripe;
			}

			if (rbio->raid_map[failb] == RAID5_P_STRIPE) {
				raid6_datap_recov(rbio->bbio->num_stripes,
						  PAGE_SIZE, faila, pointers);
			} else {
				raid6_2data_recov(rbio->bbio->num_stripes,
						  PAGE_SIZE, faila, failb,
						  pointers);
			}
		} else {
			void *p;

			/* rebuild from P stripe here (raid5 or raid6) */
			BUG_ON(failb != -1);
pstripe:
			/* Copy parity block into failed block to start with */
			memcpy(pointers[faila],
			       pointers[rbio->nr_data],
			       PAGE_CACHE_SIZE);

			/* rearrange the pointer array */
			p = pointers[faila];
			for (stripe = faila; stripe < rbio->nr_data - 1; stripe++)
				pointers[stripe] = pointers[stripe + 1];
			pointers[rbio->nr_data - 1] = p;

			/* xor in the rest */
			run_xor(pointers, rbio->nr_data - 1, PAGE_CACHE_SIZE);
		}
		/* if we're doing this rebuild as part of an rmw, go through
		 * and set all of our private rbio pages in the
		 * failed stripes as uptodate.  This way finish_rmw will
		 * know they can be trusted.  If this was a read reconstruction,
		 * other endio functions will fiddle the uptodate bits
		 */
		if (!rbio->read_rebuild) {
			for (i = 0;  i < nr_pages; i++) {
				if (faila != -1) {
					page = rbio_stripe_page(rbio, faila, i);
					SetPageUptodate(page);
				}
				if (failb != -1) {
					page = rbio_stripe_page(rbio, failb, i);
					SetPageUptodate(page);
				}
			}
		}
		for (stripe = 0; stripe < rbio->bbio->num_stripes; stripe++) {
			/*
			 * if we're rebuilding a read, we have to use
			 * pages from the bio list
			 */
			if (rbio->read_rebuild &&
			    (stripe == faila || stripe == failb)) {
				page = page_in_rbio(rbio, stripe, pagenr, 0);
			} else {
				page = rbio_stripe_page(rbio, stripe, pagenr);
			}
			kunmap(page);
		}
	}

	err = 0;
cleanup:
	kfree(pointers);

cleanup_io:

	if (rbio->read_rebuild) {
		rbio_orig_end_io(rbio, err, err == 0);
	} else if (err == 0) {
		rbio->faila = -1;
		rbio->failb = -1;
		finish_rmw(rbio);
	} else {
		rbio_orig_end_io(rbio, err, 0);
	}
}

/*
 * This is called only for stripes we've read from disk to
 * reconstruct the parity.
 */
static void raid_recover_end_io(struct bio *bio, int err)
{
	struct btrfs_raid_bio *rbio = bio->bi_private;

	/*
	 * we only read stripe pages off the disk, set them
	 * up to date if there were no errors
	 */
	if (err)
		fail_bio_stripe(rbio, bio);
	else
		set_bio_pages_uptodate(bio);
	bio_put(bio);

	if (!atomic_dec_and_test(&rbio->bbio->stripes_pending))
		return;

	if (atomic_read(&rbio->bbio->error) > rbio->bbio->max_errors)
		rbio_orig_end_io(rbio, -EIO, 0);
	else
		__raid_recover_end_io(rbio);
}

/*
 * reads everything we need off the disk to reconstruct
 * the parity. endio handlers trigger final reconstruction
 * when the IO is done.
 *
 * This is used both for reads from the higher layers and for
 * parity construction required to finish a rmw cycle.
 */
static int __raid56_parity_recover(struct btrfs_raid_bio *rbio)
{
	int bios_to_read = 0;
	struct btrfs_bio *bbio = rbio->bbio;
	struct bio_list bio_list;
	int ret;
	int nr_pages = (rbio->stripe_len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
	int pagenr;
	int stripe;
	struct bio *bio;

	bio_list_init(&bio_list);

	ret = alloc_rbio_pages(rbio);
	if (ret)
		goto cleanup;

	atomic_set(&rbio->bbio->error, 0);

	/*
	 * read everything that hasn't failed.
	 */
	for (stripe = 0; stripe < bbio->num_stripes; stripe++) {
		if (rbio->faila == stripe ||
		    rbio->failb == stripe)
			continue;

		for (pagenr = 0; pagenr < nr_pages; pagenr++) {
			struct page *p;

			/*
			 * the rmw code may have already read this
			 * page in
			 */
			p = rbio_stripe_page(rbio, stripe, pagenr);
			if (PageUptodate(p))
				continue;

			ret = rbio_add_io_page(rbio, &bio_list,
				       rbio_stripe_page(rbio, stripe, pagenr),
				       stripe, pagenr, rbio->stripe_len);
			if (ret < 0)
				goto cleanup;
		}
	}

	bios_to_read = bio_list_size(&bio_list);
	if (!bios_to_read) {
		/*
		 * we might have no bios to read just because the pages
		 * were up to date, or we might have no bios to read because
		 * the devices were gone.
		 */
		if (atomic_read(&rbio->bbio->error) <= rbio->bbio->max_errors) {
			__raid_recover_end_io(rbio);
			goto out;
		} else {
			goto cleanup;
		}
	}

	/*
	 * the bbio may be freed once we submit the last bio.  Make sure
	 * not to touch it after that
	 */
	atomic_set(&bbio->stripes_pending, bios_to_read);
	while (1) {
		bio = bio_list_pop(&bio_list);
		if (!bio)
			break;

		bio->bi_private = rbio;
		bio->bi_end_io = raid_recover_end_io;

		btrfs_bio_wq_end_io(rbio->fs_info, bio,
				    BTRFS_WQ_ENDIO_RAID56);

		BUG_ON(!test_bit(BIO_UPTODATE, &bio->bi_flags));
		submit_bio(READ, bio);
	}
out:
	return 0;

cleanup:
	if (rbio->read_rebuild)
		rbio_orig_end_io(rbio, -EIO, 0);
	return -EIO;
}

/*
 * the main entry point for reads from the higher layers.  This
 * is really only called when the normal read path had a failure,
 * so we assume the bio they send down corresponds to a failed part
 * of the drive.
 */
int raid56_parity_recover(struct btrfs_root *root, struct bio *bio,
			  struct btrfs_bio *bbio, u64 *raid_map,
			  u64 stripe_len, int mirror_num)
{
	struct btrfs_raid_bio *rbio;
	int ret;

	rbio = alloc_rbio(root, bbio, raid_map, stripe_len);
	if (IS_ERR(rbio)) {
		return PTR_ERR(rbio);
	}

	rbio->read_rebuild = 1;
	bio_list_add(&rbio->bio_list, bio);
	rbio->bio_list_bytes = bio->bi_size;

	rbio->faila = find_logical_bio_stripe(rbio, bio);
	if (rbio->faila == -1) {
		BUG();
		kfree(rbio);
		return -EIO;
	}

	/*
	 * reconstruct from the q stripe if they are
	 * asking for mirror 3
	 */
	if (mirror_num == 3)
		rbio->failb = bbio->num_stripes - 2;

	ret = lock_stripe_add(rbio);

	/*
	 * __raid56_parity_recover will end the bio with
	 * any errors it hits.  We don't want to return
	 * its error value up the stack because our caller
	 * will end up calling bio_endio with any nonzero
	 * return
	 */
	if (ret == 0)
		__raid56_parity_recover(rbio);
	/*
	 * our rbio has been added to the list of
	 * rbios that will be handled after the
	 * currently lock owner is done
	 */
	return 0;

}

static void rmw_work(struct btrfs_work *work)
{
	struct btrfs_raid_bio *rbio;

	rbio = container_of(work, struct btrfs_raid_bio, work);
	raid56_rmw_stripe(rbio);
}

static void read_rebuild_work(struct btrfs_work *work)
{
	struct btrfs_raid_bio *rbio;

	rbio = container_of(work, struct btrfs_raid_bio, work);
	__raid56_parity_recover(rbio);
}