12d1e580aSAlasdair G Kergon /* 22d1e580aSAlasdair G Kergon * Copyright (C) 2002 Sistina Software (UK) Limited. 32d1e580aSAlasdair G Kergon * Copyright (C) 2006 Red Hat GmbH 42d1e580aSAlasdair G Kergon * 52d1e580aSAlasdair G Kergon * This file is released under the GPL. 62d1e580aSAlasdair G Kergon * 72d1e580aSAlasdair G Kergon * Kcopyd provides a simple interface for copying an area of one 82d1e580aSAlasdair G Kergon * block-device to one or more other block-devices, with an asynchronous 92d1e580aSAlasdair G Kergon * completion notification. 102d1e580aSAlasdair G Kergon */ 112d1e580aSAlasdair G Kergon 122d1e580aSAlasdair G Kergon #include <linux/types.h> 132d1e580aSAlasdair G Kergon #include <asm/atomic.h> 142d1e580aSAlasdair G Kergon #include <linux/blkdev.h> 152d1e580aSAlasdair G Kergon #include <linux/fs.h> 162d1e580aSAlasdair G Kergon #include <linux/init.h> 172d1e580aSAlasdair G Kergon #include <linux/list.h> 182d1e580aSAlasdair G Kergon #include <linux/mempool.h> 192d1e580aSAlasdair G Kergon #include <linux/module.h> 202d1e580aSAlasdair G Kergon #include <linux/pagemap.h> 212d1e580aSAlasdair G Kergon #include <linux/slab.h> 222d1e580aSAlasdair G Kergon #include <linux/vmalloc.h> 232d1e580aSAlasdair G Kergon #include <linux/workqueue.h> 242d1e580aSAlasdair G Kergon #include <linux/mutex.h> 25586e80e6SMikulas Patocka #include <linux/device-mapper.h> 26a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 272d1e580aSAlasdair G Kergon 282d1e580aSAlasdair G Kergon #include "dm.h" 292d1e580aSAlasdair G Kergon 30c6ea41fbSMikulas Patocka #define SUB_JOB_SIZE 128 31c6ea41fbSMikulas Patocka #define SPLIT_COUNT 8 32c6ea41fbSMikulas Patocka #define MIN_JOBS 8 335f43ba29SMikulas Patocka #define RESERVE_PAGES (DIV_ROUND_UP(SUB_JOB_SIZE << SECTOR_SHIFT, PAGE_SIZE)) 34c6ea41fbSMikulas Patocka 352d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 362d1e580aSAlasdair G Kergon * Each kcopyd client has its own little pool of preallocated 372d1e580aSAlasdair G Kergon * pages for kcopyd io. 382d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 392d1e580aSAlasdair G Kergon struct dm_kcopyd_client { 402d1e580aSAlasdair G Kergon struct page_list *pages; 41d0471458SMikulas Patocka unsigned nr_reserved_pages; 42d0471458SMikulas Patocka unsigned nr_free_pages; 432d1e580aSAlasdair G Kergon 442d1e580aSAlasdair G Kergon struct dm_io_client *io_client; 452d1e580aSAlasdair G Kergon 462d1e580aSAlasdair G Kergon wait_queue_head_t destroyq; 472d1e580aSAlasdair G Kergon atomic_t nr_jobs; 482d1e580aSAlasdair G Kergon 492d1e580aSAlasdair G Kergon mempool_t *job_pool; 502d1e580aSAlasdair G Kergon 512d1e580aSAlasdair G Kergon struct workqueue_struct *kcopyd_wq; 522d1e580aSAlasdair G Kergon struct work_struct kcopyd_work; 532d1e580aSAlasdair G Kergon 542d1e580aSAlasdair G Kergon /* 552d1e580aSAlasdair G Kergon * We maintain three lists of jobs: 562d1e580aSAlasdair G Kergon * 572d1e580aSAlasdair G Kergon * i) jobs waiting for pages 582d1e580aSAlasdair G Kergon * ii) jobs that have pages, and are waiting for the io to be issued. 592d1e580aSAlasdair G Kergon * iii) jobs that have completed. 602d1e580aSAlasdair G Kergon * 612d1e580aSAlasdair G Kergon * All three of these are protected by job_lock. 622d1e580aSAlasdair G Kergon */ 632d1e580aSAlasdair G Kergon spinlock_t job_lock; 642d1e580aSAlasdair G Kergon struct list_head complete_jobs; 652d1e580aSAlasdair G Kergon struct list_head io_jobs; 662d1e580aSAlasdair G Kergon struct list_head pages_jobs; 672d1e580aSAlasdair G Kergon }; 682d1e580aSAlasdair G Kergon 692d1e580aSAlasdair G Kergon static void wake(struct dm_kcopyd_client *kc) 702d1e580aSAlasdair G Kergon { 712d1e580aSAlasdair G Kergon queue_work(kc->kcopyd_wq, &kc->kcopyd_work); 722d1e580aSAlasdair G Kergon } 732d1e580aSAlasdair G Kergon 74d0471458SMikulas Patocka /* 75d0471458SMikulas Patocka * Obtain one page for the use of kcopyd. 76d0471458SMikulas Patocka */ 77f99b55eeSMikulas Patocka static struct page_list *alloc_pl(gfp_t gfp) 782d1e580aSAlasdair G Kergon { 792d1e580aSAlasdair G Kergon struct page_list *pl; 802d1e580aSAlasdair G Kergon 81f99b55eeSMikulas Patocka pl = kmalloc(sizeof(*pl), gfp); 822d1e580aSAlasdair G Kergon if (!pl) 832d1e580aSAlasdair G Kergon return NULL; 842d1e580aSAlasdair G Kergon 85f99b55eeSMikulas Patocka pl->page = alloc_page(gfp); 862d1e580aSAlasdair G Kergon if (!pl->page) { 872d1e580aSAlasdair G Kergon kfree(pl); 882d1e580aSAlasdair G Kergon return NULL; 892d1e580aSAlasdair G Kergon } 902d1e580aSAlasdair G Kergon 912d1e580aSAlasdair G Kergon return pl; 922d1e580aSAlasdair G Kergon } 932d1e580aSAlasdair G Kergon 942d1e580aSAlasdair G Kergon static void free_pl(struct page_list *pl) 952d1e580aSAlasdair G Kergon { 962d1e580aSAlasdair G Kergon __free_page(pl->page); 972d1e580aSAlasdair G Kergon kfree(pl); 982d1e580aSAlasdair G Kergon } 992d1e580aSAlasdair G Kergon 100d0471458SMikulas Patocka /* 101d0471458SMikulas Patocka * Add the provided pages to a client's free page list, releasing 102d0471458SMikulas Patocka * back to the system any beyond the reserved_pages limit. 103d0471458SMikulas Patocka */ 104d0471458SMikulas Patocka static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) 105d0471458SMikulas Patocka { 106d0471458SMikulas Patocka struct page_list *next; 107d0471458SMikulas Patocka 108d0471458SMikulas Patocka do { 109d0471458SMikulas Patocka next = pl->next; 110d0471458SMikulas Patocka 111d0471458SMikulas Patocka if (kc->nr_free_pages >= kc->nr_reserved_pages) 112d0471458SMikulas Patocka free_pl(pl); 113d0471458SMikulas Patocka else { 114d0471458SMikulas Patocka pl->next = kc->pages; 115d0471458SMikulas Patocka kc->pages = pl; 116d0471458SMikulas Patocka kc->nr_free_pages++; 117d0471458SMikulas Patocka } 118d0471458SMikulas Patocka 119d0471458SMikulas Patocka pl = next; 120d0471458SMikulas Patocka } while (pl); 121d0471458SMikulas Patocka } 122d0471458SMikulas Patocka 1232d1e580aSAlasdair G Kergon static int kcopyd_get_pages(struct dm_kcopyd_client *kc, 1242d1e580aSAlasdair G Kergon unsigned int nr, struct page_list **pages) 1252d1e580aSAlasdair G Kergon { 1262d1e580aSAlasdair G Kergon struct page_list *pl; 1272d1e580aSAlasdair G Kergon 128d0471458SMikulas Patocka *pages = NULL; 1292d1e580aSAlasdair G Kergon 130d0471458SMikulas Patocka do { 131d0471458SMikulas Patocka pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY); 132d0471458SMikulas Patocka if (unlikely(!pl)) { 133d0471458SMikulas Patocka /* Use reserved pages */ 134d0471458SMikulas Patocka pl = kc->pages; 135d0471458SMikulas Patocka if (unlikely(!pl)) 136d0471458SMikulas Patocka goto out_of_memory; 1372d1e580aSAlasdair G Kergon kc->pages = pl->next; 138d0471458SMikulas Patocka kc->nr_free_pages--; 139d0471458SMikulas Patocka } 140d0471458SMikulas Patocka pl->next = *pages; 141d0471458SMikulas Patocka *pages = pl; 142d0471458SMikulas Patocka } while (--nr); 1432d1e580aSAlasdair G Kergon 1442d1e580aSAlasdair G Kergon return 0; 1452d1e580aSAlasdair G Kergon 146d0471458SMikulas Patocka out_of_memory: 147d0471458SMikulas Patocka if (*pages) 148d0471458SMikulas Patocka kcopyd_put_pages(kc, *pages); 149d0471458SMikulas Patocka return -ENOMEM; 1502d1e580aSAlasdair G Kergon } 1512d1e580aSAlasdair G Kergon 1522d1e580aSAlasdair G Kergon /* 1532d1e580aSAlasdair G Kergon * These three functions resize the page pool. 1542d1e580aSAlasdair G Kergon */ 1552d1e580aSAlasdair G Kergon static void drop_pages(struct page_list *pl) 1562d1e580aSAlasdair G Kergon { 1572d1e580aSAlasdair G Kergon struct page_list *next; 1582d1e580aSAlasdair G Kergon 1592d1e580aSAlasdair G Kergon while (pl) { 1602d1e580aSAlasdair G Kergon next = pl->next; 1612d1e580aSAlasdair G Kergon free_pl(pl); 1622d1e580aSAlasdair G Kergon pl = next; 1632d1e580aSAlasdair G Kergon } 1642d1e580aSAlasdair G Kergon } 1652d1e580aSAlasdair G Kergon 166d0471458SMikulas Patocka /* 167d0471458SMikulas Patocka * Allocate and reserve nr_pages for the use of a specific client. 168d0471458SMikulas Patocka */ 169d0471458SMikulas Patocka static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) 1702d1e580aSAlasdair G Kergon { 171d0471458SMikulas Patocka unsigned i; 1722d1e580aSAlasdair G Kergon struct page_list *pl = NULL, *next; 1732d1e580aSAlasdair G Kergon 174d0471458SMikulas Patocka for (i = 0; i < nr_pages; i++) { 175f99b55eeSMikulas Patocka next = alloc_pl(GFP_KERNEL); 1762d1e580aSAlasdair G Kergon if (!next) { 1772d1e580aSAlasdair G Kergon if (pl) 1782d1e580aSAlasdair G Kergon drop_pages(pl); 1792d1e580aSAlasdair G Kergon return -ENOMEM; 1802d1e580aSAlasdair G Kergon } 1812d1e580aSAlasdair G Kergon next->next = pl; 1822d1e580aSAlasdair G Kergon pl = next; 1832d1e580aSAlasdair G Kergon } 1842d1e580aSAlasdair G Kergon 185d0471458SMikulas Patocka kc->nr_reserved_pages += nr_pages; 1862d1e580aSAlasdair G Kergon kcopyd_put_pages(kc, pl); 187d0471458SMikulas Patocka 1882d1e580aSAlasdair G Kergon return 0; 1892d1e580aSAlasdair G Kergon } 1902d1e580aSAlasdair G Kergon 1912d1e580aSAlasdair G Kergon static void client_free_pages(struct dm_kcopyd_client *kc) 1922d1e580aSAlasdair G Kergon { 193d0471458SMikulas Patocka BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages); 1942d1e580aSAlasdair G Kergon drop_pages(kc->pages); 1952d1e580aSAlasdair G Kergon kc->pages = NULL; 196d0471458SMikulas Patocka kc->nr_free_pages = kc->nr_reserved_pages = 0; 1972d1e580aSAlasdair G Kergon } 1982d1e580aSAlasdair G Kergon 1992d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 2002d1e580aSAlasdair G Kergon * kcopyd_jobs need to be allocated by the *clients* of kcopyd, 2012d1e580aSAlasdair G Kergon * for this reason we use a mempool to prevent the client from 2022d1e580aSAlasdair G Kergon * ever having to do io (which could cause a deadlock). 2032d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 2042d1e580aSAlasdair G Kergon struct kcopyd_job { 2052d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc; 2062d1e580aSAlasdair G Kergon struct list_head list; 2072d1e580aSAlasdair G Kergon unsigned long flags; 2082d1e580aSAlasdair G Kergon 2092d1e580aSAlasdair G Kergon /* 2102d1e580aSAlasdair G Kergon * Error state of the job. 2112d1e580aSAlasdair G Kergon */ 2122d1e580aSAlasdair G Kergon int read_err; 2132d1e580aSAlasdair G Kergon unsigned long write_err; 2142d1e580aSAlasdair G Kergon 2152d1e580aSAlasdair G Kergon /* 2162d1e580aSAlasdair G Kergon * Either READ or WRITE 2172d1e580aSAlasdair G Kergon */ 2182d1e580aSAlasdair G Kergon int rw; 2192d1e580aSAlasdair G Kergon struct dm_io_region source; 2202d1e580aSAlasdair G Kergon 2212d1e580aSAlasdair G Kergon /* 2222d1e580aSAlasdair G Kergon * The destinations for the transfer. 2232d1e580aSAlasdair G Kergon */ 2242d1e580aSAlasdair G Kergon unsigned int num_dests; 2252d1e580aSAlasdair G Kergon struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS]; 2262d1e580aSAlasdair G Kergon 2272d1e580aSAlasdair G Kergon sector_t offset; 2282d1e580aSAlasdair G Kergon unsigned int nr_pages; 2292d1e580aSAlasdair G Kergon struct page_list *pages; 2302d1e580aSAlasdair G Kergon 2312d1e580aSAlasdair G Kergon /* 2322d1e580aSAlasdair G Kergon * Set this to ensure you are notified when the job has 2332d1e580aSAlasdair G Kergon * completed. 'context' is for callback to use. 2342d1e580aSAlasdair G Kergon */ 2352d1e580aSAlasdair G Kergon dm_kcopyd_notify_fn fn; 2362d1e580aSAlasdair G Kergon void *context; 2372d1e580aSAlasdair G Kergon 2382d1e580aSAlasdair G Kergon /* 2392d1e580aSAlasdair G Kergon * These fields are only used if the job has been split 2402d1e580aSAlasdair G Kergon * into more manageable parts. 2412d1e580aSAlasdair G Kergon */ 2422d1e580aSAlasdair G Kergon struct mutex lock; 2432d1e580aSAlasdair G Kergon atomic_t sub_jobs; 2442d1e580aSAlasdair G Kergon sector_t progress; 2452d1e580aSAlasdair G Kergon 246c6ea41fbSMikulas Patocka struct kcopyd_job *master_job; 247c6ea41fbSMikulas Patocka }; 2482d1e580aSAlasdair G Kergon 2492d1e580aSAlasdair G Kergon static struct kmem_cache *_job_cache; 2502d1e580aSAlasdair G Kergon 2512d1e580aSAlasdair G Kergon int __init dm_kcopyd_init(void) 2522d1e580aSAlasdair G Kergon { 253c6ea41fbSMikulas Patocka _job_cache = kmem_cache_create("kcopyd_job", 254c6ea41fbSMikulas Patocka sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1), 255c6ea41fbSMikulas Patocka __alignof__(struct kcopyd_job), 0, NULL); 2562d1e580aSAlasdair G Kergon if (!_job_cache) 2572d1e580aSAlasdair G Kergon return -ENOMEM; 2582d1e580aSAlasdair G Kergon 2592d1e580aSAlasdair G Kergon return 0; 2602d1e580aSAlasdair G Kergon } 2612d1e580aSAlasdair G Kergon 2622d1e580aSAlasdair G Kergon void dm_kcopyd_exit(void) 2632d1e580aSAlasdair G Kergon { 2642d1e580aSAlasdair G Kergon kmem_cache_destroy(_job_cache); 2652d1e580aSAlasdair G Kergon _job_cache = NULL; 2662d1e580aSAlasdair G Kergon } 2672d1e580aSAlasdair G Kergon 2682d1e580aSAlasdair G Kergon /* 2692d1e580aSAlasdair G Kergon * Functions to push and pop a job onto the head of a given job 2702d1e580aSAlasdair G Kergon * list. 2712d1e580aSAlasdair G Kergon */ 2722d1e580aSAlasdair G Kergon static struct kcopyd_job *pop(struct list_head *jobs, 2732d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc) 2742d1e580aSAlasdair G Kergon { 2752d1e580aSAlasdair G Kergon struct kcopyd_job *job = NULL; 2762d1e580aSAlasdair G Kergon unsigned long flags; 2772d1e580aSAlasdair G Kergon 2782d1e580aSAlasdair G Kergon spin_lock_irqsave(&kc->job_lock, flags); 2792d1e580aSAlasdair G Kergon 2802d1e580aSAlasdair G Kergon if (!list_empty(jobs)) { 2812d1e580aSAlasdair G Kergon job = list_entry(jobs->next, struct kcopyd_job, list); 2822d1e580aSAlasdair G Kergon list_del(&job->list); 2832d1e580aSAlasdair G Kergon } 2842d1e580aSAlasdair G Kergon spin_unlock_irqrestore(&kc->job_lock, flags); 2852d1e580aSAlasdair G Kergon 2862d1e580aSAlasdair G Kergon return job; 2872d1e580aSAlasdair G Kergon } 2882d1e580aSAlasdair G Kergon 2892d1e580aSAlasdair G Kergon static void push(struct list_head *jobs, struct kcopyd_job *job) 2902d1e580aSAlasdair G Kergon { 2912d1e580aSAlasdair G Kergon unsigned long flags; 2922d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 2932d1e580aSAlasdair G Kergon 2942d1e580aSAlasdair G Kergon spin_lock_irqsave(&kc->job_lock, flags); 2952d1e580aSAlasdair G Kergon list_add_tail(&job->list, jobs); 2962d1e580aSAlasdair G Kergon spin_unlock_irqrestore(&kc->job_lock, flags); 2972d1e580aSAlasdair G Kergon } 2982d1e580aSAlasdair G Kergon 299b673c3a8SKazuo Ito 300b673c3a8SKazuo Ito static void push_head(struct list_head *jobs, struct kcopyd_job *job) 301b673c3a8SKazuo Ito { 302b673c3a8SKazuo Ito unsigned long flags; 303b673c3a8SKazuo Ito struct dm_kcopyd_client *kc = job->kc; 304b673c3a8SKazuo Ito 305b673c3a8SKazuo Ito spin_lock_irqsave(&kc->job_lock, flags); 306b673c3a8SKazuo Ito list_add(&job->list, jobs); 307b673c3a8SKazuo Ito spin_unlock_irqrestore(&kc->job_lock, flags); 308b673c3a8SKazuo Ito } 309b673c3a8SKazuo Ito 3102d1e580aSAlasdair G Kergon /* 3112d1e580aSAlasdair G Kergon * These three functions process 1 item from the corresponding 3122d1e580aSAlasdair G Kergon * job list. 3132d1e580aSAlasdair G Kergon * 3142d1e580aSAlasdair G Kergon * They return: 3152d1e580aSAlasdair G Kergon * < 0: error 3162d1e580aSAlasdair G Kergon * 0: success 3172d1e580aSAlasdair G Kergon * > 0: can't process yet. 3182d1e580aSAlasdair G Kergon */ 3192d1e580aSAlasdair G Kergon static int run_complete_job(struct kcopyd_job *job) 3202d1e580aSAlasdair G Kergon { 3212d1e580aSAlasdair G Kergon void *context = job->context; 3222d1e580aSAlasdair G Kergon int read_err = job->read_err; 3232d1e580aSAlasdair G Kergon unsigned long write_err = job->write_err; 3242d1e580aSAlasdair G Kergon dm_kcopyd_notify_fn fn = job->fn; 3252d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 3262d1e580aSAlasdair G Kergon 32773830857SMikulas Patocka if (job->pages) 3282d1e580aSAlasdair G Kergon kcopyd_put_pages(kc, job->pages); 329c6ea41fbSMikulas Patocka /* 330c6ea41fbSMikulas Patocka * If this is the master job, the sub jobs have already 331c6ea41fbSMikulas Patocka * completed so we can free everything. 332c6ea41fbSMikulas Patocka */ 333c6ea41fbSMikulas Patocka if (job->master_job == job) 3342d1e580aSAlasdair G Kergon mempool_free(job, kc->job_pool); 3352d1e580aSAlasdair G Kergon fn(read_err, write_err, context); 3362d1e580aSAlasdair G Kergon 3372d1e580aSAlasdair G Kergon if (atomic_dec_and_test(&kc->nr_jobs)) 3382d1e580aSAlasdair G Kergon wake_up(&kc->destroyq); 3392d1e580aSAlasdair G Kergon 3402d1e580aSAlasdair G Kergon return 0; 3412d1e580aSAlasdair G Kergon } 3422d1e580aSAlasdair G Kergon 3432d1e580aSAlasdair G Kergon static void complete_io(unsigned long error, void *context) 3442d1e580aSAlasdair G Kergon { 3452d1e580aSAlasdair G Kergon struct kcopyd_job *job = (struct kcopyd_job *) context; 3462d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 3472d1e580aSAlasdair G Kergon 3482d1e580aSAlasdair G Kergon if (error) { 3492d1e580aSAlasdair G Kergon if (job->rw == WRITE) 3502d1e580aSAlasdair G Kergon job->write_err |= error; 3512d1e580aSAlasdair G Kergon else 3522d1e580aSAlasdair G Kergon job->read_err = 1; 3532d1e580aSAlasdair G Kergon 3542d1e580aSAlasdair G Kergon if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) { 3552d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 3562d1e580aSAlasdair G Kergon wake(kc); 3572d1e580aSAlasdair G Kergon return; 3582d1e580aSAlasdair G Kergon } 3592d1e580aSAlasdair G Kergon } 3602d1e580aSAlasdair G Kergon 3612d1e580aSAlasdair G Kergon if (job->rw == WRITE) 3622d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 3632d1e580aSAlasdair G Kergon 3642d1e580aSAlasdair G Kergon else { 3652d1e580aSAlasdair G Kergon job->rw = WRITE; 3662d1e580aSAlasdair G Kergon push(&kc->io_jobs, job); 3672d1e580aSAlasdair G Kergon } 3682d1e580aSAlasdair G Kergon 3692d1e580aSAlasdair G Kergon wake(kc); 3702d1e580aSAlasdair G Kergon } 3712d1e580aSAlasdair G Kergon 3722d1e580aSAlasdair G Kergon /* 3732d1e580aSAlasdair G Kergon * Request io on as many buffer heads as we can currently get for 3742d1e580aSAlasdair G Kergon * a particular job. 3752d1e580aSAlasdair G Kergon */ 3762d1e580aSAlasdair G Kergon static int run_io_job(struct kcopyd_job *job) 3772d1e580aSAlasdair G Kergon { 3782d1e580aSAlasdair G Kergon int r; 3792d1e580aSAlasdair G Kergon struct dm_io_request io_req = { 3808d35d3e3SMikulas Patocka .bi_rw = job->rw, 3812d1e580aSAlasdair G Kergon .mem.type = DM_IO_PAGE_LIST, 3822d1e580aSAlasdair G Kergon .mem.ptr.pl = job->pages, 3832d1e580aSAlasdair G Kergon .mem.offset = job->offset, 3842d1e580aSAlasdair G Kergon .notify.fn = complete_io, 3852d1e580aSAlasdair G Kergon .notify.context = job, 3862d1e580aSAlasdair G Kergon .client = job->kc->io_client, 3872d1e580aSAlasdair G Kergon }; 3882d1e580aSAlasdair G Kergon 3897eaceaccSJens Axboe if (job->rw == READ) 3902d1e580aSAlasdair G Kergon r = dm_io(&io_req, 1, &job->source, NULL); 391721a9602SJens Axboe else 3922d1e580aSAlasdair G Kergon r = dm_io(&io_req, job->num_dests, job->dests, NULL); 3932d1e580aSAlasdair G Kergon 3942d1e580aSAlasdair G Kergon return r; 3952d1e580aSAlasdair G Kergon } 3962d1e580aSAlasdair G Kergon 3972d1e580aSAlasdair G Kergon static int run_pages_job(struct kcopyd_job *job) 3982d1e580aSAlasdair G Kergon { 3992d1e580aSAlasdair G Kergon int r; 4002d1e580aSAlasdair G Kergon 4012d1e580aSAlasdair G Kergon job->nr_pages = dm_div_up(job->dests[0].count + job->offset, 4022d1e580aSAlasdair G Kergon PAGE_SIZE >> 9); 4032d1e580aSAlasdair G Kergon r = kcopyd_get_pages(job->kc, job->nr_pages, &job->pages); 4042d1e580aSAlasdair G Kergon if (!r) { 4052d1e580aSAlasdair G Kergon /* this job is ready for io */ 4062d1e580aSAlasdair G Kergon push(&job->kc->io_jobs, job); 4072d1e580aSAlasdair G Kergon return 0; 4082d1e580aSAlasdair G Kergon } 4092d1e580aSAlasdair G Kergon 4102d1e580aSAlasdair G Kergon if (r == -ENOMEM) 4112d1e580aSAlasdair G Kergon /* can't complete now */ 4122d1e580aSAlasdair G Kergon return 1; 4132d1e580aSAlasdair G Kergon 4142d1e580aSAlasdair G Kergon return r; 4152d1e580aSAlasdair G Kergon } 4162d1e580aSAlasdair G Kergon 4172d1e580aSAlasdair G Kergon /* 4182d1e580aSAlasdair G Kergon * Run through a list for as long as possible. Returns the count 4192d1e580aSAlasdair G Kergon * of successful jobs. 4202d1e580aSAlasdair G Kergon */ 4212d1e580aSAlasdair G Kergon static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, 4222d1e580aSAlasdair G Kergon int (*fn) (struct kcopyd_job *)) 4232d1e580aSAlasdair G Kergon { 4242d1e580aSAlasdair G Kergon struct kcopyd_job *job; 4252d1e580aSAlasdair G Kergon int r, count = 0; 4262d1e580aSAlasdair G Kergon 4272d1e580aSAlasdair G Kergon while ((job = pop(jobs, kc))) { 4282d1e580aSAlasdair G Kergon 4292d1e580aSAlasdair G Kergon r = fn(job); 4302d1e580aSAlasdair G Kergon 4312d1e580aSAlasdair G Kergon if (r < 0) { 4322d1e580aSAlasdair G Kergon /* error this rogue job */ 4332d1e580aSAlasdair G Kergon if (job->rw == WRITE) 4342d1e580aSAlasdair G Kergon job->write_err = (unsigned long) -1L; 4352d1e580aSAlasdair G Kergon else 4362d1e580aSAlasdair G Kergon job->read_err = 1; 4372d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 4382d1e580aSAlasdair G Kergon break; 4392d1e580aSAlasdair G Kergon } 4402d1e580aSAlasdair G Kergon 4412d1e580aSAlasdair G Kergon if (r > 0) { 4422d1e580aSAlasdair G Kergon /* 4432d1e580aSAlasdair G Kergon * We couldn't service this job ATM, so 4442d1e580aSAlasdair G Kergon * push this job back onto the list. 4452d1e580aSAlasdair G Kergon */ 446b673c3a8SKazuo Ito push_head(jobs, job); 4472d1e580aSAlasdair G Kergon break; 4482d1e580aSAlasdair G Kergon } 4492d1e580aSAlasdair G Kergon 4502d1e580aSAlasdair G Kergon count++; 4512d1e580aSAlasdair G Kergon } 4522d1e580aSAlasdair G Kergon 4532d1e580aSAlasdair G Kergon return count; 4542d1e580aSAlasdair G Kergon } 4552d1e580aSAlasdair G Kergon 4562d1e580aSAlasdair G Kergon /* 4572d1e580aSAlasdair G Kergon * kcopyd does this every time it's woken up. 4582d1e580aSAlasdair G Kergon */ 4592d1e580aSAlasdair G Kergon static void do_work(struct work_struct *work) 4602d1e580aSAlasdair G Kergon { 4612d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = container_of(work, 4622d1e580aSAlasdair G Kergon struct dm_kcopyd_client, kcopyd_work); 4637eaceaccSJens Axboe struct blk_plug plug; 4642d1e580aSAlasdair G Kergon 4652d1e580aSAlasdair G Kergon /* 4662d1e580aSAlasdair G Kergon * The order that these are called is *very* important. 4672d1e580aSAlasdair G Kergon * complete jobs can free some pages for pages jobs. 4682d1e580aSAlasdair G Kergon * Pages jobs when successful will jump onto the io jobs 4692d1e580aSAlasdair G Kergon * list. io jobs call wake when they complete and it all 4702d1e580aSAlasdair G Kergon * starts again. 4712d1e580aSAlasdair G Kergon */ 4727eaceaccSJens Axboe blk_start_plug(&plug); 4732d1e580aSAlasdair G Kergon process_jobs(&kc->complete_jobs, kc, run_complete_job); 4742d1e580aSAlasdair G Kergon process_jobs(&kc->pages_jobs, kc, run_pages_job); 4752d1e580aSAlasdair G Kergon process_jobs(&kc->io_jobs, kc, run_io_job); 4767eaceaccSJens Axboe blk_finish_plug(&plug); 4772d1e580aSAlasdair G Kergon } 4782d1e580aSAlasdair G Kergon 4792d1e580aSAlasdair G Kergon /* 4802d1e580aSAlasdair G Kergon * If we are copying a small region we just dispatch a single job 4812d1e580aSAlasdair G Kergon * to do the copy, otherwise the io has to be split up into many 4822d1e580aSAlasdair G Kergon * jobs. 4832d1e580aSAlasdair G Kergon */ 4842d1e580aSAlasdair G Kergon static void dispatch_job(struct kcopyd_job *job) 4852d1e580aSAlasdair G Kergon { 4862d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 4872d1e580aSAlasdair G Kergon atomic_inc(&kc->nr_jobs); 4889ca170a3SMikulas Patocka if (unlikely(!job->source.count)) 4899ca170a3SMikulas Patocka push(&kc->complete_jobs, job); 4909ca170a3SMikulas Patocka else 4912d1e580aSAlasdair G Kergon push(&kc->pages_jobs, job); 4922d1e580aSAlasdair G Kergon wake(kc); 4932d1e580aSAlasdair G Kergon } 4942d1e580aSAlasdair G Kergon 4952d1e580aSAlasdair G Kergon static void segment_complete(int read_err, unsigned long write_err, 4962d1e580aSAlasdair G Kergon void *context) 4972d1e580aSAlasdair G Kergon { 4982d1e580aSAlasdair G Kergon /* FIXME: tidy this function */ 4992d1e580aSAlasdair G Kergon sector_t progress = 0; 5002d1e580aSAlasdair G Kergon sector_t count = 0; 501c6ea41fbSMikulas Patocka struct kcopyd_job *sub_job = (struct kcopyd_job *) context; 502c6ea41fbSMikulas Patocka struct kcopyd_job *job = sub_job->master_job; 50373830857SMikulas Patocka struct dm_kcopyd_client *kc = job->kc; 5042d1e580aSAlasdair G Kergon 5052d1e580aSAlasdair G Kergon mutex_lock(&job->lock); 5062d1e580aSAlasdair G Kergon 5072d1e580aSAlasdair G Kergon /* update the error */ 5082d1e580aSAlasdair G Kergon if (read_err) 5092d1e580aSAlasdair G Kergon job->read_err = 1; 5102d1e580aSAlasdair G Kergon 5112d1e580aSAlasdair G Kergon if (write_err) 5122d1e580aSAlasdair G Kergon job->write_err |= write_err; 5132d1e580aSAlasdair G Kergon 5142d1e580aSAlasdair G Kergon /* 5152d1e580aSAlasdair G Kergon * Only dispatch more work if there hasn't been an error. 5162d1e580aSAlasdair G Kergon */ 5172d1e580aSAlasdair G Kergon if ((!job->read_err && !job->write_err) || 5182d1e580aSAlasdair G Kergon test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) { 5192d1e580aSAlasdair G Kergon /* get the next chunk of work */ 5202d1e580aSAlasdair G Kergon progress = job->progress; 5212d1e580aSAlasdair G Kergon count = job->source.count - progress; 5222d1e580aSAlasdair G Kergon if (count) { 5232d1e580aSAlasdair G Kergon if (count > SUB_JOB_SIZE) 5242d1e580aSAlasdair G Kergon count = SUB_JOB_SIZE; 5252d1e580aSAlasdair G Kergon 5262d1e580aSAlasdair G Kergon job->progress += count; 5272d1e580aSAlasdair G Kergon } 5282d1e580aSAlasdair G Kergon } 5292d1e580aSAlasdair G Kergon mutex_unlock(&job->lock); 5302d1e580aSAlasdair G Kergon 5312d1e580aSAlasdair G Kergon if (count) { 5322d1e580aSAlasdair G Kergon int i; 5332d1e580aSAlasdair G Kergon 5342d1e580aSAlasdair G Kergon *sub_job = *job; 5352d1e580aSAlasdair G Kergon sub_job->source.sector += progress; 5362d1e580aSAlasdair G Kergon sub_job->source.count = count; 5372d1e580aSAlasdair G Kergon 5382d1e580aSAlasdair G Kergon for (i = 0; i < job->num_dests; i++) { 5392d1e580aSAlasdair G Kergon sub_job->dests[i].sector += progress; 5402d1e580aSAlasdair G Kergon sub_job->dests[i].count = count; 5412d1e580aSAlasdair G Kergon } 5422d1e580aSAlasdair G Kergon 5432d1e580aSAlasdair G Kergon sub_job->fn = segment_complete; 544c6ea41fbSMikulas Patocka sub_job->context = sub_job; 5452d1e580aSAlasdair G Kergon dispatch_job(sub_job); 5462d1e580aSAlasdair G Kergon 5472d1e580aSAlasdair G Kergon } else if (atomic_dec_and_test(&job->sub_jobs)) { 5482d1e580aSAlasdair G Kergon 5492d1e580aSAlasdair G Kergon /* 550340cd444SMikulas Patocka * Queue the completion callback to the kcopyd thread. 551340cd444SMikulas Patocka * 552340cd444SMikulas Patocka * Some callers assume that all the completions are called 553340cd444SMikulas Patocka * from a single thread and don't race with each other. 554340cd444SMikulas Patocka * 555340cd444SMikulas Patocka * We must not call the callback directly here because this 556340cd444SMikulas Patocka * code may not be executing in the thread. 5572d1e580aSAlasdair G Kergon */ 558340cd444SMikulas Patocka push(&kc->complete_jobs, job); 559340cd444SMikulas Patocka wake(kc); 5602d1e580aSAlasdair G Kergon } 5612d1e580aSAlasdair G Kergon } 5622d1e580aSAlasdair G Kergon 5632d1e580aSAlasdair G Kergon /* 564c6ea41fbSMikulas Patocka * Create some sub jobs to share the work between them. 5652d1e580aSAlasdair G Kergon */ 566c6ea41fbSMikulas Patocka static void split_job(struct kcopyd_job *master_job) 5672d1e580aSAlasdair G Kergon { 5682d1e580aSAlasdair G Kergon int i; 5692d1e580aSAlasdair G Kergon 570c6ea41fbSMikulas Patocka atomic_inc(&master_job->kc->nr_jobs); 571340cd444SMikulas Patocka 572c6ea41fbSMikulas Patocka atomic_set(&master_job->sub_jobs, SPLIT_COUNT); 573c6ea41fbSMikulas Patocka for (i = 0; i < SPLIT_COUNT; i++) { 574c6ea41fbSMikulas Patocka master_job[i + 1].master_job = master_job; 575c6ea41fbSMikulas Patocka segment_complete(0, 0u, &master_job[i + 1]); 576c6ea41fbSMikulas Patocka } 5772d1e580aSAlasdair G Kergon } 5782d1e580aSAlasdair G Kergon 5792d1e580aSAlasdair G Kergon int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, 5802d1e580aSAlasdair G Kergon unsigned int num_dests, struct dm_io_region *dests, 5812d1e580aSAlasdair G Kergon unsigned int flags, dm_kcopyd_notify_fn fn, void *context) 5822d1e580aSAlasdair G Kergon { 5832d1e580aSAlasdair G Kergon struct kcopyd_job *job; 5842d1e580aSAlasdair G Kergon 5852d1e580aSAlasdair G Kergon /* 586c6ea41fbSMikulas Patocka * Allocate an array of jobs consisting of one master job 587c6ea41fbSMikulas Patocka * followed by SPLIT_COUNT sub jobs. 5882d1e580aSAlasdair G Kergon */ 5892d1e580aSAlasdair G Kergon job = mempool_alloc(kc->job_pool, GFP_NOIO); 5902d1e580aSAlasdair G Kergon 5912d1e580aSAlasdair G Kergon /* 5922d1e580aSAlasdair G Kergon * set up for the read. 5932d1e580aSAlasdair G Kergon */ 5942d1e580aSAlasdair G Kergon job->kc = kc; 5952d1e580aSAlasdair G Kergon job->flags = flags; 5962d1e580aSAlasdair G Kergon job->read_err = 0; 5972d1e580aSAlasdair G Kergon job->write_err = 0; 5982d1e580aSAlasdair G Kergon job->rw = READ; 5992d1e580aSAlasdair G Kergon 6002d1e580aSAlasdair G Kergon job->source = *from; 6012d1e580aSAlasdair G Kergon 6022d1e580aSAlasdair G Kergon job->num_dests = num_dests; 6032d1e580aSAlasdair G Kergon memcpy(&job->dests, dests, sizeof(*dests) * num_dests); 6042d1e580aSAlasdair G Kergon 6052d1e580aSAlasdair G Kergon job->offset = 0; 6062d1e580aSAlasdair G Kergon job->nr_pages = 0; 6072d1e580aSAlasdair G Kergon job->pages = NULL; 6082d1e580aSAlasdair G Kergon 6092d1e580aSAlasdair G Kergon job->fn = fn; 6102d1e580aSAlasdair G Kergon job->context = context; 611c6ea41fbSMikulas Patocka job->master_job = job; 6122d1e580aSAlasdair G Kergon 613a705a34aSMikulas Patocka if (job->source.count <= SUB_JOB_SIZE) 6142d1e580aSAlasdair G Kergon dispatch_job(job); 6152d1e580aSAlasdair G Kergon else { 6162d1e580aSAlasdair G Kergon mutex_init(&job->lock); 6172d1e580aSAlasdair G Kergon job->progress = 0; 6182d1e580aSAlasdair G Kergon split_job(job); 6192d1e580aSAlasdair G Kergon } 6202d1e580aSAlasdair G Kergon 6212d1e580aSAlasdair G Kergon return 0; 6222d1e580aSAlasdair G Kergon } 6232d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_copy); 6242d1e580aSAlasdair G Kergon 6252d1e580aSAlasdair G Kergon /* 6262d1e580aSAlasdair G Kergon * Cancels a kcopyd job, eg. someone might be deactivating a 6272d1e580aSAlasdair G Kergon * mirror. 6282d1e580aSAlasdair G Kergon */ 6292d1e580aSAlasdair G Kergon #if 0 6302d1e580aSAlasdair G Kergon int kcopyd_cancel(struct kcopyd_job *job, int block) 6312d1e580aSAlasdair G Kergon { 6322d1e580aSAlasdair G Kergon /* FIXME: finish */ 6332d1e580aSAlasdair G Kergon return -1; 6342d1e580aSAlasdair G Kergon } 6352d1e580aSAlasdair G Kergon #endif /* 0 */ 6362d1e580aSAlasdair G Kergon 6372d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 6382d1e580aSAlasdair G Kergon * Client setup 6392d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 640*fa34ce73SMikulas Patocka struct dm_kcopyd_client *dm_kcopyd_client_create(void) 6412d1e580aSAlasdair G Kergon { 6422d1e580aSAlasdair G Kergon int r = -ENOMEM; 6432d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc; 6442d1e580aSAlasdair G Kergon 6452d1e580aSAlasdair G Kergon kc = kmalloc(sizeof(*kc), GFP_KERNEL); 6462d1e580aSAlasdair G Kergon if (!kc) 647*fa34ce73SMikulas Patocka return ERR_PTR(-ENOMEM); 6482d1e580aSAlasdair G Kergon 6492d1e580aSAlasdair G Kergon spin_lock_init(&kc->job_lock); 6502d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->complete_jobs); 6512d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->io_jobs); 6522d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->pages_jobs); 6532d1e580aSAlasdair G Kergon 6542d1e580aSAlasdair G Kergon kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache); 6552d1e580aSAlasdair G Kergon if (!kc->job_pool) 6562d1e580aSAlasdair G Kergon goto bad_slab; 6572d1e580aSAlasdair G Kergon 6582d1e580aSAlasdair G Kergon INIT_WORK(&kc->kcopyd_work, do_work); 6599c4376deSTejun Heo kc->kcopyd_wq = alloc_workqueue("kcopyd", 6609c4376deSTejun Heo WQ_NON_REENTRANT | WQ_MEM_RECLAIM, 0); 6612d1e580aSAlasdair G Kergon if (!kc->kcopyd_wq) 6622d1e580aSAlasdair G Kergon goto bad_workqueue; 6632d1e580aSAlasdair G Kergon 6642d1e580aSAlasdair G Kergon kc->pages = NULL; 665d0471458SMikulas Patocka kc->nr_reserved_pages = kc->nr_free_pages = 0; 6665f43ba29SMikulas Patocka r = client_reserve_pages(kc, RESERVE_PAGES); 6672d1e580aSAlasdair G Kergon if (r) 6682d1e580aSAlasdair G Kergon goto bad_client_pages; 6692d1e580aSAlasdair G Kergon 670bda8efecSMikulas Patocka kc->io_client = dm_io_client_create(); 6712d1e580aSAlasdair G Kergon if (IS_ERR(kc->io_client)) { 6722d1e580aSAlasdair G Kergon r = PTR_ERR(kc->io_client); 6732d1e580aSAlasdair G Kergon goto bad_io_client; 6742d1e580aSAlasdair G Kergon } 6752d1e580aSAlasdair G Kergon 6762d1e580aSAlasdair G Kergon init_waitqueue_head(&kc->destroyq); 6772d1e580aSAlasdair G Kergon atomic_set(&kc->nr_jobs, 0); 6782d1e580aSAlasdair G Kergon 679*fa34ce73SMikulas Patocka return kc; 6802d1e580aSAlasdair G Kergon 6812d1e580aSAlasdair G Kergon bad_io_client: 6822d1e580aSAlasdair G Kergon client_free_pages(kc); 6832d1e580aSAlasdair G Kergon bad_client_pages: 6842d1e580aSAlasdair G Kergon destroy_workqueue(kc->kcopyd_wq); 6852d1e580aSAlasdair G Kergon bad_workqueue: 6862d1e580aSAlasdair G Kergon mempool_destroy(kc->job_pool); 6872d1e580aSAlasdair G Kergon bad_slab: 6882d1e580aSAlasdair G Kergon kfree(kc); 6892d1e580aSAlasdair G Kergon 690*fa34ce73SMikulas Patocka return ERR_PTR(r); 6912d1e580aSAlasdair G Kergon } 6922d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_client_create); 6932d1e580aSAlasdair G Kergon 6942d1e580aSAlasdair G Kergon void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc) 6952d1e580aSAlasdair G Kergon { 6962d1e580aSAlasdair G Kergon /* Wait for completion of all jobs submitted by this client. */ 6972d1e580aSAlasdair G Kergon wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); 6982d1e580aSAlasdair G Kergon 6992d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->complete_jobs)); 7002d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->io_jobs)); 7012d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->pages_jobs)); 7022d1e580aSAlasdair G Kergon destroy_workqueue(kc->kcopyd_wq); 7032d1e580aSAlasdair G Kergon dm_io_client_destroy(kc->io_client); 7042d1e580aSAlasdair G Kergon client_free_pages(kc); 7052d1e580aSAlasdair G Kergon mempool_destroy(kc->job_pool); 7062d1e580aSAlasdair G Kergon kfree(kc); 7072d1e580aSAlasdair G Kergon } 7082d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_client_destroy); 709