12d1e580aSAlasdair G Kergon /* 22d1e580aSAlasdair G Kergon * Copyright (C) 2002 Sistina Software (UK) Limited. 32d1e580aSAlasdair G Kergon * Copyright (C) 2006 Red Hat GmbH 42d1e580aSAlasdair G Kergon * 52d1e580aSAlasdair G Kergon * This file is released under the GPL. 62d1e580aSAlasdair G Kergon * 72d1e580aSAlasdair G Kergon * Kcopyd provides a simple interface for copying an area of one 82d1e580aSAlasdair G Kergon * block-device to one or more other block-devices, with an asynchronous 92d1e580aSAlasdair G Kergon * completion notification. 102d1e580aSAlasdair G Kergon */ 112d1e580aSAlasdair G Kergon 122d1e580aSAlasdair G Kergon #include <linux/types.h> 1360063497SArun Sharma #include <linux/atomic.h> 142d1e580aSAlasdair G Kergon #include <linux/blkdev.h> 152d1e580aSAlasdair G Kergon #include <linux/fs.h> 162d1e580aSAlasdair G Kergon #include <linux/init.h> 172d1e580aSAlasdair G Kergon #include <linux/list.h> 182d1e580aSAlasdair G Kergon #include <linux/mempool.h> 192d1e580aSAlasdair G Kergon #include <linux/module.h> 202d1e580aSAlasdair G Kergon #include <linux/pagemap.h> 212d1e580aSAlasdair G Kergon #include <linux/slab.h> 222d1e580aSAlasdair G Kergon #include <linux/vmalloc.h> 232d1e580aSAlasdair G Kergon #include <linux/workqueue.h> 242d1e580aSAlasdair G Kergon #include <linux/mutex.h> 25df5d2e90SMikulas Patocka #include <linux/delay.h> 26586e80e6SMikulas Patocka #include <linux/device-mapper.h> 27a765e20eSAlasdair G Kergon #include <linux/dm-kcopyd.h> 282d1e580aSAlasdair G Kergon 292d1e580aSAlasdair G Kergon #include "dm.h" 302d1e580aSAlasdair G Kergon 31c6ea41fbSMikulas Patocka #define SUB_JOB_SIZE 128 32c6ea41fbSMikulas Patocka #define SPLIT_COUNT 8 33c6ea41fbSMikulas Patocka #define MIN_JOBS 8 345f43ba29SMikulas Patocka #define RESERVE_PAGES (DIV_ROUND_UP(SUB_JOB_SIZE << SECTOR_SHIFT, PAGE_SIZE)) 35c6ea41fbSMikulas Patocka 362d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 372d1e580aSAlasdair G Kergon * Each kcopyd client has its own little pool of preallocated 382d1e580aSAlasdair G Kergon * pages for kcopyd io. 392d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 402d1e580aSAlasdair G Kergon struct dm_kcopyd_client { 412d1e580aSAlasdair G Kergon struct page_list *pages; 42d0471458SMikulas Patocka unsigned nr_reserved_pages; 43d0471458SMikulas Patocka unsigned nr_free_pages; 442d1e580aSAlasdair G Kergon 452d1e580aSAlasdair G Kergon struct dm_io_client *io_client; 462d1e580aSAlasdair G Kergon 472d1e580aSAlasdair G Kergon wait_queue_head_t destroyq; 482d1e580aSAlasdair G Kergon atomic_t nr_jobs; 492d1e580aSAlasdair G Kergon 502d1e580aSAlasdair G Kergon mempool_t *job_pool; 512d1e580aSAlasdair G Kergon 522d1e580aSAlasdair G Kergon struct workqueue_struct *kcopyd_wq; 532d1e580aSAlasdair G Kergon struct work_struct kcopyd_work; 542d1e580aSAlasdair G Kergon 55df5d2e90SMikulas Patocka struct dm_kcopyd_throttle *throttle; 56df5d2e90SMikulas Patocka 572d1e580aSAlasdair G Kergon /* 582d1e580aSAlasdair G Kergon * We maintain three lists of jobs: 592d1e580aSAlasdair G Kergon * 602d1e580aSAlasdair G Kergon * i) jobs waiting for pages 612d1e580aSAlasdair G Kergon * ii) jobs that have pages, and are waiting for the io to be issued. 622d1e580aSAlasdair G Kergon * iii) jobs that have completed. 632d1e580aSAlasdair G Kergon * 642d1e580aSAlasdair G Kergon * All three of these are protected by job_lock. 652d1e580aSAlasdair G Kergon */ 662d1e580aSAlasdair G Kergon spinlock_t job_lock; 672d1e580aSAlasdair G Kergon struct list_head complete_jobs; 682d1e580aSAlasdair G Kergon struct list_head io_jobs; 692d1e580aSAlasdair G Kergon struct list_head pages_jobs; 702d1e580aSAlasdair G Kergon }; 712d1e580aSAlasdair G Kergon 727f069653SMikulas Patocka static struct page_list zero_page_list; 737f069653SMikulas Patocka 74df5d2e90SMikulas Patocka static DEFINE_SPINLOCK(throttle_spinlock); 75df5d2e90SMikulas Patocka 76df5d2e90SMikulas Patocka /* 77df5d2e90SMikulas Patocka * IO/IDLE accounting slowly decays after (1 << ACCOUNT_INTERVAL_SHIFT) period. 78df5d2e90SMikulas Patocka * When total_period >= (1 << ACCOUNT_INTERVAL_SHIFT) the counters are divided 79df5d2e90SMikulas Patocka * by 2. 80df5d2e90SMikulas Patocka */ 81df5d2e90SMikulas Patocka #define ACCOUNT_INTERVAL_SHIFT SHIFT_HZ 82df5d2e90SMikulas Patocka 83df5d2e90SMikulas Patocka /* 84df5d2e90SMikulas Patocka * Sleep this number of milliseconds. 85df5d2e90SMikulas Patocka * 86df5d2e90SMikulas Patocka * The value was decided experimentally. 87df5d2e90SMikulas Patocka * Smaller values seem to cause an increased copy rate above the limit. 88df5d2e90SMikulas Patocka * The reason for this is unknown but possibly due to jiffies rounding errors 89df5d2e90SMikulas Patocka * or read/write cache inside the disk. 90df5d2e90SMikulas Patocka */ 91df5d2e90SMikulas Patocka #define SLEEP_MSEC 100 92df5d2e90SMikulas Patocka 93df5d2e90SMikulas Patocka /* 94df5d2e90SMikulas Patocka * Maximum number of sleep events. There is a theoretical livelock if more 95df5d2e90SMikulas Patocka * kcopyd clients do work simultaneously which this limit avoids. 96df5d2e90SMikulas Patocka */ 97df5d2e90SMikulas Patocka #define MAX_SLEEPS 10 98df5d2e90SMikulas Patocka 99df5d2e90SMikulas Patocka static void io_job_start(struct dm_kcopyd_throttle *t) 100df5d2e90SMikulas Patocka { 101df5d2e90SMikulas Patocka unsigned throttle, now, difference; 102df5d2e90SMikulas Patocka int slept = 0, skew; 103df5d2e90SMikulas Patocka 104df5d2e90SMikulas Patocka if (unlikely(!t)) 105df5d2e90SMikulas Patocka return; 106df5d2e90SMikulas Patocka 107df5d2e90SMikulas Patocka try_again: 108df5d2e90SMikulas Patocka spin_lock_irq(&throttle_spinlock); 109df5d2e90SMikulas Patocka 110df5d2e90SMikulas Patocka throttle = ACCESS_ONCE(t->throttle); 111df5d2e90SMikulas Patocka 112df5d2e90SMikulas Patocka if (likely(throttle >= 100)) 113df5d2e90SMikulas Patocka goto skip_limit; 114df5d2e90SMikulas Patocka 115df5d2e90SMikulas Patocka now = jiffies; 116df5d2e90SMikulas Patocka difference = now - t->last_jiffies; 117df5d2e90SMikulas Patocka t->last_jiffies = now; 118df5d2e90SMikulas Patocka if (t->num_io_jobs) 119df5d2e90SMikulas Patocka t->io_period += difference; 120df5d2e90SMikulas Patocka t->total_period += difference; 121df5d2e90SMikulas Patocka 122df5d2e90SMikulas Patocka /* 123df5d2e90SMikulas Patocka * Maintain sane values if we got a temporary overflow. 124df5d2e90SMikulas Patocka */ 125df5d2e90SMikulas Patocka if (unlikely(t->io_period > t->total_period)) 126df5d2e90SMikulas Patocka t->io_period = t->total_period; 127df5d2e90SMikulas Patocka 128df5d2e90SMikulas Patocka if (unlikely(t->total_period >= (1 << ACCOUNT_INTERVAL_SHIFT))) { 129df5d2e90SMikulas Patocka int shift = fls(t->total_period >> ACCOUNT_INTERVAL_SHIFT); 130df5d2e90SMikulas Patocka t->total_period >>= shift; 131df5d2e90SMikulas Patocka t->io_period >>= shift; 132df5d2e90SMikulas Patocka } 133df5d2e90SMikulas Patocka 134df5d2e90SMikulas Patocka skew = t->io_period - throttle * t->total_period / 100; 135df5d2e90SMikulas Patocka 136df5d2e90SMikulas Patocka if (unlikely(skew > 0) && slept < MAX_SLEEPS) { 137df5d2e90SMikulas Patocka slept++; 138df5d2e90SMikulas Patocka spin_unlock_irq(&throttle_spinlock); 139df5d2e90SMikulas Patocka msleep(SLEEP_MSEC); 140df5d2e90SMikulas Patocka goto try_again; 141df5d2e90SMikulas Patocka } 142df5d2e90SMikulas Patocka 143df5d2e90SMikulas Patocka skip_limit: 144df5d2e90SMikulas Patocka t->num_io_jobs++; 145df5d2e90SMikulas Patocka 146df5d2e90SMikulas Patocka spin_unlock_irq(&throttle_spinlock); 147df5d2e90SMikulas Patocka } 148df5d2e90SMikulas Patocka 149df5d2e90SMikulas Patocka static void io_job_finish(struct dm_kcopyd_throttle *t) 150df5d2e90SMikulas Patocka { 151df5d2e90SMikulas Patocka unsigned long flags; 152df5d2e90SMikulas Patocka 153df5d2e90SMikulas Patocka if (unlikely(!t)) 154df5d2e90SMikulas Patocka return; 155df5d2e90SMikulas Patocka 156df5d2e90SMikulas Patocka spin_lock_irqsave(&throttle_spinlock, flags); 157df5d2e90SMikulas Patocka 158df5d2e90SMikulas Patocka t->num_io_jobs--; 159df5d2e90SMikulas Patocka 160df5d2e90SMikulas Patocka if (likely(ACCESS_ONCE(t->throttle) >= 100)) 161df5d2e90SMikulas Patocka goto skip_limit; 162df5d2e90SMikulas Patocka 163df5d2e90SMikulas Patocka if (!t->num_io_jobs) { 164df5d2e90SMikulas Patocka unsigned now, difference; 165df5d2e90SMikulas Patocka 166df5d2e90SMikulas Patocka now = jiffies; 167df5d2e90SMikulas Patocka difference = now - t->last_jiffies; 168df5d2e90SMikulas Patocka t->last_jiffies = now; 169df5d2e90SMikulas Patocka 170df5d2e90SMikulas Patocka t->io_period += difference; 171df5d2e90SMikulas Patocka t->total_period += difference; 172df5d2e90SMikulas Patocka 173df5d2e90SMikulas Patocka /* 174df5d2e90SMikulas Patocka * Maintain sane values if we got a temporary overflow. 175df5d2e90SMikulas Patocka */ 176df5d2e90SMikulas Patocka if (unlikely(t->io_period > t->total_period)) 177df5d2e90SMikulas Patocka t->io_period = t->total_period; 178df5d2e90SMikulas Patocka } 179df5d2e90SMikulas Patocka 180df5d2e90SMikulas Patocka skip_limit: 181df5d2e90SMikulas Patocka spin_unlock_irqrestore(&throttle_spinlock, flags); 182df5d2e90SMikulas Patocka } 183df5d2e90SMikulas Patocka 184df5d2e90SMikulas Patocka 1852d1e580aSAlasdair G Kergon static void wake(struct dm_kcopyd_client *kc) 1862d1e580aSAlasdair G Kergon { 1872d1e580aSAlasdair G Kergon queue_work(kc->kcopyd_wq, &kc->kcopyd_work); 1882d1e580aSAlasdair G Kergon } 1892d1e580aSAlasdair G Kergon 190d0471458SMikulas Patocka /* 191d0471458SMikulas Patocka * Obtain one page for the use of kcopyd. 192d0471458SMikulas Patocka */ 193f99b55eeSMikulas Patocka static struct page_list *alloc_pl(gfp_t gfp) 1942d1e580aSAlasdair G Kergon { 1952d1e580aSAlasdair G Kergon struct page_list *pl; 1962d1e580aSAlasdair G Kergon 197f99b55eeSMikulas Patocka pl = kmalloc(sizeof(*pl), gfp); 1982d1e580aSAlasdair G Kergon if (!pl) 1992d1e580aSAlasdair G Kergon return NULL; 2002d1e580aSAlasdair G Kergon 201f99b55eeSMikulas Patocka pl->page = alloc_page(gfp); 2022d1e580aSAlasdair G Kergon if (!pl->page) { 2032d1e580aSAlasdair G Kergon kfree(pl); 2042d1e580aSAlasdair G Kergon return NULL; 2052d1e580aSAlasdair G Kergon } 2062d1e580aSAlasdair G Kergon 2072d1e580aSAlasdair G Kergon return pl; 2082d1e580aSAlasdair G Kergon } 2092d1e580aSAlasdair G Kergon 2102d1e580aSAlasdair G Kergon static void free_pl(struct page_list *pl) 2112d1e580aSAlasdair G Kergon { 2122d1e580aSAlasdair G Kergon __free_page(pl->page); 2132d1e580aSAlasdair G Kergon kfree(pl); 2142d1e580aSAlasdair G Kergon } 2152d1e580aSAlasdair G Kergon 216d0471458SMikulas Patocka /* 217d0471458SMikulas Patocka * Add the provided pages to a client's free page list, releasing 218d0471458SMikulas Patocka * back to the system any beyond the reserved_pages limit. 219d0471458SMikulas Patocka */ 220d0471458SMikulas Patocka static void kcopyd_put_pages(struct dm_kcopyd_client *kc, struct page_list *pl) 221d0471458SMikulas Patocka { 222d0471458SMikulas Patocka struct page_list *next; 223d0471458SMikulas Patocka 224d0471458SMikulas Patocka do { 225d0471458SMikulas Patocka next = pl->next; 226d0471458SMikulas Patocka 227d0471458SMikulas Patocka if (kc->nr_free_pages >= kc->nr_reserved_pages) 228d0471458SMikulas Patocka free_pl(pl); 229d0471458SMikulas Patocka else { 230d0471458SMikulas Patocka pl->next = kc->pages; 231d0471458SMikulas Patocka kc->pages = pl; 232d0471458SMikulas Patocka kc->nr_free_pages++; 233d0471458SMikulas Patocka } 234d0471458SMikulas Patocka 235d0471458SMikulas Patocka pl = next; 236d0471458SMikulas Patocka } while (pl); 237d0471458SMikulas Patocka } 238d0471458SMikulas Patocka 2392d1e580aSAlasdair G Kergon static int kcopyd_get_pages(struct dm_kcopyd_client *kc, 2402d1e580aSAlasdair G Kergon unsigned int nr, struct page_list **pages) 2412d1e580aSAlasdair G Kergon { 2422d1e580aSAlasdair G Kergon struct page_list *pl; 2432d1e580aSAlasdair G Kergon 244d0471458SMikulas Patocka *pages = NULL; 2452d1e580aSAlasdair G Kergon 246d0471458SMikulas Patocka do { 247d0164adcSMel Gorman pl = alloc_pl(__GFP_NOWARN | __GFP_NORETRY | __GFP_KSWAPD_RECLAIM); 248d0471458SMikulas Patocka if (unlikely(!pl)) { 249d0471458SMikulas Patocka /* Use reserved pages */ 250d0471458SMikulas Patocka pl = kc->pages; 251d0471458SMikulas Patocka if (unlikely(!pl)) 252d0471458SMikulas Patocka goto out_of_memory; 2532d1e580aSAlasdair G Kergon kc->pages = pl->next; 254d0471458SMikulas Patocka kc->nr_free_pages--; 255d0471458SMikulas Patocka } 256d0471458SMikulas Patocka pl->next = *pages; 257d0471458SMikulas Patocka *pages = pl; 258d0471458SMikulas Patocka } while (--nr); 2592d1e580aSAlasdair G Kergon 2602d1e580aSAlasdair G Kergon return 0; 2612d1e580aSAlasdair G Kergon 262d0471458SMikulas Patocka out_of_memory: 263d0471458SMikulas Patocka if (*pages) 264d0471458SMikulas Patocka kcopyd_put_pages(kc, *pages); 265d0471458SMikulas Patocka return -ENOMEM; 2662d1e580aSAlasdair G Kergon } 2672d1e580aSAlasdair G Kergon 2682d1e580aSAlasdair G Kergon /* 2692d1e580aSAlasdair G Kergon * These three functions resize the page pool. 2702d1e580aSAlasdair G Kergon */ 2712d1e580aSAlasdair G Kergon static void drop_pages(struct page_list *pl) 2722d1e580aSAlasdair G Kergon { 2732d1e580aSAlasdair G Kergon struct page_list *next; 2742d1e580aSAlasdair G Kergon 2752d1e580aSAlasdair G Kergon while (pl) { 2762d1e580aSAlasdair G Kergon next = pl->next; 2772d1e580aSAlasdair G Kergon free_pl(pl); 2782d1e580aSAlasdair G Kergon pl = next; 2792d1e580aSAlasdair G Kergon } 2802d1e580aSAlasdair G Kergon } 2812d1e580aSAlasdair G Kergon 282d0471458SMikulas Patocka /* 283d0471458SMikulas Patocka * Allocate and reserve nr_pages for the use of a specific client. 284d0471458SMikulas Patocka */ 285d0471458SMikulas Patocka static int client_reserve_pages(struct dm_kcopyd_client *kc, unsigned nr_pages) 2862d1e580aSAlasdair G Kergon { 287d0471458SMikulas Patocka unsigned i; 2882d1e580aSAlasdair G Kergon struct page_list *pl = NULL, *next; 2892d1e580aSAlasdair G Kergon 290d0471458SMikulas Patocka for (i = 0; i < nr_pages; i++) { 291f99b55eeSMikulas Patocka next = alloc_pl(GFP_KERNEL); 2922d1e580aSAlasdair G Kergon if (!next) { 2932d1e580aSAlasdair G Kergon if (pl) 2942d1e580aSAlasdair G Kergon drop_pages(pl); 2952d1e580aSAlasdair G Kergon return -ENOMEM; 2962d1e580aSAlasdair G Kergon } 2972d1e580aSAlasdair G Kergon next->next = pl; 2982d1e580aSAlasdair G Kergon pl = next; 2992d1e580aSAlasdair G Kergon } 3002d1e580aSAlasdair G Kergon 301d0471458SMikulas Patocka kc->nr_reserved_pages += nr_pages; 3022d1e580aSAlasdair G Kergon kcopyd_put_pages(kc, pl); 303d0471458SMikulas Patocka 3042d1e580aSAlasdair G Kergon return 0; 3052d1e580aSAlasdair G Kergon } 3062d1e580aSAlasdair G Kergon 3072d1e580aSAlasdair G Kergon static void client_free_pages(struct dm_kcopyd_client *kc) 3082d1e580aSAlasdair G Kergon { 309d0471458SMikulas Patocka BUG_ON(kc->nr_free_pages != kc->nr_reserved_pages); 3102d1e580aSAlasdair G Kergon drop_pages(kc->pages); 3112d1e580aSAlasdair G Kergon kc->pages = NULL; 312d0471458SMikulas Patocka kc->nr_free_pages = kc->nr_reserved_pages = 0; 3132d1e580aSAlasdair G Kergon } 3142d1e580aSAlasdair G Kergon 3152d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 3162d1e580aSAlasdair G Kergon * kcopyd_jobs need to be allocated by the *clients* of kcopyd, 3172d1e580aSAlasdair G Kergon * for this reason we use a mempool to prevent the client from 3182d1e580aSAlasdair G Kergon * ever having to do io (which could cause a deadlock). 3192d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 3202d1e580aSAlasdair G Kergon struct kcopyd_job { 3212d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc; 3222d1e580aSAlasdair G Kergon struct list_head list; 3232d1e580aSAlasdair G Kergon unsigned long flags; 3242d1e580aSAlasdair G Kergon 3252d1e580aSAlasdair G Kergon /* 3262d1e580aSAlasdair G Kergon * Error state of the job. 3272d1e580aSAlasdair G Kergon */ 3282d1e580aSAlasdair G Kergon int read_err; 3292d1e580aSAlasdair G Kergon unsigned long write_err; 3302d1e580aSAlasdair G Kergon 3312d1e580aSAlasdair G Kergon /* 3322d1e580aSAlasdair G Kergon * Either READ or WRITE 3332d1e580aSAlasdair G Kergon */ 3342d1e580aSAlasdair G Kergon int rw; 3352d1e580aSAlasdair G Kergon struct dm_io_region source; 3362d1e580aSAlasdair G Kergon 3372d1e580aSAlasdair G Kergon /* 3382d1e580aSAlasdair G Kergon * The destinations for the transfer. 3392d1e580aSAlasdair G Kergon */ 3402d1e580aSAlasdair G Kergon unsigned int num_dests; 3412d1e580aSAlasdair G Kergon struct dm_io_region dests[DM_KCOPYD_MAX_REGIONS]; 3422d1e580aSAlasdair G Kergon 3432d1e580aSAlasdair G Kergon struct page_list *pages; 3442d1e580aSAlasdair G Kergon 3452d1e580aSAlasdair G Kergon /* 3462d1e580aSAlasdair G Kergon * Set this to ensure you are notified when the job has 3472d1e580aSAlasdair G Kergon * completed. 'context' is for callback to use. 3482d1e580aSAlasdair G Kergon */ 3492d1e580aSAlasdair G Kergon dm_kcopyd_notify_fn fn; 3502d1e580aSAlasdair G Kergon void *context; 3512d1e580aSAlasdair G Kergon 3522d1e580aSAlasdair G Kergon /* 3532d1e580aSAlasdair G Kergon * These fields are only used if the job has been split 3542d1e580aSAlasdair G Kergon * into more manageable parts. 3552d1e580aSAlasdair G Kergon */ 3562d1e580aSAlasdair G Kergon struct mutex lock; 3572d1e580aSAlasdair G Kergon atomic_t sub_jobs; 3582d1e580aSAlasdair G Kergon sector_t progress; 3592d1e580aSAlasdair G Kergon 360c6ea41fbSMikulas Patocka struct kcopyd_job *master_job; 361c6ea41fbSMikulas Patocka }; 3622d1e580aSAlasdair G Kergon 3632d1e580aSAlasdair G Kergon static struct kmem_cache *_job_cache; 3642d1e580aSAlasdair G Kergon 3652d1e580aSAlasdair G Kergon int __init dm_kcopyd_init(void) 3662d1e580aSAlasdair G Kergon { 367c6ea41fbSMikulas Patocka _job_cache = kmem_cache_create("kcopyd_job", 368c6ea41fbSMikulas Patocka sizeof(struct kcopyd_job) * (SPLIT_COUNT + 1), 369c6ea41fbSMikulas Patocka __alignof__(struct kcopyd_job), 0, NULL); 3702d1e580aSAlasdair G Kergon if (!_job_cache) 3712d1e580aSAlasdair G Kergon return -ENOMEM; 3722d1e580aSAlasdair G Kergon 3737f069653SMikulas Patocka zero_page_list.next = &zero_page_list; 3747f069653SMikulas Patocka zero_page_list.page = ZERO_PAGE(0); 3757f069653SMikulas Patocka 3762d1e580aSAlasdair G Kergon return 0; 3772d1e580aSAlasdair G Kergon } 3782d1e580aSAlasdair G Kergon 3792d1e580aSAlasdair G Kergon void dm_kcopyd_exit(void) 3802d1e580aSAlasdair G Kergon { 3812d1e580aSAlasdair G Kergon kmem_cache_destroy(_job_cache); 3822d1e580aSAlasdair G Kergon _job_cache = NULL; 3832d1e580aSAlasdair G Kergon } 3842d1e580aSAlasdair G Kergon 3852d1e580aSAlasdair G Kergon /* 3862d1e580aSAlasdair G Kergon * Functions to push and pop a job onto the head of a given job 3872d1e580aSAlasdair G Kergon * list. 3882d1e580aSAlasdair G Kergon */ 3892d1e580aSAlasdair G Kergon static struct kcopyd_job *pop(struct list_head *jobs, 3902d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc) 3912d1e580aSAlasdair G Kergon { 3922d1e580aSAlasdair G Kergon struct kcopyd_job *job = NULL; 3932d1e580aSAlasdair G Kergon unsigned long flags; 3942d1e580aSAlasdair G Kergon 3952d1e580aSAlasdair G Kergon spin_lock_irqsave(&kc->job_lock, flags); 3962d1e580aSAlasdair G Kergon 3972d1e580aSAlasdair G Kergon if (!list_empty(jobs)) { 3982d1e580aSAlasdair G Kergon job = list_entry(jobs->next, struct kcopyd_job, list); 3992d1e580aSAlasdair G Kergon list_del(&job->list); 4002d1e580aSAlasdair G Kergon } 4012d1e580aSAlasdair G Kergon spin_unlock_irqrestore(&kc->job_lock, flags); 4022d1e580aSAlasdair G Kergon 4032d1e580aSAlasdair G Kergon return job; 4042d1e580aSAlasdair G Kergon } 4052d1e580aSAlasdair G Kergon 4062d1e580aSAlasdair G Kergon static void push(struct list_head *jobs, struct kcopyd_job *job) 4072d1e580aSAlasdair G Kergon { 4082d1e580aSAlasdair G Kergon unsigned long flags; 4092d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 4102d1e580aSAlasdair G Kergon 4112d1e580aSAlasdair G Kergon spin_lock_irqsave(&kc->job_lock, flags); 4122d1e580aSAlasdair G Kergon list_add_tail(&job->list, jobs); 4132d1e580aSAlasdair G Kergon spin_unlock_irqrestore(&kc->job_lock, flags); 4142d1e580aSAlasdair G Kergon } 4152d1e580aSAlasdair G Kergon 416b673c3a8SKazuo Ito 417b673c3a8SKazuo Ito static void push_head(struct list_head *jobs, struct kcopyd_job *job) 418b673c3a8SKazuo Ito { 419b673c3a8SKazuo Ito unsigned long flags; 420b673c3a8SKazuo Ito struct dm_kcopyd_client *kc = job->kc; 421b673c3a8SKazuo Ito 422b673c3a8SKazuo Ito spin_lock_irqsave(&kc->job_lock, flags); 423b673c3a8SKazuo Ito list_add(&job->list, jobs); 424b673c3a8SKazuo Ito spin_unlock_irqrestore(&kc->job_lock, flags); 425b673c3a8SKazuo Ito } 426b673c3a8SKazuo Ito 4272d1e580aSAlasdair G Kergon /* 4282d1e580aSAlasdair G Kergon * These three functions process 1 item from the corresponding 4292d1e580aSAlasdair G Kergon * job list. 4302d1e580aSAlasdair G Kergon * 4312d1e580aSAlasdair G Kergon * They return: 4322d1e580aSAlasdair G Kergon * < 0: error 4332d1e580aSAlasdair G Kergon * 0: success 4342d1e580aSAlasdair G Kergon * > 0: can't process yet. 4352d1e580aSAlasdair G Kergon */ 4362d1e580aSAlasdair G Kergon static int run_complete_job(struct kcopyd_job *job) 4372d1e580aSAlasdair G Kergon { 4382d1e580aSAlasdair G Kergon void *context = job->context; 4392d1e580aSAlasdair G Kergon int read_err = job->read_err; 4402d1e580aSAlasdair G Kergon unsigned long write_err = job->write_err; 4412d1e580aSAlasdair G Kergon dm_kcopyd_notify_fn fn = job->fn; 4422d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 4432d1e580aSAlasdair G Kergon 4447f069653SMikulas Patocka if (job->pages && job->pages != &zero_page_list) 4452d1e580aSAlasdair G Kergon kcopyd_put_pages(kc, job->pages); 446c6ea41fbSMikulas Patocka /* 447c6ea41fbSMikulas Patocka * If this is the master job, the sub jobs have already 448c6ea41fbSMikulas Patocka * completed so we can free everything. 449c6ea41fbSMikulas Patocka */ 450c6ea41fbSMikulas Patocka if (job->master_job == job) 4512d1e580aSAlasdair G Kergon mempool_free(job, kc->job_pool); 4522d1e580aSAlasdair G Kergon fn(read_err, write_err, context); 4532d1e580aSAlasdair G Kergon 4542d1e580aSAlasdair G Kergon if (atomic_dec_and_test(&kc->nr_jobs)) 4552d1e580aSAlasdair G Kergon wake_up(&kc->destroyq); 4562d1e580aSAlasdair G Kergon 4572d1e580aSAlasdair G Kergon return 0; 4582d1e580aSAlasdair G Kergon } 4592d1e580aSAlasdair G Kergon 4602d1e580aSAlasdair G Kergon static void complete_io(unsigned long error, void *context) 4612d1e580aSAlasdair G Kergon { 4622d1e580aSAlasdair G Kergon struct kcopyd_job *job = (struct kcopyd_job *) context; 4632d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 4642d1e580aSAlasdair G Kergon 465df5d2e90SMikulas Patocka io_job_finish(kc->throttle); 466df5d2e90SMikulas Patocka 4672d1e580aSAlasdair G Kergon if (error) { 46851111666SMike Christie if (op_is_write(job->rw)) 4692d1e580aSAlasdair G Kergon job->write_err |= error; 4702d1e580aSAlasdair G Kergon else 4712d1e580aSAlasdair G Kergon job->read_err = 1; 4722d1e580aSAlasdair G Kergon 4732d1e580aSAlasdair G Kergon if (!test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) { 4742d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 4752d1e580aSAlasdair G Kergon wake(kc); 4762d1e580aSAlasdair G Kergon return; 4772d1e580aSAlasdair G Kergon } 4782d1e580aSAlasdair G Kergon } 4792d1e580aSAlasdair G Kergon 48051111666SMike Christie if (op_is_write(job->rw)) 4812d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 4822d1e580aSAlasdair G Kergon 4832d1e580aSAlasdair G Kergon else { 4842d1e580aSAlasdair G Kergon job->rw = WRITE; 4852d1e580aSAlasdair G Kergon push(&kc->io_jobs, job); 4862d1e580aSAlasdair G Kergon } 4872d1e580aSAlasdair G Kergon 4882d1e580aSAlasdair G Kergon wake(kc); 4892d1e580aSAlasdair G Kergon } 4902d1e580aSAlasdair G Kergon 4912d1e580aSAlasdair G Kergon /* 4922d1e580aSAlasdair G Kergon * Request io on as many buffer heads as we can currently get for 4932d1e580aSAlasdair G Kergon * a particular job. 4942d1e580aSAlasdair G Kergon */ 4952d1e580aSAlasdair G Kergon static int run_io_job(struct kcopyd_job *job) 4962d1e580aSAlasdair G Kergon { 4972d1e580aSAlasdair G Kergon int r; 4982d1e580aSAlasdair G Kergon struct dm_io_request io_req = { 499*e6047149SMike Christie .bi_op = job->rw, 500*e6047149SMike Christie .bi_op_flags = 0, 5012d1e580aSAlasdair G Kergon .mem.type = DM_IO_PAGE_LIST, 5022d1e580aSAlasdair G Kergon .mem.ptr.pl = job->pages, 5034622afb3SMikulas Patocka .mem.offset = 0, 5042d1e580aSAlasdair G Kergon .notify.fn = complete_io, 5052d1e580aSAlasdair G Kergon .notify.context = job, 5062d1e580aSAlasdair G Kergon .client = job->kc->io_client, 5072d1e580aSAlasdair G Kergon }; 5082d1e580aSAlasdair G Kergon 509df5d2e90SMikulas Patocka io_job_start(job->kc->throttle); 510df5d2e90SMikulas Patocka 5117eaceaccSJens Axboe if (job->rw == READ) 5122d1e580aSAlasdair G Kergon r = dm_io(&io_req, 1, &job->source, NULL); 513721a9602SJens Axboe else 5142d1e580aSAlasdair G Kergon r = dm_io(&io_req, job->num_dests, job->dests, NULL); 5152d1e580aSAlasdair G Kergon 5162d1e580aSAlasdair G Kergon return r; 5172d1e580aSAlasdair G Kergon } 5182d1e580aSAlasdair G Kergon 5192d1e580aSAlasdair G Kergon static int run_pages_job(struct kcopyd_job *job) 5202d1e580aSAlasdair G Kergon { 5212d1e580aSAlasdair G Kergon int r; 5225bf45a3dSMikulas Patocka unsigned nr_pages = dm_div_up(job->dests[0].count, PAGE_SIZE >> 9); 5232d1e580aSAlasdair G Kergon 5245bf45a3dSMikulas Patocka r = kcopyd_get_pages(job->kc, nr_pages, &job->pages); 5252d1e580aSAlasdair G Kergon if (!r) { 5262d1e580aSAlasdair G Kergon /* this job is ready for io */ 5272d1e580aSAlasdair G Kergon push(&job->kc->io_jobs, job); 5282d1e580aSAlasdair G Kergon return 0; 5292d1e580aSAlasdair G Kergon } 5302d1e580aSAlasdair G Kergon 5312d1e580aSAlasdair G Kergon if (r == -ENOMEM) 5322d1e580aSAlasdair G Kergon /* can't complete now */ 5332d1e580aSAlasdair G Kergon return 1; 5342d1e580aSAlasdair G Kergon 5352d1e580aSAlasdair G Kergon return r; 5362d1e580aSAlasdair G Kergon } 5372d1e580aSAlasdair G Kergon 5382d1e580aSAlasdair G Kergon /* 5392d1e580aSAlasdair G Kergon * Run through a list for as long as possible. Returns the count 5402d1e580aSAlasdair G Kergon * of successful jobs. 5412d1e580aSAlasdair G Kergon */ 5422d1e580aSAlasdair G Kergon static int process_jobs(struct list_head *jobs, struct dm_kcopyd_client *kc, 5432d1e580aSAlasdair G Kergon int (*fn) (struct kcopyd_job *)) 5442d1e580aSAlasdair G Kergon { 5452d1e580aSAlasdair G Kergon struct kcopyd_job *job; 5462d1e580aSAlasdair G Kergon int r, count = 0; 5472d1e580aSAlasdair G Kergon 5482d1e580aSAlasdair G Kergon while ((job = pop(jobs, kc))) { 5492d1e580aSAlasdair G Kergon 5502d1e580aSAlasdair G Kergon r = fn(job); 5512d1e580aSAlasdair G Kergon 5522d1e580aSAlasdair G Kergon if (r < 0) { 5532d1e580aSAlasdair G Kergon /* error this rogue job */ 55451111666SMike Christie if (op_is_write(job->rw)) 5552d1e580aSAlasdair G Kergon job->write_err = (unsigned long) -1L; 5562d1e580aSAlasdair G Kergon else 5572d1e580aSAlasdair G Kergon job->read_err = 1; 5582d1e580aSAlasdair G Kergon push(&kc->complete_jobs, job); 5592d1e580aSAlasdair G Kergon break; 5602d1e580aSAlasdair G Kergon } 5612d1e580aSAlasdair G Kergon 5622d1e580aSAlasdair G Kergon if (r > 0) { 5632d1e580aSAlasdair G Kergon /* 5642d1e580aSAlasdair G Kergon * We couldn't service this job ATM, so 5652d1e580aSAlasdair G Kergon * push this job back onto the list. 5662d1e580aSAlasdair G Kergon */ 567b673c3a8SKazuo Ito push_head(jobs, job); 5682d1e580aSAlasdair G Kergon break; 5692d1e580aSAlasdair G Kergon } 5702d1e580aSAlasdair G Kergon 5712d1e580aSAlasdair G Kergon count++; 5722d1e580aSAlasdair G Kergon } 5732d1e580aSAlasdair G Kergon 5742d1e580aSAlasdair G Kergon return count; 5752d1e580aSAlasdair G Kergon } 5762d1e580aSAlasdair G Kergon 5772d1e580aSAlasdair G Kergon /* 5782d1e580aSAlasdair G Kergon * kcopyd does this every time it's woken up. 5792d1e580aSAlasdair G Kergon */ 5802d1e580aSAlasdair G Kergon static void do_work(struct work_struct *work) 5812d1e580aSAlasdair G Kergon { 5822d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = container_of(work, 5832d1e580aSAlasdair G Kergon struct dm_kcopyd_client, kcopyd_work); 5847eaceaccSJens Axboe struct blk_plug plug; 5852d1e580aSAlasdair G Kergon 5862d1e580aSAlasdair G Kergon /* 5872d1e580aSAlasdair G Kergon * The order that these are called is *very* important. 5882d1e580aSAlasdair G Kergon * complete jobs can free some pages for pages jobs. 5892d1e580aSAlasdair G Kergon * Pages jobs when successful will jump onto the io jobs 5902d1e580aSAlasdair G Kergon * list. io jobs call wake when they complete and it all 5912d1e580aSAlasdair G Kergon * starts again. 5922d1e580aSAlasdair G Kergon */ 5937eaceaccSJens Axboe blk_start_plug(&plug); 5942d1e580aSAlasdair G Kergon process_jobs(&kc->complete_jobs, kc, run_complete_job); 5952d1e580aSAlasdair G Kergon process_jobs(&kc->pages_jobs, kc, run_pages_job); 5962d1e580aSAlasdair G Kergon process_jobs(&kc->io_jobs, kc, run_io_job); 5977eaceaccSJens Axboe blk_finish_plug(&plug); 5982d1e580aSAlasdair G Kergon } 5992d1e580aSAlasdair G Kergon 6002d1e580aSAlasdair G Kergon /* 6012d1e580aSAlasdair G Kergon * If we are copying a small region we just dispatch a single job 6022d1e580aSAlasdair G Kergon * to do the copy, otherwise the io has to be split up into many 6032d1e580aSAlasdair G Kergon * jobs. 6042d1e580aSAlasdair G Kergon */ 6052d1e580aSAlasdair G Kergon static void dispatch_job(struct kcopyd_job *job) 6062d1e580aSAlasdair G Kergon { 6072d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc = job->kc; 6082d1e580aSAlasdair G Kergon atomic_inc(&kc->nr_jobs); 6099ca170a3SMikulas Patocka if (unlikely(!job->source.count)) 6109ca170a3SMikulas Patocka push(&kc->complete_jobs, job); 6117f069653SMikulas Patocka else if (job->pages == &zero_page_list) 6127f069653SMikulas Patocka push(&kc->io_jobs, job); 6139ca170a3SMikulas Patocka else 6142d1e580aSAlasdair G Kergon push(&kc->pages_jobs, job); 6152d1e580aSAlasdair G Kergon wake(kc); 6162d1e580aSAlasdair G Kergon } 6172d1e580aSAlasdair G Kergon 6182d1e580aSAlasdair G Kergon static void segment_complete(int read_err, unsigned long write_err, 6192d1e580aSAlasdair G Kergon void *context) 6202d1e580aSAlasdair G Kergon { 6212d1e580aSAlasdair G Kergon /* FIXME: tidy this function */ 6222d1e580aSAlasdair G Kergon sector_t progress = 0; 6232d1e580aSAlasdair G Kergon sector_t count = 0; 624c6ea41fbSMikulas Patocka struct kcopyd_job *sub_job = (struct kcopyd_job *) context; 625c6ea41fbSMikulas Patocka struct kcopyd_job *job = sub_job->master_job; 62673830857SMikulas Patocka struct dm_kcopyd_client *kc = job->kc; 6272d1e580aSAlasdair G Kergon 6282d1e580aSAlasdair G Kergon mutex_lock(&job->lock); 6292d1e580aSAlasdair G Kergon 6302d1e580aSAlasdair G Kergon /* update the error */ 6312d1e580aSAlasdair G Kergon if (read_err) 6322d1e580aSAlasdair G Kergon job->read_err = 1; 6332d1e580aSAlasdair G Kergon 6342d1e580aSAlasdair G Kergon if (write_err) 6352d1e580aSAlasdair G Kergon job->write_err |= write_err; 6362d1e580aSAlasdair G Kergon 6372d1e580aSAlasdair G Kergon /* 6382d1e580aSAlasdair G Kergon * Only dispatch more work if there hasn't been an error. 6392d1e580aSAlasdair G Kergon */ 6402d1e580aSAlasdair G Kergon if ((!job->read_err && !job->write_err) || 6412d1e580aSAlasdair G Kergon test_bit(DM_KCOPYD_IGNORE_ERROR, &job->flags)) { 6422d1e580aSAlasdair G Kergon /* get the next chunk of work */ 6432d1e580aSAlasdair G Kergon progress = job->progress; 6442d1e580aSAlasdair G Kergon count = job->source.count - progress; 6452d1e580aSAlasdair G Kergon if (count) { 6462d1e580aSAlasdair G Kergon if (count > SUB_JOB_SIZE) 6472d1e580aSAlasdair G Kergon count = SUB_JOB_SIZE; 6482d1e580aSAlasdair G Kergon 6492d1e580aSAlasdair G Kergon job->progress += count; 6502d1e580aSAlasdair G Kergon } 6512d1e580aSAlasdair G Kergon } 6522d1e580aSAlasdair G Kergon mutex_unlock(&job->lock); 6532d1e580aSAlasdair G Kergon 6542d1e580aSAlasdair G Kergon if (count) { 6552d1e580aSAlasdair G Kergon int i; 6562d1e580aSAlasdair G Kergon 6572d1e580aSAlasdair G Kergon *sub_job = *job; 6582d1e580aSAlasdair G Kergon sub_job->source.sector += progress; 6592d1e580aSAlasdair G Kergon sub_job->source.count = count; 6602d1e580aSAlasdair G Kergon 6612d1e580aSAlasdair G Kergon for (i = 0; i < job->num_dests; i++) { 6622d1e580aSAlasdair G Kergon sub_job->dests[i].sector += progress; 6632d1e580aSAlasdair G Kergon sub_job->dests[i].count = count; 6642d1e580aSAlasdair G Kergon } 6652d1e580aSAlasdair G Kergon 6662d1e580aSAlasdair G Kergon sub_job->fn = segment_complete; 667c6ea41fbSMikulas Patocka sub_job->context = sub_job; 6682d1e580aSAlasdair G Kergon dispatch_job(sub_job); 6692d1e580aSAlasdair G Kergon 6702d1e580aSAlasdair G Kergon } else if (atomic_dec_and_test(&job->sub_jobs)) { 6712d1e580aSAlasdair G Kergon 6722d1e580aSAlasdair G Kergon /* 673340cd444SMikulas Patocka * Queue the completion callback to the kcopyd thread. 674340cd444SMikulas Patocka * 675340cd444SMikulas Patocka * Some callers assume that all the completions are called 676340cd444SMikulas Patocka * from a single thread and don't race with each other. 677340cd444SMikulas Patocka * 678340cd444SMikulas Patocka * We must not call the callback directly here because this 679340cd444SMikulas Patocka * code may not be executing in the thread. 6802d1e580aSAlasdair G Kergon */ 681340cd444SMikulas Patocka push(&kc->complete_jobs, job); 682340cd444SMikulas Patocka wake(kc); 6832d1e580aSAlasdair G Kergon } 6842d1e580aSAlasdair G Kergon } 6852d1e580aSAlasdair G Kergon 6862d1e580aSAlasdair G Kergon /* 687c6ea41fbSMikulas Patocka * Create some sub jobs to share the work between them. 6882d1e580aSAlasdair G Kergon */ 689c6ea41fbSMikulas Patocka static void split_job(struct kcopyd_job *master_job) 6902d1e580aSAlasdair G Kergon { 6912d1e580aSAlasdair G Kergon int i; 6922d1e580aSAlasdair G Kergon 693c6ea41fbSMikulas Patocka atomic_inc(&master_job->kc->nr_jobs); 694340cd444SMikulas Patocka 695c6ea41fbSMikulas Patocka atomic_set(&master_job->sub_jobs, SPLIT_COUNT); 696c6ea41fbSMikulas Patocka for (i = 0; i < SPLIT_COUNT; i++) { 697c6ea41fbSMikulas Patocka master_job[i + 1].master_job = master_job; 698c6ea41fbSMikulas Patocka segment_complete(0, 0u, &master_job[i + 1]); 699c6ea41fbSMikulas Patocka } 7002d1e580aSAlasdair G Kergon } 7012d1e580aSAlasdair G Kergon 7022d1e580aSAlasdair G Kergon int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, 7032d1e580aSAlasdair G Kergon unsigned int num_dests, struct dm_io_region *dests, 7042d1e580aSAlasdair G Kergon unsigned int flags, dm_kcopyd_notify_fn fn, void *context) 7052d1e580aSAlasdair G Kergon { 7062d1e580aSAlasdair G Kergon struct kcopyd_job *job; 70770d6c400SMike Snitzer int i; 7082d1e580aSAlasdair G Kergon 7092d1e580aSAlasdair G Kergon /* 710c6ea41fbSMikulas Patocka * Allocate an array of jobs consisting of one master job 711c6ea41fbSMikulas Patocka * followed by SPLIT_COUNT sub jobs. 7122d1e580aSAlasdair G Kergon */ 7132d1e580aSAlasdair G Kergon job = mempool_alloc(kc->job_pool, GFP_NOIO); 7142d1e580aSAlasdair G Kergon 7152d1e580aSAlasdair G Kergon /* 7162d1e580aSAlasdair G Kergon * set up for the read. 7172d1e580aSAlasdair G Kergon */ 7182d1e580aSAlasdair G Kergon job->kc = kc; 7192d1e580aSAlasdair G Kergon job->flags = flags; 7202d1e580aSAlasdair G Kergon job->read_err = 0; 7212d1e580aSAlasdair G Kergon job->write_err = 0; 7222d1e580aSAlasdair G Kergon 7232d1e580aSAlasdair G Kergon job->num_dests = num_dests; 7242d1e580aSAlasdair G Kergon memcpy(&job->dests, dests, sizeof(*dests) * num_dests); 7252d1e580aSAlasdair G Kergon 7267f069653SMikulas Patocka if (from) { 7277f069653SMikulas Patocka job->source = *from; 7282d1e580aSAlasdair G Kergon job->pages = NULL; 7297f069653SMikulas Patocka job->rw = READ; 7307f069653SMikulas Patocka } else { 7317f069653SMikulas Patocka memset(&job->source, 0, sizeof job->source); 7327f069653SMikulas Patocka job->source.count = job->dests[0].count; 7337f069653SMikulas Patocka job->pages = &zero_page_list; 73470d6c400SMike Snitzer 73570d6c400SMike Snitzer /* 73670d6c400SMike Snitzer * Use WRITE SAME to optimize zeroing if all dests support it. 73770d6c400SMike Snitzer */ 738*e6047149SMike Christie job->rw = REQ_OP_WRITE_SAME; 73970d6c400SMike Snitzer for (i = 0; i < job->num_dests; i++) 74070d6c400SMike Snitzer if (!bdev_write_same(job->dests[i].bdev)) { 7417f069653SMikulas Patocka job->rw = WRITE; 74270d6c400SMike Snitzer break; 74370d6c400SMike Snitzer } 7447f069653SMikulas Patocka } 7452d1e580aSAlasdair G Kergon 7462d1e580aSAlasdair G Kergon job->fn = fn; 7472d1e580aSAlasdair G Kergon job->context = context; 748c6ea41fbSMikulas Patocka job->master_job = job; 7492d1e580aSAlasdair G Kergon 750a705a34aSMikulas Patocka if (job->source.count <= SUB_JOB_SIZE) 7512d1e580aSAlasdair G Kergon dispatch_job(job); 7522d1e580aSAlasdair G Kergon else { 7532d1e580aSAlasdair G Kergon mutex_init(&job->lock); 7542d1e580aSAlasdair G Kergon job->progress = 0; 7552d1e580aSAlasdair G Kergon split_job(job); 7562d1e580aSAlasdair G Kergon } 7572d1e580aSAlasdair G Kergon 7582d1e580aSAlasdair G Kergon return 0; 7592d1e580aSAlasdair G Kergon } 7602d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_copy); 7612d1e580aSAlasdair G Kergon 7627f069653SMikulas Patocka int dm_kcopyd_zero(struct dm_kcopyd_client *kc, 7637f069653SMikulas Patocka unsigned num_dests, struct dm_io_region *dests, 7647f069653SMikulas Patocka unsigned flags, dm_kcopyd_notify_fn fn, void *context) 7657f069653SMikulas Patocka { 7667f069653SMikulas Patocka return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); 7677f069653SMikulas Patocka } 7687f069653SMikulas Patocka EXPORT_SYMBOL(dm_kcopyd_zero); 7697f069653SMikulas Patocka 770a6e50b40SMikulas Patocka void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc, 771a6e50b40SMikulas Patocka dm_kcopyd_notify_fn fn, void *context) 772a6e50b40SMikulas Patocka { 773a6e50b40SMikulas Patocka struct kcopyd_job *job; 774a6e50b40SMikulas Patocka 775a6e50b40SMikulas Patocka job = mempool_alloc(kc->job_pool, GFP_NOIO); 776a6e50b40SMikulas Patocka 777a6e50b40SMikulas Patocka memset(job, 0, sizeof(struct kcopyd_job)); 778a6e50b40SMikulas Patocka job->kc = kc; 779a6e50b40SMikulas Patocka job->fn = fn; 780a6e50b40SMikulas Patocka job->context = context; 781d136f2efSAlasdair G Kergon job->master_job = job; 782a6e50b40SMikulas Patocka 783a6e50b40SMikulas Patocka atomic_inc(&kc->nr_jobs); 784a6e50b40SMikulas Patocka 785a6e50b40SMikulas Patocka return job; 786a6e50b40SMikulas Patocka } 787a6e50b40SMikulas Patocka EXPORT_SYMBOL(dm_kcopyd_prepare_callback); 788a6e50b40SMikulas Patocka 789a6e50b40SMikulas Patocka void dm_kcopyd_do_callback(void *j, int read_err, unsigned long write_err) 790a6e50b40SMikulas Patocka { 791a6e50b40SMikulas Patocka struct kcopyd_job *job = j; 792a6e50b40SMikulas Patocka struct dm_kcopyd_client *kc = job->kc; 793a6e50b40SMikulas Patocka 794a6e50b40SMikulas Patocka job->read_err = read_err; 795a6e50b40SMikulas Patocka job->write_err = write_err; 796a6e50b40SMikulas Patocka 797a6e50b40SMikulas Patocka push(&kc->complete_jobs, job); 798a6e50b40SMikulas Patocka wake(kc); 799a6e50b40SMikulas Patocka } 800a6e50b40SMikulas Patocka EXPORT_SYMBOL(dm_kcopyd_do_callback); 801a6e50b40SMikulas Patocka 8022d1e580aSAlasdair G Kergon /* 8032d1e580aSAlasdair G Kergon * Cancels a kcopyd job, eg. someone might be deactivating a 8042d1e580aSAlasdair G Kergon * mirror. 8052d1e580aSAlasdair G Kergon */ 8062d1e580aSAlasdair G Kergon #if 0 8072d1e580aSAlasdair G Kergon int kcopyd_cancel(struct kcopyd_job *job, int block) 8082d1e580aSAlasdair G Kergon { 8092d1e580aSAlasdair G Kergon /* FIXME: finish */ 8102d1e580aSAlasdair G Kergon return -1; 8112d1e580aSAlasdair G Kergon } 8122d1e580aSAlasdair G Kergon #endif /* 0 */ 8132d1e580aSAlasdair G Kergon 8142d1e580aSAlasdair G Kergon /*----------------------------------------------------------------- 8152d1e580aSAlasdair G Kergon * Client setup 8162d1e580aSAlasdair G Kergon *---------------------------------------------------------------*/ 817df5d2e90SMikulas Patocka struct dm_kcopyd_client *dm_kcopyd_client_create(struct dm_kcopyd_throttle *throttle) 8182d1e580aSAlasdair G Kergon { 8192d1e580aSAlasdair G Kergon int r = -ENOMEM; 8202d1e580aSAlasdair G Kergon struct dm_kcopyd_client *kc; 8212d1e580aSAlasdair G Kergon 8222d1e580aSAlasdair G Kergon kc = kmalloc(sizeof(*kc), GFP_KERNEL); 8232d1e580aSAlasdair G Kergon if (!kc) 824fa34ce73SMikulas Patocka return ERR_PTR(-ENOMEM); 8252d1e580aSAlasdair G Kergon 8262d1e580aSAlasdair G Kergon spin_lock_init(&kc->job_lock); 8272d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->complete_jobs); 8282d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->io_jobs); 8292d1e580aSAlasdair G Kergon INIT_LIST_HEAD(&kc->pages_jobs); 830df5d2e90SMikulas Patocka kc->throttle = throttle; 8312d1e580aSAlasdair G Kergon 8322d1e580aSAlasdair G Kergon kc->job_pool = mempool_create_slab_pool(MIN_JOBS, _job_cache); 8332d1e580aSAlasdair G Kergon if (!kc->job_pool) 8342d1e580aSAlasdair G Kergon goto bad_slab; 8352d1e580aSAlasdair G Kergon 8362d1e580aSAlasdair G Kergon INIT_WORK(&kc->kcopyd_work, do_work); 837670368a8STejun Heo kc->kcopyd_wq = alloc_workqueue("kcopyd", WQ_MEM_RECLAIM, 0); 8382d1e580aSAlasdair G Kergon if (!kc->kcopyd_wq) 8392d1e580aSAlasdair G Kergon goto bad_workqueue; 8402d1e580aSAlasdair G Kergon 8412d1e580aSAlasdair G Kergon kc->pages = NULL; 842d0471458SMikulas Patocka kc->nr_reserved_pages = kc->nr_free_pages = 0; 8435f43ba29SMikulas Patocka r = client_reserve_pages(kc, RESERVE_PAGES); 8442d1e580aSAlasdair G Kergon if (r) 8452d1e580aSAlasdair G Kergon goto bad_client_pages; 8462d1e580aSAlasdair G Kergon 847bda8efecSMikulas Patocka kc->io_client = dm_io_client_create(); 8482d1e580aSAlasdair G Kergon if (IS_ERR(kc->io_client)) { 8492d1e580aSAlasdair G Kergon r = PTR_ERR(kc->io_client); 8502d1e580aSAlasdair G Kergon goto bad_io_client; 8512d1e580aSAlasdair G Kergon } 8522d1e580aSAlasdair G Kergon 8532d1e580aSAlasdair G Kergon init_waitqueue_head(&kc->destroyq); 8542d1e580aSAlasdair G Kergon atomic_set(&kc->nr_jobs, 0); 8552d1e580aSAlasdair G Kergon 856fa34ce73SMikulas Patocka return kc; 8572d1e580aSAlasdair G Kergon 8582d1e580aSAlasdair G Kergon bad_io_client: 8592d1e580aSAlasdair G Kergon client_free_pages(kc); 8602d1e580aSAlasdair G Kergon bad_client_pages: 8612d1e580aSAlasdair G Kergon destroy_workqueue(kc->kcopyd_wq); 8622d1e580aSAlasdair G Kergon bad_workqueue: 8632d1e580aSAlasdair G Kergon mempool_destroy(kc->job_pool); 8642d1e580aSAlasdair G Kergon bad_slab: 8652d1e580aSAlasdair G Kergon kfree(kc); 8662d1e580aSAlasdair G Kergon 867fa34ce73SMikulas Patocka return ERR_PTR(r); 8682d1e580aSAlasdair G Kergon } 8692d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_client_create); 8702d1e580aSAlasdair G Kergon 8712d1e580aSAlasdair G Kergon void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc) 8722d1e580aSAlasdair G Kergon { 8732d1e580aSAlasdair G Kergon /* Wait for completion of all jobs submitted by this client. */ 8742d1e580aSAlasdair G Kergon wait_event(kc->destroyq, !atomic_read(&kc->nr_jobs)); 8752d1e580aSAlasdair G Kergon 8762d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->complete_jobs)); 8772d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->io_jobs)); 8782d1e580aSAlasdair G Kergon BUG_ON(!list_empty(&kc->pages_jobs)); 8792d1e580aSAlasdair G Kergon destroy_workqueue(kc->kcopyd_wq); 8802d1e580aSAlasdair G Kergon dm_io_client_destroy(kc->io_client); 8812d1e580aSAlasdair G Kergon client_free_pages(kc); 8822d1e580aSAlasdair G Kergon mempool_destroy(kc->job_pool); 8832d1e580aSAlasdair G Kergon kfree(kc); 8842d1e580aSAlasdair G Kergon } 8852d1e580aSAlasdair G Kergon EXPORT_SYMBOL(dm_kcopyd_client_destroy); 886