11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/net/sunrpc/sched.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Scheduling for synchronous and asynchronous RPC requests. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de> 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * TCP NFS related read + write fixes 91da177e4SLinus Torvalds * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds #include <linux/sched.h> 151da177e4SLinus Torvalds #include <linux/interrupt.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/smp.h> 191da177e4SLinus Torvalds #include <linux/smp_lock.h> 201da177e4SLinus Torvalds #include <linux/spinlock.h> 214a3e2f71SArjan van de Ven #include <linux/mutex.h> 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/sunrpc/clnt.h> 241da177e4SLinus Torvalds 251da177e4SLinus Torvalds #ifdef RPC_DEBUG 261da177e4SLinus Torvalds #define RPCDBG_FACILITY RPCDBG_SCHED 271da177e4SLinus Torvalds #define RPC_TASK_MAGIC_ID 0xf00baa 281da177e4SLinus Torvalds static int rpc_task_id; 291da177e4SLinus Torvalds #endif 301da177e4SLinus Torvalds 311da177e4SLinus Torvalds /* 321da177e4SLinus Torvalds * RPC slabs and memory pools 331da177e4SLinus Torvalds */ 341da177e4SLinus Torvalds #define RPC_BUFFER_MAXSIZE (2048) 351da177e4SLinus Torvalds #define RPC_BUFFER_POOLSIZE (8) 361da177e4SLinus Torvalds #define RPC_TASK_POOLSIZE (8) 37e18b890bSChristoph Lameter static struct kmem_cache *rpc_task_slabp __read_mostly; 38e18b890bSChristoph Lameter static struct kmem_cache *rpc_buffer_slabp __read_mostly; 39ba89966cSEric Dumazet static mempool_t *rpc_task_mempool __read_mostly; 40ba89966cSEric Dumazet static mempool_t *rpc_buffer_mempool __read_mostly; 411da177e4SLinus Torvalds 421da177e4SLinus Torvalds static void __rpc_default_timer(struct rpc_task *task); 431da177e4SLinus Torvalds static void rpciod_killall(void); 4465f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *); 45bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task); 461da177e4SLinus Torvalds 471da177e4SLinus Torvalds /* 481da177e4SLinus Torvalds * RPC tasks sit here while waiting for conditions to improve. 491da177e4SLinus Torvalds */ 501da177e4SLinus Torvalds static RPC_WAITQ(delay_queue, "delayq"); 511da177e4SLinus Torvalds 521da177e4SLinus Torvalds /* 531da177e4SLinus Torvalds * All RPC tasks are linked into this list 541da177e4SLinus Torvalds */ 551da177e4SLinus Torvalds static LIST_HEAD(all_tasks); 561da177e4SLinus Torvalds 571da177e4SLinus Torvalds /* 581da177e4SLinus Torvalds * rpciod-related stuff 591da177e4SLinus Torvalds */ 604a3e2f71SArjan van de Ven static DEFINE_MUTEX(rpciod_mutex); 611da177e4SLinus Torvalds static unsigned int rpciod_users; 6224c5d9d7STrond Myklebust struct workqueue_struct *rpciod_workqueue; 631da177e4SLinus Torvalds 641da177e4SLinus Torvalds /* 651da177e4SLinus Torvalds * Spinlock for other critical sections of code. 661da177e4SLinus Torvalds */ 671da177e4SLinus Torvalds static DEFINE_SPINLOCK(rpc_sched_lock); 681da177e4SLinus Torvalds 691da177e4SLinus Torvalds /* 701da177e4SLinus Torvalds * Disable the timer for a given RPC task. Should be called with 711da177e4SLinus Torvalds * queue->lock and bh_disabled in order to avoid races within 721da177e4SLinus Torvalds * rpc_run_timer(). 731da177e4SLinus Torvalds */ 741da177e4SLinus Torvalds static inline void 751da177e4SLinus Torvalds __rpc_disable_timer(struct rpc_task *task) 761da177e4SLinus Torvalds { 7746121cf7SChuck Lever dprintk("RPC: %5u disabling timer\n", task->tk_pid); 781da177e4SLinus Torvalds task->tk_timeout_fn = NULL; 791da177e4SLinus Torvalds task->tk_timeout = 0; 801da177e4SLinus Torvalds } 811da177e4SLinus Torvalds 821da177e4SLinus Torvalds /* 831da177e4SLinus Torvalds * Run a timeout function. 841da177e4SLinus Torvalds * We use the callback in order to allow __rpc_wake_up_task() 851da177e4SLinus Torvalds * and friends to disable the timer synchronously on SMP systems 861da177e4SLinus Torvalds * without calling del_timer_sync(). The latter could cause a 871da177e4SLinus Torvalds * deadlock if called while we're holding spinlocks... 881da177e4SLinus Torvalds */ 891da177e4SLinus Torvalds static void rpc_run_timer(struct rpc_task *task) 901da177e4SLinus Torvalds { 911da177e4SLinus Torvalds void (*callback)(struct rpc_task *); 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds callback = task->tk_timeout_fn; 941da177e4SLinus Torvalds task->tk_timeout_fn = NULL; 951da177e4SLinus Torvalds if (callback && RPC_IS_QUEUED(task)) { 9646121cf7SChuck Lever dprintk("RPC: %5u running timer\n", task->tk_pid); 971da177e4SLinus Torvalds callback(task); 981da177e4SLinus Torvalds } 991da177e4SLinus Torvalds smp_mb__before_clear_bit(); 1001da177e4SLinus Torvalds clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate); 1011da177e4SLinus Torvalds smp_mb__after_clear_bit(); 1021da177e4SLinus Torvalds } 1031da177e4SLinus Torvalds 1041da177e4SLinus Torvalds /* 1051da177e4SLinus Torvalds * Set up a timer for the current task. 1061da177e4SLinus Torvalds */ 1071da177e4SLinus Torvalds static inline void 1081da177e4SLinus Torvalds __rpc_add_timer(struct rpc_task *task, rpc_action timer) 1091da177e4SLinus Torvalds { 1101da177e4SLinus Torvalds if (!task->tk_timeout) 1111da177e4SLinus Torvalds return; 1121da177e4SLinus Torvalds 11346121cf7SChuck Lever dprintk("RPC: %5u setting alarm for %lu ms\n", 1141da177e4SLinus Torvalds task->tk_pid, task->tk_timeout * 1000 / HZ); 1151da177e4SLinus Torvalds 1161da177e4SLinus Torvalds if (timer) 1171da177e4SLinus Torvalds task->tk_timeout_fn = timer; 1181da177e4SLinus Torvalds else 1191da177e4SLinus Torvalds task->tk_timeout_fn = __rpc_default_timer; 1201da177e4SLinus Torvalds set_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate); 1211da177e4SLinus Torvalds mod_timer(&task->tk_timer, jiffies + task->tk_timeout); 1221da177e4SLinus Torvalds } 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds /* 1251da177e4SLinus Torvalds * Delete any timer for the current task. Because we use del_timer_sync(), 1261da177e4SLinus Torvalds * this function should never be called while holding queue->lock. 1271da177e4SLinus Torvalds */ 1281da177e4SLinus Torvalds static void 1291da177e4SLinus Torvalds rpc_delete_timer(struct rpc_task *task) 1301da177e4SLinus Torvalds { 1311da177e4SLinus Torvalds if (RPC_IS_QUEUED(task)) 1321da177e4SLinus Torvalds return; 1331da177e4SLinus Torvalds if (test_and_clear_bit(RPC_TASK_HAS_TIMER, &task->tk_runstate)) { 1341da177e4SLinus Torvalds del_singleshot_timer_sync(&task->tk_timer); 13546121cf7SChuck Lever dprintk("RPC: %5u deleting timer\n", task->tk_pid); 1361da177e4SLinus Torvalds } 1371da177e4SLinus Torvalds } 1381da177e4SLinus Torvalds 1391da177e4SLinus Torvalds /* 1401da177e4SLinus Torvalds * Add new request to a priority queue. 1411da177e4SLinus Torvalds */ 1421da177e4SLinus Torvalds static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task) 1431da177e4SLinus Torvalds { 1441da177e4SLinus Torvalds struct list_head *q; 1451da177e4SLinus Torvalds struct rpc_task *t; 1461da177e4SLinus Torvalds 1471da177e4SLinus Torvalds INIT_LIST_HEAD(&task->u.tk_wait.links); 1481da177e4SLinus Torvalds q = &queue->tasks[task->tk_priority]; 1491da177e4SLinus Torvalds if (unlikely(task->tk_priority > queue->maxpriority)) 1501da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 1511da177e4SLinus Torvalds list_for_each_entry(t, q, u.tk_wait.list) { 1521da177e4SLinus Torvalds if (t->tk_cookie == task->tk_cookie) { 1531da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links); 1541da177e4SLinus Torvalds return; 1551da177e4SLinus Torvalds } 1561da177e4SLinus Torvalds } 1571da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, q); 1581da177e4SLinus Torvalds } 1591da177e4SLinus Torvalds 1601da177e4SLinus Torvalds /* 1611da177e4SLinus Torvalds * Add new request to wait queue. 1621da177e4SLinus Torvalds * 1631da177e4SLinus Torvalds * Swapper tasks always get inserted at the head of the queue. 1641da177e4SLinus Torvalds * This should avoid many nasty memory deadlocks and hopefully 1651da177e4SLinus Torvalds * improve overall performance. 1661da177e4SLinus Torvalds * Everyone else gets appended to the queue to ensure proper FIFO behavior. 1671da177e4SLinus Torvalds */ 1681da177e4SLinus Torvalds static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) 1691da177e4SLinus Torvalds { 1701da177e4SLinus Torvalds BUG_ON (RPC_IS_QUEUED(task)); 1711da177e4SLinus Torvalds 1721da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 1731da177e4SLinus Torvalds __rpc_add_wait_queue_priority(queue, task); 1741da177e4SLinus Torvalds else if (RPC_IS_SWAPPER(task)) 1751da177e4SLinus Torvalds list_add(&task->u.tk_wait.list, &queue->tasks[0]); 1761da177e4SLinus Torvalds else 1771da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); 1781da177e4SLinus Torvalds task->u.tk_wait.rpc_waitq = queue; 179e19b63daSChuck Lever queue->qlen++; 1801da177e4SLinus Torvalds rpc_set_queued(task); 1811da177e4SLinus Torvalds 18246121cf7SChuck Lever dprintk("RPC: %5u added to queue %p \"%s\"\n", 1831da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 1841da177e4SLinus Torvalds } 1851da177e4SLinus Torvalds 1861da177e4SLinus Torvalds /* 1871da177e4SLinus Torvalds * Remove request from a priority queue. 1881da177e4SLinus Torvalds */ 1891da177e4SLinus Torvalds static void __rpc_remove_wait_queue_priority(struct rpc_task *task) 1901da177e4SLinus Torvalds { 1911da177e4SLinus Torvalds struct rpc_task *t; 1921da177e4SLinus Torvalds 1931da177e4SLinus Torvalds if (!list_empty(&task->u.tk_wait.links)) { 1941da177e4SLinus Torvalds t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list); 1951da177e4SLinus Torvalds list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); 1961da177e4SLinus Torvalds list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); 1971da177e4SLinus Torvalds } 1981da177e4SLinus Torvalds list_del(&task->u.tk_wait.list); 1991da177e4SLinus Torvalds } 2001da177e4SLinus Torvalds 2011da177e4SLinus Torvalds /* 2021da177e4SLinus Torvalds * Remove request from queue. 2031da177e4SLinus Torvalds * Note: must be called with spin lock held. 2041da177e4SLinus Torvalds */ 2051da177e4SLinus Torvalds static void __rpc_remove_wait_queue(struct rpc_task *task) 2061da177e4SLinus Torvalds { 2071da177e4SLinus Torvalds struct rpc_wait_queue *queue; 2081da177e4SLinus Torvalds queue = task->u.tk_wait.rpc_waitq; 2091da177e4SLinus Torvalds 2101da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 2111da177e4SLinus Torvalds __rpc_remove_wait_queue_priority(task); 2121da177e4SLinus Torvalds else 2131da177e4SLinus Torvalds list_del(&task->u.tk_wait.list); 214e19b63daSChuck Lever queue->qlen--; 21546121cf7SChuck Lever dprintk("RPC: %5u removed from queue %p \"%s\"\n", 2161da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 2171da177e4SLinus Torvalds } 2181da177e4SLinus Torvalds 2191da177e4SLinus Torvalds static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) 2201da177e4SLinus Torvalds { 2211da177e4SLinus Torvalds queue->priority = priority; 2221da177e4SLinus Torvalds queue->count = 1 << (priority * 2); 2231da177e4SLinus Torvalds } 2241da177e4SLinus Torvalds 2251da177e4SLinus Torvalds static inline void rpc_set_waitqueue_cookie(struct rpc_wait_queue *queue, unsigned long cookie) 2261da177e4SLinus Torvalds { 2271da177e4SLinus Torvalds queue->cookie = cookie; 2281da177e4SLinus Torvalds queue->nr = RPC_BATCH_COUNT; 2291da177e4SLinus Torvalds } 2301da177e4SLinus Torvalds 2311da177e4SLinus Torvalds static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) 2321da177e4SLinus Torvalds { 2331da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, queue->maxpriority); 2341da177e4SLinus Torvalds rpc_set_waitqueue_cookie(queue, 0); 2351da177e4SLinus Torvalds } 2361da177e4SLinus Torvalds 2371da177e4SLinus Torvalds static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, int maxprio) 2381da177e4SLinus Torvalds { 2391da177e4SLinus Torvalds int i; 2401da177e4SLinus Torvalds 2411da177e4SLinus Torvalds spin_lock_init(&queue->lock); 2421da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) 2431da177e4SLinus Torvalds INIT_LIST_HEAD(&queue->tasks[i]); 2441da177e4SLinus Torvalds queue->maxpriority = maxprio; 2451da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 2461da177e4SLinus Torvalds #ifdef RPC_DEBUG 2471da177e4SLinus Torvalds queue->name = qname; 2481da177e4SLinus Torvalds #endif 2491da177e4SLinus Torvalds } 2501da177e4SLinus Torvalds 2511da177e4SLinus Torvalds void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2521da177e4SLinus Torvalds { 2531da177e4SLinus Torvalds __rpc_init_priority_wait_queue(queue, qname, RPC_PRIORITY_HIGH); 2541da177e4SLinus Torvalds } 2551da177e4SLinus Torvalds 2561da177e4SLinus Torvalds void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2571da177e4SLinus Torvalds { 2581da177e4SLinus Torvalds __rpc_init_priority_wait_queue(queue, qname, 0); 2591da177e4SLinus Torvalds } 2601da177e4SLinus Torvalds EXPORT_SYMBOL(rpc_init_wait_queue); 2611da177e4SLinus Torvalds 26244c28873STrond Myklebust static int rpc_wait_bit_interruptible(void *word) 26344c28873STrond Myklebust { 26444c28873STrond Myklebust if (signal_pending(current)) 26544c28873STrond Myklebust return -ERESTARTSYS; 26644c28873STrond Myklebust schedule(); 26744c28873STrond Myklebust return 0; 26844c28873STrond Myklebust } 26944c28873STrond Myklebust 270e6b3c4dbSTrond Myklebust static void rpc_set_active(struct rpc_task *task) 271e6b3c4dbSTrond Myklebust { 272e6b3c4dbSTrond Myklebust if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0) 273e6b3c4dbSTrond Myklebust return; 274e6b3c4dbSTrond Myklebust spin_lock(&rpc_sched_lock); 275e6b3c4dbSTrond Myklebust #ifdef RPC_DEBUG 276e6b3c4dbSTrond Myklebust task->tk_magic = RPC_TASK_MAGIC_ID; 277e6b3c4dbSTrond Myklebust task->tk_pid = rpc_task_id++; 278e6b3c4dbSTrond Myklebust #endif 279e6b3c4dbSTrond Myklebust /* Add to global list of all tasks */ 280e6b3c4dbSTrond Myklebust list_add_tail(&task->tk_task, &all_tasks); 281e6b3c4dbSTrond Myklebust spin_unlock(&rpc_sched_lock); 282e6b3c4dbSTrond Myklebust } 283e6b3c4dbSTrond Myklebust 28444c28873STrond Myklebust /* 28544c28873STrond Myklebust * Mark an RPC call as having completed by clearing the 'active' bit 28644c28873STrond Myklebust */ 287e6b3c4dbSTrond Myklebust static void rpc_mark_complete_task(struct rpc_task *task) 28844c28873STrond Myklebust { 289e6b3c4dbSTrond Myklebust smp_mb__before_clear_bit(); 290e6b3c4dbSTrond Myklebust clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 291e6b3c4dbSTrond Myklebust smp_mb__after_clear_bit(); 29244c28873STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE); 29344c28873STrond Myklebust } 29444c28873STrond Myklebust 29544c28873STrond Myklebust /* 29644c28873STrond Myklebust * Allow callers to wait for completion of an RPC call 29744c28873STrond Myklebust */ 29844c28873STrond Myklebust int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *)) 29944c28873STrond Myklebust { 30044c28873STrond Myklebust if (action == NULL) 30144c28873STrond Myklebust action = rpc_wait_bit_interruptible; 30244c28873STrond Myklebust return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, 30344c28873STrond Myklebust action, TASK_INTERRUPTIBLE); 30444c28873STrond Myklebust } 30544c28873STrond Myklebust EXPORT_SYMBOL(__rpc_wait_for_completion_task); 30644c28873STrond Myklebust 3071da177e4SLinus Torvalds /* 3081da177e4SLinus Torvalds * Make an RPC task runnable. 3091da177e4SLinus Torvalds * 3101da177e4SLinus Torvalds * Note: If the task is ASYNC, this must be called with 3111da177e4SLinus Torvalds * the spinlock held to protect the wait queue operation. 3121da177e4SLinus Torvalds */ 3131da177e4SLinus Torvalds static void rpc_make_runnable(struct rpc_task *task) 3141da177e4SLinus Torvalds { 3151da177e4SLinus Torvalds BUG_ON(task->tk_timeout_fn); 3161da177e4SLinus Torvalds rpc_clear_queued(task); 317cc4dc59eSChristophe Saout if (rpc_test_and_set_running(task)) 3181da177e4SLinus Torvalds return; 319cc4dc59eSChristophe Saout /* We might have raced */ 320cc4dc59eSChristophe Saout if (RPC_IS_QUEUED(task)) { 321cc4dc59eSChristophe Saout rpc_clear_running(task); 322cc4dc59eSChristophe Saout return; 323cc4dc59eSChristophe Saout } 3241da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 3251da177e4SLinus Torvalds int status; 3261da177e4SLinus Torvalds 32765f27f38SDavid Howells INIT_WORK(&task->u.tk_work, rpc_async_schedule); 3281da177e4SLinus Torvalds status = queue_work(task->tk_workqueue, &task->u.tk_work); 3291da177e4SLinus Torvalds if (status < 0) { 3301da177e4SLinus Torvalds printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status); 3311da177e4SLinus Torvalds task->tk_status = status; 3321da177e4SLinus Torvalds return; 3331da177e4SLinus Torvalds } 3341da177e4SLinus Torvalds } else 33596651ab3STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); 3361da177e4SLinus Torvalds } 3371da177e4SLinus Torvalds 3381da177e4SLinus Torvalds /* 3391da177e4SLinus Torvalds * Prepare for sleeping on a wait queue. 3401da177e4SLinus Torvalds * By always appending tasks to the list we ensure FIFO behavior. 3411da177e4SLinus Torvalds * NB: An RPC task will only receive interrupt-driven events as long 3421da177e4SLinus Torvalds * as it's on a wait queue. 3431da177e4SLinus Torvalds */ 3441da177e4SLinus Torvalds static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 3451da177e4SLinus Torvalds rpc_action action, rpc_action timer) 3461da177e4SLinus Torvalds { 34746121cf7SChuck Lever dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n", 34846121cf7SChuck Lever task->tk_pid, rpc_qname(q), jiffies); 3491da177e4SLinus Torvalds 3501da177e4SLinus Torvalds if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) { 3511da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n"); 3521da177e4SLinus Torvalds return; 3531da177e4SLinus Torvalds } 3541da177e4SLinus Torvalds 3551da177e4SLinus Torvalds __rpc_add_wait_queue(q, task); 3561da177e4SLinus Torvalds 3571da177e4SLinus Torvalds BUG_ON(task->tk_callback != NULL); 3581da177e4SLinus Torvalds task->tk_callback = action; 3591da177e4SLinus Torvalds __rpc_add_timer(task, timer); 3601da177e4SLinus Torvalds } 3611da177e4SLinus Torvalds 3621da177e4SLinus Torvalds void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 3631da177e4SLinus Torvalds rpc_action action, rpc_action timer) 3641da177e4SLinus Torvalds { 365e6b3c4dbSTrond Myklebust /* Mark the task as being activated if so needed */ 366e6b3c4dbSTrond Myklebust rpc_set_active(task); 367e6b3c4dbSTrond Myklebust 3681da177e4SLinus Torvalds /* 3691da177e4SLinus Torvalds * Protect the queue operations. 3701da177e4SLinus Torvalds */ 3711da177e4SLinus Torvalds spin_lock_bh(&q->lock); 3721da177e4SLinus Torvalds __rpc_sleep_on(q, task, action, timer); 3731da177e4SLinus Torvalds spin_unlock_bh(&q->lock); 3741da177e4SLinus Torvalds } 3751da177e4SLinus Torvalds 3761da177e4SLinus Torvalds /** 3771da177e4SLinus Torvalds * __rpc_do_wake_up_task - wake up a single rpc_task 3781da177e4SLinus Torvalds * @task: task to be woken up 3791da177e4SLinus Torvalds * 3801da177e4SLinus Torvalds * Caller must hold queue->lock, and have cleared the task queued flag. 3811da177e4SLinus Torvalds */ 3821da177e4SLinus Torvalds static void __rpc_do_wake_up_task(struct rpc_task *task) 3831da177e4SLinus Torvalds { 38446121cf7SChuck Lever dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n", 38546121cf7SChuck Lever task->tk_pid, jiffies); 3861da177e4SLinus Torvalds 3871da177e4SLinus Torvalds #ifdef RPC_DEBUG 3881da177e4SLinus Torvalds BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID); 3891da177e4SLinus Torvalds #endif 3901da177e4SLinus Torvalds /* Has the task been executed yet? If not, we cannot wake it up! */ 3911da177e4SLinus Torvalds if (!RPC_IS_ACTIVATED(task)) { 3921da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); 3931da177e4SLinus Torvalds return; 3941da177e4SLinus Torvalds } 3951da177e4SLinus Torvalds 3961da177e4SLinus Torvalds __rpc_disable_timer(task); 3971da177e4SLinus Torvalds __rpc_remove_wait_queue(task); 3981da177e4SLinus Torvalds 3991da177e4SLinus Torvalds rpc_make_runnable(task); 4001da177e4SLinus Torvalds 4011da177e4SLinus Torvalds dprintk("RPC: __rpc_wake_up_task done\n"); 4021da177e4SLinus Torvalds } 4031da177e4SLinus Torvalds 4041da177e4SLinus Torvalds /* 4051da177e4SLinus Torvalds * Wake up the specified task 4061da177e4SLinus Torvalds */ 4071da177e4SLinus Torvalds static void __rpc_wake_up_task(struct rpc_task *task) 4081da177e4SLinus Torvalds { 4091da177e4SLinus Torvalds if (rpc_start_wakeup(task)) { 4101da177e4SLinus Torvalds if (RPC_IS_QUEUED(task)) 4111da177e4SLinus Torvalds __rpc_do_wake_up_task(task); 4121da177e4SLinus Torvalds rpc_finish_wakeup(task); 4131da177e4SLinus Torvalds } 4141da177e4SLinus Torvalds } 4151da177e4SLinus Torvalds 4161da177e4SLinus Torvalds /* 4171da177e4SLinus Torvalds * Default timeout handler if none specified by user 4181da177e4SLinus Torvalds */ 4191da177e4SLinus Torvalds static void 4201da177e4SLinus Torvalds __rpc_default_timer(struct rpc_task *task) 4211da177e4SLinus Torvalds { 42246121cf7SChuck Lever dprintk("RPC: %5u timeout (default timer)\n", task->tk_pid); 4231da177e4SLinus Torvalds task->tk_status = -ETIMEDOUT; 4241da177e4SLinus Torvalds rpc_wake_up_task(task); 4251da177e4SLinus Torvalds } 4261da177e4SLinus Torvalds 4271da177e4SLinus Torvalds /* 4281da177e4SLinus Torvalds * Wake up the specified task 4291da177e4SLinus Torvalds */ 4301da177e4SLinus Torvalds void rpc_wake_up_task(struct rpc_task *task) 4311da177e4SLinus Torvalds { 4328aca67f0STrond Myklebust rcu_read_lock_bh(); 4331da177e4SLinus Torvalds if (rpc_start_wakeup(task)) { 4341da177e4SLinus Torvalds if (RPC_IS_QUEUED(task)) { 4351da177e4SLinus Torvalds struct rpc_wait_queue *queue = task->u.tk_wait.rpc_waitq; 4361da177e4SLinus Torvalds 4378aca67f0STrond Myklebust /* Note: we're already in a bh-safe context */ 4388aca67f0STrond Myklebust spin_lock(&queue->lock); 4391da177e4SLinus Torvalds __rpc_do_wake_up_task(task); 4408aca67f0STrond Myklebust spin_unlock(&queue->lock); 4411da177e4SLinus Torvalds } 4421da177e4SLinus Torvalds rpc_finish_wakeup(task); 4431da177e4SLinus Torvalds } 4448aca67f0STrond Myklebust rcu_read_unlock_bh(); 4451da177e4SLinus Torvalds } 4461da177e4SLinus Torvalds 4471da177e4SLinus Torvalds /* 4481da177e4SLinus Torvalds * Wake up the next task on a priority queue. 4491da177e4SLinus Torvalds */ 4501da177e4SLinus Torvalds static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue) 4511da177e4SLinus Torvalds { 4521da177e4SLinus Torvalds struct list_head *q; 4531da177e4SLinus Torvalds struct rpc_task *task; 4541da177e4SLinus Torvalds 4551da177e4SLinus Torvalds /* 4561da177e4SLinus Torvalds * Service a batch of tasks from a single cookie. 4571da177e4SLinus Torvalds */ 4581da177e4SLinus Torvalds q = &queue->tasks[queue->priority]; 4591da177e4SLinus Torvalds if (!list_empty(q)) { 4601da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4611da177e4SLinus Torvalds if (queue->cookie == task->tk_cookie) { 4621da177e4SLinus Torvalds if (--queue->nr) 4631da177e4SLinus Torvalds goto out; 4641da177e4SLinus Torvalds list_move_tail(&task->u.tk_wait.list, q); 4651da177e4SLinus Torvalds } 4661da177e4SLinus Torvalds /* 4671da177e4SLinus Torvalds * Check if we need to switch queues. 4681da177e4SLinus Torvalds */ 4691da177e4SLinus Torvalds if (--queue->count) 4701da177e4SLinus Torvalds goto new_cookie; 4711da177e4SLinus Torvalds } 4721da177e4SLinus Torvalds 4731da177e4SLinus Torvalds /* 4741da177e4SLinus Torvalds * Service the next queue. 4751da177e4SLinus Torvalds */ 4761da177e4SLinus Torvalds do { 4771da177e4SLinus Torvalds if (q == &queue->tasks[0]) 4781da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 4791da177e4SLinus Torvalds else 4801da177e4SLinus Torvalds q = q - 1; 4811da177e4SLinus Torvalds if (!list_empty(q)) { 4821da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4831da177e4SLinus Torvalds goto new_queue; 4841da177e4SLinus Torvalds } 4851da177e4SLinus Torvalds } while (q != &queue->tasks[queue->priority]); 4861da177e4SLinus Torvalds 4871da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 4881da177e4SLinus Torvalds return NULL; 4891da177e4SLinus Torvalds 4901da177e4SLinus Torvalds new_queue: 4911da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); 4921da177e4SLinus Torvalds new_cookie: 4931da177e4SLinus Torvalds rpc_set_waitqueue_cookie(queue, task->tk_cookie); 4941da177e4SLinus Torvalds out: 4951da177e4SLinus Torvalds __rpc_wake_up_task(task); 4961da177e4SLinus Torvalds return task; 4971da177e4SLinus Torvalds } 4981da177e4SLinus Torvalds 4991da177e4SLinus Torvalds /* 5001da177e4SLinus Torvalds * Wake up the next task on the wait queue. 5011da177e4SLinus Torvalds */ 5021da177e4SLinus Torvalds struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue) 5031da177e4SLinus Torvalds { 5041da177e4SLinus Torvalds struct rpc_task *task = NULL; 5051da177e4SLinus Torvalds 50646121cf7SChuck Lever dprintk("RPC: wake_up_next(%p \"%s\")\n", 50746121cf7SChuck Lever queue, rpc_qname(queue)); 5088aca67f0STrond Myklebust rcu_read_lock_bh(); 5098aca67f0STrond Myklebust spin_lock(&queue->lock); 5101da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 5111da177e4SLinus Torvalds task = __rpc_wake_up_next_priority(queue); 5121da177e4SLinus Torvalds else { 5131da177e4SLinus Torvalds task_for_first(task, &queue->tasks[0]) 5141da177e4SLinus Torvalds __rpc_wake_up_task(task); 5151da177e4SLinus Torvalds } 5168aca67f0STrond Myklebust spin_unlock(&queue->lock); 5178aca67f0STrond Myklebust rcu_read_unlock_bh(); 5181da177e4SLinus Torvalds 5191da177e4SLinus Torvalds return task; 5201da177e4SLinus Torvalds } 5211da177e4SLinus Torvalds 5221da177e4SLinus Torvalds /** 5231da177e4SLinus Torvalds * rpc_wake_up - wake up all rpc_tasks 5241da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 5251da177e4SLinus Torvalds * 5261da177e4SLinus Torvalds * Grabs queue->lock 5271da177e4SLinus Torvalds */ 5281da177e4SLinus Torvalds void rpc_wake_up(struct rpc_wait_queue *queue) 5291da177e4SLinus Torvalds { 530e6d83d55STrond Myklebust struct rpc_task *task, *next; 5311da177e4SLinus Torvalds struct list_head *head; 532e6d83d55STrond Myklebust 5338aca67f0STrond Myklebust rcu_read_lock_bh(); 5348aca67f0STrond Myklebust spin_lock(&queue->lock); 5351da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5361da177e4SLinus Torvalds for (;;) { 537e6d83d55STrond Myklebust list_for_each_entry_safe(task, next, head, u.tk_wait.list) 5381da177e4SLinus Torvalds __rpc_wake_up_task(task); 5391da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5401da177e4SLinus Torvalds break; 5411da177e4SLinus Torvalds head--; 5421da177e4SLinus Torvalds } 5438aca67f0STrond Myklebust spin_unlock(&queue->lock); 5448aca67f0STrond Myklebust rcu_read_unlock_bh(); 5451da177e4SLinus Torvalds } 5461da177e4SLinus Torvalds 5471da177e4SLinus Torvalds /** 5481da177e4SLinus Torvalds * rpc_wake_up_status - wake up all rpc_tasks and set their status value. 5491da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 5501da177e4SLinus Torvalds * @status: status value to set 5511da177e4SLinus Torvalds * 5521da177e4SLinus Torvalds * Grabs queue->lock 5531da177e4SLinus Torvalds */ 5541da177e4SLinus Torvalds void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) 5551da177e4SLinus Torvalds { 556e6d83d55STrond Myklebust struct rpc_task *task, *next; 5571da177e4SLinus Torvalds struct list_head *head; 5581da177e4SLinus Torvalds 5598aca67f0STrond Myklebust rcu_read_lock_bh(); 5608aca67f0STrond Myklebust spin_lock(&queue->lock); 5611da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5621da177e4SLinus Torvalds for (;;) { 563e6d83d55STrond Myklebust list_for_each_entry_safe(task, next, head, u.tk_wait.list) { 5641da177e4SLinus Torvalds task->tk_status = status; 5651da177e4SLinus Torvalds __rpc_wake_up_task(task); 5661da177e4SLinus Torvalds } 5671da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5681da177e4SLinus Torvalds break; 5691da177e4SLinus Torvalds head--; 5701da177e4SLinus Torvalds } 5718aca67f0STrond Myklebust spin_unlock(&queue->lock); 5728aca67f0STrond Myklebust rcu_read_unlock_bh(); 5731da177e4SLinus Torvalds } 5741da177e4SLinus Torvalds 5758014793bSTrond Myklebust static void __rpc_atrun(struct rpc_task *task) 5768014793bSTrond Myklebust { 5778014793bSTrond Myklebust rpc_wake_up_task(task); 5788014793bSTrond Myklebust } 5798014793bSTrond Myklebust 5801da177e4SLinus Torvalds /* 5811da177e4SLinus Torvalds * Run a task at a later time 5821da177e4SLinus Torvalds */ 5838014793bSTrond Myklebust void rpc_delay(struct rpc_task *task, unsigned long delay) 5841da177e4SLinus Torvalds { 5851da177e4SLinus Torvalds task->tk_timeout = delay; 5861da177e4SLinus Torvalds rpc_sleep_on(&delay_queue, task, NULL, __rpc_atrun); 5871da177e4SLinus Torvalds } 5881da177e4SLinus Torvalds 5891da177e4SLinus Torvalds /* 5904ce70adaSTrond Myklebust * Helper to call task->tk_ops->rpc_call_prepare 5914ce70adaSTrond Myklebust */ 5924ce70adaSTrond Myklebust static void rpc_prepare_task(struct rpc_task *task) 5934ce70adaSTrond Myklebust { 5946d5fcb5aSTrond Myklebust lock_kernel(); 5954ce70adaSTrond Myklebust task->tk_ops->rpc_call_prepare(task, task->tk_calldata); 5966d5fcb5aSTrond Myklebust unlock_kernel(); 5974ce70adaSTrond Myklebust } 5984ce70adaSTrond Myklebust 5994ce70adaSTrond Myklebust /* 600963d8fe5STrond Myklebust * Helper that calls task->tk_ops->rpc_call_done if it exists 601d05fdb0cSTrond Myklebust */ 602abbcf28fSTrond Myklebust void rpc_exit_task(struct rpc_task *task) 603d05fdb0cSTrond Myklebust { 604abbcf28fSTrond Myklebust task->tk_action = NULL; 605963d8fe5STrond Myklebust if (task->tk_ops->rpc_call_done != NULL) { 6066d5fcb5aSTrond Myklebust lock_kernel(); 607963d8fe5STrond Myklebust task->tk_ops->rpc_call_done(task, task->tk_calldata); 6086d5fcb5aSTrond Myklebust unlock_kernel(); 609d05fdb0cSTrond Myklebust if (task->tk_action != NULL) { 610abbcf28fSTrond Myklebust WARN_ON(RPC_ASSASSINATED(task)); 611abbcf28fSTrond Myklebust /* Always release the RPC slot and buffer memory */ 612d05fdb0cSTrond Myklebust xprt_release(task); 613d05fdb0cSTrond Myklebust } 614d05fdb0cSTrond Myklebust } 615d05fdb0cSTrond Myklebust } 616abbcf28fSTrond Myklebust EXPORT_SYMBOL(rpc_exit_task); 617d05fdb0cSTrond Myklebust 618bbd5a1f9STrond Myklebust void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) 619bbd5a1f9STrond Myklebust { 620bbd5a1f9STrond Myklebust if (ops->rpc_release != NULL) { 621bbd5a1f9STrond Myklebust lock_kernel(); 622bbd5a1f9STrond Myklebust ops->rpc_release(calldata); 623bbd5a1f9STrond Myklebust unlock_kernel(); 624bbd5a1f9STrond Myklebust } 625bbd5a1f9STrond Myklebust } 626bbd5a1f9STrond Myklebust 627d05fdb0cSTrond Myklebust /* 6281da177e4SLinus Torvalds * This is the RPC `scheduler' (or rather, the finite state machine). 6291da177e4SLinus Torvalds */ 6302efef837STrond Myklebust static void __rpc_execute(struct rpc_task *task) 6311da177e4SLinus Torvalds { 6321da177e4SLinus Torvalds int status = 0; 6331da177e4SLinus Torvalds 63446121cf7SChuck Lever dprintk("RPC: %5u __rpc_execute flags=0x%x\n", 6351da177e4SLinus Torvalds task->tk_pid, task->tk_flags); 6361da177e4SLinus Torvalds 6371da177e4SLinus Torvalds BUG_ON(RPC_IS_QUEUED(task)); 6381da177e4SLinus Torvalds 639d05fdb0cSTrond Myklebust for (;;) { 6401da177e4SLinus Torvalds /* 6411da177e4SLinus Torvalds * Garbage collection of pending timers... 6421da177e4SLinus Torvalds */ 6431da177e4SLinus Torvalds rpc_delete_timer(task); 6441da177e4SLinus Torvalds 6451da177e4SLinus Torvalds /* 6461da177e4SLinus Torvalds * Execute any pending callback. 6471da177e4SLinus Torvalds */ 6481da177e4SLinus Torvalds if (RPC_DO_CALLBACK(task)) { 6491da177e4SLinus Torvalds /* Define a callback save pointer */ 6501da177e4SLinus Torvalds void (*save_callback)(struct rpc_task *); 6511da177e4SLinus Torvalds 6521da177e4SLinus Torvalds /* 6531da177e4SLinus Torvalds * If a callback exists, save it, reset it, 6541da177e4SLinus Torvalds * call it. 6551da177e4SLinus Torvalds * The save is needed to stop from resetting 6561da177e4SLinus Torvalds * another callback set within the callback handler 6571da177e4SLinus Torvalds * - Dave 6581da177e4SLinus Torvalds */ 6591da177e4SLinus Torvalds save_callback=task->tk_callback; 6601da177e4SLinus Torvalds task->tk_callback=NULL; 6611da177e4SLinus Torvalds save_callback(task); 6621da177e4SLinus Torvalds } 6631da177e4SLinus Torvalds 6641da177e4SLinus Torvalds /* 6651da177e4SLinus Torvalds * Perform the next FSM step. 6661da177e4SLinus Torvalds * tk_action may be NULL when the task has been killed 6671da177e4SLinus Torvalds * by someone else. 6681da177e4SLinus Torvalds */ 6691da177e4SLinus Torvalds if (!RPC_IS_QUEUED(task)) { 670abbcf28fSTrond Myklebust if (task->tk_action == NULL) 671abbcf28fSTrond Myklebust break; 6721da177e4SLinus Torvalds task->tk_action(task); 6731da177e4SLinus Torvalds } 6741da177e4SLinus Torvalds 6751da177e4SLinus Torvalds /* 6761da177e4SLinus Torvalds * Lockless check for whether task is sleeping or not. 6771da177e4SLinus Torvalds */ 6781da177e4SLinus Torvalds if (!RPC_IS_QUEUED(task)) 6791da177e4SLinus Torvalds continue; 6801da177e4SLinus Torvalds rpc_clear_running(task); 6811da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 6821da177e4SLinus Torvalds /* Careful! we may have raced... */ 6831da177e4SLinus Torvalds if (RPC_IS_QUEUED(task)) 6842efef837STrond Myklebust return; 6851da177e4SLinus Torvalds if (rpc_test_and_set_running(task)) 6862efef837STrond Myklebust return; 6871da177e4SLinus Torvalds continue; 6881da177e4SLinus Torvalds } 6891da177e4SLinus Torvalds 6901da177e4SLinus Torvalds /* sync task: sleep here */ 69146121cf7SChuck Lever dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid); 69296651ab3STrond Myklebust /* Note: Caller should be using rpc_clnt_sigmask() */ 69396651ab3STrond Myklebust status = out_of_line_wait_on_bit(&task->tk_runstate, 69496651ab3STrond Myklebust RPC_TASK_QUEUED, rpc_wait_bit_interruptible, 69596651ab3STrond Myklebust TASK_INTERRUPTIBLE); 69696651ab3STrond Myklebust if (status == -ERESTARTSYS) { 6971da177e4SLinus Torvalds /* 6981da177e4SLinus Torvalds * When a sync task receives a signal, it exits with 6991da177e4SLinus Torvalds * -ERESTARTSYS. In order to catch any callbacks that 7001da177e4SLinus Torvalds * clean up after sleeping on some queue, we don't 7011da177e4SLinus Torvalds * break the loop here, but go around once more. 7021da177e4SLinus Torvalds */ 70346121cf7SChuck Lever dprintk("RPC: %5u got signal\n", task->tk_pid); 7041da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_KILLED; 7051da177e4SLinus Torvalds rpc_exit(task, -ERESTARTSYS); 7061da177e4SLinus Torvalds rpc_wake_up_task(task); 7071da177e4SLinus Torvalds } 7081da177e4SLinus Torvalds rpc_set_running(task); 70946121cf7SChuck Lever dprintk("RPC: %5u sync task resuming\n", task->tk_pid); 7101da177e4SLinus Torvalds } 7111da177e4SLinus Torvalds 71246121cf7SChuck Lever dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status, 71346121cf7SChuck Lever task->tk_status); 7141da177e4SLinus Torvalds /* Release all resources associated with the task */ 7151da177e4SLinus Torvalds rpc_release_task(task); 7161da177e4SLinus Torvalds } 7171da177e4SLinus Torvalds 7181da177e4SLinus Torvalds /* 7191da177e4SLinus Torvalds * User-visible entry point to the scheduler. 7201da177e4SLinus Torvalds * 7211da177e4SLinus Torvalds * This may be called recursively if e.g. an async NFS task updates 7221da177e4SLinus Torvalds * the attributes and finds that dirty pages must be flushed. 7231da177e4SLinus Torvalds * NOTE: Upon exit of this function the task is guaranteed to be 7241da177e4SLinus Torvalds * released. In particular note that tk_release() will have 7251da177e4SLinus Torvalds * been called, so your task memory may have been freed. 7261da177e4SLinus Torvalds */ 7272efef837STrond Myklebust void rpc_execute(struct rpc_task *task) 7281da177e4SLinus Torvalds { 72944c28873STrond Myklebust rpc_set_active(task); 7301da177e4SLinus Torvalds rpc_set_running(task); 7312efef837STrond Myklebust __rpc_execute(task); 7321da177e4SLinus Torvalds } 7331da177e4SLinus Torvalds 73465f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *work) 7351da177e4SLinus Torvalds { 73665f27f38SDavid Howells __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); 7371da177e4SLinus Torvalds } 7381da177e4SLinus Torvalds 73902107148SChuck Lever /** 74002107148SChuck Lever * rpc_malloc - allocate an RPC buffer 74102107148SChuck Lever * @task: RPC task that will use this buffer 74202107148SChuck Lever * @size: requested byte size 7431da177e4SLinus Torvalds * 744*c5a4dd8bSChuck Lever * To prevent rpciod from hanging, this allocator never sleeps, 745*c5a4dd8bSChuck Lever * returning NULL if the request cannot be serviced immediately. 746*c5a4dd8bSChuck Lever * The caller can arrange to sleep in a way that is safe for rpciod. 747*c5a4dd8bSChuck Lever * 748*c5a4dd8bSChuck Lever * Most requests are 'small' (under 2KiB) and can be serviced from a 749*c5a4dd8bSChuck Lever * mempool, ensuring that NFS reads and writes can always proceed, 750*c5a4dd8bSChuck Lever * and that there is good locality of reference for these buffers. 751*c5a4dd8bSChuck Lever * 7521da177e4SLinus Torvalds * In order to avoid memory starvation triggering more writebacks of 753*c5a4dd8bSChuck Lever * NFS requests, we avoid using GFP_KERNEL. 7541da177e4SLinus Torvalds */ 75502107148SChuck Lever void *rpc_malloc(struct rpc_task *task, size_t size) 7561da177e4SLinus Torvalds { 757*c5a4dd8bSChuck Lever size_t *buf; 758*c5a4dd8bSChuck Lever gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT; 7591da177e4SLinus Torvalds 760*c5a4dd8bSChuck Lever size += sizeof(size_t); 761*c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 762*c5a4dd8bSChuck Lever buf = mempool_alloc(rpc_buffer_mempool, gfp); 7631da177e4SLinus Torvalds else 764*c5a4dd8bSChuck Lever buf = kmalloc(size, gfp); 765*c5a4dd8bSChuck Lever *buf = size; 766*c5a4dd8bSChuck Lever dprintk("RPC: %5u allocated buffer of size %u at %p\n", 767*c5a4dd8bSChuck Lever task->tk_pid, size, buf); 768*c5a4dd8bSChuck Lever return (void *) ++buf; 7691da177e4SLinus Torvalds } 7701da177e4SLinus Torvalds 77102107148SChuck Lever /** 77202107148SChuck Lever * rpc_free - free buffer allocated via rpc_malloc 773*c5a4dd8bSChuck Lever * @buffer: buffer to free 77402107148SChuck Lever * 77502107148SChuck Lever */ 776*c5a4dd8bSChuck Lever void rpc_free(void *buffer) 7771da177e4SLinus Torvalds { 778*c5a4dd8bSChuck Lever size_t size, *buf = (size_t *) buffer; 77902107148SChuck Lever 780*c5a4dd8bSChuck Lever if (!buffer) 781*c5a4dd8bSChuck Lever return; 782*c5a4dd8bSChuck Lever size = *buf; 783*c5a4dd8bSChuck Lever buf--; 784*c5a4dd8bSChuck Lever 785*c5a4dd8bSChuck Lever dprintk("RPC: freeing buffer of size %u at %p\n", 786*c5a4dd8bSChuck Lever size, buf); 787*c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 788*c5a4dd8bSChuck Lever mempool_free(buf, rpc_buffer_mempool); 7891da177e4SLinus Torvalds else 790*c5a4dd8bSChuck Lever kfree(buf); 7911da177e4SLinus Torvalds } 7921da177e4SLinus Torvalds 7931da177e4SLinus Torvalds /* 7941da177e4SLinus Torvalds * Creation and deletion of RPC task structures 7951da177e4SLinus Torvalds */ 796963d8fe5STrond Myklebust void rpc_init_task(struct rpc_task *task, struct rpc_clnt *clnt, int flags, const struct rpc_call_ops *tk_ops, void *calldata) 7971da177e4SLinus Torvalds { 7981da177e4SLinus Torvalds memset(task, 0, sizeof(*task)); 7991da177e4SLinus Torvalds init_timer(&task->tk_timer); 8001da177e4SLinus Torvalds task->tk_timer.data = (unsigned long) task; 8011da177e4SLinus Torvalds task->tk_timer.function = (void (*)(unsigned long)) rpc_run_timer; 80244c28873STrond Myklebust atomic_set(&task->tk_count, 1); 8031da177e4SLinus Torvalds task->tk_client = clnt; 8041da177e4SLinus Torvalds task->tk_flags = flags; 805963d8fe5STrond Myklebust task->tk_ops = tk_ops; 8064ce70adaSTrond Myklebust if (tk_ops->rpc_call_prepare != NULL) 8074ce70adaSTrond Myklebust task->tk_action = rpc_prepare_task; 808963d8fe5STrond Myklebust task->tk_calldata = calldata; 8091da177e4SLinus Torvalds 8101da177e4SLinus Torvalds /* Initialize retry counters */ 8111da177e4SLinus Torvalds task->tk_garb_retry = 2; 8121da177e4SLinus Torvalds task->tk_cred_retry = 2; 8131da177e4SLinus Torvalds 8141da177e4SLinus Torvalds task->tk_priority = RPC_PRIORITY_NORMAL; 8151da177e4SLinus Torvalds task->tk_cookie = (unsigned long)current; 8161da177e4SLinus Torvalds 8171da177e4SLinus Torvalds /* Initialize workqueue for async tasks */ 8181da177e4SLinus Torvalds task->tk_workqueue = rpciod_workqueue; 8191da177e4SLinus Torvalds 8201da177e4SLinus Torvalds if (clnt) { 8211da177e4SLinus Torvalds atomic_inc(&clnt->cl_users); 8221da177e4SLinus Torvalds if (clnt->cl_softrtry) 8231da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_SOFT; 8241da177e4SLinus Torvalds if (!clnt->cl_intr) 8251da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_NOINTR; 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds 828963d8fe5STrond Myklebust BUG_ON(task->tk_ops == NULL); 829963d8fe5STrond Myklebust 830ef759a2eSChuck Lever /* starting timestamp */ 831ef759a2eSChuck Lever task->tk_start = jiffies; 832ef759a2eSChuck Lever 83346121cf7SChuck Lever dprintk("RPC: new task initialized, procpid %u\n", 8341da177e4SLinus Torvalds current->pid); 8351da177e4SLinus Torvalds } 8361da177e4SLinus Torvalds 8371da177e4SLinus Torvalds static struct rpc_task * 8381da177e4SLinus Torvalds rpc_alloc_task(void) 8391da177e4SLinus Torvalds { 8401da177e4SLinus Torvalds return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS); 8411da177e4SLinus Torvalds } 8421da177e4SLinus Torvalds 8438aca67f0STrond Myklebust static void rpc_free_task(struct rcu_head *rcu) 8441da177e4SLinus Torvalds { 8458aca67f0STrond Myklebust struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu); 84646121cf7SChuck Lever dprintk("RPC: %5u freeing task\n", task->tk_pid); 8471da177e4SLinus Torvalds mempool_free(task, rpc_task_mempool); 8481da177e4SLinus Torvalds } 8491da177e4SLinus Torvalds 8501da177e4SLinus Torvalds /* 8511da177e4SLinus Torvalds * Create a new task for the specified client. We have to 8521da177e4SLinus Torvalds * clean up after an allocation failure, as the client may 8531da177e4SLinus Torvalds * have specified "oneshot". 8541da177e4SLinus Torvalds */ 855963d8fe5STrond Myklebust struct rpc_task *rpc_new_task(struct rpc_clnt *clnt, int flags, const struct rpc_call_ops *tk_ops, void *calldata) 8561da177e4SLinus Torvalds { 8571da177e4SLinus Torvalds struct rpc_task *task; 8581da177e4SLinus Torvalds 8591da177e4SLinus Torvalds task = rpc_alloc_task(); 8601da177e4SLinus Torvalds if (!task) 8611da177e4SLinus Torvalds goto cleanup; 8621da177e4SLinus Torvalds 863963d8fe5STrond Myklebust rpc_init_task(task, clnt, flags, tk_ops, calldata); 8641da177e4SLinus Torvalds 86546121cf7SChuck Lever dprintk("RPC: allocated task %p\n", task); 8661da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_DYNAMIC; 8671da177e4SLinus Torvalds out: 8681da177e4SLinus Torvalds return task; 8691da177e4SLinus Torvalds 8701da177e4SLinus Torvalds cleanup: 8711da177e4SLinus Torvalds /* Check whether to release the client */ 8721da177e4SLinus Torvalds if (clnt) { 8731da177e4SLinus Torvalds printk("rpc_new_task: failed, users=%d, oneshot=%d\n", 8741da177e4SLinus Torvalds atomic_read(&clnt->cl_users), clnt->cl_oneshot); 8751da177e4SLinus Torvalds atomic_inc(&clnt->cl_users); /* pretend we were used ... */ 8761da177e4SLinus Torvalds rpc_release_client(clnt); 8771da177e4SLinus Torvalds } 8781da177e4SLinus Torvalds goto out; 8791da177e4SLinus Torvalds } 8801da177e4SLinus Torvalds 881e6b3c4dbSTrond Myklebust 882e6b3c4dbSTrond Myklebust void rpc_put_task(struct rpc_task *task) 8831da177e4SLinus Torvalds { 884963d8fe5STrond Myklebust const struct rpc_call_ops *tk_ops = task->tk_ops; 885963d8fe5STrond Myklebust void *calldata = task->tk_calldata; 8861da177e4SLinus Torvalds 887e6b3c4dbSTrond Myklebust if (!atomic_dec_and_test(&task->tk_count)) 888e6b3c4dbSTrond Myklebust return; 889e6b3c4dbSTrond Myklebust /* Release resources */ 890e6b3c4dbSTrond Myklebust if (task->tk_rqstp) 891e6b3c4dbSTrond Myklebust xprt_release(task); 892e6b3c4dbSTrond Myklebust if (task->tk_msg.rpc_cred) 893e6b3c4dbSTrond Myklebust rpcauth_unbindcred(task); 894e6b3c4dbSTrond Myklebust if (task->tk_client) { 895e6b3c4dbSTrond Myklebust rpc_release_client(task->tk_client); 896e6b3c4dbSTrond Myklebust task->tk_client = NULL; 897e6b3c4dbSTrond Myklebust } 898e6b3c4dbSTrond Myklebust if (task->tk_flags & RPC_TASK_DYNAMIC) 8998aca67f0STrond Myklebust call_rcu_bh(&task->u.tk_rcu, rpc_free_task); 900bbd5a1f9STrond Myklebust rpc_release_calldata(tk_ops, calldata); 901e6b3c4dbSTrond Myklebust } 902e6b3c4dbSTrond Myklebust EXPORT_SYMBOL(rpc_put_task); 903e6b3c4dbSTrond Myklebust 904bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task) 905e6b3c4dbSTrond Myklebust { 9061da177e4SLinus Torvalds #ifdef RPC_DEBUG 9071da177e4SLinus Torvalds BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID); 9081da177e4SLinus Torvalds #endif 90946121cf7SChuck Lever dprintk("RPC: %5u release task\n", task->tk_pid); 9101da177e4SLinus Torvalds 9111da177e4SLinus Torvalds /* Remove from global task list */ 9121da177e4SLinus Torvalds spin_lock(&rpc_sched_lock); 9131da177e4SLinus Torvalds list_del(&task->tk_task); 9141da177e4SLinus Torvalds spin_unlock(&rpc_sched_lock); 9151da177e4SLinus Torvalds 9161da177e4SLinus Torvalds BUG_ON (RPC_IS_QUEUED(task)); 9171da177e4SLinus Torvalds 9181da177e4SLinus Torvalds /* Synchronously delete any running timer */ 9191da177e4SLinus Torvalds rpc_delete_timer(task); 9201da177e4SLinus Torvalds 9211da177e4SLinus Torvalds #ifdef RPC_DEBUG 9221da177e4SLinus Torvalds task->tk_magic = 0; 9231da177e4SLinus Torvalds #endif 924e6b3c4dbSTrond Myklebust /* Wake up anyone who is waiting for task completion */ 925e6b3c4dbSTrond Myklebust rpc_mark_complete_task(task); 926e6b3c4dbSTrond Myklebust 927e6b3c4dbSTrond Myklebust rpc_put_task(task); 9281da177e4SLinus Torvalds } 9291da177e4SLinus Torvalds 9301da177e4SLinus Torvalds /** 93144c28873STrond Myklebust * rpc_run_task - Allocate a new RPC task, then run rpc_execute against it 93299acf044SMartin Waitz * @clnt: pointer to RPC client 93399acf044SMartin Waitz * @flags: RPC flags 93499acf044SMartin Waitz * @ops: RPC call ops 93599acf044SMartin Waitz * @data: user call data 93644c28873STrond Myklebust */ 93744c28873STrond Myklebust struct rpc_task *rpc_run_task(struct rpc_clnt *clnt, int flags, 93844c28873STrond Myklebust const struct rpc_call_ops *ops, 93944c28873STrond Myklebust void *data) 94044c28873STrond Myklebust { 94144c28873STrond Myklebust struct rpc_task *task; 94244c28873STrond Myklebust task = rpc_new_task(clnt, flags, ops, data); 9437a1218a2STrond Myklebust if (task == NULL) { 944bbd5a1f9STrond Myklebust rpc_release_calldata(ops, data); 94544c28873STrond Myklebust return ERR_PTR(-ENOMEM); 9467a1218a2STrond Myklebust } 94744c28873STrond Myklebust atomic_inc(&task->tk_count); 94844c28873STrond Myklebust rpc_execute(task); 94944c28873STrond Myklebust return task; 95044c28873STrond Myklebust } 95144c28873STrond Myklebust EXPORT_SYMBOL(rpc_run_task); 95244c28873STrond Myklebust 9531da177e4SLinus Torvalds /* 9541da177e4SLinus Torvalds * Kill all tasks for the given client. 9551da177e4SLinus Torvalds * XXX: kill their descendants as well? 9561da177e4SLinus Torvalds */ 9571da177e4SLinus Torvalds void rpc_killall_tasks(struct rpc_clnt *clnt) 9581da177e4SLinus Torvalds { 9591da177e4SLinus Torvalds struct rpc_task *rovr; 9601da177e4SLinus Torvalds struct list_head *le; 9611da177e4SLinus Torvalds 9621da177e4SLinus Torvalds dprintk("RPC: killing all tasks for client %p\n", clnt); 9631da177e4SLinus Torvalds 9641da177e4SLinus Torvalds /* 9651da177e4SLinus Torvalds * Spin lock all_tasks to prevent changes... 9661da177e4SLinus Torvalds */ 9671da177e4SLinus Torvalds spin_lock(&rpc_sched_lock); 9681da177e4SLinus Torvalds alltask_for_each(rovr, le, &all_tasks) { 9691da177e4SLinus Torvalds if (! RPC_IS_ACTIVATED(rovr)) 9701da177e4SLinus Torvalds continue; 9711da177e4SLinus Torvalds if (!clnt || rovr->tk_client == clnt) { 9721da177e4SLinus Torvalds rovr->tk_flags |= RPC_TASK_KILLED; 9731da177e4SLinus Torvalds rpc_exit(rovr, -EIO); 9741da177e4SLinus Torvalds rpc_wake_up_task(rovr); 9751da177e4SLinus Torvalds } 9761da177e4SLinus Torvalds } 9771da177e4SLinus Torvalds spin_unlock(&rpc_sched_lock); 9781da177e4SLinus Torvalds } 9791da177e4SLinus Torvalds 9801da177e4SLinus Torvalds static DECLARE_MUTEX_LOCKED(rpciod_running); 9811da177e4SLinus Torvalds 9821da177e4SLinus Torvalds static void rpciod_killall(void) 9831da177e4SLinus Torvalds { 9841da177e4SLinus Torvalds unsigned long flags; 9851da177e4SLinus Torvalds 9861da177e4SLinus Torvalds while (!list_empty(&all_tasks)) { 9871da177e4SLinus Torvalds clear_thread_flag(TIF_SIGPENDING); 9881da177e4SLinus Torvalds rpc_killall_tasks(NULL); 9891da177e4SLinus Torvalds flush_workqueue(rpciod_workqueue); 9901da177e4SLinus Torvalds if (!list_empty(&all_tasks)) { 99146121cf7SChuck Lever dprintk("RPC: rpciod_killall: waiting for tasks " 99246121cf7SChuck Lever "to exit\n"); 9931da177e4SLinus Torvalds yield(); 9941da177e4SLinus Torvalds } 9951da177e4SLinus Torvalds } 9961da177e4SLinus Torvalds 9971da177e4SLinus Torvalds spin_lock_irqsave(¤t->sighand->siglock, flags); 9981da177e4SLinus Torvalds recalc_sigpending(); 9991da177e4SLinus Torvalds spin_unlock_irqrestore(¤t->sighand->siglock, flags); 10001da177e4SLinus Torvalds } 10011da177e4SLinus Torvalds 10021da177e4SLinus Torvalds /* 10031da177e4SLinus Torvalds * Start up the rpciod process if it's not already running. 10041da177e4SLinus Torvalds */ 10051da177e4SLinus Torvalds int 10061da177e4SLinus Torvalds rpciod_up(void) 10071da177e4SLinus Torvalds { 10081da177e4SLinus Torvalds struct workqueue_struct *wq; 10091da177e4SLinus Torvalds int error = 0; 10101da177e4SLinus Torvalds 10114a3e2f71SArjan van de Ven mutex_lock(&rpciod_mutex); 101246121cf7SChuck Lever dprintk("RPC: rpciod_up: users %u\n", rpciod_users); 10131da177e4SLinus Torvalds rpciod_users++; 10141da177e4SLinus Torvalds if (rpciod_workqueue) 10151da177e4SLinus Torvalds goto out; 10161da177e4SLinus Torvalds /* 10171da177e4SLinus Torvalds * If there's no pid, we should be the first user. 10181da177e4SLinus Torvalds */ 10191da177e4SLinus Torvalds if (rpciod_users > 1) 102046121cf7SChuck Lever printk(KERN_WARNING "rpciod_up: no workqueue, %u users??\n", rpciod_users); 10211da177e4SLinus Torvalds /* 10221da177e4SLinus Torvalds * Create the rpciod thread and wait for it to start. 10231da177e4SLinus Torvalds */ 10241da177e4SLinus Torvalds error = -ENOMEM; 10251da177e4SLinus Torvalds wq = create_workqueue("rpciod"); 10261da177e4SLinus Torvalds if (wq == NULL) { 10271da177e4SLinus Torvalds printk(KERN_WARNING "rpciod_up: create workqueue failed, error=%d\n", error); 10281da177e4SLinus Torvalds rpciod_users--; 10291da177e4SLinus Torvalds goto out; 10301da177e4SLinus Torvalds } 10311da177e4SLinus Torvalds rpciod_workqueue = wq; 10321da177e4SLinus Torvalds error = 0; 10331da177e4SLinus Torvalds out: 10344a3e2f71SArjan van de Ven mutex_unlock(&rpciod_mutex); 10351da177e4SLinus Torvalds return error; 10361da177e4SLinus Torvalds } 10371da177e4SLinus Torvalds 10381da177e4SLinus Torvalds void 10391da177e4SLinus Torvalds rpciod_down(void) 10401da177e4SLinus Torvalds { 10414a3e2f71SArjan van de Ven mutex_lock(&rpciod_mutex); 104246121cf7SChuck Lever dprintk("RPC: rpciod_down sema %u\n", rpciod_users); 10431da177e4SLinus Torvalds if (rpciod_users) { 10441da177e4SLinus Torvalds if (--rpciod_users) 10451da177e4SLinus Torvalds goto out; 10461da177e4SLinus Torvalds } else 10471da177e4SLinus Torvalds printk(KERN_WARNING "rpciod_down: no users??\n"); 10481da177e4SLinus Torvalds 10491da177e4SLinus Torvalds if (!rpciod_workqueue) { 105046121cf7SChuck Lever dprintk("RPC: rpciod_down: Nothing to do!\n"); 10511da177e4SLinus Torvalds goto out; 10521da177e4SLinus Torvalds } 10531da177e4SLinus Torvalds rpciod_killall(); 10541da177e4SLinus Torvalds 10551da177e4SLinus Torvalds destroy_workqueue(rpciod_workqueue); 10561da177e4SLinus Torvalds rpciod_workqueue = NULL; 10571da177e4SLinus Torvalds out: 10584a3e2f71SArjan van de Ven mutex_unlock(&rpciod_mutex); 10591da177e4SLinus Torvalds } 10601da177e4SLinus Torvalds 10611da177e4SLinus Torvalds #ifdef RPC_DEBUG 10621da177e4SLinus Torvalds void rpc_show_tasks(void) 10631da177e4SLinus Torvalds { 10641da177e4SLinus Torvalds struct list_head *le; 10651da177e4SLinus Torvalds struct rpc_task *t; 10661da177e4SLinus Torvalds 10671da177e4SLinus Torvalds spin_lock(&rpc_sched_lock); 10681da177e4SLinus Torvalds if (list_empty(&all_tasks)) { 10691da177e4SLinus Torvalds spin_unlock(&rpc_sched_lock); 10701da177e4SLinus Torvalds return; 10711da177e4SLinus Torvalds } 10721da177e4SLinus Torvalds printk("-pid- proc flgs status -client- -prog- --rqstp- -timeout " 1073963d8fe5STrond Myklebust "-rpcwait -action- ---ops--\n"); 10741da177e4SLinus Torvalds alltask_for_each(t, le, &all_tasks) { 10751da177e4SLinus Torvalds const char *rpc_waitq = "none"; 10761da177e4SLinus Torvalds 10771da177e4SLinus Torvalds if (RPC_IS_QUEUED(t)) 10781da177e4SLinus Torvalds rpc_waitq = rpc_qname(t->u.tk_wait.rpc_waitq); 10791da177e4SLinus Torvalds 108046121cf7SChuck Lever printk("%5u %04d %04x %6d %8p %6d %8p %8ld %8s %8p %8p\n", 10811da177e4SLinus Torvalds t->tk_pid, 10821da177e4SLinus Torvalds (t->tk_msg.rpc_proc ? t->tk_msg.rpc_proc->p_proc : -1), 10831da177e4SLinus Torvalds t->tk_flags, t->tk_status, 10841da177e4SLinus Torvalds t->tk_client, 10851da177e4SLinus Torvalds (t->tk_client ? t->tk_client->cl_prog : 0), 10861da177e4SLinus Torvalds t->tk_rqstp, t->tk_timeout, 10871da177e4SLinus Torvalds rpc_waitq, 1088963d8fe5STrond Myklebust t->tk_action, t->tk_ops); 10891da177e4SLinus Torvalds } 10901da177e4SLinus Torvalds spin_unlock(&rpc_sched_lock); 10911da177e4SLinus Torvalds } 10921da177e4SLinus Torvalds #endif 10931da177e4SLinus Torvalds 10941da177e4SLinus Torvalds void 10951da177e4SLinus Torvalds rpc_destroy_mempool(void) 10961da177e4SLinus Torvalds { 10971da177e4SLinus Torvalds if (rpc_buffer_mempool) 10981da177e4SLinus Torvalds mempool_destroy(rpc_buffer_mempool); 10991da177e4SLinus Torvalds if (rpc_task_mempool) 11001da177e4SLinus Torvalds mempool_destroy(rpc_task_mempool); 11011a1d92c1SAlexey Dobriyan if (rpc_task_slabp) 11021a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_task_slabp); 11031a1d92c1SAlexey Dobriyan if (rpc_buffer_slabp) 11041a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_buffer_slabp); 11051da177e4SLinus Torvalds } 11061da177e4SLinus Torvalds 11071da177e4SLinus Torvalds int 11081da177e4SLinus Torvalds rpc_init_mempool(void) 11091da177e4SLinus Torvalds { 11101da177e4SLinus Torvalds rpc_task_slabp = kmem_cache_create("rpc_tasks", 11111da177e4SLinus Torvalds sizeof(struct rpc_task), 11121da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 11131da177e4SLinus Torvalds NULL, NULL); 11141da177e4SLinus Torvalds if (!rpc_task_slabp) 11151da177e4SLinus Torvalds goto err_nomem; 11161da177e4SLinus Torvalds rpc_buffer_slabp = kmem_cache_create("rpc_buffers", 11171da177e4SLinus Torvalds RPC_BUFFER_MAXSIZE, 11181da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 11191da177e4SLinus Torvalds NULL, NULL); 11201da177e4SLinus Torvalds if (!rpc_buffer_slabp) 11211da177e4SLinus Torvalds goto err_nomem; 112293d2341cSMatthew Dobson rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, 11231da177e4SLinus Torvalds rpc_task_slabp); 11241da177e4SLinus Torvalds if (!rpc_task_mempool) 11251da177e4SLinus Torvalds goto err_nomem; 112693d2341cSMatthew Dobson rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, 11271da177e4SLinus Torvalds rpc_buffer_slabp); 11281da177e4SLinus Torvalds if (!rpc_buffer_mempool) 11291da177e4SLinus Torvalds goto err_nomem; 11301da177e4SLinus Torvalds return 0; 11311da177e4SLinus Torvalds err_nomem: 11321da177e4SLinus Torvalds rpc_destroy_mempool(); 11331da177e4SLinus Torvalds return -ENOMEM; 11341da177e4SLinus Torvalds } 1135