11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/net/sunrpc/sched.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Scheduling for synchronous and asynchronous RPC requests. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de> 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * TCP NFS related read + write fixes 91da177e4SLinus Torvalds * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds #include <linux/sched.h> 151da177e4SLinus Torvalds #include <linux/interrupt.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/smp.h> 191da177e4SLinus Torvalds #include <linux/smp_lock.h> 201da177e4SLinus Torvalds #include <linux/spinlock.h> 214a3e2f71SArjan van de Ven #include <linux/mutex.h> 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/sunrpc/clnt.h> 241da177e4SLinus Torvalds 251da177e4SLinus Torvalds #ifdef RPC_DEBUG 261da177e4SLinus Torvalds #define RPCDBG_FACILITY RPCDBG_SCHED 271da177e4SLinus Torvalds #define RPC_TASK_MAGIC_ID 0xf00baa 281da177e4SLinus Torvalds #endif 291da177e4SLinus Torvalds 301da177e4SLinus Torvalds /* 311da177e4SLinus Torvalds * RPC slabs and memory pools 321da177e4SLinus Torvalds */ 331da177e4SLinus Torvalds #define RPC_BUFFER_MAXSIZE (2048) 341da177e4SLinus Torvalds #define RPC_BUFFER_POOLSIZE (8) 351da177e4SLinus Torvalds #define RPC_TASK_POOLSIZE (8) 36e18b890bSChristoph Lameter static struct kmem_cache *rpc_task_slabp __read_mostly; 37e18b890bSChristoph Lameter static struct kmem_cache *rpc_buffer_slabp __read_mostly; 38ba89966cSEric Dumazet static mempool_t *rpc_task_mempool __read_mostly; 39ba89966cSEric Dumazet static mempool_t *rpc_buffer_mempool __read_mostly; 401da177e4SLinus Torvalds 4165f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *); 42bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task); 4336df9aaeSTrond Myklebust static void __rpc_queue_timer_fn(unsigned long ptr); 441da177e4SLinus Torvalds 451da177e4SLinus Torvalds /* 461da177e4SLinus Torvalds * RPC tasks sit here while waiting for conditions to improve. 471da177e4SLinus Torvalds */ 48a4a87499STrond Myklebust static struct rpc_wait_queue delay_queue; 491da177e4SLinus Torvalds 501da177e4SLinus Torvalds /* 511da177e4SLinus Torvalds * rpciod-related stuff 521da177e4SLinus Torvalds */ 5324c5d9d7STrond Myklebust struct workqueue_struct *rpciod_workqueue; 541da177e4SLinus Torvalds 551da177e4SLinus Torvalds /* 561da177e4SLinus Torvalds * Disable the timer for a given RPC task. Should be called with 571da177e4SLinus Torvalds * queue->lock and bh_disabled in order to avoid races within 581da177e4SLinus Torvalds * rpc_run_timer(). 591da177e4SLinus Torvalds */ 605d00837bSTrond Myklebust static void 61*eb276c0eSTrond Myklebust __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) 621da177e4SLinus Torvalds { 6336df9aaeSTrond Myklebust if (task->tk_timeout == 0) 6436df9aaeSTrond Myklebust return; 6546121cf7SChuck Lever dprintk("RPC: %5u disabling timer\n", task->tk_pid); 661da177e4SLinus Torvalds task->tk_timeout = 0; 6736df9aaeSTrond Myklebust list_del(&task->u.tk_wait.timer_list); 68*eb276c0eSTrond Myklebust if (list_empty(&queue->timer_list.list)) 69*eb276c0eSTrond Myklebust del_timer(&queue->timer_list.timer); 7036df9aaeSTrond Myklebust } 7136df9aaeSTrond Myklebust 7236df9aaeSTrond Myklebust static void 7336df9aaeSTrond Myklebust rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires) 7436df9aaeSTrond Myklebust { 7536df9aaeSTrond Myklebust queue->timer_list.expires = expires; 7636df9aaeSTrond Myklebust mod_timer(&queue->timer_list.timer, expires); 771da177e4SLinus Torvalds } 781da177e4SLinus Torvalds 791da177e4SLinus Torvalds /* 801da177e4SLinus Torvalds * Set up a timer for the current task. 811da177e4SLinus Torvalds */ 825d00837bSTrond Myklebust static void 83*eb276c0eSTrond Myklebust __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task) 841da177e4SLinus Torvalds { 851da177e4SLinus Torvalds if (!task->tk_timeout) 861da177e4SLinus Torvalds return; 871da177e4SLinus Torvalds 8846121cf7SChuck Lever dprintk("RPC: %5u setting alarm for %lu ms\n", 891da177e4SLinus Torvalds task->tk_pid, task->tk_timeout * 1000 / HZ); 901da177e4SLinus Torvalds 91*eb276c0eSTrond Myklebust task->u.tk_wait.expires = jiffies + task->tk_timeout; 92*eb276c0eSTrond Myklebust if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires)) 93*eb276c0eSTrond Myklebust rpc_set_queue_timer(queue, task->u.tk_wait.expires); 94*eb276c0eSTrond Myklebust list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); 951da177e4SLinus Torvalds } 961da177e4SLinus Torvalds 971da177e4SLinus Torvalds /* 981da177e4SLinus Torvalds * Add new request to a priority queue. 991da177e4SLinus Torvalds */ 1001da177e4SLinus Torvalds static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task) 1011da177e4SLinus Torvalds { 1021da177e4SLinus Torvalds struct list_head *q; 1031da177e4SLinus Torvalds struct rpc_task *t; 1041da177e4SLinus Torvalds 1051da177e4SLinus Torvalds INIT_LIST_HEAD(&task->u.tk_wait.links); 1061da177e4SLinus Torvalds q = &queue->tasks[task->tk_priority]; 1071da177e4SLinus Torvalds if (unlikely(task->tk_priority > queue->maxpriority)) 1081da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 1091da177e4SLinus Torvalds list_for_each_entry(t, q, u.tk_wait.list) { 1103ff7576dSTrond Myklebust if (t->tk_owner == task->tk_owner) { 1111da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links); 1121da177e4SLinus Torvalds return; 1131da177e4SLinus Torvalds } 1141da177e4SLinus Torvalds } 1151da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, q); 1161da177e4SLinus Torvalds } 1171da177e4SLinus Torvalds 1181da177e4SLinus Torvalds /* 1191da177e4SLinus Torvalds * Add new request to wait queue. 1201da177e4SLinus Torvalds * 1211da177e4SLinus Torvalds * Swapper tasks always get inserted at the head of the queue. 1221da177e4SLinus Torvalds * This should avoid many nasty memory deadlocks and hopefully 1231da177e4SLinus Torvalds * improve overall performance. 1241da177e4SLinus Torvalds * Everyone else gets appended to the queue to ensure proper FIFO behavior. 1251da177e4SLinus Torvalds */ 1261da177e4SLinus Torvalds static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) 1271da177e4SLinus Torvalds { 1281da177e4SLinus Torvalds BUG_ON (RPC_IS_QUEUED(task)); 1291da177e4SLinus Torvalds 1301da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 1311da177e4SLinus Torvalds __rpc_add_wait_queue_priority(queue, task); 1321da177e4SLinus Torvalds else if (RPC_IS_SWAPPER(task)) 1331da177e4SLinus Torvalds list_add(&task->u.tk_wait.list, &queue->tasks[0]); 1341da177e4SLinus Torvalds else 1351da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); 13696ef13b2STrond Myklebust task->tk_waitqueue = queue; 137e19b63daSChuck Lever queue->qlen++; 1381da177e4SLinus Torvalds rpc_set_queued(task); 1391da177e4SLinus Torvalds 14046121cf7SChuck Lever dprintk("RPC: %5u added to queue %p \"%s\"\n", 1411da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 1421da177e4SLinus Torvalds } 1431da177e4SLinus Torvalds 1441da177e4SLinus Torvalds /* 1451da177e4SLinus Torvalds * Remove request from a priority queue. 1461da177e4SLinus Torvalds */ 1471da177e4SLinus Torvalds static void __rpc_remove_wait_queue_priority(struct rpc_task *task) 1481da177e4SLinus Torvalds { 1491da177e4SLinus Torvalds struct rpc_task *t; 1501da177e4SLinus Torvalds 1511da177e4SLinus Torvalds if (!list_empty(&task->u.tk_wait.links)) { 1521da177e4SLinus Torvalds t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list); 1531da177e4SLinus Torvalds list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); 1541da177e4SLinus Torvalds list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); 1551da177e4SLinus Torvalds } 1561da177e4SLinus Torvalds } 1571da177e4SLinus Torvalds 1581da177e4SLinus Torvalds /* 1591da177e4SLinus Torvalds * Remove request from queue. 1601da177e4SLinus Torvalds * Note: must be called with spin lock held. 1611da177e4SLinus Torvalds */ 16296ef13b2STrond Myklebust static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) 1631da177e4SLinus Torvalds { 164*eb276c0eSTrond Myklebust __rpc_disable_timer(queue, task); 1651da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 1661da177e4SLinus Torvalds __rpc_remove_wait_queue_priority(task); 1671da177e4SLinus Torvalds list_del(&task->u.tk_wait.list); 168e19b63daSChuck Lever queue->qlen--; 16946121cf7SChuck Lever dprintk("RPC: %5u removed from queue %p \"%s\"\n", 1701da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 1711da177e4SLinus Torvalds } 1721da177e4SLinus Torvalds 1731da177e4SLinus Torvalds static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) 1741da177e4SLinus Torvalds { 1751da177e4SLinus Torvalds queue->priority = priority; 1761da177e4SLinus Torvalds queue->count = 1 << (priority * 2); 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds 1793ff7576dSTrond Myklebust static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid) 1801da177e4SLinus Torvalds { 1813ff7576dSTrond Myklebust queue->owner = pid; 1821da177e4SLinus Torvalds queue->nr = RPC_BATCH_COUNT; 1831da177e4SLinus Torvalds } 1841da177e4SLinus Torvalds 1851da177e4SLinus Torvalds static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) 1861da177e4SLinus Torvalds { 1871da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, queue->maxpriority); 1883ff7576dSTrond Myklebust rpc_set_waitqueue_owner(queue, 0); 1891da177e4SLinus Torvalds } 1901da177e4SLinus Torvalds 1913ff7576dSTrond Myklebust static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues) 1921da177e4SLinus Torvalds { 1931da177e4SLinus Torvalds int i; 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds spin_lock_init(&queue->lock); 1961da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) 1971da177e4SLinus Torvalds INIT_LIST_HEAD(&queue->tasks[i]); 1983ff7576dSTrond Myklebust queue->maxpriority = nr_queues - 1; 1991da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 20036df9aaeSTrond Myklebust queue->qlen = 0; 20136df9aaeSTrond Myklebust setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue); 20236df9aaeSTrond Myklebust INIT_LIST_HEAD(&queue->timer_list.list); 2031da177e4SLinus Torvalds #ifdef RPC_DEBUG 2041da177e4SLinus Torvalds queue->name = qname; 2051da177e4SLinus Torvalds #endif 2061da177e4SLinus Torvalds } 2071da177e4SLinus Torvalds 2081da177e4SLinus Torvalds void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2091da177e4SLinus Torvalds { 2103ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY); 2111da177e4SLinus Torvalds } 2121da177e4SLinus Torvalds 2131da177e4SLinus Torvalds void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2141da177e4SLinus Torvalds { 2153ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, 1); 2161da177e4SLinus Torvalds } 217e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_init_wait_queue); 2181da177e4SLinus Torvalds 219f6a1cc89STrond Myklebust void rpc_destroy_wait_queue(struct rpc_wait_queue *queue) 220f6a1cc89STrond Myklebust { 22136df9aaeSTrond Myklebust del_timer_sync(&queue->timer_list.timer); 222f6a1cc89STrond Myklebust } 223f6a1cc89STrond Myklebust EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue); 224f6a1cc89STrond Myklebust 225150030b7SMatthew Wilcox static int rpc_wait_bit_killable(void *word) 22644c28873STrond Myklebust { 227150030b7SMatthew Wilcox if (fatal_signal_pending(current)) 22844c28873STrond Myklebust return -ERESTARTSYS; 22944c28873STrond Myklebust schedule(); 23044c28873STrond Myklebust return 0; 23144c28873STrond Myklebust } 23244c28873STrond Myklebust 233c44fe705STrond Myklebust #ifdef RPC_DEBUG 234c44fe705STrond Myklebust static void rpc_task_set_debuginfo(struct rpc_task *task) 235c44fe705STrond Myklebust { 236c44fe705STrond Myklebust static atomic_t rpc_pid; 237c44fe705STrond Myklebust 238c44fe705STrond Myklebust task->tk_magic = RPC_TASK_MAGIC_ID; 239c44fe705STrond Myklebust task->tk_pid = atomic_inc_return(&rpc_pid); 240c44fe705STrond Myklebust } 241c44fe705STrond Myklebust #else 242c44fe705STrond Myklebust static inline void rpc_task_set_debuginfo(struct rpc_task *task) 243c44fe705STrond Myklebust { 244c44fe705STrond Myklebust } 245c44fe705STrond Myklebust #endif 246c44fe705STrond Myklebust 247e6b3c4dbSTrond Myklebust static void rpc_set_active(struct rpc_task *task) 248e6b3c4dbSTrond Myklebust { 2494bef61ffSTrond Myklebust struct rpc_clnt *clnt; 250e6b3c4dbSTrond Myklebust if (test_and_set_bit(RPC_TASK_ACTIVE, &task->tk_runstate) != 0) 251e6b3c4dbSTrond Myklebust return; 252c44fe705STrond Myklebust rpc_task_set_debuginfo(task); 253e6b3c4dbSTrond Myklebust /* Add to global list of all tasks */ 2544bef61ffSTrond Myklebust clnt = task->tk_client; 2554bef61ffSTrond Myklebust if (clnt != NULL) { 2564bef61ffSTrond Myklebust spin_lock(&clnt->cl_lock); 2574bef61ffSTrond Myklebust list_add_tail(&task->tk_task, &clnt->cl_tasks); 2584bef61ffSTrond Myklebust spin_unlock(&clnt->cl_lock); 2594bef61ffSTrond Myklebust } 260e6b3c4dbSTrond Myklebust } 261e6b3c4dbSTrond Myklebust 26244c28873STrond Myklebust /* 26344c28873STrond Myklebust * Mark an RPC call as having completed by clearing the 'active' bit 26444c28873STrond Myklebust */ 265e6b3c4dbSTrond Myklebust static void rpc_mark_complete_task(struct rpc_task *task) 26644c28873STrond Myklebust { 267e6b3c4dbSTrond Myklebust smp_mb__before_clear_bit(); 268e6b3c4dbSTrond Myklebust clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 269e6b3c4dbSTrond Myklebust smp_mb__after_clear_bit(); 27044c28873STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_ACTIVE); 27144c28873STrond Myklebust } 27244c28873STrond Myklebust 27344c28873STrond Myklebust /* 27444c28873STrond Myklebust * Allow callers to wait for completion of an RPC call 27544c28873STrond Myklebust */ 27644c28873STrond Myklebust int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *)) 27744c28873STrond Myklebust { 27844c28873STrond Myklebust if (action == NULL) 279150030b7SMatthew Wilcox action = rpc_wait_bit_killable; 28044c28873STrond Myklebust return wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, 281150030b7SMatthew Wilcox action, TASK_KILLABLE); 28244c28873STrond Myklebust } 283e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); 28444c28873STrond Myklebust 2851da177e4SLinus Torvalds /* 2861da177e4SLinus Torvalds * Make an RPC task runnable. 2871da177e4SLinus Torvalds * 2881da177e4SLinus Torvalds * Note: If the task is ASYNC, this must be called with 2891da177e4SLinus Torvalds * the spinlock held to protect the wait queue operation. 2901da177e4SLinus Torvalds */ 2911da177e4SLinus Torvalds static void rpc_make_runnable(struct rpc_task *task) 2921da177e4SLinus Torvalds { 2931da177e4SLinus Torvalds rpc_clear_queued(task); 294cc4dc59eSChristophe Saout if (rpc_test_and_set_running(task)) 2951da177e4SLinus Torvalds return; 296cc4dc59eSChristophe Saout /* We might have raced */ 297cc4dc59eSChristophe Saout if (RPC_IS_QUEUED(task)) { 298cc4dc59eSChristophe Saout rpc_clear_running(task); 299cc4dc59eSChristophe Saout return; 300cc4dc59eSChristophe Saout } 3011da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 3021da177e4SLinus Torvalds int status; 3031da177e4SLinus Torvalds 30465f27f38SDavid Howells INIT_WORK(&task->u.tk_work, rpc_async_schedule); 30532bfb5c0STrond Myklebust status = queue_work(rpciod_workqueue, &task->u.tk_work); 3061da177e4SLinus Torvalds if (status < 0) { 3071da177e4SLinus Torvalds printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status); 3081da177e4SLinus Torvalds task->tk_status = status; 3091da177e4SLinus Torvalds return; 3101da177e4SLinus Torvalds } 3111da177e4SLinus Torvalds } else 31296651ab3STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); 3131da177e4SLinus Torvalds } 3141da177e4SLinus Torvalds 3151da177e4SLinus Torvalds /* 3161da177e4SLinus Torvalds * Prepare for sleeping on a wait queue. 3171da177e4SLinus Torvalds * By always appending tasks to the list we ensure FIFO behavior. 3181da177e4SLinus Torvalds * NB: An RPC task will only receive interrupt-driven events as long 3191da177e4SLinus Torvalds * as it's on a wait queue. 3201da177e4SLinus Torvalds */ 3211da177e4SLinus Torvalds static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 3225d00837bSTrond Myklebust rpc_action action) 3231da177e4SLinus Torvalds { 32446121cf7SChuck Lever dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n", 32546121cf7SChuck Lever task->tk_pid, rpc_qname(q), jiffies); 3261da177e4SLinus Torvalds 3271da177e4SLinus Torvalds if (!RPC_IS_ASYNC(task) && !RPC_IS_ACTIVATED(task)) { 3281da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive synchronous task put to sleep!\n"); 3291da177e4SLinus Torvalds return; 3301da177e4SLinus Torvalds } 3311da177e4SLinus Torvalds 3321da177e4SLinus Torvalds __rpc_add_wait_queue(q, task); 3331da177e4SLinus Torvalds 3341da177e4SLinus Torvalds BUG_ON(task->tk_callback != NULL); 3351da177e4SLinus Torvalds task->tk_callback = action; 336*eb276c0eSTrond Myklebust __rpc_add_timer(q, task); 3371da177e4SLinus Torvalds } 3381da177e4SLinus Torvalds 3391da177e4SLinus Torvalds void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 3405d00837bSTrond Myklebust rpc_action action) 3411da177e4SLinus Torvalds { 342e6b3c4dbSTrond Myklebust /* Mark the task as being activated if so needed */ 343e6b3c4dbSTrond Myklebust rpc_set_active(task); 344e6b3c4dbSTrond Myklebust 3451da177e4SLinus Torvalds /* 3461da177e4SLinus Torvalds * Protect the queue operations. 3471da177e4SLinus Torvalds */ 3481da177e4SLinus Torvalds spin_lock_bh(&q->lock); 3495d00837bSTrond Myklebust __rpc_sleep_on(q, task, action); 3501da177e4SLinus Torvalds spin_unlock_bh(&q->lock); 3511da177e4SLinus Torvalds } 352e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on); 3531da177e4SLinus Torvalds 3541da177e4SLinus Torvalds /** 3551da177e4SLinus Torvalds * __rpc_do_wake_up_task - wake up a single rpc_task 35696ef13b2STrond Myklebust * @queue: wait queue 3571da177e4SLinus Torvalds * @task: task to be woken up 3581da177e4SLinus Torvalds * 3591da177e4SLinus Torvalds * Caller must hold queue->lock, and have cleared the task queued flag. 3601da177e4SLinus Torvalds */ 36196ef13b2STrond Myklebust static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task) 3621da177e4SLinus Torvalds { 36346121cf7SChuck Lever dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n", 36446121cf7SChuck Lever task->tk_pid, jiffies); 3651da177e4SLinus Torvalds 3661da177e4SLinus Torvalds #ifdef RPC_DEBUG 3671da177e4SLinus Torvalds BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID); 3681da177e4SLinus Torvalds #endif 3691da177e4SLinus Torvalds /* Has the task been executed yet? If not, we cannot wake it up! */ 3701da177e4SLinus Torvalds if (!RPC_IS_ACTIVATED(task)) { 3711da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); 3721da177e4SLinus Torvalds return; 3731da177e4SLinus Torvalds } 3741da177e4SLinus Torvalds 37596ef13b2STrond Myklebust __rpc_remove_wait_queue(queue, task); 3761da177e4SLinus Torvalds 3771da177e4SLinus Torvalds rpc_make_runnable(task); 3781da177e4SLinus Torvalds 3791da177e4SLinus Torvalds dprintk("RPC: __rpc_wake_up_task done\n"); 3801da177e4SLinus Torvalds } 3811da177e4SLinus Torvalds 3821da177e4SLinus Torvalds /* 38396ef13b2STrond Myklebust * Wake up a queued task while the queue lock is being held 3841da177e4SLinus Torvalds */ 38596ef13b2STrond Myklebust static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task) 3861da177e4SLinus Torvalds { 38796ef13b2STrond Myklebust if (!RPC_IS_QUEUED(task) || task->tk_waitqueue != queue) 38896ef13b2STrond Myklebust return; 3891da177e4SLinus Torvalds if (rpc_start_wakeup(task)) { 39096ef13b2STrond Myklebust __rpc_do_wake_up_task(queue, task); 3911da177e4SLinus Torvalds rpc_finish_wakeup(task); 3921da177e4SLinus Torvalds } 3931da177e4SLinus Torvalds } 3941da177e4SLinus Torvalds 3951da177e4SLinus Torvalds /* 39696ef13b2STrond Myklebust * Wake up a task on a specific queue 39796ef13b2STrond Myklebust */ 39896ef13b2STrond Myklebust void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) 39996ef13b2STrond Myklebust { 40096ef13b2STrond Myklebust rcu_read_lock_bh(); 40196ef13b2STrond Myklebust spin_lock(&queue->lock); 40296ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 40396ef13b2STrond Myklebust spin_unlock(&queue->lock); 40496ef13b2STrond Myklebust rcu_read_unlock_bh(); 40596ef13b2STrond Myklebust } 40696ef13b2STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task); 40796ef13b2STrond Myklebust 40896ef13b2STrond Myklebust /* 4091da177e4SLinus Torvalds * Wake up the specified task 4101da177e4SLinus Torvalds */ 411fda13939STrond Myklebust static void rpc_wake_up_task(struct rpc_task *task) 4121da177e4SLinus Torvalds { 41396ef13b2STrond Myklebust rpc_wake_up_queued_task(task->tk_waitqueue, task); 4141da177e4SLinus Torvalds } 4151da177e4SLinus Torvalds 4161da177e4SLinus Torvalds /* 4171da177e4SLinus Torvalds * Wake up the next task on a priority queue. 4181da177e4SLinus Torvalds */ 4191da177e4SLinus Torvalds static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue) 4201da177e4SLinus Torvalds { 4211da177e4SLinus Torvalds struct list_head *q; 4221da177e4SLinus Torvalds struct rpc_task *task; 4231da177e4SLinus Torvalds 4241da177e4SLinus Torvalds /* 4253ff7576dSTrond Myklebust * Service a batch of tasks from a single owner. 4261da177e4SLinus Torvalds */ 4271da177e4SLinus Torvalds q = &queue->tasks[queue->priority]; 4281da177e4SLinus Torvalds if (!list_empty(q)) { 4291da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4303ff7576dSTrond Myklebust if (queue->owner == task->tk_owner) { 4311da177e4SLinus Torvalds if (--queue->nr) 4321da177e4SLinus Torvalds goto out; 4331da177e4SLinus Torvalds list_move_tail(&task->u.tk_wait.list, q); 4341da177e4SLinus Torvalds } 4351da177e4SLinus Torvalds /* 4361da177e4SLinus Torvalds * Check if we need to switch queues. 4371da177e4SLinus Torvalds */ 4381da177e4SLinus Torvalds if (--queue->count) 4393ff7576dSTrond Myklebust goto new_owner; 4401da177e4SLinus Torvalds } 4411da177e4SLinus Torvalds 4421da177e4SLinus Torvalds /* 4431da177e4SLinus Torvalds * Service the next queue. 4441da177e4SLinus Torvalds */ 4451da177e4SLinus Torvalds do { 4461da177e4SLinus Torvalds if (q == &queue->tasks[0]) 4471da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 4481da177e4SLinus Torvalds else 4491da177e4SLinus Torvalds q = q - 1; 4501da177e4SLinus Torvalds if (!list_empty(q)) { 4511da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4521da177e4SLinus Torvalds goto new_queue; 4531da177e4SLinus Torvalds } 4541da177e4SLinus Torvalds } while (q != &queue->tasks[queue->priority]); 4551da177e4SLinus Torvalds 4561da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 4571da177e4SLinus Torvalds return NULL; 4581da177e4SLinus Torvalds 4591da177e4SLinus Torvalds new_queue: 4601da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); 4613ff7576dSTrond Myklebust new_owner: 4623ff7576dSTrond Myklebust rpc_set_waitqueue_owner(queue, task->tk_owner); 4631da177e4SLinus Torvalds out: 46496ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 4651da177e4SLinus Torvalds return task; 4661da177e4SLinus Torvalds } 4671da177e4SLinus Torvalds 4681da177e4SLinus Torvalds /* 4691da177e4SLinus Torvalds * Wake up the next task on the wait queue. 4701da177e4SLinus Torvalds */ 4711da177e4SLinus Torvalds struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue) 4721da177e4SLinus Torvalds { 4731da177e4SLinus Torvalds struct rpc_task *task = NULL; 4741da177e4SLinus Torvalds 47546121cf7SChuck Lever dprintk("RPC: wake_up_next(%p \"%s\")\n", 47646121cf7SChuck Lever queue, rpc_qname(queue)); 4778aca67f0STrond Myklebust rcu_read_lock_bh(); 4788aca67f0STrond Myklebust spin_lock(&queue->lock); 4791da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 4801da177e4SLinus Torvalds task = __rpc_wake_up_next_priority(queue); 4811da177e4SLinus Torvalds else { 4821da177e4SLinus Torvalds task_for_first(task, &queue->tasks[0]) 48396ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 4841da177e4SLinus Torvalds } 4858aca67f0STrond Myklebust spin_unlock(&queue->lock); 4868aca67f0STrond Myklebust rcu_read_unlock_bh(); 4871da177e4SLinus Torvalds 4881da177e4SLinus Torvalds return task; 4891da177e4SLinus Torvalds } 490e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_next); 4911da177e4SLinus Torvalds 4921da177e4SLinus Torvalds /** 4931da177e4SLinus Torvalds * rpc_wake_up - wake up all rpc_tasks 4941da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 4951da177e4SLinus Torvalds * 4961da177e4SLinus Torvalds * Grabs queue->lock 4971da177e4SLinus Torvalds */ 4981da177e4SLinus Torvalds void rpc_wake_up(struct rpc_wait_queue *queue) 4991da177e4SLinus Torvalds { 500e6d83d55STrond Myklebust struct rpc_task *task, *next; 5011da177e4SLinus Torvalds struct list_head *head; 502e6d83d55STrond Myklebust 5038aca67f0STrond Myklebust rcu_read_lock_bh(); 5048aca67f0STrond Myklebust spin_lock(&queue->lock); 5051da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5061da177e4SLinus Torvalds for (;;) { 507e6d83d55STrond Myklebust list_for_each_entry_safe(task, next, head, u.tk_wait.list) 50896ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 5091da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5101da177e4SLinus Torvalds break; 5111da177e4SLinus Torvalds head--; 5121da177e4SLinus Torvalds } 5138aca67f0STrond Myklebust spin_unlock(&queue->lock); 5148aca67f0STrond Myklebust rcu_read_unlock_bh(); 5151da177e4SLinus Torvalds } 516e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up); 5171da177e4SLinus Torvalds 5181da177e4SLinus Torvalds /** 5191da177e4SLinus Torvalds * rpc_wake_up_status - wake up all rpc_tasks and set their status value. 5201da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 5211da177e4SLinus Torvalds * @status: status value to set 5221da177e4SLinus Torvalds * 5231da177e4SLinus Torvalds * Grabs queue->lock 5241da177e4SLinus Torvalds */ 5251da177e4SLinus Torvalds void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) 5261da177e4SLinus Torvalds { 527e6d83d55STrond Myklebust struct rpc_task *task, *next; 5281da177e4SLinus Torvalds struct list_head *head; 5291da177e4SLinus Torvalds 5308aca67f0STrond Myklebust rcu_read_lock_bh(); 5318aca67f0STrond Myklebust spin_lock(&queue->lock); 5321da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5331da177e4SLinus Torvalds for (;;) { 534e6d83d55STrond Myklebust list_for_each_entry_safe(task, next, head, u.tk_wait.list) { 5351da177e4SLinus Torvalds task->tk_status = status; 53696ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 5371da177e4SLinus Torvalds } 5381da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5391da177e4SLinus Torvalds break; 5401da177e4SLinus Torvalds head--; 5411da177e4SLinus Torvalds } 5428aca67f0STrond Myklebust spin_unlock(&queue->lock); 5438aca67f0STrond Myklebust rcu_read_unlock_bh(); 5441da177e4SLinus Torvalds } 545e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_status); 5461da177e4SLinus Torvalds 54736df9aaeSTrond Myklebust static void __rpc_queue_timer_fn(unsigned long ptr) 54836df9aaeSTrond Myklebust { 54936df9aaeSTrond Myklebust struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr; 55036df9aaeSTrond Myklebust struct rpc_task *task, *n; 55136df9aaeSTrond Myklebust unsigned long expires, now, timeo; 55236df9aaeSTrond Myklebust 55336df9aaeSTrond Myklebust spin_lock(&queue->lock); 55436df9aaeSTrond Myklebust expires = now = jiffies; 55536df9aaeSTrond Myklebust list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { 55636df9aaeSTrond Myklebust timeo = task->u.tk_wait.expires; 55736df9aaeSTrond Myklebust if (time_after_eq(now, timeo)) { 55836df9aaeSTrond Myklebust dprintk("RPC: %5u timeout\n", task->tk_pid); 55936df9aaeSTrond Myklebust task->tk_status = -ETIMEDOUT; 56036df9aaeSTrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 56136df9aaeSTrond Myklebust continue; 56236df9aaeSTrond Myklebust } 56336df9aaeSTrond Myklebust if (expires == now || time_after(expires, timeo)) 56436df9aaeSTrond Myklebust expires = timeo; 56536df9aaeSTrond Myklebust } 56636df9aaeSTrond Myklebust if (!list_empty(&queue->timer_list.list)) 56736df9aaeSTrond Myklebust rpc_set_queue_timer(queue, expires); 56836df9aaeSTrond Myklebust spin_unlock(&queue->lock); 56936df9aaeSTrond Myklebust } 57036df9aaeSTrond Myklebust 5718014793bSTrond Myklebust static void __rpc_atrun(struct rpc_task *task) 5728014793bSTrond Myklebust { 5735d00837bSTrond Myklebust task->tk_status = 0; 5748014793bSTrond Myklebust } 5758014793bSTrond Myklebust 5761da177e4SLinus Torvalds /* 5771da177e4SLinus Torvalds * Run a task at a later time 5781da177e4SLinus Torvalds */ 5798014793bSTrond Myklebust void rpc_delay(struct rpc_task *task, unsigned long delay) 5801da177e4SLinus Torvalds { 5811da177e4SLinus Torvalds task->tk_timeout = delay; 5825d00837bSTrond Myklebust rpc_sleep_on(&delay_queue, task, __rpc_atrun); 5831da177e4SLinus Torvalds } 584e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_delay); 5851da177e4SLinus Torvalds 5861da177e4SLinus Torvalds /* 5874ce70adaSTrond Myklebust * Helper to call task->tk_ops->rpc_call_prepare 5884ce70adaSTrond Myklebust */ 5894ce70adaSTrond Myklebust static void rpc_prepare_task(struct rpc_task *task) 5904ce70adaSTrond Myklebust { 5916d5fcb5aSTrond Myklebust lock_kernel(); 5924ce70adaSTrond Myklebust task->tk_ops->rpc_call_prepare(task, task->tk_calldata); 5936d5fcb5aSTrond Myklebust unlock_kernel(); 5944ce70adaSTrond Myklebust } 5954ce70adaSTrond Myklebust 5964ce70adaSTrond Myklebust /* 597963d8fe5STrond Myklebust * Helper that calls task->tk_ops->rpc_call_done if it exists 598d05fdb0cSTrond Myklebust */ 599abbcf28fSTrond Myklebust void rpc_exit_task(struct rpc_task *task) 600d05fdb0cSTrond Myklebust { 601abbcf28fSTrond Myklebust task->tk_action = NULL; 602963d8fe5STrond Myklebust if (task->tk_ops->rpc_call_done != NULL) { 6036d5fcb5aSTrond Myklebust lock_kernel(); 604963d8fe5STrond Myklebust task->tk_ops->rpc_call_done(task, task->tk_calldata); 6056d5fcb5aSTrond Myklebust unlock_kernel(); 606d05fdb0cSTrond Myklebust if (task->tk_action != NULL) { 607abbcf28fSTrond Myklebust WARN_ON(RPC_ASSASSINATED(task)); 608abbcf28fSTrond Myklebust /* Always release the RPC slot and buffer memory */ 609d05fdb0cSTrond Myklebust xprt_release(task); 610d05fdb0cSTrond Myklebust } 611d05fdb0cSTrond Myklebust } 612d05fdb0cSTrond Myklebust } 613e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_exit_task); 614d05fdb0cSTrond Myklebust 615bbd5a1f9STrond Myklebust void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) 616bbd5a1f9STrond Myklebust { 617bbd5a1f9STrond Myklebust if (ops->rpc_release != NULL) { 618bbd5a1f9STrond Myklebust lock_kernel(); 619bbd5a1f9STrond Myklebust ops->rpc_release(calldata); 620bbd5a1f9STrond Myklebust unlock_kernel(); 621bbd5a1f9STrond Myklebust } 622bbd5a1f9STrond Myklebust } 623bbd5a1f9STrond Myklebust 624d05fdb0cSTrond Myklebust /* 6251da177e4SLinus Torvalds * This is the RPC `scheduler' (or rather, the finite state machine). 6261da177e4SLinus Torvalds */ 6272efef837STrond Myklebust static void __rpc_execute(struct rpc_task *task) 6281da177e4SLinus Torvalds { 6291da177e4SLinus Torvalds int status = 0; 6301da177e4SLinus Torvalds 63146121cf7SChuck Lever dprintk("RPC: %5u __rpc_execute flags=0x%x\n", 6321da177e4SLinus Torvalds task->tk_pid, task->tk_flags); 6331da177e4SLinus Torvalds 6341da177e4SLinus Torvalds BUG_ON(RPC_IS_QUEUED(task)); 6351da177e4SLinus Torvalds 636d05fdb0cSTrond Myklebust for (;;) { 6371da177e4SLinus Torvalds 6381da177e4SLinus Torvalds /* 6391da177e4SLinus Torvalds * Execute any pending callback. 6401da177e4SLinus Torvalds */ 6411da177e4SLinus Torvalds if (RPC_DO_CALLBACK(task)) { 6421da177e4SLinus Torvalds /* Define a callback save pointer */ 6431da177e4SLinus Torvalds void (*save_callback)(struct rpc_task *); 6441da177e4SLinus Torvalds 6451da177e4SLinus Torvalds /* 6461da177e4SLinus Torvalds * If a callback exists, save it, reset it, 6471da177e4SLinus Torvalds * call it. 6481da177e4SLinus Torvalds * The save is needed to stop from resetting 6491da177e4SLinus Torvalds * another callback set within the callback handler 6501da177e4SLinus Torvalds * - Dave 6511da177e4SLinus Torvalds */ 6521da177e4SLinus Torvalds save_callback=task->tk_callback; 6531da177e4SLinus Torvalds task->tk_callback=NULL; 6541da177e4SLinus Torvalds save_callback(task); 6551da177e4SLinus Torvalds } 6561da177e4SLinus Torvalds 6571da177e4SLinus Torvalds /* 6581da177e4SLinus Torvalds * Perform the next FSM step. 6591da177e4SLinus Torvalds * tk_action may be NULL when the task has been killed 6601da177e4SLinus Torvalds * by someone else. 6611da177e4SLinus Torvalds */ 6621da177e4SLinus Torvalds if (!RPC_IS_QUEUED(task)) { 663abbcf28fSTrond Myklebust if (task->tk_action == NULL) 664abbcf28fSTrond Myklebust break; 6651da177e4SLinus Torvalds task->tk_action(task); 6661da177e4SLinus Torvalds } 6671da177e4SLinus Torvalds 6681da177e4SLinus Torvalds /* 6691da177e4SLinus Torvalds * Lockless check for whether task is sleeping or not. 6701da177e4SLinus Torvalds */ 6711da177e4SLinus Torvalds if (!RPC_IS_QUEUED(task)) 6721da177e4SLinus Torvalds continue; 6731da177e4SLinus Torvalds rpc_clear_running(task); 6741da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 6751da177e4SLinus Torvalds /* Careful! we may have raced... */ 6761da177e4SLinus Torvalds if (RPC_IS_QUEUED(task)) 6772efef837STrond Myklebust return; 6781da177e4SLinus Torvalds if (rpc_test_and_set_running(task)) 6792efef837STrond Myklebust return; 6801da177e4SLinus Torvalds continue; 6811da177e4SLinus Torvalds } 6821da177e4SLinus Torvalds 6831da177e4SLinus Torvalds /* sync task: sleep here */ 68446121cf7SChuck Lever dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid); 68596651ab3STrond Myklebust status = out_of_line_wait_on_bit(&task->tk_runstate, 686150030b7SMatthew Wilcox RPC_TASK_QUEUED, rpc_wait_bit_killable, 687150030b7SMatthew Wilcox TASK_KILLABLE); 68896651ab3STrond Myklebust if (status == -ERESTARTSYS) { 6891da177e4SLinus Torvalds /* 6901da177e4SLinus Torvalds * When a sync task receives a signal, it exits with 6911da177e4SLinus Torvalds * -ERESTARTSYS. In order to catch any callbacks that 6921da177e4SLinus Torvalds * clean up after sleeping on some queue, we don't 6931da177e4SLinus Torvalds * break the loop here, but go around once more. 6941da177e4SLinus Torvalds */ 69546121cf7SChuck Lever dprintk("RPC: %5u got signal\n", task->tk_pid); 6961da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_KILLED; 6971da177e4SLinus Torvalds rpc_exit(task, -ERESTARTSYS); 6981da177e4SLinus Torvalds rpc_wake_up_task(task); 6991da177e4SLinus Torvalds } 7001da177e4SLinus Torvalds rpc_set_running(task); 70146121cf7SChuck Lever dprintk("RPC: %5u sync task resuming\n", task->tk_pid); 7021da177e4SLinus Torvalds } 7031da177e4SLinus Torvalds 70446121cf7SChuck Lever dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status, 70546121cf7SChuck Lever task->tk_status); 7061da177e4SLinus Torvalds /* Release all resources associated with the task */ 7071da177e4SLinus Torvalds rpc_release_task(task); 7081da177e4SLinus Torvalds } 7091da177e4SLinus Torvalds 7101da177e4SLinus Torvalds /* 7111da177e4SLinus Torvalds * User-visible entry point to the scheduler. 7121da177e4SLinus Torvalds * 7131da177e4SLinus Torvalds * This may be called recursively if e.g. an async NFS task updates 7141da177e4SLinus Torvalds * the attributes and finds that dirty pages must be flushed. 7151da177e4SLinus Torvalds * NOTE: Upon exit of this function the task is guaranteed to be 7161da177e4SLinus Torvalds * released. In particular note that tk_release() will have 7171da177e4SLinus Torvalds * been called, so your task memory may have been freed. 7181da177e4SLinus Torvalds */ 7192efef837STrond Myklebust void rpc_execute(struct rpc_task *task) 7201da177e4SLinus Torvalds { 72144c28873STrond Myklebust rpc_set_active(task); 7221da177e4SLinus Torvalds rpc_set_running(task); 7232efef837STrond Myklebust __rpc_execute(task); 7241da177e4SLinus Torvalds } 7251da177e4SLinus Torvalds 72665f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *work) 7271da177e4SLinus Torvalds { 72865f27f38SDavid Howells __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); 7291da177e4SLinus Torvalds } 7301da177e4SLinus Torvalds 731aa3d1faeSChuck Lever struct rpc_buffer { 732aa3d1faeSChuck Lever size_t len; 733aa3d1faeSChuck Lever char data[]; 734aa3d1faeSChuck Lever }; 735aa3d1faeSChuck Lever 73602107148SChuck Lever /** 73702107148SChuck Lever * rpc_malloc - allocate an RPC buffer 73802107148SChuck Lever * @task: RPC task that will use this buffer 73902107148SChuck Lever * @size: requested byte size 7401da177e4SLinus Torvalds * 741c5a4dd8bSChuck Lever * To prevent rpciod from hanging, this allocator never sleeps, 742c5a4dd8bSChuck Lever * returning NULL if the request cannot be serviced immediately. 743c5a4dd8bSChuck Lever * The caller can arrange to sleep in a way that is safe for rpciod. 744c5a4dd8bSChuck Lever * 745c5a4dd8bSChuck Lever * Most requests are 'small' (under 2KiB) and can be serviced from a 746c5a4dd8bSChuck Lever * mempool, ensuring that NFS reads and writes can always proceed, 747c5a4dd8bSChuck Lever * and that there is good locality of reference for these buffers. 748c5a4dd8bSChuck Lever * 7491da177e4SLinus Torvalds * In order to avoid memory starvation triggering more writebacks of 750c5a4dd8bSChuck Lever * NFS requests, we avoid using GFP_KERNEL. 7511da177e4SLinus Torvalds */ 75202107148SChuck Lever void *rpc_malloc(struct rpc_task *task, size_t size) 7531da177e4SLinus Torvalds { 754aa3d1faeSChuck Lever struct rpc_buffer *buf; 755c5a4dd8bSChuck Lever gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT; 7561da177e4SLinus Torvalds 757aa3d1faeSChuck Lever size += sizeof(struct rpc_buffer); 758c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 759c5a4dd8bSChuck Lever buf = mempool_alloc(rpc_buffer_mempool, gfp); 7601da177e4SLinus Torvalds else 761c5a4dd8bSChuck Lever buf = kmalloc(size, gfp); 762ddce40dfSPeter Zijlstra 763ddce40dfSPeter Zijlstra if (!buf) 764ddce40dfSPeter Zijlstra return NULL; 765ddce40dfSPeter Zijlstra 766aa3d1faeSChuck Lever buf->len = size; 767215d0678SGeert Uytterhoeven dprintk("RPC: %5u allocated buffer of size %zu at %p\n", 768c5a4dd8bSChuck Lever task->tk_pid, size, buf); 769aa3d1faeSChuck Lever return &buf->data; 7701da177e4SLinus Torvalds } 77112444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_malloc); 7721da177e4SLinus Torvalds 77302107148SChuck Lever /** 77402107148SChuck Lever * rpc_free - free buffer allocated via rpc_malloc 775c5a4dd8bSChuck Lever * @buffer: buffer to free 77602107148SChuck Lever * 77702107148SChuck Lever */ 778c5a4dd8bSChuck Lever void rpc_free(void *buffer) 7791da177e4SLinus Torvalds { 780aa3d1faeSChuck Lever size_t size; 781aa3d1faeSChuck Lever struct rpc_buffer *buf; 78202107148SChuck Lever 783c5a4dd8bSChuck Lever if (!buffer) 784c5a4dd8bSChuck Lever return; 785aa3d1faeSChuck Lever 786aa3d1faeSChuck Lever buf = container_of(buffer, struct rpc_buffer, data); 787aa3d1faeSChuck Lever size = buf->len; 788c5a4dd8bSChuck Lever 789215d0678SGeert Uytterhoeven dprintk("RPC: freeing buffer of size %zu at %p\n", 790c5a4dd8bSChuck Lever size, buf); 791aa3d1faeSChuck Lever 792c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 793c5a4dd8bSChuck Lever mempool_free(buf, rpc_buffer_mempool); 7941da177e4SLinus Torvalds else 795c5a4dd8bSChuck Lever kfree(buf); 7961da177e4SLinus Torvalds } 79712444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_free); 7981da177e4SLinus Torvalds 7991da177e4SLinus Torvalds /* 8001da177e4SLinus Torvalds * Creation and deletion of RPC task structures 8011da177e4SLinus Torvalds */ 80247fe0648STrond Myklebust static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) 8031da177e4SLinus Torvalds { 8041da177e4SLinus Torvalds memset(task, 0, sizeof(*task)); 80544c28873STrond Myklebust atomic_set(&task->tk_count, 1); 80684115e1cSTrond Myklebust task->tk_flags = task_setup_data->flags; 80784115e1cSTrond Myklebust task->tk_ops = task_setup_data->callback_ops; 80884115e1cSTrond Myklebust task->tk_calldata = task_setup_data->callback_data; 8096529eba0STrond Myklebust INIT_LIST_HEAD(&task->tk_task); 8101da177e4SLinus Torvalds 8111da177e4SLinus Torvalds /* Initialize retry counters */ 8121da177e4SLinus Torvalds task->tk_garb_retry = 2; 8131da177e4SLinus Torvalds task->tk_cred_retry = 2; 8141da177e4SLinus Torvalds 8153ff7576dSTrond Myklebust task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; 8163ff7576dSTrond Myklebust task->tk_owner = current->tgid; 8171da177e4SLinus Torvalds 8181da177e4SLinus Torvalds /* Initialize workqueue for async tasks */ 81932bfb5c0STrond Myklebust task->tk_workqueue = task_setup_data->workqueue; 8201da177e4SLinus Torvalds 82184115e1cSTrond Myklebust task->tk_client = task_setup_data->rpc_client; 82284115e1cSTrond Myklebust if (task->tk_client != NULL) { 82384115e1cSTrond Myklebust kref_get(&task->tk_client->cl_kref); 82484115e1cSTrond Myklebust if (task->tk_client->cl_softrtry) 8251da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_SOFT; 8261da177e4SLinus Torvalds } 8271da177e4SLinus Torvalds 82884115e1cSTrond Myklebust if (task->tk_ops->rpc_call_prepare != NULL) 82984115e1cSTrond Myklebust task->tk_action = rpc_prepare_task; 830963d8fe5STrond Myklebust 831b3ef8b3bSTrond Myklebust if (task_setup_data->rpc_message != NULL) { 832b3ef8b3bSTrond Myklebust memcpy(&task->tk_msg, task_setup_data->rpc_message, sizeof(task->tk_msg)); 833b3ef8b3bSTrond Myklebust /* Bind the user cred */ 834b3ef8b3bSTrond Myklebust if (task->tk_msg.rpc_cred != NULL) 835b3ef8b3bSTrond Myklebust rpcauth_holdcred(task); 836b3ef8b3bSTrond Myklebust else 837b3ef8b3bSTrond Myklebust rpcauth_bindcred(task); 838b3ef8b3bSTrond Myklebust if (task->tk_action == NULL) 839b3ef8b3bSTrond Myklebust rpc_call_start(task); 840b3ef8b3bSTrond Myklebust } 841b3ef8b3bSTrond Myklebust 842ef759a2eSChuck Lever /* starting timestamp */ 843ef759a2eSChuck Lever task->tk_start = jiffies; 844ef759a2eSChuck Lever 84546121cf7SChuck Lever dprintk("RPC: new task initialized, procpid %u\n", 846ba25f9dcSPavel Emelyanov task_pid_nr(current)); 8471da177e4SLinus Torvalds } 8481da177e4SLinus Torvalds 8491da177e4SLinus Torvalds static struct rpc_task * 8501da177e4SLinus Torvalds rpc_alloc_task(void) 8511da177e4SLinus Torvalds { 8521da177e4SLinus Torvalds return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS); 8531da177e4SLinus Torvalds } 8541da177e4SLinus Torvalds 85532bfb5c0STrond Myklebust static void rpc_free_task_rcu(struct rcu_head *rcu) 8561da177e4SLinus Torvalds { 8578aca67f0STrond Myklebust struct rpc_task *task = container_of(rcu, struct rpc_task, u.tk_rcu); 85846121cf7SChuck Lever dprintk("RPC: %5u freeing task\n", task->tk_pid); 8591da177e4SLinus Torvalds mempool_free(task, rpc_task_mempool); 8601da177e4SLinus Torvalds } 8611da177e4SLinus Torvalds 8621da177e4SLinus Torvalds /* 86390c5755fSTrond Myklebust * Create a new task for the specified client. 8641da177e4SLinus Torvalds */ 86584115e1cSTrond Myklebust struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data) 8661da177e4SLinus Torvalds { 867e8f5d77cSTrond Myklebust struct rpc_task *task = setup_data->task; 868e8f5d77cSTrond Myklebust unsigned short flags = 0; 8691da177e4SLinus Torvalds 870e8f5d77cSTrond Myklebust if (task == NULL) { 8711da177e4SLinus Torvalds task = rpc_alloc_task(); 872e8f5d77cSTrond Myklebust if (task == NULL) 87390c5755fSTrond Myklebust goto out; 874e8f5d77cSTrond Myklebust flags = RPC_TASK_DYNAMIC; 875e8f5d77cSTrond Myklebust } 8761da177e4SLinus Torvalds 87784115e1cSTrond Myklebust rpc_init_task(task, setup_data); 8781da177e4SLinus Torvalds 879e8f5d77cSTrond Myklebust task->tk_flags |= flags; 88046121cf7SChuck Lever dprintk("RPC: allocated task %p\n", task); 8811da177e4SLinus Torvalds out: 8821da177e4SLinus Torvalds return task; 8831da177e4SLinus Torvalds } 8841da177e4SLinus Torvalds 88532bfb5c0STrond Myklebust static void rpc_free_task(struct rpc_task *task) 8861da177e4SLinus Torvalds { 887963d8fe5STrond Myklebust const struct rpc_call_ops *tk_ops = task->tk_ops; 888963d8fe5STrond Myklebust void *calldata = task->tk_calldata; 8891da177e4SLinus Torvalds 89032bfb5c0STrond Myklebust if (task->tk_flags & RPC_TASK_DYNAMIC) 89132bfb5c0STrond Myklebust call_rcu_bh(&task->u.tk_rcu, rpc_free_task_rcu); 89232bfb5c0STrond Myklebust rpc_release_calldata(tk_ops, calldata); 89332bfb5c0STrond Myklebust } 89432bfb5c0STrond Myklebust 89532bfb5c0STrond Myklebust static void rpc_async_release(struct work_struct *work) 89632bfb5c0STrond Myklebust { 89732bfb5c0STrond Myklebust rpc_free_task(container_of(work, struct rpc_task, u.tk_work)); 89832bfb5c0STrond Myklebust } 89932bfb5c0STrond Myklebust 90032bfb5c0STrond Myklebust void rpc_put_task(struct rpc_task *task) 90132bfb5c0STrond Myklebust { 902e6b3c4dbSTrond Myklebust if (!atomic_dec_and_test(&task->tk_count)) 903e6b3c4dbSTrond Myklebust return; 904e6b3c4dbSTrond Myklebust /* Release resources */ 905e6b3c4dbSTrond Myklebust if (task->tk_rqstp) 906e6b3c4dbSTrond Myklebust xprt_release(task); 907e6b3c4dbSTrond Myklebust if (task->tk_msg.rpc_cred) 908e6b3c4dbSTrond Myklebust rpcauth_unbindcred(task); 909e6b3c4dbSTrond Myklebust if (task->tk_client) { 910e6b3c4dbSTrond Myklebust rpc_release_client(task->tk_client); 911e6b3c4dbSTrond Myklebust task->tk_client = NULL; 912e6b3c4dbSTrond Myklebust } 91332bfb5c0STrond Myklebust if (task->tk_workqueue != NULL) { 91432bfb5c0STrond Myklebust INIT_WORK(&task->u.tk_work, rpc_async_release); 91532bfb5c0STrond Myklebust queue_work(task->tk_workqueue, &task->u.tk_work); 91632bfb5c0STrond Myklebust } else 91732bfb5c0STrond Myklebust rpc_free_task(task); 918e6b3c4dbSTrond Myklebust } 919e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_put_task); 920e6b3c4dbSTrond Myklebust 921bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task) 922e6b3c4dbSTrond Myklebust { 9231da177e4SLinus Torvalds #ifdef RPC_DEBUG 9241da177e4SLinus Torvalds BUG_ON(task->tk_magic != RPC_TASK_MAGIC_ID); 9251da177e4SLinus Torvalds #endif 92646121cf7SChuck Lever dprintk("RPC: %5u release task\n", task->tk_pid); 9271da177e4SLinus Torvalds 9286529eba0STrond Myklebust if (!list_empty(&task->tk_task)) { 9294bef61ffSTrond Myklebust struct rpc_clnt *clnt = task->tk_client; 9306529eba0STrond Myklebust /* Remove from client task list */ 9314bef61ffSTrond Myklebust spin_lock(&clnt->cl_lock); 9321da177e4SLinus Torvalds list_del(&task->tk_task); 9334bef61ffSTrond Myklebust spin_unlock(&clnt->cl_lock); 9346529eba0STrond Myklebust } 9351da177e4SLinus Torvalds BUG_ON (RPC_IS_QUEUED(task)); 9361da177e4SLinus Torvalds 9371da177e4SLinus Torvalds #ifdef RPC_DEBUG 9381da177e4SLinus Torvalds task->tk_magic = 0; 9391da177e4SLinus Torvalds #endif 940e6b3c4dbSTrond Myklebust /* Wake up anyone who is waiting for task completion */ 941e6b3c4dbSTrond Myklebust rpc_mark_complete_task(task); 942e6b3c4dbSTrond Myklebust 943e6b3c4dbSTrond Myklebust rpc_put_task(task); 9441da177e4SLinus Torvalds } 9451da177e4SLinus Torvalds 9461da177e4SLinus Torvalds /* 9471da177e4SLinus Torvalds * Kill all tasks for the given client. 9481da177e4SLinus Torvalds * XXX: kill their descendants as well? 9491da177e4SLinus Torvalds */ 9504bef61ffSTrond Myklebust void rpc_killall_tasks(struct rpc_clnt *clnt) 9511da177e4SLinus Torvalds { 9521da177e4SLinus Torvalds struct rpc_task *rovr; 9531da177e4SLinus Torvalds 9541da177e4SLinus Torvalds 9554bef61ffSTrond Myklebust if (list_empty(&clnt->cl_tasks)) 9564bef61ffSTrond Myklebust return; 9574bef61ffSTrond Myklebust dprintk("RPC: killing all tasks for client %p\n", clnt); 9584bef61ffSTrond Myklebust /* 9594bef61ffSTrond Myklebust * Spin lock all_tasks to prevent changes... 9604bef61ffSTrond Myklebust */ 9614bef61ffSTrond Myklebust spin_lock(&clnt->cl_lock); 9624bef61ffSTrond Myklebust list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) { 9631da177e4SLinus Torvalds if (! RPC_IS_ACTIVATED(rovr)) 9641da177e4SLinus Torvalds continue; 9656529eba0STrond Myklebust if (!(rovr->tk_flags & RPC_TASK_KILLED)) { 9661da177e4SLinus Torvalds rovr->tk_flags |= RPC_TASK_KILLED; 9671da177e4SLinus Torvalds rpc_exit(rovr, -EIO); 9681da177e4SLinus Torvalds rpc_wake_up_task(rovr); 9691da177e4SLinus Torvalds } 9701da177e4SLinus Torvalds } 9714bef61ffSTrond Myklebust spin_unlock(&clnt->cl_lock); 9721da177e4SLinus Torvalds } 973e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_killall_tasks); 9741da177e4SLinus Torvalds 975b247bbf1STrond Myklebust int rpciod_up(void) 976b247bbf1STrond Myklebust { 977b247bbf1STrond Myklebust return try_module_get(THIS_MODULE) ? 0 : -EINVAL; 978b247bbf1STrond Myklebust } 979b247bbf1STrond Myklebust 980b247bbf1STrond Myklebust void rpciod_down(void) 981b247bbf1STrond Myklebust { 982b247bbf1STrond Myklebust module_put(THIS_MODULE); 983b247bbf1STrond Myklebust } 984b247bbf1STrond Myklebust 9851da177e4SLinus Torvalds /* 986b247bbf1STrond Myklebust * Start up the rpciod workqueue. 9871da177e4SLinus Torvalds */ 988b247bbf1STrond Myklebust static int rpciod_start(void) 9891da177e4SLinus Torvalds { 9901da177e4SLinus Torvalds struct workqueue_struct *wq; 9911da177e4SLinus Torvalds 9921da177e4SLinus Torvalds /* 9931da177e4SLinus Torvalds * Create the rpciod thread and wait for it to start. 9941da177e4SLinus Torvalds */ 995ab418d70STrond Myklebust dprintk("RPC: creating workqueue rpciod\n"); 9961da177e4SLinus Torvalds wq = create_workqueue("rpciod"); 9971da177e4SLinus Torvalds rpciod_workqueue = wq; 998b247bbf1STrond Myklebust return rpciod_workqueue != NULL; 9991da177e4SLinus Torvalds } 10001da177e4SLinus Torvalds 1001b247bbf1STrond Myklebust static void rpciod_stop(void) 10021da177e4SLinus Torvalds { 1003b247bbf1STrond Myklebust struct workqueue_struct *wq = NULL; 1004ab418d70STrond Myklebust 1005b247bbf1STrond Myklebust if (rpciod_workqueue == NULL) 1006b247bbf1STrond Myklebust return; 1007ab418d70STrond Myklebust dprintk("RPC: destroying workqueue rpciod\n"); 10081da177e4SLinus Torvalds 1009b247bbf1STrond Myklebust wq = rpciod_workqueue; 10101da177e4SLinus Torvalds rpciod_workqueue = NULL; 1011b247bbf1STrond Myklebust destroy_workqueue(wq); 10121da177e4SLinus Torvalds } 10131da177e4SLinus Torvalds 10141da177e4SLinus Torvalds void 10151da177e4SLinus Torvalds rpc_destroy_mempool(void) 10161da177e4SLinus Torvalds { 1017b247bbf1STrond Myklebust rpciod_stop(); 10181da177e4SLinus Torvalds if (rpc_buffer_mempool) 10191da177e4SLinus Torvalds mempool_destroy(rpc_buffer_mempool); 10201da177e4SLinus Torvalds if (rpc_task_mempool) 10211da177e4SLinus Torvalds mempool_destroy(rpc_task_mempool); 10221a1d92c1SAlexey Dobriyan if (rpc_task_slabp) 10231a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_task_slabp); 10241a1d92c1SAlexey Dobriyan if (rpc_buffer_slabp) 10251a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_buffer_slabp); 1026f6a1cc89STrond Myklebust rpc_destroy_wait_queue(&delay_queue); 10271da177e4SLinus Torvalds } 10281da177e4SLinus Torvalds 10291da177e4SLinus Torvalds int 10301da177e4SLinus Torvalds rpc_init_mempool(void) 10311da177e4SLinus Torvalds { 1032f6a1cc89STrond Myklebust /* 1033f6a1cc89STrond Myklebust * The following is not strictly a mempool initialisation, 1034f6a1cc89STrond Myklebust * but there is no harm in doing it here 1035f6a1cc89STrond Myklebust */ 1036f6a1cc89STrond Myklebust rpc_init_wait_queue(&delay_queue, "delayq"); 1037f6a1cc89STrond Myklebust if (!rpciod_start()) 1038f6a1cc89STrond Myklebust goto err_nomem; 1039f6a1cc89STrond Myklebust 10401da177e4SLinus Torvalds rpc_task_slabp = kmem_cache_create("rpc_tasks", 10411da177e4SLinus Torvalds sizeof(struct rpc_task), 10421da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 104320c2df83SPaul Mundt NULL); 10441da177e4SLinus Torvalds if (!rpc_task_slabp) 10451da177e4SLinus Torvalds goto err_nomem; 10461da177e4SLinus Torvalds rpc_buffer_slabp = kmem_cache_create("rpc_buffers", 10471da177e4SLinus Torvalds RPC_BUFFER_MAXSIZE, 10481da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 104920c2df83SPaul Mundt NULL); 10501da177e4SLinus Torvalds if (!rpc_buffer_slabp) 10511da177e4SLinus Torvalds goto err_nomem; 105293d2341cSMatthew Dobson rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, 10531da177e4SLinus Torvalds rpc_task_slabp); 10541da177e4SLinus Torvalds if (!rpc_task_mempool) 10551da177e4SLinus Torvalds goto err_nomem; 105693d2341cSMatthew Dobson rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, 10571da177e4SLinus Torvalds rpc_buffer_slabp); 10581da177e4SLinus Torvalds if (!rpc_buffer_mempool) 10591da177e4SLinus Torvalds goto err_nomem; 10601da177e4SLinus Torvalds return 0; 10611da177e4SLinus Torvalds err_nomem: 10621da177e4SLinus Torvalds rpc_destroy_mempool(); 10631da177e4SLinus Torvalds return -ENOMEM; 10641da177e4SLinus Torvalds } 1065