11da177e4SLinus Torvalds /* 21da177e4SLinus Torvalds * linux/net/sunrpc/sched.c 31da177e4SLinus Torvalds * 41da177e4SLinus Torvalds * Scheduling for synchronous and asynchronous RPC requests. 51da177e4SLinus Torvalds * 61da177e4SLinus Torvalds * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de> 71da177e4SLinus Torvalds * 81da177e4SLinus Torvalds * TCP NFS related read + write fixes 91da177e4SLinus Torvalds * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 101da177e4SLinus Torvalds */ 111da177e4SLinus Torvalds 121da177e4SLinus Torvalds #include <linux/module.h> 131da177e4SLinus Torvalds 141da177e4SLinus Torvalds #include <linux/sched.h> 151da177e4SLinus Torvalds #include <linux/interrupt.h> 161da177e4SLinus Torvalds #include <linux/slab.h> 171da177e4SLinus Torvalds #include <linux/mempool.h> 181da177e4SLinus Torvalds #include <linux/smp.h> 191da177e4SLinus Torvalds #include <linux/spinlock.h> 204a3e2f71SArjan van de Ven #include <linux/mutex.h> 21d310310cSJeff Layton #include <linux/freezer.h> 221da177e4SLinus Torvalds 231da177e4SLinus Torvalds #include <linux/sunrpc/clnt.h> 241da177e4SLinus Torvalds 256951867bSBenny Halevy #include "sunrpc.h" 266951867bSBenny Halevy 271da177e4SLinus Torvalds #ifdef RPC_DEBUG 281da177e4SLinus Torvalds #define RPCDBG_FACILITY RPCDBG_SCHED 291da177e4SLinus Torvalds #endif 301da177e4SLinus Torvalds 3182b0a4c3STrond Myklebust #define CREATE_TRACE_POINTS 3282b0a4c3STrond Myklebust #include <trace/events/sunrpc.h> 3382b0a4c3STrond Myklebust 341da177e4SLinus Torvalds /* 351da177e4SLinus Torvalds * RPC slabs and memory pools 361da177e4SLinus Torvalds */ 371da177e4SLinus Torvalds #define RPC_BUFFER_MAXSIZE (2048) 381da177e4SLinus Torvalds #define RPC_BUFFER_POOLSIZE (8) 391da177e4SLinus Torvalds #define RPC_TASK_POOLSIZE (8) 40e18b890bSChristoph Lameter static struct kmem_cache *rpc_task_slabp __read_mostly; 41e18b890bSChristoph Lameter static struct kmem_cache *rpc_buffer_slabp __read_mostly; 42ba89966cSEric Dumazet static mempool_t *rpc_task_mempool __read_mostly; 43ba89966cSEric Dumazet static mempool_t *rpc_buffer_mempool __read_mostly; 441da177e4SLinus Torvalds 4565f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *); 46bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task); 4736df9aaeSTrond Myklebust static void __rpc_queue_timer_fn(unsigned long ptr); 481da177e4SLinus Torvalds 491da177e4SLinus Torvalds /* 501da177e4SLinus Torvalds * RPC tasks sit here while waiting for conditions to improve. 511da177e4SLinus Torvalds */ 52a4a87499STrond Myklebust static struct rpc_wait_queue delay_queue; 531da177e4SLinus Torvalds 541da177e4SLinus Torvalds /* 551da177e4SLinus Torvalds * rpciod-related stuff 561da177e4SLinus Torvalds */ 5724c5d9d7STrond Myklebust struct workqueue_struct *rpciod_workqueue; 581da177e4SLinus Torvalds 591da177e4SLinus Torvalds /* 601da177e4SLinus Torvalds * Disable the timer for a given RPC task. Should be called with 611da177e4SLinus Torvalds * queue->lock and bh_disabled in order to avoid races within 621da177e4SLinus Torvalds * rpc_run_timer(). 631da177e4SLinus Torvalds */ 645d00837bSTrond Myklebust static void 65eb276c0eSTrond Myklebust __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) 661da177e4SLinus Torvalds { 6736df9aaeSTrond Myklebust if (task->tk_timeout == 0) 6836df9aaeSTrond Myklebust return; 6946121cf7SChuck Lever dprintk("RPC: %5u disabling timer\n", task->tk_pid); 701da177e4SLinus Torvalds task->tk_timeout = 0; 7136df9aaeSTrond Myklebust list_del(&task->u.tk_wait.timer_list); 72eb276c0eSTrond Myklebust if (list_empty(&queue->timer_list.list)) 73eb276c0eSTrond Myklebust del_timer(&queue->timer_list.timer); 7436df9aaeSTrond Myklebust } 7536df9aaeSTrond Myklebust 7636df9aaeSTrond Myklebust static void 7736df9aaeSTrond Myklebust rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires) 7836df9aaeSTrond Myklebust { 7936df9aaeSTrond Myklebust queue->timer_list.expires = expires; 8036df9aaeSTrond Myklebust mod_timer(&queue->timer_list.timer, expires); 811da177e4SLinus Torvalds } 821da177e4SLinus Torvalds 831da177e4SLinus Torvalds /* 841da177e4SLinus Torvalds * Set up a timer for the current task. 851da177e4SLinus Torvalds */ 865d00837bSTrond Myklebust static void 87eb276c0eSTrond Myklebust __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task) 881da177e4SLinus Torvalds { 891da177e4SLinus Torvalds if (!task->tk_timeout) 901da177e4SLinus Torvalds return; 911da177e4SLinus Torvalds 9246121cf7SChuck Lever dprintk("RPC: %5u setting alarm for %lu ms\n", 931da177e4SLinus Torvalds task->tk_pid, task->tk_timeout * 1000 / HZ); 941da177e4SLinus Torvalds 95eb276c0eSTrond Myklebust task->u.tk_wait.expires = jiffies + task->tk_timeout; 96eb276c0eSTrond Myklebust if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires)) 97eb276c0eSTrond Myklebust rpc_set_queue_timer(queue, task->u.tk_wait.expires); 98eb276c0eSTrond Myklebust list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); 991da177e4SLinus Torvalds } 1001da177e4SLinus Torvalds 1011da177e4SLinus Torvalds /* 1021da177e4SLinus Torvalds * Add new request to a priority queue. 1031da177e4SLinus Torvalds */ 1043b27bad7STrond Myklebust static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, 1053b27bad7STrond Myklebust struct rpc_task *task, 1063b27bad7STrond Myklebust unsigned char queue_priority) 1071da177e4SLinus Torvalds { 1081da177e4SLinus Torvalds struct list_head *q; 1091da177e4SLinus Torvalds struct rpc_task *t; 1101da177e4SLinus Torvalds 1111da177e4SLinus Torvalds INIT_LIST_HEAD(&task->u.tk_wait.links); 1123b27bad7STrond Myklebust q = &queue->tasks[queue_priority]; 1133b27bad7STrond Myklebust if (unlikely(queue_priority > queue->maxpriority)) 1141da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 1151da177e4SLinus Torvalds list_for_each_entry(t, q, u.tk_wait.list) { 1163ff7576dSTrond Myklebust if (t->tk_owner == task->tk_owner) { 1171da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links); 1181da177e4SLinus Torvalds return; 1191da177e4SLinus Torvalds } 1201da177e4SLinus Torvalds } 1211da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, q); 1221da177e4SLinus Torvalds } 1231da177e4SLinus Torvalds 1241da177e4SLinus Torvalds /* 1251da177e4SLinus Torvalds * Add new request to wait queue. 1261da177e4SLinus Torvalds * 1271da177e4SLinus Torvalds * Swapper tasks always get inserted at the head of the queue. 1281da177e4SLinus Torvalds * This should avoid many nasty memory deadlocks and hopefully 1291da177e4SLinus Torvalds * improve overall performance. 1301da177e4SLinus Torvalds * Everyone else gets appended to the queue to ensure proper FIFO behavior. 1311da177e4SLinus Torvalds */ 1323b27bad7STrond Myklebust static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, 1333b27bad7STrond Myklebust struct rpc_task *task, 1343b27bad7STrond Myklebust unsigned char queue_priority) 1351da177e4SLinus Torvalds { 1362bd4eef8SWeston Andros Adamson WARN_ON_ONCE(RPC_IS_QUEUED(task)); 1372bd4eef8SWeston Andros Adamson if (RPC_IS_QUEUED(task)) 1382bd4eef8SWeston Andros Adamson return; 1391da177e4SLinus Torvalds 1401da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 1413b27bad7STrond Myklebust __rpc_add_wait_queue_priority(queue, task, queue_priority); 1421da177e4SLinus Torvalds else if (RPC_IS_SWAPPER(task)) 1431da177e4SLinus Torvalds list_add(&task->u.tk_wait.list, &queue->tasks[0]); 1441da177e4SLinus Torvalds else 1451da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); 14696ef13b2STrond Myklebust task->tk_waitqueue = queue; 147e19b63daSChuck Lever queue->qlen++; 1481da177e4SLinus Torvalds rpc_set_queued(task); 1491da177e4SLinus Torvalds 15046121cf7SChuck Lever dprintk("RPC: %5u added to queue %p \"%s\"\n", 1511da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 1521da177e4SLinus Torvalds } 1531da177e4SLinus Torvalds 1541da177e4SLinus Torvalds /* 1551da177e4SLinus Torvalds * Remove request from a priority queue. 1561da177e4SLinus Torvalds */ 1571da177e4SLinus Torvalds static void __rpc_remove_wait_queue_priority(struct rpc_task *task) 1581da177e4SLinus Torvalds { 1591da177e4SLinus Torvalds struct rpc_task *t; 1601da177e4SLinus Torvalds 1611da177e4SLinus Torvalds if (!list_empty(&task->u.tk_wait.links)) { 1621da177e4SLinus Torvalds t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list); 1631da177e4SLinus Torvalds list_move(&t->u.tk_wait.list, &task->u.tk_wait.list); 1641da177e4SLinus Torvalds list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links); 1651da177e4SLinus Torvalds } 1661da177e4SLinus Torvalds } 1671da177e4SLinus Torvalds 1681da177e4SLinus Torvalds /* 1691da177e4SLinus Torvalds * Remove request from queue. 1701da177e4SLinus Torvalds * Note: must be called with spin lock held. 1711da177e4SLinus Torvalds */ 17296ef13b2STrond Myklebust static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) 1731da177e4SLinus Torvalds { 174eb276c0eSTrond Myklebust __rpc_disable_timer(queue, task); 1751da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 1761da177e4SLinus Torvalds __rpc_remove_wait_queue_priority(task); 1771da177e4SLinus Torvalds list_del(&task->u.tk_wait.list); 178e19b63daSChuck Lever queue->qlen--; 17946121cf7SChuck Lever dprintk("RPC: %5u removed from queue %p \"%s\"\n", 1801da177e4SLinus Torvalds task->tk_pid, queue, rpc_qname(queue)); 1811da177e4SLinus Torvalds } 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) 1841da177e4SLinus Torvalds { 1851da177e4SLinus Torvalds queue->priority = priority; 1861da177e4SLinus Torvalds queue->count = 1 << (priority * 2); 1871da177e4SLinus Torvalds } 1881da177e4SLinus Torvalds 1893ff7576dSTrond Myklebust static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid) 1901da177e4SLinus Torvalds { 1913ff7576dSTrond Myklebust queue->owner = pid; 1921da177e4SLinus Torvalds queue->nr = RPC_BATCH_COUNT; 1931da177e4SLinus Torvalds } 1941da177e4SLinus Torvalds 1951da177e4SLinus Torvalds static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) 1961da177e4SLinus Torvalds { 1971da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, queue->maxpriority); 1983ff7576dSTrond Myklebust rpc_set_waitqueue_owner(queue, 0); 1991da177e4SLinus Torvalds } 2001da177e4SLinus Torvalds 2013ff7576dSTrond Myklebust static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues) 2021da177e4SLinus Torvalds { 2031da177e4SLinus Torvalds int i; 2041da177e4SLinus Torvalds 2051da177e4SLinus Torvalds spin_lock_init(&queue->lock); 2061da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) 2071da177e4SLinus Torvalds INIT_LIST_HEAD(&queue->tasks[i]); 2083ff7576dSTrond Myklebust queue->maxpriority = nr_queues - 1; 2091da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 21036df9aaeSTrond Myklebust queue->qlen = 0; 21136df9aaeSTrond Myklebust setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue); 21236df9aaeSTrond Myklebust INIT_LIST_HEAD(&queue->timer_list.list); 2132f09c242STrond Myklebust rpc_assign_waitqueue_name(queue, qname); 2141da177e4SLinus Torvalds } 2151da177e4SLinus Torvalds 2161da177e4SLinus Torvalds void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2171da177e4SLinus Torvalds { 2183ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY); 2191da177e4SLinus Torvalds } 220689cf5c1SAlexandros Batsakis EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue); 2211da177e4SLinus Torvalds 2221da177e4SLinus Torvalds void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2231da177e4SLinus Torvalds { 2243ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, 1); 2251da177e4SLinus Torvalds } 226e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_init_wait_queue); 2271da177e4SLinus Torvalds 228f6a1cc89STrond Myklebust void rpc_destroy_wait_queue(struct rpc_wait_queue *queue) 229f6a1cc89STrond Myklebust { 23036df9aaeSTrond Myklebust del_timer_sync(&queue->timer_list.timer); 231f6a1cc89STrond Myklebust } 232f6a1cc89STrond Myklebust EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue); 233f6a1cc89STrond Myklebust 234150030b7SMatthew Wilcox static int rpc_wait_bit_killable(void *word) 23544c28873STrond Myklebust { 236150030b7SMatthew Wilcox if (fatal_signal_pending(current)) 23744c28873STrond Myklebust return -ERESTARTSYS; 238d310310cSJeff Layton freezable_schedule(); 23944c28873STrond Myklebust return 0; 24044c28873STrond Myklebust } 24144c28873STrond Myklebust 242c44fe705STrond Myklebust #ifdef RPC_DEBUG 243c44fe705STrond Myklebust static void rpc_task_set_debuginfo(struct rpc_task *task) 244c44fe705STrond Myklebust { 245c44fe705STrond Myklebust static atomic_t rpc_pid; 246c44fe705STrond Myklebust 247c44fe705STrond Myklebust task->tk_pid = atomic_inc_return(&rpc_pid); 248c44fe705STrond Myklebust } 249c44fe705STrond Myklebust #else 250c44fe705STrond Myklebust static inline void rpc_task_set_debuginfo(struct rpc_task *task) 251c44fe705STrond Myklebust { 252c44fe705STrond Myklebust } 253c44fe705STrond Myklebust #endif 254c44fe705STrond Myklebust 255e6b3c4dbSTrond Myklebust static void rpc_set_active(struct rpc_task *task) 256e6b3c4dbSTrond Myklebust { 25782b0a4c3STrond Myklebust trace_rpc_task_begin(task->tk_client, task, NULL); 25882b0a4c3STrond Myklebust 259c44fe705STrond Myklebust rpc_task_set_debuginfo(task); 26058f9612cSTrond Myklebust set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 261e6b3c4dbSTrond Myklebust } 262e6b3c4dbSTrond Myklebust 26344c28873STrond Myklebust /* 26444c28873STrond Myklebust * Mark an RPC call as having completed by clearing the 'active' bit 265bf294b41STrond Myklebust * and then waking up all tasks that were sleeping. 26644c28873STrond Myklebust */ 267bf294b41STrond Myklebust static int rpc_complete_task(struct rpc_task *task) 26844c28873STrond Myklebust { 269bf294b41STrond Myklebust void *m = &task->tk_runstate; 270bf294b41STrond Myklebust wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE); 271bf294b41STrond Myklebust struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE); 272bf294b41STrond Myklebust unsigned long flags; 273bf294b41STrond Myklebust int ret; 274bf294b41STrond Myklebust 27582b0a4c3STrond Myklebust trace_rpc_task_complete(task->tk_client, task, NULL); 27682b0a4c3STrond Myklebust 277bf294b41STrond Myklebust spin_lock_irqsave(&wq->lock, flags); 278e6b3c4dbSTrond Myklebust clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 279bf294b41STrond Myklebust ret = atomic_dec_and_test(&task->tk_count); 280bf294b41STrond Myklebust if (waitqueue_active(wq)) 281bf294b41STrond Myklebust __wake_up_locked_key(wq, TASK_NORMAL, &k); 282bf294b41STrond Myklebust spin_unlock_irqrestore(&wq->lock, flags); 283bf294b41STrond Myklebust return ret; 28444c28873STrond Myklebust } 28544c28873STrond Myklebust 28644c28873STrond Myklebust /* 28744c28873STrond Myklebust * Allow callers to wait for completion of an RPC call 288bf294b41STrond Myklebust * 289bf294b41STrond Myklebust * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit() 290bf294b41STrond Myklebust * to enforce taking of the wq->lock and hence avoid races with 291bf294b41STrond Myklebust * rpc_complete_task(). 29244c28873STrond Myklebust */ 29344c28873STrond Myklebust int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *)) 29444c28873STrond Myklebust { 29544c28873STrond Myklebust if (action == NULL) 296150030b7SMatthew Wilcox action = rpc_wait_bit_killable; 297bf294b41STrond Myklebust return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, 298150030b7SMatthew Wilcox action, TASK_KILLABLE); 29944c28873STrond Myklebust } 300e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); 30144c28873STrond Myklebust 3021da177e4SLinus Torvalds /* 3031da177e4SLinus Torvalds * Make an RPC task runnable. 3041da177e4SLinus Torvalds * 305506026c3SJeff Layton * Note: If the task is ASYNC, and is being made runnable after sitting on an 306506026c3SJeff Layton * rpc_wait_queue, this must be called with the queue spinlock held to protect 307506026c3SJeff Layton * the wait queue operation. 3081da177e4SLinus Torvalds */ 3091da177e4SLinus Torvalds static void rpc_make_runnable(struct rpc_task *task) 3101da177e4SLinus Torvalds { 3111da177e4SLinus Torvalds rpc_clear_queued(task); 312cc4dc59eSChristophe Saout if (rpc_test_and_set_running(task)) 3131da177e4SLinus Torvalds return; 3141da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 31565f27f38SDavid Howells INIT_WORK(&task->u.tk_work, rpc_async_schedule); 316cee6a537SFred Isaman queue_work(rpciod_workqueue, &task->u.tk_work); 3171da177e4SLinus Torvalds } else 31896651ab3STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); 3191da177e4SLinus Torvalds } 3201da177e4SLinus Torvalds 3211da177e4SLinus Torvalds /* 3221da177e4SLinus Torvalds * Prepare for sleeping on a wait queue. 3231da177e4SLinus Torvalds * By always appending tasks to the list we ensure FIFO behavior. 3241da177e4SLinus Torvalds * NB: An RPC task will only receive interrupt-driven events as long 3251da177e4SLinus Torvalds * as it's on a wait queue. 3261da177e4SLinus Torvalds */ 3273b27bad7STrond Myklebust static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, 3283b27bad7STrond Myklebust struct rpc_task *task, 3293b27bad7STrond Myklebust rpc_action action, 3303b27bad7STrond Myklebust unsigned char queue_priority) 3311da177e4SLinus Torvalds { 33246121cf7SChuck Lever dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n", 33346121cf7SChuck Lever task->tk_pid, rpc_qname(q), jiffies); 3341da177e4SLinus Torvalds 33582b0a4c3STrond Myklebust trace_rpc_task_sleep(task->tk_client, task, q); 33682b0a4c3STrond Myklebust 3373b27bad7STrond Myklebust __rpc_add_wait_queue(q, task, queue_priority); 3381da177e4SLinus Torvalds 339f50ad428SWeston Andros Adamson WARN_ON_ONCE(task->tk_callback != NULL); 3401da177e4SLinus Torvalds task->tk_callback = action; 341eb276c0eSTrond Myklebust __rpc_add_timer(q, task); 3421da177e4SLinus Torvalds } 3431da177e4SLinus Torvalds 3441da177e4SLinus Torvalds void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 3455d00837bSTrond Myklebust rpc_action action) 3461da177e4SLinus Torvalds { 34758f9612cSTrond Myklebust /* We shouldn't ever put an inactive task to sleep */ 348e454a7a8SWeston Andros Adamson WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); 349e454a7a8SWeston Andros Adamson if (!RPC_IS_ACTIVATED(task)) { 350e454a7a8SWeston Andros Adamson task->tk_status = -EIO; 351e454a7a8SWeston Andros Adamson rpc_put_task_async(task); 352e454a7a8SWeston Andros Adamson return; 353e454a7a8SWeston Andros Adamson } 354e6b3c4dbSTrond Myklebust 3551da177e4SLinus Torvalds /* 3561da177e4SLinus Torvalds * Protect the queue operations. 3571da177e4SLinus Torvalds */ 3581da177e4SLinus Torvalds spin_lock_bh(&q->lock); 3593b27bad7STrond Myklebust __rpc_sleep_on_priority(q, task, action, task->tk_priority); 3601da177e4SLinus Torvalds spin_unlock_bh(&q->lock); 3611da177e4SLinus Torvalds } 362e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on); 3631da177e4SLinus Torvalds 3643b27bad7STrond Myklebust void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, 3653b27bad7STrond Myklebust rpc_action action, int priority) 3663b27bad7STrond Myklebust { 3673b27bad7STrond Myklebust /* We shouldn't ever put an inactive task to sleep */ 368e454a7a8SWeston Andros Adamson WARN_ON_ONCE(!RPC_IS_ACTIVATED(task)); 369e454a7a8SWeston Andros Adamson if (!RPC_IS_ACTIVATED(task)) { 370e454a7a8SWeston Andros Adamson task->tk_status = -EIO; 371e454a7a8SWeston Andros Adamson rpc_put_task_async(task); 372e454a7a8SWeston Andros Adamson return; 373e454a7a8SWeston Andros Adamson } 3743b27bad7STrond Myklebust 3753b27bad7STrond Myklebust /* 3763b27bad7STrond Myklebust * Protect the queue operations. 3773b27bad7STrond Myklebust */ 3783b27bad7STrond Myklebust spin_lock_bh(&q->lock); 3793b27bad7STrond Myklebust __rpc_sleep_on_priority(q, task, action, priority - RPC_PRIORITY_LOW); 3803b27bad7STrond Myklebust spin_unlock_bh(&q->lock); 3813b27bad7STrond Myklebust } 3823b27bad7STrond Myklebust 3831da177e4SLinus Torvalds /** 3841da177e4SLinus Torvalds * __rpc_do_wake_up_task - wake up a single rpc_task 38596ef13b2STrond Myklebust * @queue: wait queue 3861da177e4SLinus Torvalds * @task: task to be woken up 3871da177e4SLinus Torvalds * 3881da177e4SLinus Torvalds * Caller must hold queue->lock, and have cleared the task queued flag. 3891da177e4SLinus Torvalds */ 39096ef13b2STrond Myklebust static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task) 3911da177e4SLinus Torvalds { 39246121cf7SChuck Lever dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n", 39346121cf7SChuck Lever task->tk_pid, jiffies); 3941da177e4SLinus Torvalds 3951da177e4SLinus Torvalds /* Has the task been executed yet? If not, we cannot wake it up! */ 3961da177e4SLinus Torvalds if (!RPC_IS_ACTIVATED(task)) { 3971da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); 3981da177e4SLinus Torvalds return; 3991da177e4SLinus Torvalds } 4001da177e4SLinus Torvalds 40182b0a4c3STrond Myklebust trace_rpc_task_wakeup(task->tk_client, task, queue); 40282b0a4c3STrond Myklebust 40396ef13b2STrond Myklebust __rpc_remove_wait_queue(queue, task); 4041da177e4SLinus Torvalds 4051da177e4SLinus Torvalds rpc_make_runnable(task); 4061da177e4SLinus Torvalds 4071da177e4SLinus Torvalds dprintk("RPC: __rpc_wake_up_task done\n"); 4081da177e4SLinus Torvalds } 4091da177e4SLinus Torvalds 4101da177e4SLinus Torvalds /* 41196ef13b2STrond Myklebust * Wake up a queued task while the queue lock is being held 4121da177e4SLinus Torvalds */ 41396ef13b2STrond Myklebust static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task) 4141da177e4SLinus Torvalds { 415f5fb7b06STrond Myklebust if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue) 41696ef13b2STrond Myklebust __rpc_do_wake_up_task(queue, task); 4171da177e4SLinus Torvalds } 4181da177e4SLinus Torvalds 4191da177e4SLinus Torvalds /* 42048f18612SAlexandros Batsakis * Tests whether rpc queue is empty 42148f18612SAlexandros Batsakis */ 42248f18612SAlexandros Batsakis int rpc_queue_empty(struct rpc_wait_queue *queue) 42348f18612SAlexandros Batsakis { 42448f18612SAlexandros Batsakis int res; 42548f18612SAlexandros Batsakis 42648f18612SAlexandros Batsakis spin_lock_bh(&queue->lock); 42748f18612SAlexandros Batsakis res = queue->qlen; 42848f18612SAlexandros Batsakis spin_unlock_bh(&queue->lock); 429a02cec21SEric Dumazet return res == 0; 43048f18612SAlexandros Batsakis } 43148f18612SAlexandros Batsakis EXPORT_SYMBOL_GPL(rpc_queue_empty); 43248f18612SAlexandros Batsakis 43348f18612SAlexandros Batsakis /* 43496ef13b2STrond Myklebust * Wake up a task on a specific queue 43596ef13b2STrond Myklebust */ 43696ef13b2STrond Myklebust void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) 43796ef13b2STrond Myklebust { 4385e4424afSTrond Myklebust spin_lock_bh(&queue->lock); 43996ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 4405e4424afSTrond Myklebust spin_unlock_bh(&queue->lock); 44196ef13b2STrond Myklebust } 44296ef13b2STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task); 44396ef13b2STrond Myklebust 44496ef13b2STrond Myklebust /* 4451da177e4SLinus Torvalds * Wake up the next task on a priority queue. 4461da177e4SLinus Torvalds */ 447961a828dSTrond Myklebust static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue) 4481da177e4SLinus Torvalds { 4491da177e4SLinus Torvalds struct list_head *q; 4501da177e4SLinus Torvalds struct rpc_task *task; 4511da177e4SLinus Torvalds 4521da177e4SLinus Torvalds /* 4533ff7576dSTrond Myklebust * Service a batch of tasks from a single owner. 4541da177e4SLinus Torvalds */ 4551da177e4SLinus Torvalds q = &queue->tasks[queue->priority]; 4561da177e4SLinus Torvalds if (!list_empty(q)) { 4571da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4583ff7576dSTrond Myklebust if (queue->owner == task->tk_owner) { 4591da177e4SLinus Torvalds if (--queue->nr) 4601da177e4SLinus Torvalds goto out; 4611da177e4SLinus Torvalds list_move_tail(&task->u.tk_wait.list, q); 4621da177e4SLinus Torvalds } 4631da177e4SLinus Torvalds /* 4641da177e4SLinus Torvalds * Check if we need to switch queues. 4651da177e4SLinus Torvalds */ 4661da177e4SLinus Torvalds if (--queue->count) 4673ff7576dSTrond Myklebust goto new_owner; 4681da177e4SLinus Torvalds } 4691da177e4SLinus Torvalds 4701da177e4SLinus Torvalds /* 4711da177e4SLinus Torvalds * Service the next queue. 4721da177e4SLinus Torvalds */ 4731da177e4SLinus Torvalds do { 4741da177e4SLinus Torvalds if (q == &queue->tasks[0]) 4751da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 4761da177e4SLinus Torvalds else 4771da177e4SLinus Torvalds q = q - 1; 4781da177e4SLinus Torvalds if (!list_empty(q)) { 4791da177e4SLinus Torvalds task = list_entry(q->next, struct rpc_task, u.tk_wait.list); 4801da177e4SLinus Torvalds goto new_queue; 4811da177e4SLinus Torvalds } 4821da177e4SLinus Torvalds } while (q != &queue->tasks[queue->priority]); 4831da177e4SLinus Torvalds 4841da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 4851da177e4SLinus Torvalds return NULL; 4861da177e4SLinus Torvalds 4871da177e4SLinus Torvalds new_queue: 4881da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); 4893ff7576dSTrond Myklebust new_owner: 4903ff7576dSTrond Myklebust rpc_set_waitqueue_owner(queue, task->tk_owner); 4911da177e4SLinus Torvalds out: 4921da177e4SLinus Torvalds return task; 4931da177e4SLinus Torvalds } 4941da177e4SLinus Torvalds 495961a828dSTrond Myklebust static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue) 496961a828dSTrond Myklebust { 497961a828dSTrond Myklebust if (RPC_IS_PRIORITY(queue)) 498961a828dSTrond Myklebust return __rpc_find_next_queued_priority(queue); 499961a828dSTrond Myklebust if (!list_empty(&queue->tasks[0])) 500961a828dSTrond Myklebust return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list); 501961a828dSTrond Myklebust return NULL; 502961a828dSTrond Myklebust } 503961a828dSTrond Myklebust 504961a828dSTrond Myklebust /* 505961a828dSTrond Myklebust * Wake up the first task on the wait queue. 506961a828dSTrond Myklebust */ 507961a828dSTrond Myklebust struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue, 508961a828dSTrond Myklebust bool (*func)(struct rpc_task *, void *), void *data) 509961a828dSTrond Myklebust { 510961a828dSTrond Myklebust struct rpc_task *task = NULL; 511961a828dSTrond Myklebust 512961a828dSTrond Myklebust dprintk("RPC: wake_up_first(%p \"%s\")\n", 513961a828dSTrond Myklebust queue, rpc_qname(queue)); 514961a828dSTrond Myklebust spin_lock_bh(&queue->lock); 515961a828dSTrond Myklebust task = __rpc_find_next_queued(queue); 516961a828dSTrond Myklebust if (task != NULL) { 517961a828dSTrond Myklebust if (func(task, data)) 518961a828dSTrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 519961a828dSTrond Myklebust else 520961a828dSTrond Myklebust task = NULL; 521961a828dSTrond Myklebust } 522961a828dSTrond Myklebust spin_unlock_bh(&queue->lock); 523961a828dSTrond Myklebust 524961a828dSTrond Myklebust return task; 525961a828dSTrond Myklebust } 526961a828dSTrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_first); 527961a828dSTrond Myklebust 528961a828dSTrond Myklebust static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) 529961a828dSTrond Myklebust { 530961a828dSTrond Myklebust return true; 531961a828dSTrond Myklebust } 532961a828dSTrond Myklebust 5331da177e4SLinus Torvalds /* 5341da177e4SLinus Torvalds * Wake up the next task on the wait queue. 5351da177e4SLinus Torvalds */ 5361da177e4SLinus Torvalds struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue) 5371da177e4SLinus Torvalds { 538961a828dSTrond Myklebust return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL); 5391da177e4SLinus Torvalds } 540e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_next); 5411da177e4SLinus Torvalds 5421da177e4SLinus Torvalds /** 5431da177e4SLinus Torvalds * rpc_wake_up - wake up all rpc_tasks 5441da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 5451da177e4SLinus Torvalds * 5461da177e4SLinus Torvalds * Grabs queue->lock 5471da177e4SLinus Torvalds */ 5481da177e4SLinus Torvalds void rpc_wake_up(struct rpc_wait_queue *queue) 5491da177e4SLinus Torvalds { 5501da177e4SLinus Torvalds struct list_head *head; 551e6d83d55STrond Myklebust 5525e4424afSTrond Myklebust spin_lock_bh(&queue->lock); 5531da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5541da177e4SLinus Torvalds for (;;) { 555540a0f75STrond Myklebust while (!list_empty(head)) { 556540a0f75STrond Myklebust struct rpc_task *task; 557540a0f75STrond Myklebust task = list_first_entry(head, 558540a0f75STrond Myklebust struct rpc_task, 559540a0f75STrond Myklebust u.tk_wait.list); 56096ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 561540a0f75STrond Myklebust } 5621da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5631da177e4SLinus Torvalds break; 5641da177e4SLinus Torvalds head--; 5651da177e4SLinus Torvalds } 5665e4424afSTrond Myklebust spin_unlock_bh(&queue->lock); 5671da177e4SLinus Torvalds } 568e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up); 5691da177e4SLinus Torvalds 5701da177e4SLinus Torvalds /** 5711da177e4SLinus Torvalds * rpc_wake_up_status - wake up all rpc_tasks and set their status value. 5721da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 5731da177e4SLinus Torvalds * @status: status value to set 5741da177e4SLinus Torvalds * 5751da177e4SLinus Torvalds * Grabs queue->lock 5761da177e4SLinus Torvalds */ 5771da177e4SLinus Torvalds void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) 5781da177e4SLinus Torvalds { 5791da177e4SLinus Torvalds struct list_head *head; 5801da177e4SLinus Torvalds 5815e4424afSTrond Myklebust spin_lock_bh(&queue->lock); 5821da177e4SLinus Torvalds head = &queue->tasks[queue->maxpriority]; 5831da177e4SLinus Torvalds for (;;) { 584540a0f75STrond Myklebust while (!list_empty(head)) { 585540a0f75STrond Myklebust struct rpc_task *task; 586540a0f75STrond Myklebust task = list_first_entry(head, 587540a0f75STrond Myklebust struct rpc_task, 588540a0f75STrond Myklebust u.tk_wait.list); 5891da177e4SLinus Torvalds task->tk_status = status; 59096ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 5911da177e4SLinus Torvalds } 5921da177e4SLinus Torvalds if (head == &queue->tasks[0]) 5931da177e4SLinus Torvalds break; 5941da177e4SLinus Torvalds head--; 5951da177e4SLinus Torvalds } 5965e4424afSTrond Myklebust spin_unlock_bh(&queue->lock); 5971da177e4SLinus Torvalds } 598e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_status); 5991da177e4SLinus Torvalds 60036df9aaeSTrond Myklebust static void __rpc_queue_timer_fn(unsigned long ptr) 60136df9aaeSTrond Myklebust { 60236df9aaeSTrond Myklebust struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr; 60336df9aaeSTrond Myklebust struct rpc_task *task, *n; 60436df9aaeSTrond Myklebust unsigned long expires, now, timeo; 60536df9aaeSTrond Myklebust 60636df9aaeSTrond Myklebust spin_lock(&queue->lock); 60736df9aaeSTrond Myklebust expires = now = jiffies; 60836df9aaeSTrond Myklebust list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { 60936df9aaeSTrond Myklebust timeo = task->u.tk_wait.expires; 61036df9aaeSTrond Myklebust if (time_after_eq(now, timeo)) { 61136df9aaeSTrond Myklebust dprintk("RPC: %5u timeout\n", task->tk_pid); 61236df9aaeSTrond Myklebust task->tk_status = -ETIMEDOUT; 61336df9aaeSTrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 61436df9aaeSTrond Myklebust continue; 61536df9aaeSTrond Myklebust } 61636df9aaeSTrond Myklebust if (expires == now || time_after(expires, timeo)) 61736df9aaeSTrond Myklebust expires = timeo; 61836df9aaeSTrond Myklebust } 61936df9aaeSTrond Myklebust if (!list_empty(&queue->timer_list.list)) 62036df9aaeSTrond Myklebust rpc_set_queue_timer(queue, expires); 62136df9aaeSTrond Myklebust spin_unlock(&queue->lock); 62236df9aaeSTrond Myklebust } 62336df9aaeSTrond Myklebust 6248014793bSTrond Myklebust static void __rpc_atrun(struct rpc_task *task) 6258014793bSTrond Myklebust { 6265d00837bSTrond Myklebust task->tk_status = 0; 6278014793bSTrond Myklebust } 6288014793bSTrond Myklebust 6291da177e4SLinus Torvalds /* 6301da177e4SLinus Torvalds * Run a task at a later time 6311da177e4SLinus Torvalds */ 6328014793bSTrond Myklebust void rpc_delay(struct rpc_task *task, unsigned long delay) 6331da177e4SLinus Torvalds { 6341da177e4SLinus Torvalds task->tk_timeout = delay; 6355d00837bSTrond Myklebust rpc_sleep_on(&delay_queue, task, __rpc_atrun); 6361da177e4SLinus Torvalds } 637e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_delay); 6381da177e4SLinus Torvalds 6391da177e4SLinus Torvalds /* 6404ce70adaSTrond Myklebust * Helper to call task->tk_ops->rpc_call_prepare 6414ce70adaSTrond Myklebust */ 642aae2006eSAndy Adamson void rpc_prepare_task(struct rpc_task *task) 6434ce70adaSTrond Myklebust { 6444ce70adaSTrond Myklebust task->tk_ops->rpc_call_prepare(task, task->tk_calldata); 6454ce70adaSTrond Myklebust } 6464ce70adaSTrond Myklebust 6477fdcf13bSTrond Myklebust static void 6487fdcf13bSTrond Myklebust rpc_init_task_statistics(struct rpc_task *task) 6497fdcf13bSTrond Myklebust { 6507fdcf13bSTrond Myklebust /* Initialize retry counters */ 6517fdcf13bSTrond Myklebust task->tk_garb_retry = 2; 6527fdcf13bSTrond Myklebust task->tk_cred_retry = 2; 6537fdcf13bSTrond Myklebust task->tk_rebind_retry = 2; 6547fdcf13bSTrond Myklebust 6557fdcf13bSTrond Myklebust /* starting timestamp */ 6567fdcf13bSTrond Myklebust task->tk_start = ktime_get(); 6577fdcf13bSTrond Myklebust } 6587fdcf13bSTrond Myklebust 6597fdcf13bSTrond Myklebust static void 6607fdcf13bSTrond Myklebust rpc_reset_task_statistics(struct rpc_task *task) 6617fdcf13bSTrond Myklebust { 6627fdcf13bSTrond Myklebust task->tk_timeouts = 0; 6637fdcf13bSTrond Myklebust task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_KILLED|RPC_TASK_SENT); 6647fdcf13bSTrond Myklebust 6657fdcf13bSTrond Myklebust rpc_init_task_statistics(task); 6667fdcf13bSTrond Myklebust } 6677fdcf13bSTrond Myklebust 6684ce70adaSTrond Myklebust /* 669963d8fe5STrond Myklebust * Helper that calls task->tk_ops->rpc_call_done if it exists 670d05fdb0cSTrond Myklebust */ 671abbcf28fSTrond Myklebust void rpc_exit_task(struct rpc_task *task) 672d05fdb0cSTrond Myklebust { 673abbcf28fSTrond Myklebust task->tk_action = NULL; 674963d8fe5STrond Myklebust if (task->tk_ops->rpc_call_done != NULL) { 675963d8fe5STrond Myklebust task->tk_ops->rpc_call_done(task, task->tk_calldata); 676d05fdb0cSTrond Myklebust if (task->tk_action != NULL) { 677abbcf28fSTrond Myklebust WARN_ON(RPC_ASSASSINATED(task)); 678abbcf28fSTrond Myklebust /* Always release the RPC slot and buffer memory */ 679d05fdb0cSTrond Myklebust xprt_release(task); 6807fdcf13bSTrond Myklebust rpc_reset_task_statistics(task); 681d05fdb0cSTrond Myklebust } 682d05fdb0cSTrond Myklebust } 683d05fdb0cSTrond Myklebust } 684d9b6cd94STrond Myklebust 685d9b6cd94STrond Myklebust void rpc_exit(struct rpc_task *task, int status) 686d9b6cd94STrond Myklebust { 687d9b6cd94STrond Myklebust task->tk_status = status; 688d9b6cd94STrond Myklebust task->tk_action = rpc_exit_task; 689d9b6cd94STrond Myklebust if (RPC_IS_QUEUED(task)) 690d9b6cd94STrond Myklebust rpc_wake_up_queued_task(task->tk_waitqueue, task); 691d9b6cd94STrond Myklebust } 692d9b6cd94STrond Myklebust EXPORT_SYMBOL_GPL(rpc_exit); 693d05fdb0cSTrond Myklebust 694bbd5a1f9STrond Myklebust void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) 695bbd5a1f9STrond Myklebust { 696a86dc496STrond Myklebust if (ops->rpc_release != NULL) 697bbd5a1f9STrond Myklebust ops->rpc_release(calldata); 698bbd5a1f9STrond Myklebust } 699bbd5a1f9STrond Myklebust 700d05fdb0cSTrond Myklebust /* 7011da177e4SLinus Torvalds * This is the RPC `scheduler' (or rather, the finite state machine). 7021da177e4SLinus Torvalds */ 7032efef837STrond Myklebust static void __rpc_execute(struct rpc_task *task) 7041da177e4SLinus Torvalds { 705eb9b55abSTrond Myklebust struct rpc_wait_queue *queue; 706eb9b55abSTrond Myklebust int task_is_async = RPC_IS_ASYNC(task); 7071da177e4SLinus Torvalds int status = 0; 7081da177e4SLinus Torvalds 70946121cf7SChuck Lever dprintk("RPC: %5u __rpc_execute flags=0x%x\n", 7101da177e4SLinus Torvalds task->tk_pid, task->tk_flags); 7111da177e4SLinus Torvalds 7122bd4eef8SWeston Andros Adamson WARN_ON_ONCE(RPC_IS_QUEUED(task)); 7132bd4eef8SWeston Andros Adamson if (RPC_IS_QUEUED(task)) 7142bd4eef8SWeston Andros Adamson return; 7151da177e4SLinus Torvalds 716d05fdb0cSTrond Myklebust for (;;) { 717b55c5989STrond Myklebust void (*do_action)(struct rpc_task *); 7181da177e4SLinus Torvalds 7191da177e4SLinus Torvalds /* 720b55c5989STrond Myklebust * Execute any pending callback first. 7211da177e4SLinus Torvalds */ 722b55c5989STrond Myklebust do_action = task->tk_callback; 7231da177e4SLinus Torvalds task->tk_callback = NULL; 724b55c5989STrond Myklebust if (do_action == NULL) { 7251da177e4SLinus Torvalds /* 7261da177e4SLinus Torvalds * Perform the next FSM step. 727b55c5989STrond Myklebust * tk_action may be NULL if the task has been killed. 728b55c5989STrond Myklebust * In particular, note that rpc_killall_tasks may 729b55c5989STrond Myklebust * do this at any time, so beware when dereferencing. 7301da177e4SLinus Torvalds */ 731b55c5989STrond Myklebust do_action = task->tk_action; 732b55c5989STrond Myklebust if (do_action == NULL) 733abbcf28fSTrond Myklebust break; 7341da177e4SLinus Torvalds } 73582b0a4c3STrond Myklebust trace_rpc_task_run_action(task->tk_client, task, task->tk_action); 736b55c5989STrond Myklebust do_action(task); 7371da177e4SLinus Torvalds 7381da177e4SLinus Torvalds /* 7391da177e4SLinus Torvalds * Lockless check for whether task is sleeping or not. 7401da177e4SLinus Torvalds */ 7411da177e4SLinus Torvalds if (!RPC_IS_QUEUED(task)) 7421da177e4SLinus Torvalds continue; 743eb9b55abSTrond Myklebust /* 744eb9b55abSTrond Myklebust * The queue->lock protects against races with 745eb9b55abSTrond Myklebust * rpc_make_runnable(). 746eb9b55abSTrond Myklebust * 747eb9b55abSTrond Myklebust * Note that once we clear RPC_TASK_RUNNING on an asynchronous 748eb9b55abSTrond Myklebust * rpc_task, rpc_make_runnable() can assign it to a 749eb9b55abSTrond Myklebust * different workqueue. We therefore cannot assume that the 750eb9b55abSTrond Myklebust * rpc_task pointer may still be dereferenced. 751eb9b55abSTrond Myklebust */ 752eb9b55abSTrond Myklebust queue = task->tk_waitqueue; 753eb9b55abSTrond Myklebust spin_lock_bh(&queue->lock); 754eb9b55abSTrond Myklebust if (!RPC_IS_QUEUED(task)) { 755eb9b55abSTrond Myklebust spin_unlock_bh(&queue->lock); 7561da177e4SLinus Torvalds continue; 7571da177e4SLinus Torvalds } 758eb9b55abSTrond Myklebust rpc_clear_running(task); 759eb9b55abSTrond Myklebust spin_unlock_bh(&queue->lock); 760eb9b55abSTrond Myklebust if (task_is_async) 761eb9b55abSTrond Myklebust return; 7621da177e4SLinus Torvalds 7631da177e4SLinus Torvalds /* sync task: sleep here */ 76446121cf7SChuck Lever dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid); 76596651ab3STrond Myklebust status = out_of_line_wait_on_bit(&task->tk_runstate, 766150030b7SMatthew Wilcox RPC_TASK_QUEUED, rpc_wait_bit_killable, 767150030b7SMatthew Wilcox TASK_KILLABLE); 76896651ab3STrond Myklebust if (status == -ERESTARTSYS) { 7691da177e4SLinus Torvalds /* 7701da177e4SLinus Torvalds * When a sync task receives a signal, it exits with 7711da177e4SLinus Torvalds * -ERESTARTSYS. In order to catch any callbacks that 7721da177e4SLinus Torvalds * clean up after sleeping on some queue, we don't 7731da177e4SLinus Torvalds * break the loop here, but go around once more. 7741da177e4SLinus Torvalds */ 77546121cf7SChuck Lever dprintk("RPC: %5u got signal\n", task->tk_pid); 7761da177e4SLinus Torvalds task->tk_flags |= RPC_TASK_KILLED; 7771da177e4SLinus Torvalds rpc_exit(task, -ERESTARTSYS); 7781da177e4SLinus Torvalds } 7791da177e4SLinus Torvalds rpc_set_running(task); 78046121cf7SChuck Lever dprintk("RPC: %5u sync task resuming\n", task->tk_pid); 7811da177e4SLinus Torvalds } 7821da177e4SLinus Torvalds 78346121cf7SChuck Lever dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status, 78446121cf7SChuck Lever task->tk_status); 7851da177e4SLinus Torvalds /* Release all resources associated with the task */ 7861da177e4SLinus Torvalds rpc_release_task(task); 7871da177e4SLinus Torvalds } 7881da177e4SLinus Torvalds 7891da177e4SLinus Torvalds /* 7901da177e4SLinus Torvalds * User-visible entry point to the scheduler. 7911da177e4SLinus Torvalds * 7921da177e4SLinus Torvalds * This may be called recursively if e.g. an async NFS task updates 7931da177e4SLinus Torvalds * the attributes and finds that dirty pages must be flushed. 7941da177e4SLinus Torvalds * NOTE: Upon exit of this function the task is guaranteed to be 7951da177e4SLinus Torvalds * released. In particular note that tk_release() will have 7961da177e4SLinus Torvalds * been called, so your task memory may have been freed. 7971da177e4SLinus Torvalds */ 7982efef837STrond Myklebust void rpc_execute(struct rpc_task *task) 7991da177e4SLinus Torvalds { 80044c28873STrond Myklebust rpc_set_active(task); 801d6a1ed08STrond Myklebust rpc_make_runnable(task); 802d6a1ed08STrond Myklebust if (!RPC_IS_ASYNC(task)) 8032efef837STrond Myklebust __rpc_execute(task); 8041da177e4SLinus Torvalds } 8051da177e4SLinus Torvalds 80665f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *work) 8071da177e4SLinus Torvalds { 8085cf02d09SJeff Layton current->flags |= PF_FSTRANS; 80965f27f38SDavid Howells __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); 8105cf02d09SJeff Layton current->flags &= ~PF_FSTRANS; 8111da177e4SLinus Torvalds } 8121da177e4SLinus Torvalds 81302107148SChuck Lever /** 81402107148SChuck Lever * rpc_malloc - allocate an RPC buffer 81502107148SChuck Lever * @task: RPC task that will use this buffer 81602107148SChuck Lever * @size: requested byte size 8171da177e4SLinus Torvalds * 818c5a4dd8bSChuck Lever * To prevent rpciod from hanging, this allocator never sleeps, 819c5a4dd8bSChuck Lever * returning NULL if the request cannot be serviced immediately. 820c5a4dd8bSChuck Lever * The caller can arrange to sleep in a way that is safe for rpciod. 821c5a4dd8bSChuck Lever * 822c5a4dd8bSChuck Lever * Most requests are 'small' (under 2KiB) and can be serviced from a 823c5a4dd8bSChuck Lever * mempool, ensuring that NFS reads and writes can always proceed, 824c5a4dd8bSChuck Lever * and that there is good locality of reference for these buffers. 825c5a4dd8bSChuck Lever * 8261da177e4SLinus Torvalds * In order to avoid memory starvation triggering more writebacks of 827c5a4dd8bSChuck Lever * NFS requests, we avoid using GFP_KERNEL. 8281da177e4SLinus Torvalds */ 82902107148SChuck Lever void *rpc_malloc(struct rpc_task *task, size_t size) 8301da177e4SLinus Torvalds { 831aa3d1faeSChuck Lever struct rpc_buffer *buf; 832a564b8f0SMel Gorman gfp_t gfp = GFP_NOWAIT; 833a564b8f0SMel Gorman 834a564b8f0SMel Gorman if (RPC_IS_SWAPPER(task)) 835a564b8f0SMel Gorman gfp |= __GFP_MEMALLOC; 8361da177e4SLinus Torvalds 837aa3d1faeSChuck Lever size += sizeof(struct rpc_buffer); 838c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 839c5a4dd8bSChuck Lever buf = mempool_alloc(rpc_buffer_mempool, gfp); 8401da177e4SLinus Torvalds else 841c5a4dd8bSChuck Lever buf = kmalloc(size, gfp); 842ddce40dfSPeter Zijlstra 843ddce40dfSPeter Zijlstra if (!buf) 844ddce40dfSPeter Zijlstra return NULL; 845ddce40dfSPeter Zijlstra 846aa3d1faeSChuck Lever buf->len = size; 847215d0678SGeert Uytterhoeven dprintk("RPC: %5u allocated buffer of size %zu at %p\n", 848c5a4dd8bSChuck Lever task->tk_pid, size, buf); 849aa3d1faeSChuck Lever return &buf->data; 8501da177e4SLinus Torvalds } 85112444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_malloc); 8521da177e4SLinus Torvalds 85302107148SChuck Lever /** 85402107148SChuck Lever * rpc_free - free buffer allocated via rpc_malloc 855c5a4dd8bSChuck Lever * @buffer: buffer to free 85602107148SChuck Lever * 85702107148SChuck Lever */ 858c5a4dd8bSChuck Lever void rpc_free(void *buffer) 8591da177e4SLinus Torvalds { 860aa3d1faeSChuck Lever size_t size; 861aa3d1faeSChuck Lever struct rpc_buffer *buf; 86202107148SChuck Lever 863c5a4dd8bSChuck Lever if (!buffer) 864c5a4dd8bSChuck Lever return; 865aa3d1faeSChuck Lever 866aa3d1faeSChuck Lever buf = container_of(buffer, struct rpc_buffer, data); 867aa3d1faeSChuck Lever size = buf->len; 868c5a4dd8bSChuck Lever 869215d0678SGeert Uytterhoeven dprintk("RPC: freeing buffer of size %zu at %p\n", 870c5a4dd8bSChuck Lever size, buf); 871aa3d1faeSChuck Lever 872c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 873c5a4dd8bSChuck Lever mempool_free(buf, rpc_buffer_mempool); 8741da177e4SLinus Torvalds else 875c5a4dd8bSChuck Lever kfree(buf); 8761da177e4SLinus Torvalds } 87712444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_free); 8781da177e4SLinus Torvalds 8791da177e4SLinus Torvalds /* 8801da177e4SLinus Torvalds * Creation and deletion of RPC task structures 8811da177e4SLinus Torvalds */ 88247fe0648STrond Myklebust static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) 8831da177e4SLinus Torvalds { 8841da177e4SLinus Torvalds memset(task, 0, sizeof(*task)); 88544c28873STrond Myklebust atomic_set(&task->tk_count, 1); 88684115e1cSTrond Myklebust task->tk_flags = task_setup_data->flags; 88784115e1cSTrond Myklebust task->tk_ops = task_setup_data->callback_ops; 88884115e1cSTrond Myklebust task->tk_calldata = task_setup_data->callback_data; 8896529eba0STrond Myklebust INIT_LIST_HEAD(&task->tk_task); 8901da177e4SLinus Torvalds 8913ff7576dSTrond Myklebust task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; 8923ff7576dSTrond Myklebust task->tk_owner = current->tgid; 8931da177e4SLinus Torvalds 8941da177e4SLinus Torvalds /* Initialize workqueue for async tasks */ 89532bfb5c0STrond Myklebust task->tk_workqueue = task_setup_data->workqueue; 8961da177e4SLinus Torvalds 89784115e1cSTrond Myklebust if (task->tk_ops->rpc_call_prepare != NULL) 89884115e1cSTrond Myklebust task->tk_action = rpc_prepare_task; 899963d8fe5STrond Myklebust 9007fdcf13bSTrond Myklebust rpc_init_task_statistics(task); 901ef759a2eSChuck Lever 90246121cf7SChuck Lever dprintk("RPC: new task initialized, procpid %u\n", 903ba25f9dcSPavel Emelyanov task_pid_nr(current)); 9041da177e4SLinus Torvalds } 9051da177e4SLinus Torvalds 9061da177e4SLinus Torvalds static struct rpc_task * 9071da177e4SLinus Torvalds rpc_alloc_task(void) 9081da177e4SLinus Torvalds { 909a564b8f0SMel Gorman return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOIO); 9101da177e4SLinus Torvalds } 9111da177e4SLinus Torvalds 9121da177e4SLinus Torvalds /* 91390c5755fSTrond Myklebust * Create a new task for the specified client. 9141da177e4SLinus Torvalds */ 91584115e1cSTrond Myklebust struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data) 9161da177e4SLinus Torvalds { 917e8f5d77cSTrond Myklebust struct rpc_task *task = setup_data->task; 918e8f5d77cSTrond Myklebust unsigned short flags = 0; 9191da177e4SLinus Torvalds 920e8f5d77cSTrond Myklebust if (task == NULL) { 9211da177e4SLinus Torvalds task = rpc_alloc_task(); 92219445b99STrond Myklebust if (task == NULL) { 92319445b99STrond Myklebust rpc_release_calldata(setup_data->callback_ops, 92419445b99STrond Myklebust setup_data->callback_data); 92519445b99STrond Myklebust return ERR_PTR(-ENOMEM); 92619445b99STrond Myklebust } 927e8f5d77cSTrond Myklebust flags = RPC_TASK_DYNAMIC; 928e8f5d77cSTrond Myklebust } 9291da177e4SLinus Torvalds 93084115e1cSTrond Myklebust rpc_init_task(task, setup_data); 931e8f5d77cSTrond Myklebust task->tk_flags |= flags; 93246121cf7SChuck Lever dprintk("RPC: allocated task %p\n", task); 9331da177e4SLinus Torvalds return task; 9341da177e4SLinus Torvalds } 9351da177e4SLinus Torvalds 93632bfb5c0STrond Myklebust static void rpc_free_task(struct rpc_task *task) 9371da177e4SLinus Torvalds { 938963d8fe5STrond Myklebust const struct rpc_call_ops *tk_ops = task->tk_ops; 939963d8fe5STrond Myklebust void *calldata = task->tk_calldata; 9401da177e4SLinus Torvalds 9415e4424afSTrond Myklebust if (task->tk_flags & RPC_TASK_DYNAMIC) { 9425e4424afSTrond Myklebust dprintk("RPC: %5u freeing task\n", task->tk_pid); 9435e4424afSTrond Myklebust mempool_free(task, rpc_task_mempool); 9445e4424afSTrond Myklebust } 94532bfb5c0STrond Myklebust rpc_release_calldata(tk_ops, calldata); 94632bfb5c0STrond Myklebust } 94732bfb5c0STrond Myklebust 94832bfb5c0STrond Myklebust static void rpc_async_release(struct work_struct *work) 94932bfb5c0STrond Myklebust { 95032bfb5c0STrond Myklebust rpc_free_task(container_of(work, struct rpc_task, u.tk_work)); 95132bfb5c0STrond Myklebust } 95232bfb5c0STrond Myklebust 953bf294b41STrond Myklebust static void rpc_release_resources_task(struct rpc_task *task) 95432bfb5c0STrond Myklebust { 955e6b3c4dbSTrond Myklebust if (task->tk_rqstp) 956e6b3c4dbSTrond Myklebust xprt_release(task); 957a271c5a0SOGAWA Hirofumi if (task->tk_msg.rpc_cred) { 958a17c2153STrond Myklebust put_rpccred(task->tk_msg.rpc_cred); 959a271c5a0SOGAWA Hirofumi task->tk_msg.rpc_cred = NULL; 960a271c5a0SOGAWA Hirofumi } 96158f9612cSTrond Myklebust rpc_task_release_client(task); 962bf294b41STrond Myklebust } 963bf294b41STrond Myklebust 964bf294b41STrond Myklebust static void rpc_final_put_task(struct rpc_task *task, 965bf294b41STrond Myklebust struct workqueue_struct *q) 966bf294b41STrond Myklebust { 967bf294b41STrond Myklebust if (q != NULL) { 96832bfb5c0STrond Myklebust INIT_WORK(&task->u.tk_work, rpc_async_release); 969bf294b41STrond Myklebust queue_work(q, &task->u.tk_work); 97032bfb5c0STrond Myklebust } else 97132bfb5c0STrond Myklebust rpc_free_task(task); 972e6b3c4dbSTrond Myklebust } 973bf294b41STrond Myklebust 974bf294b41STrond Myklebust static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) 975bf294b41STrond Myklebust { 976bf294b41STrond Myklebust if (atomic_dec_and_test(&task->tk_count)) { 977bf294b41STrond Myklebust rpc_release_resources_task(task); 978bf294b41STrond Myklebust rpc_final_put_task(task, q); 979bf294b41STrond Myklebust } 980bf294b41STrond Myklebust } 981bf294b41STrond Myklebust 982bf294b41STrond Myklebust void rpc_put_task(struct rpc_task *task) 983bf294b41STrond Myklebust { 984bf294b41STrond Myklebust rpc_do_put_task(task, NULL); 985bf294b41STrond Myklebust } 986e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_put_task); 987e6b3c4dbSTrond Myklebust 988bf294b41STrond Myklebust void rpc_put_task_async(struct rpc_task *task) 989bf294b41STrond Myklebust { 990bf294b41STrond Myklebust rpc_do_put_task(task, task->tk_workqueue); 991bf294b41STrond Myklebust } 992bf294b41STrond Myklebust EXPORT_SYMBOL_GPL(rpc_put_task_async); 993bf294b41STrond Myklebust 994bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task) 995e6b3c4dbSTrond Myklebust { 99646121cf7SChuck Lever dprintk("RPC: %5u release task\n", task->tk_pid); 9971da177e4SLinus Torvalds 998*0a0c2a57SWeston Andros Adamson WARN_ON_ONCE(RPC_IS_QUEUED(task)); 9991da177e4SLinus Torvalds 1000bf294b41STrond Myklebust rpc_release_resources_task(task); 1001e6b3c4dbSTrond Myklebust 1002bf294b41STrond Myklebust /* 1003bf294b41STrond Myklebust * Note: at this point we have been removed from rpc_clnt->cl_tasks, 1004bf294b41STrond Myklebust * so it should be safe to use task->tk_count as a test for whether 1005bf294b41STrond Myklebust * or not any other processes still hold references to our rpc_task. 1006bf294b41STrond Myklebust */ 1007bf294b41STrond Myklebust if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { 1008bf294b41STrond Myklebust /* Wake up anyone who may be waiting for task completion */ 1009bf294b41STrond Myklebust if (!rpc_complete_task(task)) 1010bf294b41STrond Myklebust return; 1011bf294b41STrond Myklebust } else { 1012bf294b41STrond Myklebust if (!atomic_dec_and_test(&task->tk_count)) 1013bf294b41STrond Myklebust return; 1014bf294b41STrond Myklebust } 1015bf294b41STrond Myklebust rpc_final_put_task(task, task->tk_workqueue); 10161da177e4SLinus Torvalds } 10171da177e4SLinus Torvalds 1018b247bbf1STrond Myklebust int rpciod_up(void) 1019b247bbf1STrond Myklebust { 1020b247bbf1STrond Myklebust return try_module_get(THIS_MODULE) ? 0 : -EINVAL; 1021b247bbf1STrond Myklebust } 1022b247bbf1STrond Myklebust 1023b247bbf1STrond Myklebust void rpciod_down(void) 1024b247bbf1STrond Myklebust { 1025b247bbf1STrond Myklebust module_put(THIS_MODULE); 1026b247bbf1STrond Myklebust } 1027b247bbf1STrond Myklebust 10281da177e4SLinus Torvalds /* 1029b247bbf1STrond Myklebust * Start up the rpciod workqueue. 10301da177e4SLinus Torvalds */ 1031b247bbf1STrond Myklebust static int rpciod_start(void) 10321da177e4SLinus Torvalds { 10331da177e4SLinus Torvalds struct workqueue_struct *wq; 10341da177e4SLinus Torvalds 10351da177e4SLinus Torvalds /* 10361da177e4SLinus Torvalds * Create the rpciod thread and wait for it to start. 10371da177e4SLinus Torvalds */ 1038ab418d70STrond Myklebust dprintk("RPC: creating workqueue rpciod\n"); 10399b96ce71STrond Myklebust wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM, 1); 10401da177e4SLinus Torvalds rpciod_workqueue = wq; 1041b247bbf1STrond Myklebust return rpciod_workqueue != NULL; 10421da177e4SLinus Torvalds } 10431da177e4SLinus Torvalds 1044b247bbf1STrond Myklebust static void rpciod_stop(void) 10451da177e4SLinus Torvalds { 1046b247bbf1STrond Myklebust struct workqueue_struct *wq = NULL; 1047ab418d70STrond Myklebust 1048b247bbf1STrond Myklebust if (rpciod_workqueue == NULL) 1049b247bbf1STrond Myklebust return; 1050ab418d70STrond Myklebust dprintk("RPC: destroying workqueue rpciod\n"); 10511da177e4SLinus Torvalds 1052b247bbf1STrond Myklebust wq = rpciod_workqueue; 10531da177e4SLinus Torvalds rpciod_workqueue = NULL; 1054b247bbf1STrond Myklebust destroy_workqueue(wq); 10551da177e4SLinus Torvalds } 10561da177e4SLinus Torvalds 10571da177e4SLinus Torvalds void 10581da177e4SLinus Torvalds rpc_destroy_mempool(void) 10591da177e4SLinus Torvalds { 1060b247bbf1STrond Myklebust rpciod_stop(); 10611da177e4SLinus Torvalds if (rpc_buffer_mempool) 10621da177e4SLinus Torvalds mempool_destroy(rpc_buffer_mempool); 10631da177e4SLinus Torvalds if (rpc_task_mempool) 10641da177e4SLinus Torvalds mempool_destroy(rpc_task_mempool); 10651a1d92c1SAlexey Dobriyan if (rpc_task_slabp) 10661a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_task_slabp); 10671a1d92c1SAlexey Dobriyan if (rpc_buffer_slabp) 10681a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_buffer_slabp); 1069f6a1cc89STrond Myklebust rpc_destroy_wait_queue(&delay_queue); 10701da177e4SLinus Torvalds } 10711da177e4SLinus Torvalds 10721da177e4SLinus Torvalds int 10731da177e4SLinus Torvalds rpc_init_mempool(void) 10741da177e4SLinus Torvalds { 1075f6a1cc89STrond Myklebust /* 1076f6a1cc89STrond Myklebust * The following is not strictly a mempool initialisation, 1077f6a1cc89STrond Myklebust * but there is no harm in doing it here 1078f6a1cc89STrond Myklebust */ 1079f6a1cc89STrond Myklebust rpc_init_wait_queue(&delay_queue, "delayq"); 1080f6a1cc89STrond Myklebust if (!rpciod_start()) 1081f6a1cc89STrond Myklebust goto err_nomem; 1082f6a1cc89STrond Myklebust 10831da177e4SLinus Torvalds rpc_task_slabp = kmem_cache_create("rpc_tasks", 10841da177e4SLinus Torvalds sizeof(struct rpc_task), 10851da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 108620c2df83SPaul Mundt NULL); 10871da177e4SLinus Torvalds if (!rpc_task_slabp) 10881da177e4SLinus Torvalds goto err_nomem; 10891da177e4SLinus Torvalds rpc_buffer_slabp = kmem_cache_create("rpc_buffers", 10901da177e4SLinus Torvalds RPC_BUFFER_MAXSIZE, 10911da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 109220c2df83SPaul Mundt NULL); 10931da177e4SLinus Torvalds if (!rpc_buffer_slabp) 10941da177e4SLinus Torvalds goto err_nomem; 109593d2341cSMatthew Dobson rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, 10961da177e4SLinus Torvalds rpc_task_slabp); 10971da177e4SLinus Torvalds if (!rpc_task_mempool) 10981da177e4SLinus Torvalds goto err_nomem; 109993d2341cSMatthew Dobson rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, 11001da177e4SLinus Torvalds rpc_buffer_slabp); 11011da177e4SLinus Torvalds if (!rpc_buffer_mempool) 11021da177e4SLinus Torvalds goto err_nomem; 11031da177e4SLinus Torvalds return 0; 11041da177e4SLinus Torvalds err_nomem: 11051da177e4SLinus Torvalds rpc_destroy_mempool(); 11061da177e4SLinus Torvalds return -ENOMEM; 11071da177e4SLinus Torvalds } 1108