1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 21da177e4SLinus Torvalds /* 31da177e4SLinus Torvalds * linux/net/sunrpc/sched.c 41da177e4SLinus Torvalds * 51da177e4SLinus Torvalds * Scheduling for synchronous and asynchronous RPC requests. 61da177e4SLinus Torvalds * 71da177e4SLinus Torvalds * Copyright (C) 1996 Olaf Kirch, <okir@monad.swb.de> 81da177e4SLinus Torvalds * 91da177e4SLinus Torvalds * TCP NFS related read + write fixes 101da177e4SLinus Torvalds * (C) 1999 Dave Airlie, University of Limerick, Ireland <airlied@linux.ie> 111da177e4SLinus Torvalds */ 121da177e4SLinus Torvalds 131da177e4SLinus Torvalds #include <linux/module.h> 141da177e4SLinus Torvalds 151da177e4SLinus Torvalds #include <linux/sched.h> 161da177e4SLinus Torvalds #include <linux/interrupt.h> 171da177e4SLinus Torvalds #include <linux/slab.h> 181da177e4SLinus Torvalds #include <linux/mempool.h> 191da177e4SLinus Torvalds #include <linux/smp.h> 201da177e4SLinus Torvalds #include <linux/spinlock.h> 214a3e2f71SArjan van de Ven #include <linux/mutex.h> 22d310310cSJeff Layton #include <linux/freezer.h> 23a1231fdaSTrond Myklebust #include <linux/sched/mm.h> 241da177e4SLinus Torvalds 251da177e4SLinus Torvalds #include <linux/sunrpc/clnt.h> 269dfe52a9SDave Wysochanski #include <linux/sunrpc/metrics.h> 271da177e4SLinus Torvalds 286951867bSBenny Halevy #include "sunrpc.h" 296951867bSBenny Halevy 3082b0a4c3STrond Myklebust #define CREATE_TRACE_POINTS 3182b0a4c3STrond Myklebust #include <trace/events/sunrpc.h> 3282b0a4c3STrond Myklebust 331da177e4SLinus Torvalds /* 341da177e4SLinus Torvalds * RPC slabs and memory pools 351da177e4SLinus Torvalds */ 361da177e4SLinus Torvalds #define RPC_BUFFER_MAXSIZE (2048) 371da177e4SLinus Torvalds #define RPC_BUFFER_POOLSIZE (8) 381da177e4SLinus Torvalds #define RPC_TASK_POOLSIZE (8) 39e18b890bSChristoph Lameter static struct kmem_cache *rpc_task_slabp __read_mostly; 40e18b890bSChristoph Lameter static struct kmem_cache *rpc_buffer_slabp __read_mostly; 41ba89966cSEric Dumazet static mempool_t *rpc_task_mempool __read_mostly; 42ba89966cSEric Dumazet static mempool_t *rpc_buffer_mempool __read_mostly; 431da177e4SLinus Torvalds 4465f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *); 45bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task); 467e0a0e38STrond Myklebust static void __rpc_queue_timer_fn(struct work_struct *); 471da177e4SLinus Torvalds 481da177e4SLinus Torvalds /* 491da177e4SLinus Torvalds * RPC tasks sit here while waiting for conditions to improve. 501da177e4SLinus Torvalds */ 51a4a87499STrond Myklebust static struct rpc_wait_queue delay_queue; 521da177e4SLinus Torvalds 531da177e4SLinus Torvalds /* 541da177e4SLinus Torvalds * rpciod-related stuff 551da177e4SLinus Torvalds */ 5640a5f1b1STrond Myklebust struct workqueue_struct *rpciod_workqueue __read_mostly; 5740a5f1b1STrond Myklebust struct workqueue_struct *xprtiod_workqueue __read_mostly; 58675dd90aSChuck Lever EXPORT_SYMBOL_GPL(xprtiod_workqueue); 591da177e4SLinus Torvalds 605efd1876STrond Myklebust unsigned long 615efd1876STrond Myklebust rpc_task_timeout(const struct rpc_task *task) 625efd1876STrond Myklebust { 635efd1876STrond Myklebust unsigned long timeout = READ_ONCE(task->tk_timeout); 645efd1876STrond Myklebust 655efd1876STrond Myklebust if (timeout != 0) { 665efd1876STrond Myklebust unsigned long now = jiffies; 675efd1876STrond Myklebust if (time_before(now, timeout)) 685efd1876STrond Myklebust return timeout - now; 695efd1876STrond Myklebust } 705efd1876STrond Myklebust return 0; 715efd1876STrond Myklebust } 725efd1876STrond Myklebust EXPORT_SYMBOL_GPL(rpc_task_timeout); 735efd1876STrond Myklebust 741da177e4SLinus Torvalds /* 751da177e4SLinus Torvalds * Disable the timer for a given RPC task. Should be called with 761da177e4SLinus Torvalds * queue->lock and bh_disabled in order to avoid races within 771da177e4SLinus Torvalds * rpc_run_timer(). 781da177e4SLinus Torvalds */ 795d00837bSTrond Myklebust static void 80eb276c0eSTrond Myklebust __rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task) 811da177e4SLinus Torvalds { 826b2e6856STrond Myklebust if (list_empty(&task->u.tk_wait.timer_list)) 8336df9aaeSTrond Myklebust return; 841da177e4SLinus Torvalds task->tk_timeout = 0; 8536df9aaeSTrond Myklebust list_del(&task->u.tk_wait.timer_list); 86eb276c0eSTrond Myklebust if (list_empty(&queue->timer_list.list)) 877e0a0e38STrond Myklebust cancel_delayed_work(&queue->timer_list.dwork); 8836df9aaeSTrond Myklebust } 8936df9aaeSTrond Myklebust 9036df9aaeSTrond Myklebust static void 9136df9aaeSTrond Myklebust rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires) 9236df9aaeSTrond Myklebust { 937e0a0e38STrond Myklebust unsigned long now = jiffies; 947e0a0e38STrond Myklebust queue->timer_list.expires = expires; 957e0a0e38STrond Myklebust if (time_before_eq(expires, now)) 967e0a0e38STrond Myklebust expires = 0; 977e0a0e38STrond Myklebust else 987e0a0e38STrond Myklebust expires -= now; 997e0a0e38STrond Myklebust mod_delayed_work(rpciod_workqueue, &queue->timer_list.dwork, expires); 1001da177e4SLinus Torvalds } 1011da177e4SLinus Torvalds 1021da177e4SLinus Torvalds /* 1031da177e4SLinus Torvalds * Set up a timer for the current task. 1041da177e4SLinus Torvalds */ 1055d00837bSTrond Myklebust static void 1066b2e6856STrond Myklebust __rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task, 1076b2e6856STrond Myklebust unsigned long timeout) 1081da177e4SLinus Torvalds { 1096b2e6856STrond Myklebust task->tk_timeout = timeout; 1107e0a0e38STrond Myklebust if (list_empty(&queue->timer_list.list) || time_before(timeout, queue->timer_list.expires)) 1116b2e6856STrond Myklebust rpc_set_queue_timer(queue, timeout); 112eb276c0eSTrond Myklebust list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list); 1131da177e4SLinus Torvalds } 1141da177e4SLinus Torvalds 115c05eecf6STrond Myklebust static void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority) 116c05eecf6STrond Myklebust { 117edd2e36fSTrond Myklebust if (queue->priority != priority) { 118c05eecf6STrond Myklebust queue->priority = priority; 119f42f7c28STrond Myklebust queue->nr = 1U << priority; 120c05eecf6STrond Myklebust } 121edd2e36fSTrond Myklebust } 122c05eecf6STrond Myklebust 123c05eecf6STrond Myklebust static void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue) 124c05eecf6STrond Myklebust { 125c05eecf6STrond Myklebust rpc_set_waitqueue_priority(queue, queue->maxpriority); 126f42f7c28STrond Myklebust } 127f42f7c28STrond Myklebust 128f42f7c28STrond Myklebust /* 129f42f7c28STrond Myklebust * Add a request to a queue list 130f42f7c28STrond Myklebust */ 131f42f7c28STrond Myklebust static void 132f42f7c28STrond Myklebust __rpc_list_enqueue_task(struct list_head *q, struct rpc_task *task) 133f42f7c28STrond Myklebust { 134f42f7c28STrond Myklebust struct rpc_task *t; 135f42f7c28STrond Myklebust 136f42f7c28STrond Myklebust list_for_each_entry(t, q, u.tk_wait.list) { 137f42f7c28STrond Myklebust if (t->tk_owner == task->tk_owner) { 138f42f7c28STrond Myklebust list_add_tail(&task->u.tk_wait.links, 139f42f7c28STrond Myklebust &t->u.tk_wait.links); 140f42f7c28STrond Myklebust /* Cache the queue head in task->u.tk_wait.list */ 141f42f7c28STrond Myklebust task->u.tk_wait.list.next = q; 142f42f7c28STrond Myklebust task->u.tk_wait.list.prev = NULL; 143f42f7c28STrond Myklebust return; 144f42f7c28STrond Myklebust } 145f42f7c28STrond Myklebust } 146f42f7c28STrond Myklebust INIT_LIST_HEAD(&task->u.tk_wait.links); 147f42f7c28STrond Myklebust list_add_tail(&task->u.tk_wait.list, q); 148f42f7c28STrond Myklebust } 149f42f7c28STrond Myklebust 150f42f7c28STrond Myklebust /* 151f42f7c28STrond Myklebust * Remove request from a queue list 152f42f7c28STrond Myklebust */ 153f42f7c28STrond Myklebust static void 154f42f7c28STrond Myklebust __rpc_list_dequeue_task(struct rpc_task *task) 155f42f7c28STrond Myklebust { 156f42f7c28STrond Myklebust struct list_head *q; 157f42f7c28STrond Myklebust struct rpc_task *t; 158f42f7c28STrond Myklebust 159f42f7c28STrond Myklebust if (task->u.tk_wait.list.prev == NULL) { 160f42f7c28STrond Myklebust list_del(&task->u.tk_wait.links); 161f42f7c28STrond Myklebust return; 162f42f7c28STrond Myklebust } 163f42f7c28STrond Myklebust if (!list_empty(&task->u.tk_wait.links)) { 164f42f7c28STrond Myklebust t = list_first_entry(&task->u.tk_wait.links, 165f42f7c28STrond Myklebust struct rpc_task, 166f42f7c28STrond Myklebust u.tk_wait.links); 167f42f7c28STrond Myklebust /* Assume __rpc_list_enqueue_task() cached the queue head */ 168f42f7c28STrond Myklebust q = t->u.tk_wait.list.next; 169f42f7c28STrond Myklebust list_add_tail(&t->u.tk_wait.list, q); 170f42f7c28STrond Myklebust list_del(&task->u.tk_wait.links); 171f42f7c28STrond Myklebust } 172f42f7c28STrond Myklebust list_del(&task->u.tk_wait.list); 173c05eecf6STrond Myklebust } 174c05eecf6STrond Myklebust 1751da177e4SLinus Torvalds /* 1761da177e4SLinus Torvalds * Add new request to a priority queue. 1771da177e4SLinus Torvalds */ 1783b27bad7STrond Myklebust static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, 1793b27bad7STrond Myklebust struct rpc_task *task, 1803b27bad7STrond Myklebust unsigned char queue_priority) 1811da177e4SLinus Torvalds { 1823b27bad7STrond Myklebust if (unlikely(queue_priority > queue->maxpriority)) 183c05eecf6STrond Myklebust queue_priority = queue->maxpriority; 184f42f7c28STrond Myklebust __rpc_list_enqueue_task(&queue->tasks[queue_priority], task); 1851da177e4SLinus Torvalds } 1861da177e4SLinus Torvalds 1871da177e4SLinus Torvalds /* 1881da177e4SLinus Torvalds * Add new request to wait queue. 1891da177e4SLinus Torvalds * 1901da177e4SLinus Torvalds * Swapper tasks always get inserted at the head of the queue. 1911da177e4SLinus Torvalds * This should avoid many nasty memory deadlocks and hopefully 1921da177e4SLinus Torvalds * improve overall performance. 1931da177e4SLinus Torvalds * Everyone else gets appended to the queue to ensure proper FIFO behavior. 1941da177e4SLinus Torvalds */ 1953b27bad7STrond Myklebust static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, 1963b27bad7STrond Myklebust struct rpc_task *task, 1973b27bad7STrond Myklebust unsigned char queue_priority) 1981da177e4SLinus Torvalds { 1996b2e6856STrond Myklebust INIT_LIST_HEAD(&task->u.tk_wait.timer_list); 2001da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 2013b27bad7STrond Myklebust __rpc_add_wait_queue_priority(queue, task, queue_priority); 2021da177e4SLinus Torvalds else if (RPC_IS_SWAPPER(task)) 2031da177e4SLinus Torvalds list_add(&task->u.tk_wait.list, &queue->tasks[0]); 2041da177e4SLinus Torvalds else 2051da177e4SLinus Torvalds list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]); 20696ef13b2STrond Myklebust task->tk_waitqueue = queue; 207e19b63daSChuck Lever queue->qlen++; 2081166fde6STrond Myklebust /* barrier matches the read in rpc_wake_up_task_queue_locked() */ 2091166fde6STrond Myklebust smp_wmb(); 2101da177e4SLinus Torvalds rpc_set_queued(task); 2111da177e4SLinus Torvalds } 2121da177e4SLinus Torvalds 2131da177e4SLinus Torvalds /* 2141da177e4SLinus Torvalds * Remove request from a priority queue. 2151da177e4SLinus Torvalds */ 2161da177e4SLinus Torvalds static void __rpc_remove_wait_queue_priority(struct rpc_task *task) 2171da177e4SLinus Torvalds { 218f42f7c28STrond Myklebust __rpc_list_dequeue_task(task); 2191da177e4SLinus Torvalds } 2201da177e4SLinus Torvalds 2211da177e4SLinus Torvalds /* 2221da177e4SLinus Torvalds * Remove request from queue. 2231da177e4SLinus Torvalds * Note: must be called with spin lock held. 2241da177e4SLinus Torvalds */ 22596ef13b2STrond Myklebust static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task) 2261da177e4SLinus Torvalds { 227eb276c0eSTrond Myklebust __rpc_disable_timer(queue, task); 2281da177e4SLinus Torvalds if (RPC_IS_PRIORITY(queue)) 2291da177e4SLinus Torvalds __rpc_remove_wait_queue_priority(task); 230f42f7c28STrond Myklebust else 2311da177e4SLinus Torvalds list_del(&task->u.tk_wait.list); 232e19b63daSChuck Lever queue->qlen--; 2331da177e4SLinus Torvalds } 2341da177e4SLinus Torvalds 2353ff7576dSTrond Myklebust static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues) 2361da177e4SLinus Torvalds { 2371da177e4SLinus Torvalds int i; 2381da177e4SLinus Torvalds 2391da177e4SLinus Torvalds spin_lock_init(&queue->lock); 2401da177e4SLinus Torvalds for (i = 0; i < ARRAY_SIZE(queue->tasks); i++) 2411da177e4SLinus Torvalds INIT_LIST_HEAD(&queue->tasks[i]); 2423ff7576dSTrond Myklebust queue->maxpriority = nr_queues - 1; 2431da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 24436df9aaeSTrond Myklebust queue->qlen = 0; 2457e0a0e38STrond Myklebust queue->timer_list.expires = 0; 24666eb3addSTrond Myklebust INIT_DELAYED_WORK(&queue->timer_list.dwork, __rpc_queue_timer_fn); 24736df9aaeSTrond Myklebust INIT_LIST_HEAD(&queue->timer_list.list); 2482f09c242STrond Myklebust rpc_assign_waitqueue_name(queue, qname); 2491da177e4SLinus Torvalds } 2501da177e4SLinus Torvalds 2511da177e4SLinus Torvalds void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2521da177e4SLinus Torvalds { 2533ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY); 2541da177e4SLinus Torvalds } 255689cf5c1SAlexandros Batsakis EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue); 2561da177e4SLinus Torvalds 2571da177e4SLinus Torvalds void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname) 2581da177e4SLinus Torvalds { 2593ff7576dSTrond Myklebust __rpc_init_priority_wait_queue(queue, qname, 1); 2601da177e4SLinus Torvalds } 261e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_init_wait_queue); 2621da177e4SLinus Torvalds 263f6a1cc89STrond Myklebust void rpc_destroy_wait_queue(struct rpc_wait_queue *queue) 264f6a1cc89STrond Myklebust { 2657e0a0e38STrond Myklebust cancel_delayed_work_sync(&queue->timer_list.dwork); 266f6a1cc89STrond Myklebust } 267f6a1cc89STrond Myklebust EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue); 268f6a1cc89STrond Myklebust 269dfd01f02SPeter Zijlstra static int rpc_wait_bit_killable(struct wait_bit_key *key, int mode) 27044c28873STrond Myklebust { 271416ad3c9SColin Cross freezable_schedule_unsafe(); 272dfd01f02SPeter Zijlstra if (signal_pending_state(mode, current)) 273dfd01f02SPeter Zijlstra return -ERESTARTSYS; 27444c28873STrond Myklebust return 0; 27544c28873STrond Myklebust } 27644c28873STrond Myklebust 2771306729bSJeff Layton #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) || IS_ENABLED(CONFIG_TRACEPOINTS) 278c44fe705STrond Myklebust static void rpc_task_set_debuginfo(struct rpc_task *task) 279c44fe705STrond Myklebust { 280*0392dd51SChuck Lever struct rpc_clnt *clnt = task->tk_client; 281*0392dd51SChuck Lever 282*0392dd51SChuck Lever /* Might be a task carrying a reverse-direction operation */ 283*0392dd51SChuck Lever if (!clnt) { 284c44fe705STrond Myklebust static atomic_t rpc_pid; 285c44fe705STrond Myklebust 286c44fe705STrond Myklebust task->tk_pid = atomic_inc_return(&rpc_pid); 287*0392dd51SChuck Lever return; 288*0392dd51SChuck Lever } 289*0392dd51SChuck Lever 290*0392dd51SChuck Lever task->tk_pid = atomic_inc_return(&clnt->cl_pid); 291c44fe705STrond Myklebust } 292c44fe705STrond Myklebust #else 293c44fe705STrond Myklebust static inline void rpc_task_set_debuginfo(struct rpc_task *task) 294c44fe705STrond Myklebust { 295c44fe705STrond Myklebust } 296c44fe705STrond Myklebust #endif 297c44fe705STrond Myklebust 298e6b3c4dbSTrond Myklebust static void rpc_set_active(struct rpc_task *task) 299e6b3c4dbSTrond Myklebust { 300c44fe705STrond Myklebust rpc_task_set_debuginfo(task); 30158f9612cSTrond Myklebust set_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 302e671edb9SChuck Lever trace_rpc_task_begin(task, NULL); 303e6b3c4dbSTrond Myklebust } 304e6b3c4dbSTrond Myklebust 30544c28873STrond Myklebust /* 30644c28873STrond Myklebust * Mark an RPC call as having completed by clearing the 'active' bit 307bf294b41STrond Myklebust * and then waking up all tasks that were sleeping. 30844c28873STrond Myklebust */ 309bf294b41STrond Myklebust static int rpc_complete_task(struct rpc_task *task) 31044c28873STrond Myklebust { 311bf294b41STrond Myklebust void *m = &task->tk_runstate; 312bf294b41STrond Myklebust wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE); 313bf294b41STrond Myklebust struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE); 314bf294b41STrond Myklebust unsigned long flags; 315bf294b41STrond Myklebust int ret; 316bf294b41STrond Myklebust 317e671edb9SChuck Lever trace_rpc_task_complete(task, NULL); 31882b0a4c3STrond Myklebust 319bf294b41STrond Myklebust spin_lock_irqsave(&wq->lock, flags); 320e6b3c4dbSTrond Myklebust clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate); 321bf294b41STrond Myklebust ret = atomic_dec_and_test(&task->tk_count); 322bf294b41STrond Myklebust if (waitqueue_active(wq)) 323ac5be6b4SAndrea Arcangeli __wake_up_locked_key(wq, TASK_NORMAL, &k); 324bf294b41STrond Myklebust spin_unlock_irqrestore(&wq->lock, flags); 325bf294b41STrond Myklebust return ret; 32644c28873STrond Myklebust } 32744c28873STrond Myklebust 32844c28873STrond Myklebust /* 32944c28873STrond Myklebust * Allow callers to wait for completion of an RPC call 330bf294b41STrond Myklebust * 331bf294b41STrond Myklebust * Note the use of out_of_line_wait_on_bit() rather than wait_on_bit() 332bf294b41STrond Myklebust * to enforce taking of the wq->lock and hence avoid races with 333bf294b41STrond Myklebust * rpc_complete_task(). 33444c28873STrond Myklebust */ 335c1221321SNeilBrown int __rpc_wait_for_completion_task(struct rpc_task *task, wait_bit_action_f *action) 33644c28873STrond Myklebust { 33744c28873STrond Myklebust if (action == NULL) 338150030b7SMatthew Wilcox action = rpc_wait_bit_killable; 339bf294b41STrond Myklebust return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE, 340150030b7SMatthew Wilcox action, TASK_KILLABLE); 34144c28873STrond Myklebust } 342e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task); 34344c28873STrond Myklebust 3441da177e4SLinus Torvalds /* 3451da177e4SLinus Torvalds * Make an RPC task runnable. 3461da177e4SLinus Torvalds * 347506026c3SJeff Layton * Note: If the task is ASYNC, and is being made runnable after sitting on an 348506026c3SJeff Layton * rpc_wait_queue, this must be called with the queue spinlock held to protect 349506026c3SJeff Layton * the wait queue operation. 350a3c3cac5STrond Myklebust * Note the ordering of rpc_test_and_set_running() and rpc_clear_queued(), 351a3c3cac5STrond Myklebust * which is needed to ensure that __rpc_execute() doesn't loop (due to the 352a3c3cac5STrond Myklebust * lockless RPC_IS_QUEUED() test) before we've had a chance to test 353a3c3cac5STrond Myklebust * the RPC_TASK_RUNNING flag. 3541da177e4SLinus Torvalds */ 355f1dc237cSTrond Myklebust static void rpc_make_runnable(struct workqueue_struct *wq, 356f1dc237cSTrond Myklebust struct rpc_task *task) 3571da177e4SLinus Torvalds { 358a3c3cac5STrond Myklebust bool need_wakeup = !rpc_test_and_set_running(task); 359a3c3cac5STrond Myklebust 3601da177e4SLinus Torvalds rpc_clear_queued(task); 361a3c3cac5STrond Myklebust if (!need_wakeup) 3621da177e4SLinus Torvalds return; 3631da177e4SLinus Torvalds if (RPC_IS_ASYNC(task)) { 36465f27f38SDavid Howells INIT_WORK(&task->u.tk_work, rpc_async_schedule); 365f1dc237cSTrond Myklebust queue_work(wq, &task->u.tk_work); 3661da177e4SLinus Torvalds } else 36796651ab3STrond Myklebust wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED); 3681da177e4SLinus Torvalds } 3691da177e4SLinus Torvalds 3701da177e4SLinus Torvalds /* 3711da177e4SLinus Torvalds * Prepare for sleeping on a wait queue. 3721da177e4SLinus Torvalds * By always appending tasks to the list we ensure FIFO behavior. 3731da177e4SLinus Torvalds * NB: An RPC task will only receive interrupt-driven events as long 3741da177e4SLinus Torvalds * as it's on a wait queue. 3751da177e4SLinus Torvalds */ 3761fab7dc4STrond Myklebust static void __rpc_do_sleep_on_priority(struct rpc_wait_queue *q, 3773b27bad7STrond Myklebust struct rpc_task *task, 3783b27bad7STrond Myklebust unsigned char queue_priority) 3791da177e4SLinus Torvalds { 380e671edb9SChuck Lever trace_rpc_task_sleep(task, q); 38182b0a4c3STrond Myklebust 3823b27bad7STrond Myklebust __rpc_add_wait_queue(q, task, queue_priority); 3836b2e6856STrond Myklebust } 3846b2e6856STrond Myklebust 3851fab7dc4STrond Myklebust static void __rpc_sleep_on_priority(struct rpc_wait_queue *q, 3861fab7dc4STrond Myklebust struct rpc_task *task, 3871fab7dc4STrond Myklebust unsigned char queue_priority) 3881fab7dc4STrond Myklebust { 3891fab7dc4STrond Myklebust if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) 3901fab7dc4STrond Myklebust return; 3911fab7dc4STrond Myklebust __rpc_do_sleep_on_priority(q, task, queue_priority); 3921fab7dc4STrond Myklebust } 3931fab7dc4STrond Myklebust 3946b2e6856STrond Myklebust static void __rpc_sleep_on_priority_timeout(struct rpc_wait_queue *q, 3956b2e6856STrond Myklebust struct rpc_task *task, unsigned long timeout, 3966b2e6856STrond Myklebust unsigned char queue_priority) 3976b2e6856STrond Myklebust { 3981fab7dc4STrond Myklebust if (WARN_ON_ONCE(RPC_IS_QUEUED(task))) 3991fab7dc4STrond Myklebust return; 4006b2e6856STrond Myklebust if (time_is_after_jiffies(timeout)) { 4011fab7dc4STrond Myklebust __rpc_do_sleep_on_priority(q, task, queue_priority); 4026b2e6856STrond Myklebust __rpc_add_timer(q, task, timeout); 4036b2e6856STrond Myklebust } else 4046b2e6856STrond Myklebust task->tk_status = -ETIMEDOUT; 4051da177e4SLinus Torvalds } 4061da177e4SLinus Torvalds 40787150aaeSTrond Myklebust static void rpc_set_tk_callback(struct rpc_task *task, rpc_action action) 40887150aaeSTrond Myklebust { 40987150aaeSTrond Myklebust if (action && !WARN_ON_ONCE(task->tk_callback != NULL)) 41087150aaeSTrond Myklebust task->tk_callback = action; 41187150aaeSTrond Myklebust } 41287150aaeSTrond Myklebust 41387150aaeSTrond Myklebust static bool rpc_sleep_check_activated(struct rpc_task *task) 41487150aaeSTrond Myklebust { 41587150aaeSTrond Myklebust /* We shouldn't ever put an inactive task to sleep */ 41687150aaeSTrond Myklebust if (WARN_ON_ONCE(!RPC_IS_ACTIVATED(task))) { 41787150aaeSTrond Myklebust task->tk_status = -EIO; 41887150aaeSTrond Myklebust rpc_put_task_async(task); 41987150aaeSTrond Myklebust return false; 42087150aaeSTrond Myklebust } 42187150aaeSTrond Myklebust return true; 42287150aaeSTrond Myklebust } 42387150aaeSTrond Myklebust 4246b2e6856STrond Myklebust void rpc_sleep_on_timeout(struct rpc_wait_queue *q, struct rpc_task *task, 4256b2e6856STrond Myklebust rpc_action action, unsigned long timeout) 4261da177e4SLinus Torvalds { 42787150aaeSTrond Myklebust if (!rpc_sleep_check_activated(task)) 428e454a7a8SWeston Andros Adamson return; 42987150aaeSTrond Myklebust 43087150aaeSTrond Myklebust rpc_set_tk_callback(task, action); 431e6b3c4dbSTrond Myklebust 4321da177e4SLinus Torvalds /* 4331da177e4SLinus Torvalds * Protect the queue operations. 4341da177e4SLinus Torvalds */ 435c049f8eaSTrond Myklebust spin_lock(&q->lock); 4366b2e6856STrond Myklebust __rpc_sleep_on_priority_timeout(q, task, timeout, task->tk_priority); 437c049f8eaSTrond Myklebust spin_unlock(&q->lock); 4386b2e6856STrond Myklebust } 4396b2e6856STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on_timeout); 4406b2e6856STrond Myklebust 4416b2e6856STrond Myklebust void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task, 4426b2e6856STrond Myklebust rpc_action action) 4436b2e6856STrond Myklebust { 4446b2e6856STrond Myklebust if (!rpc_sleep_check_activated(task)) 4456b2e6856STrond Myklebust return; 4466b2e6856STrond Myklebust 4476b2e6856STrond Myklebust rpc_set_tk_callback(task, action); 4486b2e6856STrond Myklebust 4496b2e6856STrond Myklebust WARN_ON_ONCE(task->tk_timeout != 0); 4506b2e6856STrond Myklebust /* 4516b2e6856STrond Myklebust * Protect the queue operations. 4526b2e6856STrond Myklebust */ 453c049f8eaSTrond Myklebust spin_lock(&q->lock); 45487150aaeSTrond Myklebust __rpc_sleep_on_priority(q, task, task->tk_priority); 455c049f8eaSTrond Myklebust spin_unlock(&q->lock); 4561da177e4SLinus Torvalds } 457e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on); 4581da177e4SLinus Torvalds 4596b2e6856STrond Myklebust void rpc_sleep_on_priority_timeout(struct rpc_wait_queue *q, 4606b2e6856STrond Myklebust struct rpc_task *task, unsigned long timeout, int priority) 4616b2e6856STrond Myklebust { 4626b2e6856STrond Myklebust if (!rpc_sleep_check_activated(task)) 4636b2e6856STrond Myklebust return; 4646b2e6856STrond Myklebust 4656b2e6856STrond Myklebust priority -= RPC_PRIORITY_LOW; 4666b2e6856STrond Myklebust /* 4676b2e6856STrond Myklebust * Protect the queue operations. 4686b2e6856STrond Myklebust */ 469c049f8eaSTrond Myklebust spin_lock(&q->lock); 4706b2e6856STrond Myklebust __rpc_sleep_on_priority_timeout(q, task, timeout, priority); 471c049f8eaSTrond Myklebust spin_unlock(&q->lock); 4726b2e6856STrond Myklebust } 4736b2e6856STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on_priority_timeout); 4746b2e6856STrond Myklebust 4753b27bad7STrond Myklebust void rpc_sleep_on_priority(struct rpc_wait_queue *q, struct rpc_task *task, 4768357a9b6STrond Myklebust int priority) 4773b27bad7STrond Myklebust { 47887150aaeSTrond Myklebust if (!rpc_sleep_check_activated(task)) 479e454a7a8SWeston Andros Adamson return; 48087150aaeSTrond Myklebust 4816b2e6856STrond Myklebust WARN_ON_ONCE(task->tk_timeout != 0); 4828357a9b6STrond Myklebust priority -= RPC_PRIORITY_LOW; 4833b27bad7STrond Myklebust /* 4843b27bad7STrond Myklebust * Protect the queue operations. 4853b27bad7STrond Myklebust */ 486c049f8eaSTrond Myklebust spin_lock(&q->lock); 4878357a9b6STrond Myklebust __rpc_sleep_on_priority(q, task, priority); 488c049f8eaSTrond Myklebust spin_unlock(&q->lock); 4893b27bad7STrond Myklebust } 4901e1093c7STrond Myklebust EXPORT_SYMBOL_GPL(rpc_sleep_on_priority); 4913b27bad7STrond Myklebust 4921da177e4SLinus Torvalds /** 493f1dc237cSTrond Myklebust * __rpc_do_wake_up_task_on_wq - wake up a single rpc_task 494f1dc237cSTrond Myklebust * @wq: workqueue on which to run task 49596ef13b2STrond Myklebust * @queue: wait queue 4961da177e4SLinus Torvalds * @task: task to be woken up 4971da177e4SLinus Torvalds * 4981da177e4SLinus Torvalds * Caller must hold queue->lock, and have cleared the task queued flag. 4991da177e4SLinus Torvalds */ 500f1dc237cSTrond Myklebust static void __rpc_do_wake_up_task_on_wq(struct workqueue_struct *wq, 501f1dc237cSTrond Myklebust struct rpc_wait_queue *queue, 502f1dc237cSTrond Myklebust struct rpc_task *task) 5031da177e4SLinus Torvalds { 5041da177e4SLinus Torvalds /* Has the task been executed yet? If not, we cannot wake it up! */ 5051da177e4SLinus Torvalds if (!RPC_IS_ACTIVATED(task)) { 5061da177e4SLinus Torvalds printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task); 5071da177e4SLinus Torvalds return; 5081da177e4SLinus Torvalds } 5091da177e4SLinus Torvalds 510e671edb9SChuck Lever trace_rpc_task_wakeup(task, queue); 51182b0a4c3STrond Myklebust 51296ef13b2STrond Myklebust __rpc_remove_wait_queue(queue, task); 5131da177e4SLinus Torvalds 514f1dc237cSTrond Myklebust rpc_make_runnable(wq, task); 5151da177e4SLinus Torvalds } 5161da177e4SLinus Torvalds 5171da177e4SLinus Torvalds /* 51896ef13b2STrond Myklebust * Wake up a queued task while the queue lock is being held 5191da177e4SLinus Torvalds */ 520359c48c0STrond Myklebust static struct rpc_task * 521359c48c0STrond Myklebust rpc_wake_up_task_on_wq_queue_action_locked(struct workqueue_struct *wq, 522359c48c0STrond Myklebust struct rpc_wait_queue *queue, struct rpc_task *task, 523359c48c0STrond Myklebust bool (*action)(struct rpc_task *, void *), void *data) 5241da177e4SLinus Torvalds { 5251166fde6STrond Myklebust if (RPC_IS_QUEUED(task)) { 5261166fde6STrond Myklebust smp_rmb(); 527359c48c0STrond Myklebust if (task->tk_waitqueue == queue) { 528359c48c0STrond Myklebust if (action == NULL || action(task, data)) { 529f1dc237cSTrond Myklebust __rpc_do_wake_up_task_on_wq(wq, queue, task); 530359c48c0STrond Myklebust return task; 5311da177e4SLinus Torvalds } 5321166fde6STrond Myklebust } 533359c48c0STrond Myklebust } 534359c48c0STrond Myklebust return NULL; 535359c48c0STrond Myklebust } 536359c48c0STrond Myklebust 5371da177e4SLinus Torvalds /* 538f1dc237cSTrond Myklebust * Wake up a queued task while the queue lock is being held 539f1dc237cSTrond Myklebust */ 540691b45ddSChuck Lever static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, 5412275cde4STrond Myklebust struct rpc_task *task) 5422275cde4STrond Myklebust { 543691b45ddSChuck Lever rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue, 544691b45ddSChuck Lever task, NULL, NULL); 5452275cde4STrond Myklebust } 5462275cde4STrond Myklebust 5472275cde4STrond Myklebust /* 5482275cde4STrond Myklebust * Wake up a task on a specific queue 5492275cde4STrond Myklebust */ 55096ef13b2STrond Myklebust void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task) 55196ef13b2STrond Myklebust { 5525ce97039STrond Myklebust if (!RPC_IS_QUEUED(task)) 5535ce97039STrond Myklebust return; 554c049f8eaSTrond Myklebust spin_lock(&queue->lock); 55596ef13b2STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 556c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 55796ef13b2STrond Myklebust } 55896ef13b2STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task); 55996ef13b2STrond Myklebust 560359c48c0STrond Myklebust static bool rpc_task_action_set_status(struct rpc_task *task, void *status) 561359c48c0STrond Myklebust { 562359c48c0STrond Myklebust task->tk_status = *(int *)status; 563359c48c0STrond Myklebust return true; 564359c48c0STrond Myklebust } 565359c48c0STrond Myklebust 566359c48c0STrond Myklebust static void 567359c48c0STrond Myklebust rpc_wake_up_task_queue_set_status_locked(struct rpc_wait_queue *queue, 568359c48c0STrond Myklebust struct rpc_task *task, int status) 569359c48c0STrond Myklebust { 570359c48c0STrond Myklebust rpc_wake_up_task_on_wq_queue_action_locked(rpciod_workqueue, queue, 571359c48c0STrond Myklebust task, rpc_task_action_set_status, &status); 572359c48c0STrond Myklebust } 573359c48c0STrond Myklebust 574359c48c0STrond Myklebust /** 575359c48c0STrond Myklebust * rpc_wake_up_queued_task_set_status - wake up a task and set task->tk_status 576359c48c0STrond Myklebust * @queue: pointer to rpc_wait_queue 577359c48c0STrond Myklebust * @task: pointer to rpc_task 578359c48c0STrond Myklebust * @status: integer error value 579359c48c0STrond Myklebust * 580359c48c0STrond Myklebust * If @task is queued on @queue, then it is woken up, and @task->tk_status is 581359c48c0STrond Myklebust * set to the value of @status. 582359c48c0STrond Myklebust */ 583359c48c0STrond Myklebust void 584359c48c0STrond Myklebust rpc_wake_up_queued_task_set_status(struct rpc_wait_queue *queue, 585359c48c0STrond Myklebust struct rpc_task *task, int status) 586359c48c0STrond Myklebust { 587359c48c0STrond Myklebust if (!RPC_IS_QUEUED(task)) 588359c48c0STrond Myklebust return; 589c049f8eaSTrond Myklebust spin_lock(&queue->lock); 590359c48c0STrond Myklebust rpc_wake_up_task_queue_set_status_locked(queue, task, status); 591c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 592359c48c0STrond Myklebust } 593359c48c0STrond Myklebust 59496ef13b2STrond Myklebust /* 5951da177e4SLinus Torvalds * Wake up the next task on a priority queue. 5961da177e4SLinus Torvalds */ 597961a828dSTrond Myklebust static struct rpc_task *__rpc_find_next_queued_priority(struct rpc_wait_queue *queue) 5981da177e4SLinus Torvalds { 5991da177e4SLinus Torvalds struct list_head *q; 6001da177e4SLinus Torvalds struct rpc_task *task; 6011da177e4SLinus Torvalds 6021da177e4SLinus Torvalds /* 6035483b904SZhang Xiaoxu * Service the privileged queue. 6045483b904SZhang Xiaoxu */ 6055483b904SZhang Xiaoxu q = &queue->tasks[RPC_NR_PRIORITY - 1]; 6065483b904SZhang Xiaoxu if (queue->maxpriority > RPC_PRIORITY_PRIVILEGED && !list_empty(q)) { 6075483b904SZhang Xiaoxu task = list_first_entry(q, struct rpc_task, u.tk_wait.list); 6085483b904SZhang Xiaoxu goto out; 6095483b904SZhang Xiaoxu } 6105483b904SZhang Xiaoxu 6115483b904SZhang Xiaoxu /* 6123ff7576dSTrond Myklebust * Service a batch of tasks from a single owner. 6131da177e4SLinus Torvalds */ 6141da177e4SLinus Torvalds q = &queue->tasks[queue->priority]; 615fcb170a9SZhang Xiaoxu if (!list_empty(q) && queue->nr) { 616fcb170a9SZhang Xiaoxu queue->nr--; 617f42f7c28STrond Myklebust task = list_first_entry(q, struct rpc_task, u.tk_wait.list); 6181da177e4SLinus Torvalds goto out; 6191da177e4SLinus Torvalds } 6201da177e4SLinus Torvalds 6211da177e4SLinus Torvalds /* 6221da177e4SLinus Torvalds * Service the next queue. 6231da177e4SLinus Torvalds */ 6241da177e4SLinus Torvalds do { 6251da177e4SLinus Torvalds if (q == &queue->tasks[0]) 6261da177e4SLinus Torvalds q = &queue->tasks[queue->maxpriority]; 6271da177e4SLinus Torvalds else 6281da177e4SLinus Torvalds q = q - 1; 6291da177e4SLinus Torvalds if (!list_empty(q)) { 630f42f7c28STrond Myklebust task = list_first_entry(q, struct rpc_task, u.tk_wait.list); 6311da177e4SLinus Torvalds goto new_queue; 6321da177e4SLinus Torvalds } 6331da177e4SLinus Torvalds } while (q != &queue->tasks[queue->priority]); 6341da177e4SLinus Torvalds 6351da177e4SLinus Torvalds rpc_reset_waitqueue_priority(queue); 6361da177e4SLinus Torvalds return NULL; 6371da177e4SLinus Torvalds 6381da177e4SLinus Torvalds new_queue: 6391da177e4SLinus Torvalds rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0])); 6401da177e4SLinus Torvalds out: 6411da177e4SLinus Torvalds return task; 6421da177e4SLinus Torvalds } 6431da177e4SLinus Torvalds 644961a828dSTrond Myklebust static struct rpc_task *__rpc_find_next_queued(struct rpc_wait_queue *queue) 645961a828dSTrond Myklebust { 646961a828dSTrond Myklebust if (RPC_IS_PRIORITY(queue)) 647961a828dSTrond Myklebust return __rpc_find_next_queued_priority(queue); 648961a828dSTrond Myklebust if (!list_empty(&queue->tasks[0])) 649961a828dSTrond Myklebust return list_first_entry(&queue->tasks[0], struct rpc_task, u.tk_wait.list); 650961a828dSTrond Myklebust return NULL; 651961a828dSTrond Myklebust } 652961a828dSTrond Myklebust 653961a828dSTrond Myklebust /* 654961a828dSTrond Myklebust * Wake up the first task on the wait queue. 655961a828dSTrond Myklebust */ 656f1dc237cSTrond Myklebust struct rpc_task *rpc_wake_up_first_on_wq(struct workqueue_struct *wq, 657f1dc237cSTrond Myklebust struct rpc_wait_queue *queue, 658961a828dSTrond Myklebust bool (*func)(struct rpc_task *, void *), void *data) 659961a828dSTrond Myklebust { 660961a828dSTrond Myklebust struct rpc_task *task = NULL; 661961a828dSTrond Myklebust 662c049f8eaSTrond Myklebust spin_lock(&queue->lock); 663961a828dSTrond Myklebust task = __rpc_find_next_queued(queue); 664359c48c0STrond Myklebust if (task != NULL) 665359c48c0STrond Myklebust task = rpc_wake_up_task_on_wq_queue_action_locked(wq, queue, 666359c48c0STrond Myklebust task, func, data); 667c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 668961a828dSTrond Myklebust 669961a828dSTrond Myklebust return task; 670961a828dSTrond Myklebust } 671f1dc237cSTrond Myklebust 672f1dc237cSTrond Myklebust /* 673f1dc237cSTrond Myklebust * Wake up the first task on the wait queue. 674f1dc237cSTrond Myklebust */ 675f1dc237cSTrond Myklebust struct rpc_task *rpc_wake_up_first(struct rpc_wait_queue *queue, 676f1dc237cSTrond Myklebust bool (*func)(struct rpc_task *, void *), void *data) 677f1dc237cSTrond Myklebust { 678f1dc237cSTrond Myklebust return rpc_wake_up_first_on_wq(rpciod_workqueue, queue, func, data); 679f1dc237cSTrond Myklebust } 680961a828dSTrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_first); 681961a828dSTrond Myklebust 682961a828dSTrond Myklebust static bool rpc_wake_up_next_func(struct rpc_task *task, void *data) 683961a828dSTrond Myklebust { 684961a828dSTrond Myklebust return true; 685961a828dSTrond Myklebust } 686961a828dSTrond Myklebust 6871da177e4SLinus Torvalds /* 6881da177e4SLinus Torvalds * Wake up the next task on the wait queue. 6891da177e4SLinus Torvalds */ 6901da177e4SLinus Torvalds struct rpc_task *rpc_wake_up_next(struct rpc_wait_queue *queue) 6911da177e4SLinus Torvalds { 692961a828dSTrond Myklebust return rpc_wake_up_first(queue, rpc_wake_up_next_func, NULL); 6931da177e4SLinus Torvalds } 694e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_next); 6951da177e4SLinus Torvalds 6961da177e4SLinus Torvalds /** 697e4c72201STrond Myklebust * rpc_wake_up_locked - wake up all rpc_tasks 698e4c72201STrond Myklebust * @queue: rpc_wait_queue on which the tasks are sleeping 699e4c72201STrond Myklebust * 700e4c72201STrond Myklebust */ 701e4c72201STrond Myklebust static void rpc_wake_up_locked(struct rpc_wait_queue *queue) 702e4c72201STrond Myklebust { 703e4c72201STrond Myklebust struct rpc_task *task; 704e4c72201STrond Myklebust 705e4c72201STrond Myklebust for (;;) { 706e4c72201STrond Myklebust task = __rpc_find_next_queued(queue); 707e4c72201STrond Myklebust if (task == NULL) 708e4c72201STrond Myklebust break; 709e4c72201STrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 710e4c72201STrond Myklebust } 711e4c72201STrond Myklebust } 712e4c72201STrond Myklebust 713e4c72201STrond Myklebust /** 7141da177e4SLinus Torvalds * rpc_wake_up - wake up all rpc_tasks 7151da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 7161da177e4SLinus Torvalds * 7171da177e4SLinus Torvalds * Grabs queue->lock 7181da177e4SLinus Torvalds */ 7191da177e4SLinus Torvalds void rpc_wake_up(struct rpc_wait_queue *queue) 7201da177e4SLinus Torvalds { 721c049f8eaSTrond Myklebust spin_lock(&queue->lock); 722e4c72201STrond Myklebust rpc_wake_up_locked(queue); 723c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 7241da177e4SLinus Torvalds } 725e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up); 7261da177e4SLinus Torvalds 7271da177e4SLinus Torvalds /** 728e4c72201STrond Myklebust * rpc_wake_up_status_locked - wake up all rpc_tasks and set their status value. 729e4c72201STrond Myklebust * @queue: rpc_wait_queue on which the tasks are sleeping 730e4c72201STrond Myklebust * @status: status value to set 731e4c72201STrond Myklebust */ 732e4c72201STrond Myklebust static void rpc_wake_up_status_locked(struct rpc_wait_queue *queue, int status) 733e4c72201STrond Myklebust { 734e4c72201STrond Myklebust struct rpc_task *task; 735e4c72201STrond Myklebust 736e4c72201STrond Myklebust for (;;) { 737e4c72201STrond Myklebust task = __rpc_find_next_queued(queue); 738e4c72201STrond Myklebust if (task == NULL) 739e4c72201STrond Myklebust break; 740e4c72201STrond Myklebust rpc_wake_up_task_queue_set_status_locked(queue, task, status); 741e4c72201STrond Myklebust } 742e4c72201STrond Myklebust } 743e4c72201STrond Myklebust 744e4c72201STrond Myklebust /** 7451da177e4SLinus Torvalds * rpc_wake_up_status - wake up all rpc_tasks and set their status value. 7461da177e4SLinus Torvalds * @queue: rpc_wait_queue on which the tasks are sleeping 7471da177e4SLinus Torvalds * @status: status value to set 7481da177e4SLinus Torvalds * 7491da177e4SLinus Torvalds * Grabs queue->lock 7501da177e4SLinus Torvalds */ 7511da177e4SLinus Torvalds void rpc_wake_up_status(struct rpc_wait_queue *queue, int status) 7521da177e4SLinus Torvalds { 753c049f8eaSTrond Myklebust spin_lock(&queue->lock); 754e4c72201STrond Myklebust rpc_wake_up_status_locked(queue, status); 755c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 7561da177e4SLinus Torvalds } 757e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_wake_up_status); 7581da177e4SLinus Torvalds 7597e0a0e38STrond Myklebust static void __rpc_queue_timer_fn(struct work_struct *work) 76036df9aaeSTrond Myklebust { 7617e0a0e38STrond Myklebust struct rpc_wait_queue *queue = container_of(work, 7627e0a0e38STrond Myklebust struct rpc_wait_queue, 7637e0a0e38STrond Myklebust timer_list.dwork.work); 76436df9aaeSTrond Myklebust struct rpc_task *task, *n; 76536df9aaeSTrond Myklebust unsigned long expires, now, timeo; 76636df9aaeSTrond Myklebust 76736df9aaeSTrond Myklebust spin_lock(&queue->lock); 76836df9aaeSTrond Myklebust expires = now = jiffies; 76936df9aaeSTrond Myklebust list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) { 7706b2e6856STrond Myklebust timeo = task->tk_timeout; 77136df9aaeSTrond Myklebust if (time_after_eq(now, timeo)) { 772721a1d38SChuck Lever trace_rpc_task_timeout(task, task->tk_action); 77336df9aaeSTrond Myklebust task->tk_status = -ETIMEDOUT; 77436df9aaeSTrond Myklebust rpc_wake_up_task_queue_locked(queue, task); 77536df9aaeSTrond Myklebust continue; 77636df9aaeSTrond Myklebust } 77736df9aaeSTrond Myklebust if (expires == now || time_after(expires, timeo)) 77836df9aaeSTrond Myklebust expires = timeo; 77936df9aaeSTrond Myklebust } 78036df9aaeSTrond Myklebust if (!list_empty(&queue->timer_list.list)) 78136df9aaeSTrond Myklebust rpc_set_queue_timer(queue, expires); 78236df9aaeSTrond Myklebust spin_unlock(&queue->lock); 78336df9aaeSTrond Myklebust } 78436df9aaeSTrond Myklebust 7858014793bSTrond Myklebust static void __rpc_atrun(struct rpc_task *task) 7868014793bSTrond Myklebust { 7876bd14416STrond Myklebust if (task->tk_status == -ETIMEDOUT) 7885d00837bSTrond Myklebust task->tk_status = 0; 7898014793bSTrond Myklebust } 7908014793bSTrond Myklebust 7911da177e4SLinus Torvalds /* 7921da177e4SLinus Torvalds * Run a task at a later time 7931da177e4SLinus Torvalds */ 7948014793bSTrond Myklebust void rpc_delay(struct rpc_task *task, unsigned long delay) 7951da177e4SLinus Torvalds { 7966b2e6856STrond Myklebust rpc_sleep_on_timeout(&delay_queue, task, __rpc_atrun, jiffies + delay); 7971da177e4SLinus Torvalds } 798e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_delay); 7991da177e4SLinus Torvalds 8001da177e4SLinus Torvalds /* 8014ce70adaSTrond Myklebust * Helper to call task->tk_ops->rpc_call_prepare 8024ce70adaSTrond Myklebust */ 803aae2006eSAndy Adamson void rpc_prepare_task(struct rpc_task *task) 8044ce70adaSTrond Myklebust { 8054ce70adaSTrond Myklebust task->tk_ops->rpc_call_prepare(task, task->tk_calldata); 8064ce70adaSTrond Myklebust } 8074ce70adaSTrond Myklebust 8087fdcf13bSTrond Myklebust static void 8097fdcf13bSTrond Myklebust rpc_init_task_statistics(struct rpc_task *task) 8107fdcf13bSTrond Myklebust { 8117fdcf13bSTrond Myklebust /* Initialize retry counters */ 8127fdcf13bSTrond Myklebust task->tk_garb_retry = 2; 8137fdcf13bSTrond Myklebust task->tk_cred_retry = 2; 8147fdcf13bSTrond Myklebust task->tk_rebind_retry = 2; 8157fdcf13bSTrond Myklebust 8167fdcf13bSTrond Myklebust /* starting timestamp */ 8177fdcf13bSTrond Myklebust task->tk_start = ktime_get(); 8187fdcf13bSTrond Myklebust } 8197fdcf13bSTrond Myklebust 8207fdcf13bSTrond Myklebust static void 8217fdcf13bSTrond Myklebust rpc_reset_task_statistics(struct rpc_task *task) 8227fdcf13bSTrond Myklebust { 8237fdcf13bSTrond Myklebust task->tk_timeouts = 0; 824ae67bd38STrond Myklebust task->tk_flags &= ~(RPC_CALL_MAJORSEEN|RPC_TASK_SENT); 8257fdcf13bSTrond Myklebust rpc_init_task_statistics(task); 8267fdcf13bSTrond Myklebust } 8277fdcf13bSTrond Myklebust 8284ce70adaSTrond Myklebust /* 829963d8fe5STrond Myklebust * Helper that calls task->tk_ops->rpc_call_done if it exists 830d05fdb0cSTrond Myklebust */ 831abbcf28fSTrond Myklebust void rpc_exit_task(struct rpc_task *task) 832d05fdb0cSTrond Myklebust { 833a264abadSChuck Lever trace_rpc_task_end(task, task->tk_action); 834abbcf28fSTrond Myklebust task->tk_action = NULL; 8359dfe52a9SDave Wysochanski if (task->tk_ops->rpc_count_stats) 8369dfe52a9SDave Wysochanski task->tk_ops->rpc_count_stats(task, task->tk_calldata); 8379dfe52a9SDave Wysochanski else if (task->tk_client) 8389dfe52a9SDave Wysochanski rpc_count_iostats(task, task->tk_client->cl_metrics); 839963d8fe5STrond Myklebust if (task->tk_ops->rpc_call_done != NULL) { 840963d8fe5STrond Myklebust task->tk_ops->rpc_call_done(task, task->tk_calldata); 841d05fdb0cSTrond Myklebust if (task->tk_action != NULL) { 842abbcf28fSTrond Myklebust /* Always release the RPC slot and buffer memory */ 843d05fdb0cSTrond Myklebust xprt_release(task); 8447fdcf13bSTrond Myklebust rpc_reset_task_statistics(task); 845d05fdb0cSTrond Myklebust } 846d05fdb0cSTrond Myklebust } 847d05fdb0cSTrond Myklebust } 848d9b6cd94STrond Myklebust 849ae67bd38STrond Myklebust void rpc_signal_task(struct rpc_task *task) 850ae67bd38STrond Myklebust { 851ae67bd38STrond Myklebust struct rpc_wait_queue *queue; 852ae67bd38STrond Myklebust 853ae67bd38STrond Myklebust if (!RPC_IS_ACTIVATED(task)) 854ae67bd38STrond Myklebust return; 855abf8af78SChuck Lever 856abf8af78SChuck Lever trace_rpc_task_signalled(task, task->tk_action); 857ae67bd38STrond Myklebust set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); 858ae67bd38STrond Myklebust smp_mb__after_atomic(); 859ae67bd38STrond Myklebust queue = READ_ONCE(task->tk_waitqueue); 860ae67bd38STrond Myklebust if (queue) 861ae67bd38STrond Myklebust rpc_wake_up_queued_task_set_status(queue, task, -ERESTARTSYS); 862ae67bd38STrond Myklebust } 863ae67bd38STrond Myklebust 864d9b6cd94STrond Myklebust void rpc_exit(struct rpc_task *task, int status) 865d9b6cd94STrond Myklebust { 866d9b6cd94STrond Myklebust task->tk_status = status; 867d9b6cd94STrond Myklebust task->tk_action = rpc_exit_task; 868d9b6cd94STrond Myklebust rpc_wake_up_queued_task(task->tk_waitqueue, task); 869d9b6cd94STrond Myklebust } 870d9b6cd94STrond Myklebust EXPORT_SYMBOL_GPL(rpc_exit); 871d05fdb0cSTrond Myklebust 872bbd5a1f9STrond Myklebust void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata) 873bbd5a1f9STrond Myklebust { 874a86dc496STrond Myklebust if (ops->rpc_release != NULL) 875bbd5a1f9STrond Myklebust ops->rpc_release(calldata); 876bbd5a1f9STrond Myklebust } 877bbd5a1f9STrond Myklebust 878d05fdb0cSTrond Myklebust /* 8791da177e4SLinus Torvalds * This is the RPC `scheduler' (or rather, the finite state machine). 8801da177e4SLinus Torvalds */ 8812efef837STrond Myklebust static void __rpc_execute(struct rpc_task *task) 8821da177e4SLinus Torvalds { 883eb9b55abSTrond Myklebust struct rpc_wait_queue *queue; 884eb9b55abSTrond Myklebust int task_is_async = RPC_IS_ASYNC(task); 8851da177e4SLinus Torvalds int status = 0; 8861da177e4SLinus Torvalds 8872bd4eef8SWeston Andros Adamson WARN_ON_ONCE(RPC_IS_QUEUED(task)); 8882bd4eef8SWeston Andros Adamson if (RPC_IS_QUEUED(task)) 8892bd4eef8SWeston Andros Adamson return; 8901da177e4SLinus Torvalds 891d05fdb0cSTrond Myklebust for (;;) { 892b55c5989STrond Myklebust void (*do_action)(struct rpc_task *); 8931da177e4SLinus Torvalds 8941da177e4SLinus Torvalds /* 89521ead9ffSChuck Lever * Perform the next FSM step or a pending callback. 89621ead9ffSChuck Lever * 897b55c5989STrond Myklebust * tk_action may be NULL if the task has been killed. 898b55c5989STrond Myklebust * In particular, note that rpc_killall_tasks may 899b55c5989STrond Myklebust * do this at any time, so beware when dereferencing. 9001da177e4SLinus Torvalds */ 901b55c5989STrond Myklebust do_action = task->tk_action; 90221ead9ffSChuck Lever if (task->tk_callback) { 90321ead9ffSChuck Lever do_action = task->tk_callback; 90421ead9ffSChuck Lever task->tk_callback = NULL; 9051da177e4SLinus Torvalds } 90621ead9ffSChuck Lever if (!do_action) 90721ead9ffSChuck Lever break; 908e671edb9SChuck Lever trace_rpc_task_run_action(task, do_action); 909b55c5989STrond Myklebust do_action(task); 9101da177e4SLinus Torvalds 9111da177e4SLinus Torvalds /* 9121da177e4SLinus Torvalds * Lockless check for whether task is sleeping or not. 9131da177e4SLinus Torvalds */ 91447dd8796STrond Myklebust if (!RPC_IS_QUEUED(task)) { 91547dd8796STrond Myklebust cond_resched(); 9161da177e4SLinus Torvalds continue; 91747dd8796STrond Myklebust } 918ae67bd38STrond Myklebust 919ae67bd38STrond Myklebust /* 920ae67bd38STrond Myklebust * Signalled tasks should exit rather than sleep. 921ae67bd38STrond Myklebust */ 922714fbc73STrond Myklebust if (RPC_SIGNALLED(task)) { 923714fbc73STrond Myklebust task->tk_rpc_status = -ERESTARTSYS; 924ae67bd38STrond Myklebust rpc_exit(task, -ERESTARTSYS); 925714fbc73STrond Myklebust } 926ae67bd38STrond Myklebust 927eb9b55abSTrond Myklebust /* 928eb9b55abSTrond Myklebust * The queue->lock protects against races with 929eb9b55abSTrond Myklebust * rpc_make_runnable(). 930eb9b55abSTrond Myklebust * 931eb9b55abSTrond Myklebust * Note that once we clear RPC_TASK_RUNNING on an asynchronous 932eb9b55abSTrond Myklebust * rpc_task, rpc_make_runnable() can assign it to a 933eb9b55abSTrond Myklebust * different workqueue. We therefore cannot assume that the 934eb9b55abSTrond Myklebust * rpc_task pointer may still be dereferenced. 935eb9b55abSTrond Myklebust */ 936eb9b55abSTrond Myklebust queue = task->tk_waitqueue; 937c049f8eaSTrond Myklebust spin_lock(&queue->lock); 938eb9b55abSTrond Myklebust if (!RPC_IS_QUEUED(task)) { 939c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 9401da177e4SLinus Torvalds continue; 9411da177e4SLinus Torvalds } 942eb9b55abSTrond Myklebust rpc_clear_running(task); 943c049f8eaSTrond Myklebust spin_unlock(&queue->lock); 944eb9b55abSTrond Myklebust if (task_is_async) 945eb9b55abSTrond Myklebust return; 9461da177e4SLinus Torvalds 9471da177e4SLinus Torvalds /* sync task: sleep here */ 9481466c221SChuck Lever trace_rpc_task_sync_sleep(task, task->tk_action); 94996651ab3STrond Myklebust status = out_of_line_wait_on_bit(&task->tk_runstate, 950150030b7SMatthew Wilcox RPC_TASK_QUEUED, rpc_wait_bit_killable, 951150030b7SMatthew Wilcox TASK_KILLABLE); 952ae67bd38STrond Myklebust if (status < 0) { 9531da177e4SLinus Torvalds /* 9541da177e4SLinus Torvalds * When a sync task receives a signal, it exits with 9551da177e4SLinus Torvalds * -ERESTARTSYS. In order to catch any callbacks that 9561da177e4SLinus Torvalds * clean up after sleeping on some queue, we don't 9571da177e4SLinus Torvalds * break the loop here, but go around once more. 9581da177e4SLinus Torvalds */ 959abf8af78SChuck Lever trace_rpc_task_signalled(task, task->tk_action); 960ae67bd38STrond Myklebust set_bit(RPC_TASK_SIGNALLED, &task->tk_runstate); 961714fbc73STrond Myklebust task->tk_rpc_status = -ERESTARTSYS; 9621da177e4SLinus Torvalds rpc_exit(task, -ERESTARTSYS); 9631da177e4SLinus Torvalds } 9641466c221SChuck Lever trace_rpc_task_sync_wake(task, task->tk_action); 9651da177e4SLinus Torvalds } 9661da177e4SLinus Torvalds 9671da177e4SLinus Torvalds /* Release all resources associated with the task */ 9681da177e4SLinus Torvalds rpc_release_task(task); 9691da177e4SLinus Torvalds } 9701da177e4SLinus Torvalds 9711da177e4SLinus Torvalds /* 9721da177e4SLinus Torvalds * User-visible entry point to the scheduler. 9731da177e4SLinus Torvalds * 9741da177e4SLinus Torvalds * This may be called recursively if e.g. an async NFS task updates 9751da177e4SLinus Torvalds * the attributes and finds that dirty pages must be flushed. 9761da177e4SLinus Torvalds * NOTE: Upon exit of this function the task is guaranteed to be 9771da177e4SLinus Torvalds * released. In particular note that tk_release() will have 9781da177e4SLinus Torvalds * been called, so your task memory may have been freed. 9791da177e4SLinus Torvalds */ 9802efef837STrond Myklebust void rpc_execute(struct rpc_task *task) 9811da177e4SLinus Torvalds { 982a76580fbSTrond Myklebust bool is_async = RPC_IS_ASYNC(task); 983a76580fbSTrond Myklebust 98444c28873STrond Myklebust rpc_set_active(task); 985f1dc237cSTrond Myklebust rpc_make_runnable(rpciod_workqueue, task); 986f0940f4bSBenjamin Coddington if (!is_async) { 987f0940f4bSBenjamin Coddington unsigned int pflags = memalloc_nofs_save(); 9882efef837STrond Myklebust __rpc_execute(task); 989f0940f4bSBenjamin Coddington memalloc_nofs_restore(pflags); 990f0940f4bSBenjamin Coddington } 9911da177e4SLinus Torvalds } 9921da177e4SLinus Torvalds 99365f27f38SDavid Howells static void rpc_async_schedule(struct work_struct *work) 9941da177e4SLinus Torvalds { 995a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 996a1231fdaSTrond Myklebust 99765f27f38SDavid Howells __rpc_execute(container_of(work, struct rpc_task, u.tk_work)); 998a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 9991da177e4SLinus Torvalds } 10001da177e4SLinus Torvalds 100102107148SChuck Lever /** 10025fe6eaa1SChuck Lever * rpc_malloc - allocate RPC buffer resources 10035fe6eaa1SChuck Lever * @task: RPC task 10045fe6eaa1SChuck Lever * 10055fe6eaa1SChuck Lever * A single memory region is allocated, which is split between the 10065fe6eaa1SChuck Lever * RPC call and RPC reply that this task is being used for. When 10075fe6eaa1SChuck Lever * this RPC is retired, the memory is released by calling rpc_free. 10081da177e4SLinus Torvalds * 1009c5a4dd8bSChuck Lever * To prevent rpciod from hanging, this allocator never sleeps, 10105fe6eaa1SChuck Lever * returning -ENOMEM and suppressing warning if the request cannot 10115fe6eaa1SChuck Lever * be serviced immediately. The caller can arrange to sleep in a 10125fe6eaa1SChuck Lever * way that is safe for rpciod. 1013c5a4dd8bSChuck Lever * 1014c5a4dd8bSChuck Lever * Most requests are 'small' (under 2KiB) and can be serviced from a 1015c5a4dd8bSChuck Lever * mempool, ensuring that NFS reads and writes can always proceed, 1016c5a4dd8bSChuck Lever * and that there is good locality of reference for these buffers. 10171da177e4SLinus Torvalds */ 10185fe6eaa1SChuck Lever int rpc_malloc(struct rpc_task *task) 10191da177e4SLinus Torvalds { 10205fe6eaa1SChuck Lever struct rpc_rqst *rqst = task->tk_rqstp; 10215fe6eaa1SChuck Lever size_t size = rqst->rq_callsize + rqst->rq_rcvsize; 1022aa3d1faeSChuck Lever struct rpc_buffer *buf; 102312a3ad61STrond Myklebust gfp_t gfp = GFP_NOFS; 1024a564b8f0SMel Gorman 1025a564b8f0SMel Gorman if (RPC_IS_SWAPPER(task)) 1026c4a7ca77STrond Myklebust gfp = __GFP_MEMALLOC | GFP_NOWAIT | __GFP_NOWARN; 10271da177e4SLinus Torvalds 1028aa3d1faeSChuck Lever size += sizeof(struct rpc_buffer); 1029c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 1030c5a4dd8bSChuck Lever buf = mempool_alloc(rpc_buffer_mempool, gfp); 10311da177e4SLinus Torvalds else 1032c5a4dd8bSChuck Lever buf = kmalloc(size, gfp); 1033ddce40dfSPeter Zijlstra 1034ddce40dfSPeter Zijlstra if (!buf) 10355fe6eaa1SChuck Lever return -ENOMEM; 1036ddce40dfSPeter Zijlstra 1037aa3d1faeSChuck Lever buf->len = size; 10385fe6eaa1SChuck Lever rqst->rq_buffer = buf->data; 103968778945SChuck Lever rqst->rq_rbuffer = (char *)rqst->rq_buffer + rqst->rq_callsize; 10405fe6eaa1SChuck Lever return 0; 10411da177e4SLinus Torvalds } 104212444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_malloc); 10431da177e4SLinus Torvalds 104402107148SChuck Lever /** 10453435c74aSChuck Lever * rpc_free - free RPC buffer resources allocated via rpc_malloc 10463435c74aSChuck Lever * @task: RPC task 104702107148SChuck Lever * 104802107148SChuck Lever */ 10493435c74aSChuck Lever void rpc_free(struct rpc_task *task) 10501da177e4SLinus Torvalds { 10513435c74aSChuck Lever void *buffer = task->tk_rqstp->rq_buffer; 1052aa3d1faeSChuck Lever size_t size; 1053aa3d1faeSChuck Lever struct rpc_buffer *buf; 105402107148SChuck Lever 1055aa3d1faeSChuck Lever buf = container_of(buffer, struct rpc_buffer, data); 1056aa3d1faeSChuck Lever size = buf->len; 1057c5a4dd8bSChuck Lever 1058c5a4dd8bSChuck Lever if (size <= RPC_BUFFER_MAXSIZE) 1059c5a4dd8bSChuck Lever mempool_free(buf, rpc_buffer_mempool); 10601da177e4SLinus Torvalds else 1061c5a4dd8bSChuck Lever kfree(buf); 10621da177e4SLinus Torvalds } 106312444809S\"Talpey, Thomas\ EXPORT_SYMBOL_GPL(rpc_free); 10641da177e4SLinus Torvalds 10651da177e4SLinus Torvalds /* 10661da177e4SLinus Torvalds * Creation and deletion of RPC task structures 10671da177e4SLinus Torvalds */ 106847fe0648STrond Myklebust static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data) 10691da177e4SLinus Torvalds { 10701da177e4SLinus Torvalds memset(task, 0, sizeof(*task)); 107144c28873STrond Myklebust atomic_set(&task->tk_count, 1); 107284115e1cSTrond Myklebust task->tk_flags = task_setup_data->flags; 107384115e1cSTrond Myklebust task->tk_ops = task_setup_data->callback_ops; 107484115e1cSTrond Myklebust task->tk_calldata = task_setup_data->callback_data; 10756529eba0STrond Myklebust INIT_LIST_HEAD(&task->tk_task); 10761da177e4SLinus Torvalds 10773ff7576dSTrond Myklebust task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW; 10783ff7576dSTrond Myklebust task->tk_owner = current->tgid; 10791da177e4SLinus Torvalds 10801da177e4SLinus Torvalds /* Initialize workqueue for async tasks */ 108132bfb5c0STrond Myklebust task->tk_workqueue = task_setup_data->workqueue; 10821da177e4SLinus Torvalds 1083a101b043STrond Myklebust task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client, 1084a101b043STrond Myklebust xprt_get(task_setup_data->rpc_xprt)); 10859d61498dSTrond Myklebust 10861de7eea9SNeilBrown task->tk_op_cred = get_rpccred(task_setup_data->rpc_op_cred); 10871de7eea9SNeilBrown 108884115e1cSTrond Myklebust if (task->tk_ops->rpc_call_prepare != NULL) 108984115e1cSTrond Myklebust task->tk_action = rpc_prepare_task; 1090963d8fe5STrond Myklebust 10917fdcf13bSTrond Myklebust rpc_init_task_statistics(task); 10921da177e4SLinus Torvalds } 10931da177e4SLinus Torvalds 10941da177e4SLinus Torvalds static struct rpc_task * 10951da177e4SLinus Torvalds rpc_alloc_task(void) 10961da177e4SLinus Torvalds { 109712a3ad61STrond Myklebust return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS); 10981da177e4SLinus Torvalds } 10991da177e4SLinus Torvalds 11001da177e4SLinus Torvalds /* 110190c5755fSTrond Myklebust * Create a new task for the specified client. 11021da177e4SLinus Torvalds */ 110384115e1cSTrond Myklebust struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data) 11041da177e4SLinus Torvalds { 1105e8f5d77cSTrond Myklebust struct rpc_task *task = setup_data->task; 1106e8f5d77cSTrond Myklebust unsigned short flags = 0; 11071da177e4SLinus Torvalds 1108e8f5d77cSTrond Myklebust if (task == NULL) { 11091da177e4SLinus Torvalds task = rpc_alloc_task(); 1110e8f5d77cSTrond Myklebust flags = RPC_TASK_DYNAMIC; 1111e8f5d77cSTrond Myklebust } 11121da177e4SLinus Torvalds 111384115e1cSTrond Myklebust rpc_init_task(task, setup_data); 1114e8f5d77cSTrond Myklebust task->tk_flags |= flags; 11151da177e4SLinus Torvalds return task; 11161da177e4SLinus Torvalds } 11171da177e4SLinus Torvalds 1118c6567ed1STrond Myklebust /* 1119c6567ed1STrond Myklebust * rpc_free_task - release rpc task and perform cleanups 1120c6567ed1STrond Myklebust * 1121c6567ed1STrond Myklebust * Note that we free up the rpc_task _after_ rpc_release_calldata() 1122c6567ed1STrond Myklebust * in order to work around a workqueue dependency issue. 1123c6567ed1STrond Myklebust * 1124c6567ed1STrond Myklebust * Tejun Heo states: 1125c6567ed1STrond Myklebust * "Workqueue currently considers two work items to be the same if they're 1126c6567ed1STrond Myklebust * on the same address and won't execute them concurrently - ie. it 1127c6567ed1STrond Myklebust * makes a work item which is queued again while being executed wait 1128c6567ed1STrond Myklebust * for the previous execution to complete. 1129c6567ed1STrond Myklebust * 1130c6567ed1STrond Myklebust * If a work function frees the work item, and then waits for an event 1131c6567ed1STrond Myklebust * which should be performed by another work item and *that* work item 1132c6567ed1STrond Myklebust * recycles the freed work item, it can create a false dependency loop. 1133c6567ed1STrond Myklebust * There really is no reliable way to detect this short of verifying 1134c6567ed1STrond Myklebust * every memory free." 1135c6567ed1STrond Myklebust * 1136c6567ed1STrond Myklebust */ 113732bfb5c0STrond Myklebust static void rpc_free_task(struct rpc_task *task) 11381da177e4SLinus Torvalds { 1139c6567ed1STrond Myklebust unsigned short tk_flags = task->tk_flags; 11401da177e4SLinus Torvalds 11411de7eea9SNeilBrown put_rpccred(task->tk_op_cred); 1142c6567ed1STrond Myklebust rpc_release_calldata(task->tk_ops, task->tk_calldata); 1143c6567ed1STrond Myklebust 11441466c221SChuck Lever if (tk_flags & RPC_TASK_DYNAMIC) 11455e4424afSTrond Myklebust mempool_free(task, rpc_task_mempool); 11465e4424afSTrond Myklebust } 114732bfb5c0STrond Myklebust 114832bfb5c0STrond Myklebust static void rpc_async_release(struct work_struct *work) 114932bfb5c0STrond Myklebust { 1150a1231fdaSTrond Myklebust unsigned int pflags = memalloc_nofs_save(); 1151a1231fdaSTrond Myklebust 115232bfb5c0STrond Myklebust rpc_free_task(container_of(work, struct rpc_task, u.tk_work)); 1153a1231fdaSTrond Myklebust memalloc_nofs_restore(pflags); 115432bfb5c0STrond Myklebust } 115532bfb5c0STrond Myklebust 1156bf294b41STrond Myklebust static void rpc_release_resources_task(struct rpc_task *task) 115732bfb5c0STrond Myklebust { 1158e6b3c4dbSTrond Myklebust xprt_release(task); 1159a271c5a0SOGAWA Hirofumi if (task->tk_msg.rpc_cred) { 11607eac5264STrond Myklebust if (!(task->tk_flags & RPC_TASK_CRED_NOREF)) 1161a52458b4SNeilBrown put_cred(task->tk_msg.rpc_cred); 1162a271c5a0SOGAWA Hirofumi task->tk_msg.rpc_cred = NULL; 1163a271c5a0SOGAWA Hirofumi } 116458f9612cSTrond Myklebust rpc_task_release_client(task); 1165bf294b41STrond Myklebust } 1166bf294b41STrond Myklebust 1167bf294b41STrond Myklebust static void rpc_final_put_task(struct rpc_task *task, 1168bf294b41STrond Myklebust struct workqueue_struct *q) 1169bf294b41STrond Myklebust { 1170bf294b41STrond Myklebust if (q != NULL) { 117132bfb5c0STrond Myklebust INIT_WORK(&task->u.tk_work, rpc_async_release); 1172bf294b41STrond Myklebust queue_work(q, &task->u.tk_work); 117332bfb5c0STrond Myklebust } else 117432bfb5c0STrond Myklebust rpc_free_task(task); 1175e6b3c4dbSTrond Myklebust } 1176bf294b41STrond Myklebust 1177bf294b41STrond Myklebust static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q) 1178bf294b41STrond Myklebust { 1179bf294b41STrond Myklebust if (atomic_dec_and_test(&task->tk_count)) { 1180bf294b41STrond Myklebust rpc_release_resources_task(task); 1181bf294b41STrond Myklebust rpc_final_put_task(task, q); 1182bf294b41STrond Myklebust } 1183bf294b41STrond Myklebust } 1184bf294b41STrond Myklebust 1185bf294b41STrond Myklebust void rpc_put_task(struct rpc_task *task) 1186bf294b41STrond Myklebust { 1187bf294b41STrond Myklebust rpc_do_put_task(task, NULL); 1188bf294b41STrond Myklebust } 1189e8914c65STrond Myklebust EXPORT_SYMBOL_GPL(rpc_put_task); 1190e6b3c4dbSTrond Myklebust 1191bf294b41STrond Myklebust void rpc_put_task_async(struct rpc_task *task) 1192bf294b41STrond Myklebust { 1193bf294b41STrond Myklebust rpc_do_put_task(task, task->tk_workqueue); 1194bf294b41STrond Myklebust } 1195bf294b41STrond Myklebust EXPORT_SYMBOL_GPL(rpc_put_task_async); 1196bf294b41STrond Myklebust 1197bde8f00cSTrond Myklebust static void rpc_release_task(struct rpc_task *task) 1198e6b3c4dbSTrond Myklebust { 11990a0c2a57SWeston Andros Adamson WARN_ON_ONCE(RPC_IS_QUEUED(task)); 12001da177e4SLinus Torvalds 1201bf294b41STrond Myklebust rpc_release_resources_task(task); 1202e6b3c4dbSTrond Myklebust 1203bf294b41STrond Myklebust /* 1204bf294b41STrond Myklebust * Note: at this point we have been removed from rpc_clnt->cl_tasks, 1205bf294b41STrond Myklebust * so it should be safe to use task->tk_count as a test for whether 1206bf294b41STrond Myklebust * or not any other processes still hold references to our rpc_task. 1207bf294b41STrond Myklebust */ 1208bf294b41STrond Myklebust if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) { 1209bf294b41STrond Myklebust /* Wake up anyone who may be waiting for task completion */ 1210bf294b41STrond Myklebust if (!rpc_complete_task(task)) 1211bf294b41STrond Myklebust return; 1212bf294b41STrond Myklebust } else { 1213bf294b41STrond Myklebust if (!atomic_dec_and_test(&task->tk_count)) 1214bf294b41STrond Myklebust return; 1215bf294b41STrond Myklebust } 1216bf294b41STrond Myklebust rpc_final_put_task(task, task->tk_workqueue); 12171da177e4SLinus Torvalds } 12181da177e4SLinus Torvalds 1219b247bbf1STrond Myklebust int rpciod_up(void) 1220b247bbf1STrond Myklebust { 1221b247bbf1STrond Myklebust return try_module_get(THIS_MODULE) ? 0 : -EINVAL; 1222b247bbf1STrond Myklebust } 1223b247bbf1STrond Myklebust 1224b247bbf1STrond Myklebust void rpciod_down(void) 1225b247bbf1STrond Myklebust { 1226b247bbf1STrond Myklebust module_put(THIS_MODULE); 1227b247bbf1STrond Myklebust } 1228b247bbf1STrond Myklebust 12291da177e4SLinus Torvalds /* 1230b247bbf1STrond Myklebust * Start up the rpciod workqueue. 12311da177e4SLinus Torvalds */ 1232b247bbf1STrond Myklebust static int rpciod_start(void) 12331da177e4SLinus Torvalds { 12341da177e4SLinus Torvalds struct workqueue_struct *wq; 12351da177e4SLinus Torvalds 12361da177e4SLinus Torvalds /* 12371da177e4SLinus Torvalds * Create the rpciod thread and wait for it to start. 12381da177e4SLinus Torvalds */ 1239f515f86bSOlga Kornievskaia wq = alloc_workqueue("rpciod", WQ_MEM_RECLAIM | WQ_UNBOUND, 0); 124040a5f1b1STrond Myklebust if (!wq) 124140a5f1b1STrond Myklebust goto out_failed; 12421da177e4SLinus Torvalds rpciod_workqueue = wq; 12436dbcbe3fSTrond Myklebust wq = alloc_workqueue("xprtiod", WQ_UNBOUND | WQ_MEM_RECLAIM, 0); 124440a5f1b1STrond Myklebust if (!wq) 124540a5f1b1STrond Myklebust goto free_rpciod; 124640a5f1b1STrond Myklebust xprtiod_workqueue = wq; 124740a5f1b1STrond Myklebust return 1; 124840a5f1b1STrond Myklebust free_rpciod: 124940a5f1b1STrond Myklebust wq = rpciod_workqueue; 125040a5f1b1STrond Myklebust rpciod_workqueue = NULL; 125140a5f1b1STrond Myklebust destroy_workqueue(wq); 125240a5f1b1STrond Myklebust out_failed: 125340a5f1b1STrond Myklebust return 0; 12541da177e4SLinus Torvalds } 12551da177e4SLinus Torvalds 1256b247bbf1STrond Myklebust static void rpciod_stop(void) 12571da177e4SLinus Torvalds { 1258b247bbf1STrond Myklebust struct workqueue_struct *wq = NULL; 1259ab418d70STrond Myklebust 1260b247bbf1STrond Myklebust if (rpciod_workqueue == NULL) 1261b247bbf1STrond Myklebust return; 12621da177e4SLinus Torvalds 1263b247bbf1STrond Myklebust wq = rpciod_workqueue; 12641da177e4SLinus Torvalds rpciod_workqueue = NULL; 1265b247bbf1STrond Myklebust destroy_workqueue(wq); 126640a5f1b1STrond Myklebust wq = xprtiod_workqueue; 126740a5f1b1STrond Myklebust xprtiod_workqueue = NULL; 126840a5f1b1STrond Myklebust destroy_workqueue(wq); 12691da177e4SLinus Torvalds } 12701da177e4SLinus Torvalds 12711da177e4SLinus Torvalds void 12721da177e4SLinus Torvalds rpc_destroy_mempool(void) 12731da177e4SLinus Torvalds { 1274b247bbf1STrond Myklebust rpciod_stop(); 12751da177e4SLinus Torvalds mempool_destroy(rpc_buffer_mempool); 12761da177e4SLinus Torvalds mempool_destroy(rpc_task_mempool); 12771a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_task_slabp); 12781a1d92c1SAlexey Dobriyan kmem_cache_destroy(rpc_buffer_slabp); 1279f6a1cc89STrond Myklebust rpc_destroy_wait_queue(&delay_queue); 12801da177e4SLinus Torvalds } 12811da177e4SLinus Torvalds 12821da177e4SLinus Torvalds int 12831da177e4SLinus Torvalds rpc_init_mempool(void) 12841da177e4SLinus Torvalds { 1285f6a1cc89STrond Myklebust /* 1286f6a1cc89STrond Myklebust * The following is not strictly a mempool initialisation, 1287f6a1cc89STrond Myklebust * but there is no harm in doing it here 1288f6a1cc89STrond Myklebust */ 1289f6a1cc89STrond Myklebust rpc_init_wait_queue(&delay_queue, "delayq"); 1290f6a1cc89STrond Myklebust if (!rpciod_start()) 1291f6a1cc89STrond Myklebust goto err_nomem; 1292f6a1cc89STrond Myklebust 12931da177e4SLinus Torvalds rpc_task_slabp = kmem_cache_create("rpc_tasks", 12941da177e4SLinus Torvalds sizeof(struct rpc_task), 12951da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 129620c2df83SPaul Mundt NULL); 12971da177e4SLinus Torvalds if (!rpc_task_slabp) 12981da177e4SLinus Torvalds goto err_nomem; 12991da177e4SLinus Torvalds rpc_buffer_slabp = kmem_cache_create("rpc_buffers", 13001da177e4SLinus Torvalds RPC_BUFFER_MAXSIZE, 13011da177e4SLinus Torvalds 0, SLAB_HWCACHE_ALIGN, 130220c2df83SPaul Mundt NULL); 13031da177e4SLinus Torvalds if (!rpc_buffer_slabp) 13041da177e4SLinus Torvalds goto err_nomem; 130593d2341cSMatthew Dobson rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE, 13061da177e4SLinus Torvalds rpc_task_slabp); 13071da177e4SLinus Torvalds if (!rpc_task_mempool) 13081da177e4SLinus Torvalds goto err_nomem; 130993d2341cSMatthew Dobson rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE, 13101da177e4SLinus Torvalds rpc_buffer_slabp); 13111da177e4SLinus Torvalds if (!rpc_buffer_mempool) 13121da177e4SLinus Torvalds goto err_nomem; 13131da177e4SLinus Torvalds return 0; 13141da177e4SLinus Torvalds err_nomem: 13151da177e4SLinus Torvalds rpc_destroy_mempool(); 13161da177e4SLinus Torvalds return -ENOMEM; 13171da177e4SLinus Torvalds } 1318