xref: /linux/kernel/async.c (revision 9fdb04cdc5566d6ba68283a0bebe49667ca0b0e8)
122a9d645SArjan van de Ven /*
222a9d645SArjan van de Ven  * async.c: Asynchronous function calls for boot performance
322a9d645SArjan van de Ven  *
422a9d645SArjan van de Ven  * (C) Copyright 2009 Intel Corporation
522a9d645SArjan van de Ven  * Author: Arjan van de Ven <arjan@linux.intel.com>
622a9d645SArjan van de Ven  *
722a9d645SArjan van de Ven  * This program is free software; you can redistribute it and/or
822a9d645SArjan van de Ven  * modify it under the terms of the GNU General Public License
922a9d645SArjan van de Ven  * as published by the Free Software Foundation; version 2
1022a9d645SArjan van de Ven  * of the License.
1122a9d645SArjan van de Ven  */
1222a9d645SArjan van de Ven 
1322a9d645SArjan van de Ven 
1422a9d645SArjan van de Ven /*
1522a9d645SArjan van de Ven 
1622a9d645SArjan van de Ven Goals and Theory of Operation
1722a9d645SArjan van de Ven 
1822a9d645SArjan van de Ven The primary goal of this feature is to reduce the kernel boot time,
1922a9d645SArjan van de Ven by doing various independent hardware delays and discovery operations
2022a9d645SArjan van de Ven decoupled and not strictly serialized.
2122a9d645SArjan van de Ven 
2222a9d645SArjan van de Ven More specifically, the asynchronous function call concept allows
2322a9d645SArjan van de Ven certain operations (primarily during system boot) to happen
2422a9d645SArjan van de Ven asynchronously, out of order, while these operations still
2522a9d645SArjan van de Ven have their externally visible parts happen sequentially and in-order.
2622a9d645SArjan van de Ven (not unlike how out-of-order CPUs retire their instructions in order)
2722a9d645SArjan van de Ven 
2822a9d645SArjan van de Ven Key to the asynchronous function call implementation is the concept of
2922a9d645SArjan van de Ven a "sequence cookie" (which, although it has an abstracted type, can be
3022a9d645SArjan van de Ven thought of as a monotonically incrementing number).
3122a9d645SArjan van de Ven 
3222a9d645SArjan van de Ven The async core will assign each scheduled event such a sequence cookie and
3322a9d645SArjan van de Ven pass this to the called functions.
3422a9d645SArjan van de Ven 
3522a9d645SArjan van de Ven The asynchronously called function should before doing a globally visible
3622a9d645SArjan van de Ven operation, such as registering device numbers, call the
3722a9d645SArjan van de Ven async_synchronize_cookie() function and pass in its own cookie. The
3822a9d645SArjan van de Ven async_synchronize_cookie() function will make sure that all asynchronous
3922a9d645SArjan van de Ven operations that were scheduled prior to the operation corresponding with the
4022a9d645SArjan van de Ven cookie have completed.
4122a9d645SArjan van de Ven 
4222a9d645SArjan van de Ven Subsystem/driver initialization code that scheduled asynchronous probe
4322a9d645SArjan van de Ven functions, but which shares global resources with other drivers/subsystems
4422a9d645SArjan van de Ven that do not use the asynchronous call feature, need to do a full
4522a9d645SArjan van de Ven synchronization with the async_synchronize_full() function, before returning
4622a9d645SArjan van de Ven from their init function. This is to maintain strict ordering between the
4722a9d645SArjan van de Ven asynchronous and synchronous parts of the kernel.
4822a9d645SArjan van de Ven 
4922a9d645SArjan van de Ven */
5022a9d645SArjan van de Ven 
5122a9d645SArjan van de Ven #include <linux/async.h>
5284c15027SPaul McQuade #include <linux/atomic.h>
5384c15027SPaul McQuade #include <linux/ktime.h>
549984de1aSPaul Gortmaker #include <linux/export.h>
5522a9d645SArjan van de Ven #include <linux/wait.h>
5622a9d645SArjan van de Ven #include <linux/sched.h>
575a0e3ad6STejun Heo #include <linux/slab.h>
58083b804cSTejun Heo #include <linux/workqueue.h>
5922a9d645SArjan van de Ven 
6084b233adSTejun Heo #include "workqueue_internal.h"
6184b233adSTejun Heo 
6222a9d645SArjan van de Ven static async_cookie_t next_cookie = 1;
6322a9d645SArjan van de Ven 
6422a9d645SArjan van de Ven #define MAX_WORK		32768
65c68eee14STejun Heo #define ASYNC_COOKIE_MAX	ULLONG_MAX	/* infinity cookie */
6622a9d645SArjan van de Ven 
67*9fdb04cdSTejun Heo static LIST_HEAD(async_global_pending);	/* pending from all registered doms */
688723d503STejun Heo static ASYNC_DOMAIN(async_dfl_domain);
6922a9d645SArjan van de Ven static DEFINE_SPINLOCK(async_lock);
7022a9d645SArjan van de Ven 
7122a9d645SArjan van de Ven struct async_entry {
72*9fdb04cdSTejun Heo 	struct list_head	domain_list;
73*9fdb04cdSTejun Heo 	struct list_head	global_list;
74083b804cSTejun Heo 	struct work_struct	work;
7522a9d645SArjan van de Ven 	async_cookie_t		cookie;
7622a9d645SArjan van de Ven 	async_func_ptr		*func;
7722a9d645SArjan van de Ven 	void			*data;
788723d503STejun Heo 	struct async_domain	*domain;
7922a9d645SArjan van de Ven };
8022a9d645SArjan van de Ven 
8122a9d645SArjan van de Ven static DECLARE_WAIT_QUEUE_HEAD(async_done);
8222a9d645SArjan van de Ven 
8322a9d645SArjan van de Ven static atomic_t entry_count;
8422a9d645SArjan van de Ven 
858723d503STejun Heo static async_cookie_t lowest_in_progress(struct async_domain *domain)
8637a76bd4SArjan van de Ven {
87*9fdb04cdSTejun Heo 	struct async_entry *first = NULL;
8852722794STejun Heo 	async_cookie_t ret = ASYNC_COOKIE_MAX;
8937a76bd4SArjan van de Ven 	unsigned long flags;
9037a76bd4SArjan van de Ven 
9137a76bd4SArjan van de Ven 	spin_lock_irqsave(&async_lock, flags);
92*9fdb04cdSTejun Heo 
93*9fdb04cdSTejun Heo 	if (domain) {
94*9fdb04cdSTejun Heo 		if (!list_empty(&domain->pending))
95*9fdb04cdSTejun Heo 			first = list_first_entry(&domain->pending,
96*9fdb04cdSTejun Heo 					struct async_entry, domain_list);
97*9fdb04cdSTejun Heo 	} else {
98*9fdb04cdSTejun Heo 		if (!list_empty(&async_global_pending))
99*9fdb04cdSTejun Heo 			first = list_first_entry(&async_global_pending,
100*9fdb04cdSTejun Heo 					struct async_entry, global_list);
10152722794STejun Heo 	}
102*9fdb04cdSTejun Heo 
103*9fdb04cdSTejun Heo 	if (first)
104*9fdb04cdSTejun Heo 		ret = first->cookie;
105*9fdb04cdSTejun Heo 
10637a76bd4SArjan van de Ven 	spin_unlock_irqrestore(&async_lock, flags);
10737a76bd4SArjan van de Ven 	return ret;
10837a76bd4SArjan van de Ven }
109083b804cSTejun Heo 
11022a9d645SArjan van de Ven /*
11122a9d645SArjan van de Ven  * pick the first pending entry and run it
11222a9d645SArjan van de Ven  */
113083b804cSTejun Heo static void async_run_entry_fn(struct work_struct *work)
11422a9d645SArjan van de Ven {
115083b804cSTejun Heo 	struct async_entry *entry =
116083b804cSTejun Heo 		container_of(work, struct async_entry, work);
11722a9d645SArjan van de Ven 	unsigned long flags;
118124ff4e5SVitaliy Ivanov 	ktime_t uninitialized_var(calltime), delta, rettime;
11922a9d645SArjan van de Ven 
12052722794STejun Heo 	/* 1) run (and print duration) */
121ad160d23SArjan van de Ven 	if (initcall_debug && system_state == SYSTEM_BOOTING) {
12284c15027SPaul McQuade 		printk(KERN_DEBUG "calling  %lli_%pF @ %i\n",
12384c15027SPaul McQuade 			(long long)entry->cookie,
12458763a29SAndrew Morton 			entry->func, task_pid_nr(current));
12522a9d645SArjan van de Ven 		calltime = ktime_get();
12622a9d645SArjan van de Ven 	}
12722a9d645SArjan van de Ven 	entry->func(entry->data, entry->cookie);
128ad160d23SArjan van de Ven 	if (initcall_debug && system_state == SYSTEM_BOOTING) {
12922a9d645SArjan van de Ven 		rettime = ktime_get();
13022a9d645SArjan van de Ven 		delta = ktime_sub(rettime, calltime);
13184c15027SPaul McQuade 		printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n",
13258763a29SAndrew Morton 			(long long)entry->cookie,
13358763a29SAndrew Morton 			entry->func,
13458763a29SAndrew Morton 			(long long)ktime_to_ns(delta) >> 10);
13522a9d645SArjan van de Ven 	}
13622a9d645SArjan van de Ven 
13752722794STejun Heo 	/* 2) remove self from the pending queues */
13822a9d645SArjan van de Ven 	spin_lock_irqsave(&async_lock, flags);
139*9fdb04cdSTejun Heo 	list_del_init(&entry->domain_list);
140*9fdb04cdSTejun Heo 	list_del_init(&entry->global_list);
14122a9d645SArjan van de Ven 
14252722794STejun Heo 	/* 3) free the entry */
14322a9d645SArjan van de Ven 	kfree(entry);
14422a9d645SArjan van de Ven 	atomic_dec(&entry_count);
14522a9d645SArjan van de Ven 
14622a9d645SArjan van de Ven 	spin_unlock_irqrestore(&async_lock, flags);
14722a9d645SArjan van de Ven 
14852722794STejun Heo 	/* 4) wake up any waiters */
14922a9d645SArjan van de Ven 	wake_up(&async_done);
15022a9d645SArjan van de Ven }
15122a9d645SArjan van de Ven 
1528723d503STejun Heo static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *domain)
15322a9d645SArjan van de Ven {
15422a9d645SArjan van de Ven 	struct async_entry *entry;
15522a9d645SArjan van de Ven 	unsigned long flags;
15622a9d645SArjan van de Ven 	async_cookie_t newcookie;
15722a9d645SArjan van de Ven 
15822a9d645SArjan van de Ven 	/* allow irq-off callers */
15922a9d645SArjan van de Ven 	entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
16022a9d645SArjan van de Ven 
16122a9d645SArjan van de Ven 	/*
16222a9d645SArjan van de Ven 	 * If we're out of memory or if there's too much work
16322a9d645SArjan van de Ven 	 * pending already, we execute synchronously.
16422a9d645SArjan van de Ven 	 */
165083b804cSTejun Heo 	if (!entry || atomic_read(&entry_count) > MAX_WORK) {
16622a9d645SArjan van de Ven 		kfree(entry);
16722a9d645SArjan van de Ven 		spin_lock_irqsave(&async_lock, flags);
16822a9d645SArjan van de Ven 		newcookie = next_cookie++;
16922a9d645SArjan van de Ven 		spin_unlock_irqrestore(&async_lock, flags);
17022a9d645SArjan van de Ven 
17122a9d645SArjan van de Ven 		/* low on memory.. run synchronously */
17222a9d645SArjan van de Ven 		ptr(data, newcookie);
17322a9d645SArjan van de Ven 		return newcookie;
17422a9d645SArjan van de Ven 	}
175083b804cSTejun Heo 	INIT_WORK(&entry->work, async_run_entry_fn);
17622a9d645SArjan van de Ven 	entry->func = ptr;
17722a9d645SArjan van de Ven 	entry->data = data;
1788723d503STejun Heo 	entry->domain = domain;
17922a9d645SArjan van de Ven 
18022a9d645SArjan van de Ven 	spin_lock_irqsave(&async_lock, flags);
181*9fdb04cdSTejun Heo 
182*9fdb04cdSTejun Heo 	/* allocate cookie and queue */
18322a9d645SArjan van de Ven 	newcookie = entry->cookie = next_cookie++;
184*9fdb04cdSTejun Heo 
185*9fdb04cdSTejun Heo 	list_add_tail(&entry->domain_list, &domain->pending);
186*9fdb04cdSTejun Heo 	if (domain->registered)
187*9fdb04cdSTejun Heo 		list_add_tail(&entry->global_list, &async_global_pending);
188*9fdb04cdSTejun Heo 
18922a9d645SArjan van de Ven 	atomic_inc(&entry_count);
19022a9d645SArjan van de Ven 	spin_unlock_irqrestore(&async_lock, flags);
191083b804cSTejun Heo 
192774a1221STejun Heo 	/* mark that this task has queued an async job, used by module init */
193774a1221STejun Heo 	current->flags |= PF_USED_ASYNC;
194774a1221STejun Heo 
195083b804cSTejun Heo 	/* schedule for execution */
196083b804cSTejun Heo 	queue_work(system_unbound_wq, &entry->work);
197083b804cSTejun Heo 
19822a9d645SArjan van de Ven 	return newcookie;
19922a9d645SArjan van de Ven }
20022a9d645SArjan van de Ven 
201f30d5b30SCornelia Huck /**
202f30d5b30SCornelia Huck  * async_schedule - schedule a function for asynchronous execution
203f30d5b30SCornelia Huck  * @ptr: function to execute asynchronously
204f30d5b30SCornelia Huck  * @data: data pointer to pass to the function
205f30d5b30SCornelia Huck  *
206f30d5b30SCornelia Huck  * Returns an async_cookie_t that may be used for checkpointing later.
207f30d5b30SCornelia Huck  * Note: This function may be called from atomic or non-atomic contexts.
208f30d5b30SCornelia Huck  */
20922a9d645SArjan van de Ven async_cookie_t async_schedule(async_func_ptr *ptr, void *data)
21022a9d645SArjan van de Ven {
2118723d503STejun Heo 	return __async_schedule(ptr, data, &async_dfl_domain);
21222a9d645SArjan van de Ven }
21322a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_schedule);
21422a9d645SArjan van de Ven 
215f30d5b30SCornelia Huck /**
216766ccb9eSCornelia Huck  * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
217f30d5b30SCornelia Huck  * @ptr: function to execute asynchronously
218f30d5b30SCornelia Huck  * @data: data pointer to pass to the function
2198723d503STejun Heo  * @domain: the domain
220f30d5b30SCornelia Huck  *
221f30d5b30SCornelia Huck  * Returns an async_cookie_t that may be used for checkpointing later.
2228723d503STejun Heo  * @domain may be used in the async_synchronize_*_domain() functions to
2238723d503STejun Heo  * wait within a certain synchronization domain rather than globally.  A
2248723d503STejun Heo  * synchronization domain is specified via @domain.  Note: This function
2258723d503STejun Heo  * may be called from atomic or non-atomic contexts.
226f30d5b30SCornelia Huck  */
227766ccb9eSCornelia Huck async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data,
2288723d503STejun Heo 				     struct async_domain *domain)
22922a9d645SArjan van de Ven {
2308723d503STejun Heo 	return __async_schedule(ptr, data, domain);
23122a9d645SArjan van de Ven }
232766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_schedule_domain);
23322a9d645SArjan van de Ven 
234f30d5b30SCornelia Huck /**
235f30d5b30SCornelia Huck  * async_synchronize_full - synchronize all asynchronous function calls
236f30d5b30SCornelia Huck  *
237f30d5b30SCornelia Huck  * This function waits until all asynchronous function calls have been done.
238f30d5b30SCornelia Huck  */
23922a9d645SArjan van de Ven void async_synchronize_full(void)
24022a9d645SArjan van de Ven {
241*9fdb04cdSTejun Heo 	async_synchronize_full_domain(NULL);
24222a9d645SArjan van de Ven }
24322a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_synchronize_full);
24422a9d645SArjan van de Ven 
245f30d5b30SCornelia Huck /**
246a4683487SDan Williams  * async_unregister_domain - ensure no more anonymous waiters on this domain
247a4683487SDan Williams  * @domain: idle domain to flush out of any async_synchronize_full instances
248a4683487SDan Williams  *
249a4683487SDan Williams  * async_synchronize_{cookie|full}_domain() are not flushed since callers
250a4683487SDan Williams  * of these routines should know the lifetime of @domain
251a4683487SDan Williams  *
252a4683487SDan Williams  * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
253a4683487SDan Williams  */
254a4683487SDan Williams void async_unregister_domain(struct async_domain *domain)
255a4683487SDan Williams {
256a4683487SDan Williams 	spin_lock_irq(&async_lock);
257*9fdb04cdSTejun Heo 	WARN_ON(!domain->registered || !list_empty(&domain->pending));
258a4683487SDan Williams 	domain->registered = 0;
259a4683487SDan Williams 	spin_unlock_irq(&async_lock);
260a4683487SDan Williams }
261a4683487SDan Williams EXPORT_SYMBOL_GPL(async_unregister_domain);
262a4683487SDan Williams 
263a4683487SDan Williams /**
264766ccb9eSCornelia Huck  * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
2658723d503STejun Heo  * @domain: the domain to synchronize
266f30d5b30SCornelia Huck  *
267766ccb9eSCornelia Huck  * This function waits until all asynchronous function calls for the
2688723d503STejun Heo  * synchronization domain specified by @domain have been done.
269f30d5b30SCornelia Huck  */
2702955b47dSDan Williams void async_synchronize_full_domain(struct async_domain *domain)
27122a9d645SArjan van de Ven {
272c68eee14STejun Heo 	async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
27322a9d645SArjan van de Ven }
274766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
27522a9d645SArjan van de Ven 
276f30d5b30SCornelia Huck /**
277766ccb9eSCornelia Huck  * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
278f30d5b30SCornelia Huck  * @cookie: async_cookie_t to use as checkpoint
279*9fdb04cdSTejun Heo  * @domain: the domain to synchronize (%NULL for all registered domains)
280f30d5b30SCornelia Huck  *
281766ccb9eSCornelia Huck  * This function waits until all asynchronous function calls for the
2828723d503STejun Heo  * synchronization domain specified by @domain submitted prior to @cookie
2838723d503STejun Heo  * have been done.
284f30d5b30SCornelia Huck  */
2858723d503STejun Heo void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
28622a9d645SArjan van de Ven {
287124ff4e5SVitaliy Ivanov 	ktime_t uninitialized_var(starttime), delta, endtime;
28822a9d645SArjan van de Ven 
289ad160d23SArjan van de Ven 	if (initcall_debug && system_state == SYSTEM_BOOTING) {
29084c15027SPaul McQuade 		printk(KERN_DEBUG "async_waiting @ %i\n", task_pid_nr(current));
29122a9d645SArjan van de Ven 		starttime = ktime_get();
29222a9d645SArjan van de Ven 	}
29322a9d645SArjan van de Ven 
2948723d503STejun Heo 	wait_event(async_done, lowest_in_progress(domain) >= cookie);
29522a9d645SArjan van de Ven 
296ad160d23SArjan van de Ven 	if (initcall_debug && system_state == SYSTEM_BOOTING) {
29722a9d645SArjan van de Ven 		endtime = ktime_get();
29822a9d645SArjan van de Ven 		delta = ktime_sub(endtime, starttime);
29922a9d645SArjan van de Ven 
30084c15027SPaul McQuade 		printk(KERN_DEBUG "async_continuing @ %i after %lli usec\n",
30158763a29SAndrew Morton 			task_pid_nr(current),
30258763a29SAndrew Morton 			(long long)ktime_to_ns(delta) >> 10);
30322a9d645SArjan van de Ven 	}
30422a9d645SArjan van de Ven }
305766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
30622a9d645SArjan van de Ven 
307f30d5b30SCornelia Huck /**
308f30d5b30SCornelia Huck  * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
309f30d5b30SCornelia Huck  * @cookie: async_cookie_t to use as checkpoint
310f30d5b30SCornelia Huck  *
311f30d5b30SCornelia Huck  * This function waits until all asynchronous function calls prior to @cookie
312f30d5b30SCornelia Huck  * have been done.
313f30d5b30SCornelia Huck  */
31422a9d645SArjan van de Ven void async_synchronize_cookie(async_cookie_t cookie)
31522a9d645SArjan van de Ven {
3168723d503STejun Heo 	async_synchronize_cookie_domain(cookie, &async_dfl_domain);
31722a9d645SArjan van de Ven }
31822a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_synchronize_cookie);
31984b233adSTejun Heo 
32084b233adSTejun Heo /**
32184b233adSTejun Heo  * current_is_async - is %current an async worker task?
32284b233adSTejun Heo  *
32384b233adSTejun Heo  * Returns %true if %current is an async worker task.
32484b233adSTejun Heo  */
32584b233adSTejun Heo bool current_is_async(void)
32684b233adSTejun Heo {
32784b233adSTejun Heo 	struct worker *worker = current_wq_worker();
32884b233adSTejun Heo 
32984b233adSTejun Heo 	return worker && worker->current_func == async_run_entry_fn;
33084b233adSTejun Heo }
331