122a9d645SArjan van de Ven /* 222a9d645SArjan van de Ven * async.c: Asynchronous function calls for boot performance 322a9d645SArjan van de Ven * 422a9d645SArjan van de Ven * (C) Copyright 2009 Intel Corporation 522a9d645SArjan van de Ven * Author: Arjan van de Ven <arjan@linux.intel.com> 622a9d645SArjan van de Ven * 722a9d645SArjan van de Ven * This program is free software; you can redistribute it and/or 822a9d645SArjan van de Ven * modify it under the terms of the GNU General Public License 922a9d645SArjan van de Ven * as published by the Free Software Foundation; version 2 1022a9d645SArjan van de Ven * of the License. 1122a9d645SArjan van de Ven */ 1222a9d645SArjan van de Ven 1322a9d645SArjan van de Ven 1422a9d645SArjan van de Ven /* 1522a9d645SArjan van de Ven 1622a9d645SArjan van de Ven Goals and Theory of Operation 1722a9d645SArjan van de Ven 1822a9d645SArjan van de Ven The primary goal of this feature is to reduce the kernel boot time, 1922a9d645SArjan van de Ven by doing various independent hardware delays and discovery operations 2022a9d645SArjan van de Ven decoupled and not strictly serialized. 2122a9d645SArjan van de Ven 2222a9d645SArjan van de Ven More specifically, the asynchronous function call concept allows 2322a9d645SArjan van de Ven certain operations (primarily during system boot) to happen 2422a9d645SArjan van de Ven asynchronously, out of order, while these operations still 2522a9d645SArjan van de Ven have their externally visible parts happen sequentially and in-order. 2622a9d645SArjan van de Ven (not unlike how out-of-order CPUs retire their instructions in order) 2722a9d645SArjan van de Ven 2822a9d645SArjan van de Ven Key to the asynchronous function call implementation is the concept of 2922a9d645SArjan van de Ven a "sequence cookie" (which, although it has an abstracted type, can be 3022a9d645SArjan van de Ven thought of as a monotonically incrementing number). 3122a9d645SArjan van de Ven 3222a9d645SArjan van de Ven The async core will assign each scheduled event such a sequence cookie and 3322a9d645SArjan van de Ven pass this to the called functions. 3422a9d645SArjan van de Ven 3522a9d645SArjan van de Ven The asynchronously called function should before doing a globally visible 3622a9d645SArjan van de Ven operation, such as registering device numbers, call the 3722a9d645SArjan van de Ven async_synchronize_cookie() function and pass in its own cookie. The 3822a9d645SArjan van de Ven async_synchronize_cookie() function will make sure that all asynchronous 3922a9d645SArjan van de Ven operations that were scheduled prior to the operation corresponding with the 4022a9d645SArjan van de Ven cookie have completed. 4122a9d645SArjan van de Ven 4222a9d645SArjan van de Ven Subsystem/driver initialization code that scheduled asynchronous probe 4322a9d645SArjan van de Ven functions, but which shares global resources with other drivers/subsystems 4422a9d645SArjan van de Ven that do not use the asynchronous call feature, need to do a full 4522a9d645SArjan van de Ven synchronization with the async_synchronize_full() function, before returning 4622a9d645SArjan van de Ven from their init function. This is to maintain strict ordering between the 4722a9d645SArjan van de Ven asynchronous and synchronous parts of the kernel. 4822a9d645SArjan van de Ven 4922a9d645SArjan van de Ven */ 5022a9d645SArjan van de Ven 5122a9d645SArjan van de Ven #include <linux/async.h> 5284c15027SPaul McQuade #include <linux/atomic.h> 5384c15027SPaul McQuade #include <linux/ktime.h> 549984de1aSPaul Gortmaker #include <linux/export.h> 5522a9d645SArjan van de Ven #include <linux/wait.h> 5622a9d645SArjan van de Ven #include <linux/sched.h> 575a0e3ad6STejun Heo #include <linux/slab.h> 58083b804cSTejun Heo #include <linux/workqueue.h> 5922a9d645SArjan van de Ven 6084b233adSTejun Heo #include "workqueue_internal.h" 6184b233adSTejun Heo 6222a9d645SArjan van de Ven static async_cookie_t next_cookie = 1; 6322a9d645SArjan van de Ven 6422a9d645SArjan van de Ven #define MAX_WORK 32768 65c68eee14STejun Heo #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */ 6622a9d645SArjan van de Ven 67*9fdb04cdSTejun Heo static LIST_HEAD(async_global_pending); /* pending from all registered doms */ 688723d503STejun Heo static ASYNC_DOMAIN(async_dfl_domain); 6922a9d645SArjan van de Ven static DEFINE_SPINLOCK(async_lock); 7022a9d645SArjan van de Ven 7122a9d645SArjan van de Ven struct async_entry { 72*9fdb04cdSTejun Heo struct list_head domain_list; 73*9fdb04cdSTejun Heo struct list_head global_list; 74083b804cSTejun Heo struct work_struct work; 7522a9d645SArjan van de Ven async_cookie_t cookie; 7622a9d645SArjan van de Ven async_func_ptr *func; 7722a9d645SArjan van de Ven void *data; 788723d503STejun Heo struct async_domain *domain; 7922a9d645SArjan van de Ven }; 8022a9d645SArjan van de Ven 8122a9d645SArjan van de Ven static DECLARE_WAIT_QUEUE_HEAD(async_done); 8222a9d645SArjan van de Ven 8322a9d645SArjan van de Ven static atomic_t entry_count; 8422a9d645SArjan van de Ven 858723d503STejun Heo static async_cookie_t lowest_in_progress(struct async_domain *domain) 8637a76bd4SArjan van de Ven { 87*9fdb04cdSTejun Heo struct async_entry *first = NULL; 8852722794STejun Heo async_cookie_t ret = ASYNC_COOKIE_MAX; 8937a76bd4SArjan van de Ven unsigned long flags; 9037a76bd4SArjan van de Ven 9137a76bd4SArjan van de Ven spin_lock_irqsave(&async_lock, flags); 92*9fdb04cdSTejun Heo 93*9fdb04cdSTejun Heo if (domain) { 94*9fdb04cdSTejun Heo if (!list_empty(&domain->pending)) 95*9fdb04cdSTejun Heo first = list_first_entry(&domain->pending, 96*9fdb04cdSTejun Heo struct async_entry, domain_list); 97*9fdb04cdSTejun Heo } else { 98*9fdb04cdSTejun Heo if (!list_empty(&async_global_pending)) 99*9fdb04cdSTejun Heo first = list_first_entry(&async_global_pending, 100*9fdb04cdSTejun Heo struct async_entry, global_list); 10152722794STejun Heo } 102*9fdb04cdSTejun Heo 103*9fdb04cdSTejun Heo if (first) 104*9fdb04cdSTejun Heo ret = first->cookie; 105*9fdb04cdSTejun Heo 10637a76bd4SArjan van de Ven spin_unlock_irqrestore(&async_lock, flags); 10737a76bd4SArjan van de Ven return ret; 10837a76bd4SArjan van de Ven } 109083b804cSTejun Heo 11022a9d645SArjan van de Ven /* 11122a9d645SArjan van de Ven * pick the first pending entry and run it 11222a9d645SArjan van de Ven */ 113083b804cSTejun Heo static void async_run_entry_fn(struct work_struct *work) 11422a9d645SArjan van de Ven { 115083b804cSTejun Heo struct async_entry *entry = 116083b804cSTejun Heo container_of(work, struct async_entry, work); 11722a9d645SArjan van de Ven unsigned long flags; 118124ff4e5SVitaliy Ivanov ktime_t uninitialized_var(calltime), delta, rettime; 11922a9d645SArjan van de Ven 12052722794STejun Heo /* 1) run (and print duration) */ 121ad160d23SArjan van de Ven if (initcall_debug && system_state == SYSTEM_BOOTING) { 12284c15027SPaul McQuade printk(KERN_DEBUG "calling %lli_%pF @ %i\n", 12384c15027SPaul McQuade (long long)entry->cookie, 12458763a29SAndrew Morton entry->func, task_pid_nr(current)); 12522a9d645SArjan van de Ven calltime = ktime_get(); 12622a9d645SArjan van de Ven } 12722a9d645SArjan van de Ven entry->func(entry->data, entry->cookie); 128ad160d23SArjan van de Ven if (initcall_debug && system_state == SYSTEM_BOOTING) { 12922a9d645SArjan van de Ven rettime = ktime_get(); 13022a9d645SArjan van de Ven delta = ktime_sub(rettime, calltime); 13184c15027SPaul McQuade printk(KERN_DEBUG "initcall %lli_%pF returned 0 after %lld usecs\n", 13258763a29SAndrew Morton (long long)entry->cookie, 13358763a29SAndrew Morton entry->func, 13458763a29SAndrew Morton (long long)ktime_to_ns(delta) >> 10); 13522a9d645SArjan van de Ven } 13622a9d645SArjan van de Ven 13752722794STejun Heo /* 2) remove self from the pending queues */ 13822a9d645SArjan van de Ven spin_lock_irqsave(&async_lock, flags); 139*9fdb04cdSTejun Heo list_del_init(&entry->domain_list); 140*9fdb04cdSTejun Heo list_del_init(&entry->global_list); 14122a9d645SArjan van de Ven 14252722794STejun Heo /* 3) free the entry */ 14322a9d645SArjan van de Ven kfree(entry); 14422a9d645SArjan van de Ven atomic_dec(&entry_count); 14522a9d645SArjan van de Ven 14622a9d645SArjan van de Ven spin_unlock_irqrestore(&async_lock, flags); 14722a9d645SArjan van de Ven 14852722794STejun Heo /* 4) wake up any waiters */ 14922a9d645SArjan van de Ven wake_up(&async_done); 15022a9d645SArjan van de Ven } 15122a9d645SArjan van de Ven 1528723d503STejun Heo static async_cookie_t __async_schedule(async_func_ptr *ptr, void *data, struct async_domain *domain) 15322a9d645SArjan van de Ven { 15422a9d645SArjan van de Ven struct async_entry *entry; 15522a9d645SArjan van de Ven unsigned long flags; 15622a9d645SArjan van de Ven async_cookie_t newcookie; 15722a9d645SArjan van de Ven 15822a9d645SArjan van de Ven /* allow irq-off callers */ 15922a9d645SArjan van de Ven entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC); 16022a9d645SArjan van de Ven 16122a9d645SArjan van de Ven /* 16222a9d645SArjan van de Ven * If we're out of memory or if there's too much work 16322a9d645SArjan van de Ven * pending already, we execute synchronously. 16422a9d645SArjan van de Ven */ 165083b804cSTejun Heo if (!entry || atomic_read(&entry_count) > MAX_WORK) { 16622a9d645SArjan van de Ven kfree(entry); 16722a9d645SArjan van de Ven spin_lock_irqsave(&async_lock, flags); 16822a9d645SArjan van de Ven newcookie = next_cookie++; 16922a9d645SArjan van de Ven spin_unlock_irqrestore(&async_lock, flags); 17022a9d645SArjan van de Ven 17122a9d645SArjan van de Ven /* low on memory.. run synchronously */ 17222a9d645SArjan van de Ven ptr(data, newcookie); 17322a9d645SArjan van de Ven return newcookie; 17422a9d645SArjan van de Ven } 175083b804cSTejun Heo INIT_WORK(&entry->work, async_run_entry_fn); 17622a9d645SArjan van de Ven entry->func = ptr; 17722a9d645SArjan van de Ven entry->data = data; 1788723d503STejun Heo entry->domain = domain; 17922a9d645SArjan van de Ven 18022a9d645SArjan van de Ven spin_lock_irqsave(&async_lock, flags); 181*9fdb04cdSTejun Heo 182*9fdb04cdSTejun Heo /* allocate cookie and queue */ 18322a9d645SArjan van de Ven newcookie = entry->cookie = next_cookie++; 184*9fdb04cdSTejun Heo 185*9fdb04cdSTejun Heo list_add_tail(&entry->domain_list, &domain->pending); 186*9fdb04cdSTejun Heo if (domain->registered) 187*9fdb04cdSTejun Heo list_add_tail(&entry->global_list, &async_global_pending); 188*9fdb04cdSTejun Heo 18922a9d645SArjan van de Ven atomic_inc(&entry_count); 19022a9d645SArjan van de Ven spin_unlock_irqrestore(&async_lock, flags); 191083b804cSTejun Heo 192774a1221STejun Heo /* mark that this task has queued an async job, used by module init */ 193774a1221STejun Heo current->flags |= PF_USED_ASYNC; 194774a1221STejun Heo 195083b804cSTejun Heo /* schedule for execution */ 196083b804cSTejun Heo queue_work(system_unbound_wq, &entry->work); 197083b804cSTejun Heo 19822a9d645SArjan van de Ven return newcookie; 19922a9d645SArjan van de Ven } 20022a9d645SArjan van de Ven 201f30d5b30SCornelia Huck /** 202f30d5b30SCornelia Huck * async_schedule - schedule a function for asynchronous execution 203f30d5b30SCornelia Huck * @ptr: function to execute asynchronously 204f30d5b30SCornelia Huck * @data: data pointer to pass to the function 205f30d5b30SCornelia Huck * 206f30d5b30SCornelia Huck * Returns an async_cookie_t that may be used for checkpointing later. 207f30d5b30SCornelia Huck * Note: This function may be called from atomic or non-atomic contexts. 208f30d5b30SCornelia Huck */ 20922a9d645SArjan van de Ven async_cookie_t async_schedule(async_func_ptr *ptr, void *data) 21022a9d645SArjan van de Ven { 2118723d503STejun Heo return __async_schedule(ptr, data, &async_dfl_domain); 21222a9d645SArjan van de Ven } 21322a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_schedule); 21422a9d645SArjan van de Ven 215f30d5b30SCornelia Huck /** 216766ccb9eSCornelia Huck * async_schedule_domain - schedule a function for asynchronous execution within a certain domain 217f30d5b30SCornelia Huck * @ptr: function to execute asynchronously 218f30d5b30SCornelia Huck * @data: data pointer to pass to the function 2198723d503STejun Heo * @domain: the domain 220f30d5b30SCornelia Huck * 221f30d5b30SCornelia Huck * Returns an async_cookie_t that may be used for checkpointing later. 2228723d503STejun Heo * @domain may be used in the async_synchronize_*_domain() functions to 2238723d503STejun Heo * wait within a certain synchronization domain rather than globally. A 2248723d503STejun Heo * synchronization domain is specified via @domain. Note: This function 2258723d503STejun Heo * may be called from atomic or non-atomic contexts. 226f30d5b30SCornelia Huck */ 227766ccb9eSCornelia Huck async_cookie_t async_schedule_domain(async_func_ptr *ptr, void *data, 2288723d503STejun Heo struct async_domain *domain) 22922a9d645SArjan van de Ven { 2308723d503STejun Heo return __async_schedule(ptr, data, domain); 23122a9d645SArjan van de Ven } 232766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_schedule_domain); 23322a9d645SArjan van de Ven 234f30d5b30SCornelia Huck /** 235f30d5b30SCornelia Huck * async_synchronize_full - synchronize all asynchronous function calls 236f30d5b30SCornelia Huck * 237f30d5b30SCornelia Huck * This function waits until all asynchronous function calls have been done. 238f30d5b30SCornelia Huck */ 23922a9d645SArjan van de Ven void async_synchronize_full(void) 24022a9d645SArjan van de Ven { 241*9fdb04cdSTejun Heo async_synchronize_full_domain(NULL); 24222a9d645SArjan van de Ven } 24322a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_synchronize_full); 24422a9d645SArjan van de Ven 245f30d5b30SCornelia Huck /** 246a4683487SDan Williams * async_unregister_domain - ensure no more anonymous waiters on this domain 247a4683487SDan Williams * @domain: idle domain to flush out of any async_synchronize_full instances 248a4683487SDan Williams * 249a4683487SDan Williams * async_synchronize_{cookie|full}_domain() are not flushed since callers 250a4683487SDan Williams * of these routines should know the lifetime of @domain 251a4683487SDan Williams * 252a4683487SDan Williams * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing 253a4683487SDan Williams */ 254a4683487SDan Williams void async_unregister_domain(struct async_domain *domain) 255a4683487SDan Williams { 256a4683487SDan Williams spin_lock_irq(&async_lock); 257*9fdb04cdSTejun Heo WARN_ON(!domain->registered || !list_empty(&domain->pending)); 258a4683487SDan Williams domain->registered = 0; 259a4683487SDan Williams spin_unlock_irq(&async_lock); 260a4683487SDan Williams } 261a4683487SDan Williams EXPORT_SYMBOL_GPL(async_unregister_domain); 262a4683487SDan Williams 263a4683487SDan Williams /** 264766ccb9eSCornelia Huck * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain 2658723d503STejun Heo * @domain: the domain to synchronize 266f30d5b30SCornelia Huck * 267766ccb9eSCornelia Huck * This function waits until all asynchronous function calls for the 2688723d503STejun Heo * synchronization domain specified by @domain have been done. 269f30d5b30SCornelia Huck */ 2702955b47dSDan Williams void async_synchronize_full_domain(struct async_domain *domain) 27122a9d645SArjan van de Ven { 272c68eee14STejun Heo async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain); 27322a9d645SArjan van de Ven } 274766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_synchronize_full_domain); 27522a9d645SArjan van de Ven 276f30d5b30SCornelia Huck /** 277766ccb9eSCornelia Huck * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing 278f30d5b30SCornelia Huck * @cookie: async_cookie_t to use as checkpoint 279*9fdb04cdSTejun Heo * @domain: the domain to synchronize (%NULL for all registered domains) 280f30d5b30SCornelia Huck * 281766ccb9eSCornelia Huck * This function waits until all asynchronous function calls for the 2828723d503STejun Heo * synchronization domain specified by @domain submitted prior to @cookie 2838723d503STejun Heo * have been done. 284f30d5b30SCornelia Huck */ 2858723d503STejun Heo void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain) 28622a9d645SArjan van de Ven { 287124ff4e5SVitaliy Ivanov ktime_t uninitialized_var(starttime), delta, endtime; 28822a9d645SArjan van de Ven 289ad160d23SArjan van de Ven if (initcall_debug && system_state == SYSTEM_BOOTING) { 29084c15027SPaul McQuade printk(KERN_DEBUG "async_waiting @ %i\n", task_pid_nr(current)); 29122a9d645SArjan van de Ven starttime = ktime_get(); 29222a9d645SArjan van de Ven } 29322a9d645SArjan van de Ven 2948723d503STejun Heo wait_event(async_done, lowest_in_progress(domain) >= cookie); 29522a9d645SArjan van de Ven 296ad160d23SArjan van de Ven if (initcall_debug && system_state == SYSTEM_BOOTING) { 29722a9d645SArjan van de Ven endtime = ktime_get(); 29822a9d645SArjan van de Ven delta = ktime_sub(endtime, starttime); 29922a9d645SArjan van de Ven 30084c15027SPaul McQuade printk(KERN_DEBUG "async_continuing @ %i after %lli usec\n", 30158763a29SAndrew Morton task_pid_nr(current), 30258763a29SAndrew Morton (long long)ktime_to_ns(delta) >> 10); 30322a9d645SArjan van de Ven } 30422a9d645SArjan van de Ven } 305766ccb9eSCornelia Huck EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain); 30622a9d645SArjan van de Ven 307f30d5b30SCornelia Huck /** 308f30d5b30SCornelia Huck * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing 309f30d5b30SCornelia Huck * @cookie: async_cookie_t to use as checkpoint 310f30d5b30SCornelia Huck * 311f30d5b30SCornelia Huck * This function waits until all asynchronous function calls prior to @cookie 312f30d5b30SCornelia Huck * have been done. 313f30d5b30SCornelia Huck */ 31422a9d645SArjan van de Ven void async_synchronize_cookie(async_cookie_t cookie) 31522a9d645SArjan van de Ven { 3168723d503STejun Heo async_synchronize_cookie_domain(cookie, &async_dfl_domain); 31722a9d645SArjan van de Ven } 31822a9d645SArjan van de Ven EXPORT_SYMBOL_GPL(async_synchronize_cookie); 31984b233adSTejun Heo 32084b233adSTejun Heo /** 32184b233adSTejun Heo * current_is_async - is %current an async worker task? 32284b233adSTejun Heo * 32384b233adSTejun Heo * Returns %true if %current is an async worker task. 32484b233adSTejun Heo */ 32584b233adSTejun Heo bool current_is_async(void) 32684b233adSTejun Heo { 32784b233adSTejun Heo struct worker *worker = current_wq_worker(); 32884b233adSTejun Heo 32984b233adSTejun Heo return worker && worker->current_func == async_run_entry_fn; 33084b233adSTejun Heo } 331