123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334 |
- /*
- * async.c: Asynchronous function calls for boot performance
- *
- * (C) Copyright 2009 Intel Corporation
- * Author: Arjan van de Ven <arjan@linux.intel.com>
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * as published by the Free Software Foundation; version 2
- * of the License.
- */
- /*
- Goals and Theory of Operation
- The primary goal of this feature is to reduce the kernel boot time,
- by doing various independent hardware delays and discovery operations
- decoupled and not strictly serialized.
- More specifically, the asynchronous function call concept allows
- certain operations (primarily during system boot) to happen
- asynchronously, out of order, while these operations still
- have their externally visible parts happen sequentially and in-order.
- (not unlike how out-of-order CPUs retire their instructions in order)
- Key to the asynchronous function call implementation is the concept of
- a "sequence cookie" (which, although it has an abstracted type, can be
- thought of as a monotonically incrementing number).
- The async core will assign each scheduled event such a sequence cookie and
- pass this to the called functions.
- The asynchronously called function should before doing a globally visible
- operation, such as registering device numbers, call the
- async_synchronize_cookie() function and pass in its own cookie. The
- async_synchronize_cookie() function will make sure that all asynchronous
- operations that were scheduled prior to the operation corresponding with the
- cookie have completed.
- Subsystem/driver initialization code that scheduled asynchronous probe
- functions, but which shares global resources with other drivers/subsystems
- that do not use the asynchronous call feature, need to do a full
- synchronization with the async_synchronize_full() function, before returning
- from their init function. This is to maintain strict ordering between the
- asynchronous and synchronous parts of the kernel.
- */
- #include <linux/async.h>
- #include <linux/atomic.h>
- #include <linux/ktime.h>
- #include <linux/export.h>
- #include <linux/wait.h>
- #include <linux/sched.h>
- #include <linux/slab.h>
- #include <linux/workqueue.h>
- #include "workqueue_internal.h"
- static async_cookie_t next_cookie = 1;
- #define MAX_WORK 32768
- #define ASYNC_COOKIE_MAX ULLONG_MAX /* infinity cookie */
- static LIST_HEAD(async_global_pending); /* pending from all registered doms */
- static ASYNC_DOMAIN(async_dfl_domain);
- static DEFINE_SPINLOCK(async_lock);
- struct async_entry {
- struct list_head domain_list;
- struct list_head global_list;
- struct work_struct work;
- async_cookie_t cookie;
- async_func_t func;
- void *data;
- struct async_domain *domain;
- };
- static DECLARE_WAIT_QUEUE_HEAD(async_done);
- static atomic_t entry_count;
- static async_cookie_t lowest_in_progress(struct async_domain *domain)
- {
- struct async_entry *first = NULL;
- async_cookie_t ret = ASYNC_COOKIE_MAX;
- unsigned long flags;
- spin_lock_irqsave(&async_lock, flags);
- if (domain) {
- if (!list_empty(&domain->pending))
- first = list_first_entry(&domain->pending,
- struct async_entry, domain_list);
- } else {
- if (!list_empty(&async_global_pending))
- first = list_first_entry(&async_global_pending,
- struct async_entry, global_list);
- }
- if (first)
- ret = first->cookie;
- spin_unlock_irqrestore(&async_lock, flags);
- return ret;
- }
- /*
- * pick the first pending entry and run it
- */
- static void async_run_entry_fn(struct work_struct *work)
- {
- struct async_entry *entry =
- container_of(work, struct async_entry, work);
- unsigned long flags;
- ktime_t uninitialized_var(calltime), delta, rettime;
- /* 1) run (and print duration) */
- if (initcall_debug && system_state < SYSTEM_RUNNING) {
- pr_debug("calling %lli_%pF @ %i\n",
- (long long)entry->cookie,
- entry->func, task_pid_nr(current));
- calltime = ktime_get();
- }
- entry->func(entry->data, entry->cookie);
- if (initcall_debug && system_state < SYSTEM_RUNNING) {
- rettime = ktime_get();
- delta = ktime_sub(rettime, calltime);
- pr_debug("initcall %lli_%pF returned 0 after %lld usecs\n",
- (long long)entry->cookie,
- entry->func,
- (long long)ktime_to_ns(delta) >> 10);
- }
- /* 2) remove self from the pending queues */
- spin_lock_irqsave(&async_lock, flags);
- list_del_init(&entry->domain_list);
- list_del_init(&entry->global_list);
- /* 3) free the entry */
- kfree(entry);
- atomic_dec(&entry_count);
- spin_unlock_irqrestore(&async_lock, flags);
- /* 4) wake up any waiters */
- wake_up(&async_done);
- }
- static async_cookie_t __async_schedule(async_func_t func, void *data, struct async_domain *domain)
- {
- struct async_entry *entry;
- unsigned long flags;
- async_cookie_t newcookie;
- /* allow irq-off callers */
- entry = kzalloc(sizeof(struct async_entry), GFP_ATOMIC);
- /*
- * If we're out of memory or if there's too much work
- * pending already, we execute synchronously.
- */
- if (!entry || atomic_read(&entry_count) > MAX_WORK) {
- kfree(entry);
- spin_lock_irqsave(&async_lock, flags);
- newcookie = next_cookie++;
- spin_unlock_irqrestore(&async_lock, flags);
- /* low on memory.. run synchronously */
- func(data, newcookie);
- return newcookie;
- }
- INIT_LIST_HEAD(&entry->domain_list);
- INIT_LIST_HEAD(&entry->global_list);
- INIT_WORK(&entry->work, async_run_entry_fn);
- entry->func = func;
- entry->data = data;
- entry->domain = domain;
- spin_lock_irqsave(&async_lock, flags);
- /* allocate cookie and queue */
- newcookie = entry->cookie = next_cookie++;
- list_add_tail(&entry->domain_list, &domain->pending);
- if (domain->registered)
- list_add_tail(&entry->global_list, &async_global_pending);
- atomic_inc(&entry_count);
- spin_unlock_irqrestore(&async_lock, flags);
- /* mark that this task has queued an async job, used by module init */
- current->flags |= PF_USED_ASYNC;
- /* schedule for execution */
- queue_work(system_unbound_wq, &entry->work);
- return newcookie;
- }
- /**
- * async_schedule - schedule a function for asynchronous execution
- * @func: function to execute asynchronously
- * @data: data pointer to pass to the function
- *
- * Returns an async_cookie_t that may be used for checkpointing later.
- * Note: This function may be called from atomic or non-atomic contexts.
- */
- async_cookie_t async_schedule(async_func_t func, void *data)
- {
- return __async_schedule(func, data, &async_dfl_domain);
- }
- EXPORT_SYMBOL_GPL(async_schedule);
- /**
- * async_schedule_domain - schedule a function for asynchronous execution within a certain domain
- * @func: function to execute asynchronously
- * @data: data pointer to pass to the function
- * @domain: the domain
- *
- * Returns an async_cookie_t that may be used for checkpointing later.
- * @domain may be used in the async_synchronize_*_domain() functions to
- * wait within a certain synchronization domain rather than globally. A
- * synchronization domain is specified via @domain. Note: This function
- * may be called from atomic or non-atomic contexts.
- */
- async_cookie_t async_schedule_domain(async_func_t func, void *data,
- struct async_domain *domain)
- {
- return __async_schedule(func, data, domain);
- }
- EXPORT_SYMBOL_GPL(async_schedule_domain);
- /**
- * async_synchronize_full - synchronize all asynchronous function calls
- *
- * This function waits until all asynchronous function calls have been done.
- */
- void async_synchronize_full(void)
- {
- async_synchronize_full_domain(NULL);
- }
- EXPORT_SYMBOL_GPL(async_synchronize_full);
- /**
- * async_unregister_domain - ensure no more anonymous waiters on this domain
- * @domain: idle domain to flush out of any async_synchronize_full instances
- *
- * async_synchronize_{cookie|full}_domain() are not flushed since callers
- * of these routines should know the lifetime of @domain
- *
- * Prefer ASYNC_DOMAIN_EXCLUSIVE() declarations over flushing
- */
- void async_unregister_domain(struct async_domain *domain)
- {
- spin_lock_irq(&async_lock);
- WARN_ON(!domain->registered || !list_empty(&domain->pending));
- domain->registered = 0;
- spin_unlock_irq(&async_lock);
- }
- EXPORT_SYMBOL_GPL(async_unregister_domain);
- /**
- * async_synchronize_full_domain - synchronize all asynchronous function within a certain domain
- * @domain: the domain to synchronize
- *
- * This function waits until all asynchronous function calls for the
- * synchronization domain specified by @domain have been done.
- */
- void async_synchronize_full_domain(struct async_domain *domain)
- {
- async_synchronize_cookie_domain(ASYNC_COOKIE_MAX, domain);
- }
- EXPORT_SYMBOL_GPL(async_synchronize_full_domain);
- /**
- * async_synchronize_cookie_domain - synchronize asynchronous function calls within a certain domain with cookie checkpointing
- * @cookie: async_cookie_t to use as checkpoint
- * @domain: the domain to synchronize (%NULL for all registered domains)
- *
- * This function waits until all asynchronous function calls for the
- * synchronization domain specified by @domain submitted prior to @cookie
- * have been done.
- */
- void async_synchronize_cookie_domain(async_cookie_t cookie, struct async_domain *domain)
- {
- ktime_t uninitialized_var(starttime), delta, endtime;
- if (initcall_debug && system_state < SYSTEM_RUNNING) {
- pr_debug("async_waiting @ %i\n", task_pid_nr(current));
- starttime = ktime_get();
- }
- wait_event(async_done, lowest_in_progress(domain) >= cookie);
- if (initcall_debug && system_state < SYSTEM_RUNNING) {
- endtime = ktime_get();
- delta = ktime_sub(endtime, starttime);
- pr_debug("async_continuing @ %i after %lli usec\n",
- task_pid_nr(current),
- (long long)ktime_to_ns(delta) >> 10);
- }
- }
- EXPORT_SYMBOL_GPL(async_synchronize_cookie_domain);
- /**
- * async_synchronize_cookie - synchronize asynchronous function calls with cookie checkpointing
- * @cookie: async_cookie_t to use as checkpoint
- *
- * This function waits until all asynchronous function calls prior to @cookie
- * have been done.
- */
- void async_synchronize_cookie(async_cookie_t cookie)
- {
- async_synchronize_cookie_domain(cookie, &async_dfl_domain);
- }
- EXPORT_SYMBOL_GPL(async_synchronize_cookie);
- /**
- * current_is_async - is %current an async worker task?
- *
- * Returns %true if %current is an async worker task.
- */
- bool current_is_async(void)
- {
- struct worker *worker = current_wq_worker();
- return worker && worker->current_func == async_run_entry_fn;
- }
- EXPORT_SYMBOL_GPL(current_is_async);
|