41 	 * Entrires are sorted by key.  in jump_label_cmp()
72 	jea->key	= jeb->key - delta;  in jump_label_swap()
76 	jeb->key	= tmp.key + delta;  in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key)  in static_key_count()  argument
110 	int n = atomic_read(&key->enabled);  in static_key_count()
117  * static_key_fast_inc_not_disabled - adds a user for a static key
118  * @key: static key that must be already enabled
120  * The caller must make sure that the static key can't get disabled while
122  * an already enabled static key.
127 bool static_key_fast_inc_not_disabled(struct static_key *key)  in static_key_fast_inc_not_disabled()  argument
131 	STATIC_KEY_CHECK_USE(key);  in static_key_fast_inc_not_disabled()
133 	 * Negative key->enabled has a special meaning: it sends  in static_key_fast_inc_not_disabled()
141 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
145 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
151 bool static_key_slow_inc_cpuslocked(struct static_key *key)  in static_key_slow_inc_cpuslocked()  argument
160 	 * static_key_enabled(&key) for jumps to be updated properly.  in static_key_slow_inc_cpuslocked()
162 	if (static_key_fast_inc_not_disabled(key))  in static_key_slow_inc_cpuslocked()
167 	if (!atomic_cmpxchg(&key->enabled, 0, -1)) {  in static_key_slow_inc_cpuslocked()
168 		jump_label_update(key);  in static_key_slow_inc_cpuslocked()
174 		atomic_set_release(&key->enabled, 1);  in static_key_slow_inc_cpuslocked()
180 		if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key)))  in static_key_slow_inc_cpuslocked()
186 bool static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
191 	ret = static_key_slow_inc_cpuslocked(key);  in static_key_slow_inc()
197 void static_key_enable_cpuslocked(struct static_key *key)  in static_key_enable_cpuslocked()  argument
199 	STATIC_KEY_CHECK_USE(key);  in static_key_enable_cpuslocked()
202 	if (atomic_read(&key->enabled) > 0) {  in static_key_enable_cpuslocked()
203 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable_cpuslocked()
208 	if (atomic_read(&key->enabled) == 0) {  in static_key_enable_cpuslocked()
209 		atomic_set(&key->enabled, -1);  in static_key_enable_cpuslocked()
210 		jump_label_update(key);  in static_key_enable_cpuslocked()
214 		atomic_set_release(&key->enabled, 1);  in static_key_enable_cpuslocked()
220 void static_key_enable(struct static_key *key)  in static_key_enable()  argument
223 	static_key_enable_cpuslocked(key);  in static_key_enable()
228 void static_key_disable_cpuslocked(struct static_key *key)  in static_key_disable_cpuslocked()  argument
230 	STATIC_KEY_CHECK_USE(key);  in static_key_disable_cpuslocked()
233 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable_cpuslocked()
234 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable_cpuslocked()
239 	if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)  in static_key_disable_cpuslocked()
240 		jump_label_update(key);  in static_key_disable_cpuslocked()
245 void static_key_disable(struct static_key *key)  in static_key_disable()  argument
248 	static_key_disable_cpuslocked(key);  in static_key_disable()
253 static bool static_key_dec_not_one(struct static_key *key)  in static_key_dec_not_one()  argument
258 	 * Go into the slow path if key::enabled is less than or equal than  in static_key_dec_not_one()
259 	 * one. One is valid to shut down the key, anything less than one  in static_key_dec_not_one()
268 	v = atomic_read(&key->enabled);  in static_key_dec_not_one()
287 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1)));  in static_key_dec_not_one()
292 static void __static_key_slow_dec_cpuslocked(struct static_key *key)  in __static_key_slow_dec_cpuslocked()  argument
297 	if (static_key_dec_not_one(key))  in __static_key_slow_dec_cpuslocked()
301 	val = atomic_read(&key->enabled);  in __static_key_slow_dec_cpuslocked()
314 	if (atomic_dec_and_test(&key->enabled))  in __static_key_slow_dec_cpuslocked()
315 		jump_label_update(key);  in __static_key_slow_dec_cpuslocked()
318 static void __static_key_slow_dec(struct static_key *key)  in __static_key_slow_dec()  argument
321 	__static_key_slow_dec_cpuslocked(key);  in __static_key_slow_dec()
327 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
329 	__static_key_slow_dec(&key->key);  in jump_label_update_timeout()
333 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
335 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
336 	__static_key_slow_dec(key);  in static_key_slow_dec()
340 void static_key_slow_dec_cpuslocked(struct static_key *key)  in static_key_slow_dec_cpuslocked()  argument
342 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_cpuslocked()
343 	__static_key_slow_dec_cpuslocked(key);  in static_key_slow_dec_cpuslocked()
346 void __static_key_slow_dec_deferred(struct static_key *key,  in __static_key_slow_dec_deferred()  argument
350 	STATIC_KEY_CHECK_USE(key);  in __static_key_slow_dec_deferred()
352 	if (static_key_dec_not_one(key))  in __static_key_slow_dec_deferred()
359 void __static_key_deferred_flush(void *key, struct delayed_work *work)  in __static_key_deferred_flush()  argument
361 	STATIC_KEY_CHECK_USE(key);  in __static_key_deferred_flush()
366 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
369 	STATIC_KEY_CHECK_USE(key);  in jump_label_rate_limit()
370 	key->timeout = rl;  in jump_label_rate_limit()
371 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
409 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
411 	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);  in static_key_entries()
412 	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);  in static_key_entries()
415 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
417 	return key->type & JUMP_TYPE_TRUE;  in static_key_type()
420 static inline bool static_key_linked(struct static_key *key)  in static_key_linked()  argument
422 	return key->type & JUMP_TYPE_LINKED;  in static_key_linked()
425 static inline void static_key_clear_linked(struct static_key *key)  in static_key_clear_linked()  argument
427 	key->type &= ~JUMP_TYPE_LINKED;  in static_key_clear_linked()
430 static inline void static_key_set_linked(struct static_key *key)  in static_key_set_linked()  argument
432 	key->type |= JUMP_TYPE_LINKED;  in static_key_set_linked()
444 static void static_key_set_entries(struct static_key *key,  in static_key_set_entries()  argument
450 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_entries()
451 	key->entries = entries;  in static_key_set_entries()
452 	key->type |= type;  in static_key_set_entries()
457 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
458 	bool enabled = static_key_enabled(key);  in jump_label_type()
492 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
497 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
503 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
508 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
529 	struct static_key *key = NULL;  in jump_label_init()  local
560 		if (iterk == key)  in jump_label_init()
563 		key = iterk;  in jump_label_init()
564 		static_key_set_entries(key, iter);  in jump_label_init()
571 static inline bool static_key_sealed(struct static_key *key)  in static_key_sealed()  argument
573 	return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK);  in static_key_sealed()
576 static inline void static_key_seal(struct static_key *key)  in static_key_seal()  argument
578 	unsigned long type = key->type & JUMP_TYPE_TRUE;  in static_key_seal()
579 	key->type = JUMP_TYPE_LINKED | type;  in static_key_seal()
614 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
615 	bool type = static_key_type(key);  in jump_label_init_type()
628 static inline struct static_key_mod *static_key_mod(struct static_key *key)  in static_key_mod()  argument
630 	WARN_ON_ONCE(!static_key_linked(key));  in static_key_mod()
631 	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);  in static_key_mod()
635  * key->type and key->next are the same via union.
636  * This sets key->next and preserves the type bits.
640 static void static_key_set_mod(struct static_key *key,  in static_key_set_mod()  argument
646 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_mod()
647 	key->next = mod;  in static_key_set_mod()
648 	key->type |= type;  in static_key_set_mod()
675 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
679 	for (mod = static_key_mod(key); mod; mod = mod->next) {  in __jump_label_mod_update()
695 		__jump_label_update(key, mod->entries, stop,  in __jump_label_mod_update()
705 	struct static_key *key = NULL;  in jump_label_add_module()  local
722 		if (iterk == key)  in jump_label_add_module()
725 		key = iterk;  in jump_label_add_module()
726 		if (within_module((unsigned long)key, mod)) {  in jump_label_add_module()
727 			static_key_set_entries(key, iter);  in jump_label_add_module()
732 		 * If the key was sealed at init, then there's no need to keep a  in jump_label_add_module()
736 		if (static_key_sealed(key))  in jump_label_add_module()
742 		if (!static_key_linked(key)) {  in jump_label_add_module()
750 			jlm2->mod = __module_address((unsigned long)key);  in jump_label_add_module()
752 			jlm2->entries = static_key_entries(key);  in jump_label_add_module()
754 			static_key_set_mod(key, jlm2);  in jump_label_add_module()
755 			static_key_set_linked(key);  in jump_label_add_module()
759 		jlm->next = static_key_mod(key);  in jump_label_add_module()
760 		static_key_set_mod(key, jlm);  in jump_label_add_module()
761 		static_key_set_linked(key);  in jump_label_add_module()
766 			__jump_label_update(key, iter, iter_stop, true);  in jump_label_add_module()
777 	struct static_key *key = NULL;  in jump_label_del_module()  local
781 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
784 		key = jump_entry_key(iter);  in jump_label_del_module()
786 		if (within_module((unsigned long)key, mod))  in jump_label_del_module()
789 		/* No @jlm allocated because key was sealed at init. */  in jump_label_del_module()
790 		if (static_key_sealed(key))  in jump_label_del_module()
794 		if (WARN_ON(!static_key_linked(key)))  in jump_label_del_module()
797 		prev = &key->next;  in jump_label_del_module()
798 		jlm = static_key_mod(key);  in jump_label_del_module()
809 		if (prev == &key->next)  in jump_label_del_module()
810 			static_key_set_mod(key, jlm->next);  in jump_label_del_module()
816 		jlm = static_key_mod(key);  in jump_label_del_module()
819 			static_key_set_entries(key, jlm->entries);  in jump_label_del_module()
820 			static_key_clear_linked(key);  in jump_label_del_module()
896 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
904 	if (static_key_linked(key)) {  in jump_label_update()
905 		__jump_label_mod_update(key);  in jump_label_update()
910 	mod = __module_address((unsigned long)key);  in jump_label_update()
917 	entry = static_key_entries(key);  in jump_label_update()
920 		__jump_label_update(key, entry, stop, init);  in jump_label_update()
932 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
933 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
943 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
944 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()