1 // SPDX-License-Identifier: GPL-2.0-only
41 	 * Entrires are sorted by key.  in jump_label_cmp()
44 		return -1;  in jump_label_cmp()
55 		return -1;  in jump_label_cmp()
65 	long delta = (unsigned long)a - (unsigned long)b;  in jump_label_swap()
70 	jea->code	= jeb->code - delta;  in jump_label_swap()
71 	jea->target	= jeb->target - delta;  in jump_label_swap()
72 	jea->key	= jeb->key - delta;  in jump_label_swap()
74 	jeb->code	= tmp.code + delta;  in jump_label_swap()
75 	jeb->target	= tmp.target + delta;  in jump_label_swap()
76 	jeb->key	= tmp.key + delta;  in jump_label_swap()
88 	size = (((unsigned long)stop - (unsigned long)start)  in jump_label_sort_entries()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key)  in static_key_count()  argument
107 	 * -1 means the first static_key_slow_inc() is in progress.  in static_key_count()
110 	int n = atomic_read(&key->enabled);  in static_key_count()
117  * static_key_fast_inc_not_disabled - adds a user for a static key
118  * @key: static key that must be already enabled
120  * The caller must make sure that the static key can't get disabled while
122  * an already enabled static key.
127 bool static_key_fast_inc_not_disabled(struct static_key *key)  in static_key_fast_inc_not_disabled()  argument
131 	STATIC_KEY_CHECK_USE(key);  in static_key_fast_inc_not_disabled()
133 	 * Negative key->enabled has a special meaning: it sends  in static_key_fast_inc_not_disabled()
134 	 * static_key_slow_inc/dec() down the slow path, and it is non-zero  in static_key_fast_inc_not_disabled()
141 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
145 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
151 bool static_key_slow_inc_cpuslocked(struct static_key *key)  in static_key_slow_inc_cpuslocked()  argument
160 	 * static_key_enabled(&key) for jumps to be updated properly.  in static_key_slow_inc_cpuslocked()
162 	if (static_key_fast_inc_not_disabled(key))  in static_key_slow_inc_cpuslocked()
167 	if (!atomic_cmpxchg(&key->enabled, 0, -1)) {  in static_key_slow_inc_cpuslocked()
168 		jump_label_update(key);  in static_key_slow_inc_cpuslocked()
174 		atomic_set_release(&key->enabled, 1);  in static_key_slow_inc_cpuslocked()
180 		if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key)))  in static_key_slow_inc_cpuslocked()
186 bool static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
191 	ret = static_key_slow_inc_cpuslocked(key);  in static_key_slow_inc()
197 void static_key_enable_cpuslocked(struct static_key *key)  in static_key_enable_cpuslocked()  argument
199 	STATIC_KEY_CHECK_USE(key);  in static_key_enable_cpuslocked()
202 	if (atomic_read(&key->enabled) > 0) {  in static_key_enable_cpuslocked()
203 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable_cpuslocked()
208 	if (atomic_read(&key->enabled) == 0) {  in static_key_enable_cpuslocked()
209 		atomic_set(&key->enabled, -1);  in static_key_enable_cpuslocked()
210 		jump_label_update(key);  in static_key_enable_cpuslocked()
214 		atomic_set_release(&key->enabled, 1);  in static_key_enable_cpuslocked()
220 void static_key_enable(struct static_key *key)  in static_key_enable()  argument
223 	static_key_enable_cpuslocked(key);  in static_key_enable()
228 void static_key_disable_cpuslocked(struct static_key *key)  in static_key_disable_cpuslocked()  argument
230 	STATIC_KEY_CHECK_USE(key);  in static_key_disable_cpuslocked()
233 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable_cpuslocked()
234 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable_cpuslocked()
239 	if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)  in static_key_disable_cpuslocked()
240 		jump_label_update(key);  in static_key_disable_cpuslocked()
245 void static_key_disable(struct static_key *key)  in static_key_disable()  argument
248 	static_key_disable_cpuslocked(key);  in static_key_disable()
253 static bool static_key_dec_not_one(struct static_key *key)  in static_key_dec_not_one()  argument
258 	 * Go into the slow path if key::enabled is less than or equal than  in static_key_dec_not_one()
259 	 * one. One is valid to shut down the key, anything less than one  in static_key_dec_not_one()
262 	 * That includes the special case of '-1' which is set in  in static_key_dec_not_one()
268 	v = atomic_read(&key->enabled);  in static_key_dec_not_one()
271 		 * Warn about the '-1' case though; since that means a  in static_key_dec_not_one()
272 		 * decrement is concurrent with a first (0->1) increment. IOW  in static_key_dec_not_one()
287 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1)));  in static_key_dec_not_one()
292 static void __static_key_slow_dec_cpuslocked(struct static_key *key)  in __static_key_slow_dec_cpuslocked()  argument
297 	if (static_key_dec_not_one(key))  in __static_key_slow_dec_cpuslocked()
301 	val = atomic_read(&key->enabled);  in __static_key_slow_dec_cpuslocked()
303 	 * It should be impossible to observe -1 with jump_label_mutex held,  in __static_key_slow_dec_cpuslocked()
306 	if (WARN_ON_ONCE(val == -1))  in __static_key_slow_dec_cpuslocked()
314 	if (atomic_dec_and_test(&key->enabled))  in __static_key_slow_dec_cpuslocked()
315 		jump_label_update(key);  in __static_key_slow_dec_cpuslocked()
318 static void __static_key_slow_dec(struct static_key *key)  in __static_key_slow_dec()  argument
321 	__static_key_slow_dec_cpuslocked(key);  in __static_key_slow_dec()
327 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
329 	__static_key_slow_dec(&key->key);  in jump_label_update_timeout()
333 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
335 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
336 	__static_key_slow_dec(key);  in static_key_slow_dec()
340 void static_key_slow_dec_cpuslocked(struct static_key *key)  in static_key_slow_dec_cpuslocked()  argument
342 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_cpuslocked()
343 	__static_key_slow_dec_cpuslocked(key);  in static_key_slow_dec_cpuslocked()
346 void __static_key_slow_dec_deferred(struct static_key *key,  in __static_key_slow_dec_deferred()  argument
350 	STATIC_KEY_CHECK_USE(key);  in __static_key_slow_dec_deferred()
352 	if (static_key_dec_not_one(key))  in __static_key_slow_dec_deferred()
359 void __static_key_deferred_flush(void *key, struct delayed_work *work)  in __static_key_deferred_flush()  argument
361 	STATIC_KEY_CHECK_USE(key);  in __static_key_deferred_flush()
366 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
369 	STATIC_KEY_CHECK_USE(key);  in jump_label_rate_limit()
370 	key->timeout = rl;  in jump_label_rate_limit()
371 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
409 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
411 	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);  in static_key_entries()
412 	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);  in static_key_entries()
415 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
417 	return key->type & JUMP_TYPE_TRUE;  in static_key_type()
420 static inline bool static_key_linked(struct static_key *key)  in static_key_linked()  argument
422 	return key->type & JUMP_TYPE_LINKED;  in static_key_linked()
425 static inline void static_key_clear_linked(struct static_key *key)  in static_key_clear_linked()  argument
427 	key->type &= ~JUMP_TYPE_LINKED;  in static_key_clear_linked()
430 static inline void static_key_set_linked(struct static_key *key)  in static_key_set_linked()  argument
432 	key->type |= JUMP_TYPE_LINKED;  in static_key_set_linked()
444 static void static_key_set_entries(struct static_key *key,  in static_key_set_entries()  argument
450 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_entries()
451 	key->entries = entries;  in static_key_set_entries()
452 	key->type |= type;  in static_key_set_entries()
457 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
458 	bool enabled = static_key_enabled(key);  in jump_label_type()
475 		 * This skips patching built-in __exit, which  in jump_label_can_update()
479 		 * Skipping built-in __exit is fine since it  in jump_label_can_update()
492 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
497 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
503 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
508 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
529 	struct static_key *key = NULL;  in jump_label_init()  local
560 		if (iterk == key)  in jump_label_init()
563 		key = iterk;  in jump_label_init()
564 		static_key_set_entries(key, iter);  in jump_label_init()
571 static inline bool static_key_sealed(struct static_key *key)  in static_key_sealed()  argument
573 	return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK);  in static_key_sealed()
576 static inline void static_key_seal(struct static_key *key)  in static_key_seal()  argument
578 	unsigned long type = key->type & JUMP_TYPE_TRUE;  in static_key_seal()
579 	key->type = JUMP_TYPE_LINKED | type;  in static_key_seal()
614 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
615 	bool type = static_key_type(key);  in jump_label_init_type()
628 static inline struct static_key_mod *static_key_mod(struct static_key *key)  in static_key_mod()  argument
630 	WARN_ON_ONCE(!static_key_linked(key));  in static_key_mod()
631 	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);  in static_key_mod()
635  * key->type and key->next are the same via union.
636  * This sets key->next and preserves the type bits.
640 static void static_key_set_mod(struct static_key *key,  in static_key_set_mod()  argument
646 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_mod()
647 	key->next = mod;  in static_key_set_mod()
648 	key->type |= type;  in static_key_set_mod()
666 	ret = __jump_label_text_reserved(mod->jump_entries,  in __jump_label_mod_text_reserved()
667 				mod->jump_entries + mod->num_jump_entries,  in __jump_label_mod_text_reserved()
668 				start, end, mod->state == MODULE_STATE_COMING);  in __jump_label_mod_text_reserved()
675 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
679 	for (mod = static_key_mod(key); mod; mod = mod->next) {  in __jump_label_mod_update()
687 		if (!mod->entries)  in __jump_label_mod_update()
690 		m = mod->mod;  in __jump_label_mod_update()
694 			stop = m->jump_entries + m->num_jump_entries;  in __jump_label_mod_update()
695 		__jump_label_update(key, mod->entries, stop,  in __jump_label_mod_update()
696 				    m && m->state == MODULE_STATE_COMING);  in __jump_label_mod_update()
702 	struct jump_entry *iter_start = mod->jump_entries;  in jump_label_add_module()
703 	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;  in jump_label_add_module()
705 	struct static_key *key = NULL;  in jump_label_add_module()  local
722 		if (iterk == key)  in jump_label_add_module()
725 		key = iterk;  in jump_label_add_module()
726 		if (within_module((unsigned long)key, mod)) {  in jump_label_add_module()
727 			static_key_set_entries(key, iter);  in jump_label_add_module()
732 		 * If the key was sealed at init, then there's no need to keep a  in jump_label_add_module()
733 		 * reference to its module entries - just patch them now and be  in jump_label_add_module()
736 		if (static_key_sealed(key))  in jump_label_add_module()
741 			return -ENOMEM;  in jump_label_add_module()
742 		if (!static_key_linked(key)) {  in jump_label_add_module()
747 				return -ENOMEM;  in jump_label_add_module()
750 			jlm2->mod = __module_address((unsigned long)key);  in jump_label_add_module()
752 			jlm2->entries = static_key_entries(key);  in jump_label_add_module()
753 			jlm2->next = NULL;  in jump_label_add_module()
754 			static_key_set_mod(key, jlm2);  in jump_label_add_module()
755 			static_key_set_linked(key);  in jump_label_add_module()
757 		jlm->mod = mod;  in jump_label_add_module()
758 		jlm->entries = iter;  in jump_label_add_module()
759 		jlm->next = static_key_mod(key);  in jump_label_add_module()
760 		static_key_set_mod(key, jlm);  in jump_label_add_module()
761 		static_key_set_linked(key);  in jump_label_add_module()
766 			__jump_label_update(key, iter, iter_stop, true);  in jump_label_add_module()
774 	struct jump_entry *iter_start = mod->jump_entries;  in jump_label_del_module()
775 	struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;  in jump_label_del_module()
777 	struct static_key *key = NULL;  in jump_label_del_module()  local
781 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
784 		key = jump_entry_key(iter);  in jump_label_del_module()
786 		if (within_module((unsigned long)key, mod))  in jump_label_del_module()
789 		/* No @jlm allocated because key was sealed at init. */  in jump_label_del_module()
790 		if (static_key_sealed(key))  in jump_label_del_module()
794 		if (WARN_ON(!static_key_linked(key)))  in jump_label_del_module()
797 		prev = &key->next;  in jump_label_del_module()
798 		jlm = static_key_mod(key);  in jump_label_del_module()
800 		while (jlm && jlm->mod != mod) {  in jump_label_del_module()
801 			prev = &jlm->next;  in jump_label_del_module()
802 			jlm = jlm->next;  in jump_label_del_module()
809 		if (prev == &key->next)  in jump_label_del_module()
810 			static_key_set_mod(key, jlm->next);  in jump_label_del_module()
812 			*prev = jlm->next;  in jump_label_del_module()
816 		jlm = static_key_mod(key);  in jump_label_del_module()
818 		if (jlm->next == NULL) {  in jump_label_del_module()
819 			static_key_set_entries(key, jlm->entries);  in jump_label_del_module()
820 			static_key_clear_linked(key);  in jump_label_del_module()
869  * jump_label_text_reserved - check if addr range is reserved
896 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
904 	if (static_key_linked(key)) {  in jump_label_update()
905 		__jump_label_mod_update(key);  in jump_label_update()
910 	mod = __module_address((unsigned long)key);  in jump_label_update()
912 		stop = mod->jump_entries + mod->num_jump_entries;  in jump_label_update()
913 		init = mod->state == MODULE_STATE_COMING;  in jump_label_update()
917 	entry = static_key_entries(key);  in jump_label_update()
920 		__jump_label_update(key, entry, stop, init);  in jump_label_update()
932 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
933 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
943 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
944 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()