Lines Matching +full:key +full:- +full:code
1 // SPDX-License-Identifier: GPL-2.0-only
41 * Entrires are sorted by key. in jump_label_cmp()
44 return -1; in jump_label_cmp()
50 * In the batching mode, entries should also be sorted by the code in jump_label_cmp()
55 return -1; in jump_label_cmp()
65 long delta = (unsigned long)a - (unsigned long)b; in jump_label_swap()
70 jea->code = jeb->code - delta; in jump_label_swap()
71 jea->target = jeb->target - delta; in jump_label_swap()
72 jea->key = jeb->key - delta; in jump_label_swap()
74 jeb->code = tmp.code + delta; in jump_label_swap()
75 jeb->target = tmp.target + delta; in jump_label_swap()
76 jeb->key = tmp.key + delta; in jump_label_swap()
88 size = (((unsigned long)stop - (unsigned long)start) in jump_label_sort_entries()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key) in static_key_count() argument
107 * -1 means the first static_key_slow_inc() is in progress. in static_key_count()
110 int n = atomic_read(&key->enabled); in static_key_count()
117 * static_key_fast_inc_not_disabled - adds a user for a static key
118 * @key: static key that must be already enabled
120 * The caller must make sure that the static key can't get disabled while
122 * an already enabled static key.
127 bool static_key_fast_inc_not_disabled(struct static_key *key) in static_key_fast_inc_not_disabled() argument
131 STATIC_KEY_CHECK_USE(key); in static_key_fast_inc_not_disabled()
133 * Negative key->enabled has a special meaning: it sends in static_key_fast_inc_not_disabled()
134 * static_key_slow_inc/dec() down the slow path, and it is non-zero in static_key_fast_inc_not_disabled()
138 * code to reset or detected in the slow path of in static_key_fast_inc_not_disabled()
141 v = atomic_read(&key->enabled); in static_key_fast_inc_not_disabled()
145 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1))); in static_key_fast_inc_not_disabled()
151 bool static_key_slow_inc_cpuslocked(struct static_key *key) in static_key_slow_inc_cpuslocked() argument
160 * static_key_enabled(&key) for jumps to be updated properly. in static_key_slow_inc_cpuslocked()
162 if (static_key_fast_inc_not_disabled(key)) in static_key_slow_inc_cpuslocked()
167 if (!atomic_cmpxchg(&key->enabled, 0, -1)) { in static_key_slow_inc_cpuslocked()
168 jump_label_update(key); in static_key_slow_inc_cpuslocked()
174 atomic_set_release(&key->enabled, 1); in static_key_slow_inc_cpuslocked()
180 if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key))) in static_key_slow_inc_cpuslocked()
186 bool static_key_slow_inc(struct static_key *key) in static_key_slow_inc() argument
191 ret = static_key_slow_inc_cpuslocked(key); in static_key_slow_inc()
197 void static_key_enable_cpuslocked(struct static_key *key) in static_key_enable_cpuslocked() argument
199 STATIC_KEY_CHECK_USE(key); in static_key_enable_cpuslocked()
202 if (atomic_read(&key->enabled) > 0) { in static_key_enable_cpuslocked()
203 WARN_ON_ONCE(atomic_read(&key->enabled) != 1); in static_key_enable_cpuslocked()
208 if (atomic_read(&key->enabled) == 0) { in static_key_enable_cpuslocked()
209 atomic_set(&key->enabled, -1); in static_key_enable_cpuslocked()
210 jump_label_update(key); in static_key_enable_cpuslocked()
214 atomic_set_release(&key->enabled, 1); in static_key_enable_cpuslocked()
220 void static_key_enable(struct static_key *key) in static_key_enable() argument
223 static_key_enable_cpuslocked(key); in static_key_enable()
228 void static_key_disable_cpuslocked(struct static_key *key) in static_key_disable_cpuslocked() argument
230 STATIC_KEY_CHECK_USE(key); in static_key_disable_cpuslocked()
233 if (atomic_read(&key->enabled) != 1) { in static_key_disable_cpuslocked()
234 WARN_ON_ONCE(atomic_read(&key->enabled) != 0); in static_key_disable_cpuslocked()
239 if (atomic_cmpxchg(&key->enabled, 1, 0) == 1) in static_key_disable_cpuslocked()
240 jump_label_update(key); in static_key_disable_cpuslocked()
245 void static_key_disable(struct static_key *key) in static_key_disable() argument
248 static_key_disable_cpuslocked(key); in static_key_disable()
253 static bool static_key_dec_not_one(struct static_key *key) in static_key_dec_not_one() argument
258 * Go into the slow path if key::enabled is less than or equal than in static_key_dec_not_one()
259 * one. One is valid to shut down the key, anything less than one in static_key_dec_not_one()
262 * That includes the special case of '-1' which is set in in static_key_dec_not_one()
268 v = atomic_read(&key->enabled); in static_key_dec_not_one()
271 * Warn about the '-1' case though; since that means a in static_key_dec_not_one()
272 * decrement is concurrent with a first (0->1) increment. IOW in static_key_dec_not_one()
287 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1))); in static_key_dec_not_one()
292 static void __static_key_slow_dec_cpuslocked(struct static_key *key) in __static_key_slow_dec_cpuslocked() argument
297 if (static_key_dec_not_one(key)) in __static_key_slow_dec_cpuslocked()
301 val = atomic_read(&key->enabled); in __static_key_slow_dec_cpuslocked()
303 * It should be impossible to observe -1 with jump_label_mutex held, in __static_key_slow_dec_cpuslocked()
306 if (WARN_ON_ONCE(val == -1)) in __static_key_slow_dec_cpuslocked()
314 if (atomic_dec_and_test(&key->enabled)) in __static_key_slow_dec_cpuslocked()
315 jump_label_update(key); in __static_key_slow_dec_cpuslocked()
318 static void __static_key_slow_dec(struct static_key *key) in __static_key_slow_dec() argument
321 __static_key_slow_dec_cpuslocked(key); in __static_key_slow_dec()
327 struct static_key_deferred *key = in jump_label_update_timeout() local
329 __static_key_slow_dec(&key->key); in jump_label_update_timeout()
333 void static_key_slow_dec(struct static_key *key) in static_key_slow_dec() argument
335 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec()
336 __static_key_slow_dec(key); in static_key_slow_dec()
340 void static_key_slow_dec_cpuslocked(struct static_key *key) in static_key_slow_dec_cpuslocked() argument
342 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec_cpuslocked()
343 __static_key_slow_dec_cpuslocked(key); in static_key_slow_dec_cpuslocked()
346 void __static_key_slow_dec_deferred(struct static_key *key, in __static_key_slow_dec_deferred() argument
350 STATIC_KEY_CHECK_USE(key); in __static_key_slow_dec_deferred()
352 if (static_key_dec_not_one(key)) in __static_key_slow_dec_deferred()
359 void __static_key_deferred_flush(void *key, struct delayed_work *work) in __static_key_deferred_flush() argument
361 STATIC_KEY_CHECK_USE(key); in __static_key_deferred_flush()
366 void jump_label_rate_limit(struct static_key_deferred *key, in jump_label_rate_limit() argument
369 STATIC_KEY_CHECK_USE(key); in jump_label_rate_limit()
370 key->timeout = rl; in jump_label_rate_limit()
371 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); in jump_label_rate_limit()
409 static inline struct jump_entry *static_key_entries(struct static_key *key) in static_key_entries() argument
411 WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED); in static_key_entries()
412 return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK); in static_key_entries()
415 static inline bool static_key_type(struct static_key *key) in static_key_type() argument
417 return key->type & JUMP_TYPE_TRUE; in static_key_type()
420 static inline bool static_key_linked(struct static_key *key) in static_key_linked() argument
422 return key->type & JUMP_TYPE_LINKED; in static_key_linked()
425 static inline void static_key_clear_linked(struct static_key *key) in static_key_clear_linked() argument
427 key->type &= ~JUMP_TYPE_LINKED; in static_key_clear_linked()
430 static inline void static_key_set_linked(struct static_key *key) in static_key_set_linked() argument
432 key->type |= JUMP_TYPE_LINKED; in static_key_set_linked()
444 static void static_key_set_entries(struct static_key *key, in static_key_set_entries() argument
450 type = key->type & JUMP_TYPE_MASK; in static_key_set_entries()
451 key->entries = entries; in static_key_set_entries()
452 key->type |= type; in static_key_set_entries()
457 struct static_key *key = jump_entry_key(entry); in jump_label_type() local
458 bool enabled = static_key_enabled(key); in jump_label_type()
468 * Cannot update code that was in an init text area. in jump_label_can_update()
475 * This skips patching built-in __exit, which in jump_label_can_update()
479 * Skipping built-in __exit is fine since it in jump_label_can_update()
492 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
497 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
503 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
508 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
529 struct static_key *key = NULL; in jump_label_init() local
551 if (iterk == key) in jump_label_init()
554 key = iterk; in jump_label_init()
555 static_key_set_entries(key, iter); in jump_label_init()
562 static inline bool static_key_sealed(struct static_key *key) in static_key_sealed() argument
564 return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK); in static_key_sealed()
567 static inline void static_key_seal(struct static_key *key) in static_key_seal() argument
569 unsigned long type = key->type & JUMP_TYPE_TRUE; in static_key_seal()
570 key->type = JUMP_TYPE_LINKED | type; in static_key_seal()
605 struct static_key *key = jump_entry_key(entry); in jump_label_init_type() local
606 bool type = static_key_type(key); in jump_label_init_type()
619 static inline struct static_key_mod *static_key_mod(struct static_key *key) in static_key_mod() argument
621 WARN_ON_ONCE(!static_key_linked(key)); in static_key_mod()
622 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK); in static_key_mod()
626 * key->type and key->next are the same via union.
627 * This sets key->next and preserves the type bits.
631 static void static_key_set_mod(struct static_key *key, in static_key_set_mod() argument
637 type = key->type & JUMP_TYPE_MASK; in static_key_set_mod()
638 key->next = mod; in static_key_set_mod()
639 key->type |= type; in static_key_set_mod()
656 ret = __jump_label_text_reserved(mod->jump_entries, in __jump_label_mod_text_reserved()
657 mod->jump_entries + mod->num_jump_entries, in __jump_label_mod_text_reserved()
658 start, end, mod->state == MODULE_STATE_COMING); in __jump_label_mod_text_reserved()
665 static void __jump_label_mod_update(struct static_key *key) in __jump_label_mod_update() argument
669 for (mod = static_key_mod(key); mod; mod = mod->next) { in __jump_label_mod_update()
677 if (!mod->entries) in __jump_label_mod_update()
680 m = mod->mod; in __jump_label_mod_update()
684 stop = m->jump_entries + m->num_jump_entries; in __jump_label_mod_update()
685 __jump_label_update(key, mod->entries, stop, in __jump_label_mod_update()
686 m && m->state == MODULE_STATE_COMING); in __jump_label_mod_update()
692 struct jump_entry *iter_start = mod->jump_entries; in jump_label_add_module()
693 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; in jump_label_add_module()
695 struct static_key *key = NULL; in jump_label_add_module() local
712 if (iterk == key) in jump_label_add_module()
715 key = iterk; in jump_label_add_module()
716 if (within_module((unsigned long)key, mod)) { in jump_label_add_module()
717 static_key_set_entries(key, iter); in jump_label_add_module()
722 * If the key was sealed at init, then there's no need to keep a in jump_label_add_module()
723 * reference to its module entries - just patch them now and be in jump_label_add_module()
726 if (static_key_sealed(key)) in jump_label_add_module()
731 return -ENOMEM; in jump_label_add_module()
732 if (!static_key_linked(key)) { in jump_label_add_module()
736 return -ENOMEM; in jump_label_add_module()
739 jlm2->mod = __module_address((unsigned long)key); in jump_label_add_module()
741 jlm2->entries = static_key_entries(key); in jump_label_add_module()
742 jlm2->next = NULL; in jump_label_add_module()
743 static_key_set_mod(key, jlm2); in jump_label_add_module()
744 static_key_set_linked(key); in jump_label_add_module()
746 jlm->mod = mod; in jump_label_add_module()
747 jlm->entries = iter; in jump_label_add_module()
748 jlm->next = static_key_mod(key); in jump_label_add_module()
749 static_key_set_mod(key, jlm); in jump_label_add_module()
750 static_key_set_linked(key); in jump_label_add_module()
755 __jump_label_update(key, iter, iter_stop, true); in jump_label_add_module()
763 struct jump_entry *iter_start = mod->jump_entries; in jump_label_del_module()
764 struct jump_entry *iter_stop = iter_start + mod->num_jump_entries; in jump_label_del_module()
766 struct static_key *key = NULL; in jump_label_del_module() local
770 if (jump_entry_key(iter) == key) in jump_label_del_module()
773 key = jump_entry_key(iter); in jump_label_del_module()
775 if (within_module((unsigned long)key, mod)) in jump_label_del_module()
778 /* No @jlm allocated because key was sealed at init. */ in jump_label_del_module()
779 if (static_key_sealed(key)) in jump_label_del_module()
783 if (WARN_ON(!static_key_linked(key))) in jump_label_del_module()
786 prev = &key->next; in jump_label_del_module()
787 jlm = static_key_mod(key); in jump_label_del_module()
789 while (jlm && jlm->mod != mod) { in jump_label_del_module()
790 prev = &jlm->next; in jump_label_del_module()
791 jlm = jlm->next; in jump_label_del_module()
798 if (prev == &key->next) in jump_label_del_module()
799 static_key_set_mod(key, jlm->next); in jump_label_del_module()
801 *prev = jlm->next; in jump_label_del_module()
805 jlm = static_key_mod(key); in jump_label_del_module()
807 if (jlm->next == NULL) { in jump_label_del_module()
808 static_key_set_entries(key, jlm->entries); in jump_label_del_module()
809 static_key_clear_linked(key); in jump_label_del_module()
858 * jump_label_text_reserved - check if addr range is reserved
863 * overlaps with any of the jump label patch addresses. Code
885 static void jump_label_update(struct static_key *key) in jump_label_update() argument
893 if (static_key_linked(key)) { in jump_label_update()
894 __jump_label_mod_update(key); in jump_label_update()
899 mod = __module_address((unsigned long)key); in jump_label_update()
901 stop = mod->jump_entries + mod->num_jump_entries; in jump_label_update()
902 init = mod->state == MODULE_STATE_COMING; in jump_label_update()
906 entry = static_key_entries(key); in jump_label_update()
909 __jump_label_update(key, entry, stop, init); in jump_label_update()
921 WARN_ON(static_key_enabled(&sk_true.key) != true); in jump_label_test()
922 WARN_ON(static_key_enabled(&sk_false.key) != false); in jump_label_test()
932 WARN_ON(static_key_enabled(&sk_true.key) == true); in jump_label_test()
933 WARN_ON(static_key_enabled(&sk_false.key) == false); in jump_label_test()