Lines Matching defs:key
41 * Entrires are sorted by key.
72 jea->key = jeb->key - delta;
76 jeb->key = tmp.key + delta;
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key)
110 int n = atomic_read(&key->enabled);
117 * static_key_fast_inc_not_disabled - adds a user for a static key
118 * @key: static key that must be already enabled
120 * The caller must make sure that the static key can't get disabled while
122 * an already enabled static key.
127 bool static_key_fast_inc_not_disabled(struct static_key *key)
131 STATIC_KEY_CHECK_USE(key);
133 * Negative key->enabled has a special meaning: it sends
141 v = atomic_read(&key->enabled);
145 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));
151 bool static_key_slow_inc_cpuslocked(struct static_key *key)
160 * static_key_enabled(&key) for jumps to be updated properly.
162 if (static_key_fast_inc_not_disabled(key))
167 if (!atomic_cmpxchg(&key->enabled, 0, -1)) {
168 jump_label_update(key);
174 atomic_set_release(&key->enabled, 1);
180 if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key)))
186 bool static_key_slow_inc(struct static_key *key)
191 ret = static_key_slow_inc_cpuslocked(key);
197 void static_key_enable_cpuslocked(struct static_key *key)
199 STATIC_KEY_CHECK_USE(key);
202 if (atomic_read(&key->enabled) > 0) {
203 WARN_ON_ONCE(atomic_read(&key->enabled) != 1);
208 if (atomic_read(&key->enabled) == 0) {
209 atomic_set(&key->enabled, -1);
210 jump_label_update(key);
214 atomic_set_release(&key->enabled, 1);
220 void static_key_enable(struct static_key *key)
223 static_key_enable_cpuslocked(key);
228 void static_key_disable_cpuslocked(struct static_key *key)
230 STATIC_KEY_CHECK_USE(key);
233 if (atomic_read(&key->enabled) != 1) {
234 WARN_ON_ONCE(atomic_read(&key->enabled) != 0);
239 if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)
240 jump_label_update(key);
245 void static_key_disable(struct static_key *key)
248 static_key_disable_cpuslocked(key);
253 static bool static_key_dec_not_one(struct static_key *key)
258 * Go into the slow path if key::enabled is less than or equal than
259 * one. One is valid to shut down the key, anything less than one
268 v = atomic_read(&key->enabled);
287 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1)));
292 static void __static_key_slow_dec_cpuslocked(struct static_key *key)
297 if (static_key_dec_not_one(key))
301 val = atomic_read(&key->enabled);
314 if (atomic_dec_and_test(&key->enabled))
315 jump_label_update(key);
318 static void __static_key_slow_dec(struct static_key *key)
321 __static_key_slow_dec_cpuslocked(key);
327 struct static_key_deferred *key =
329 __static_key_slow_dec(&key->key);
333 void static_key_slow_dec(struct static_key *key)
335 STATIC_KEY_CHECK_USE(key);
336 __static_key_slow_dec(key);
340 void static_key_slow_dec_cpuslocked(struct static_key *key)
342 STATIC_KEY_CHECK_USE(key);
343 __static_key_slow_dec_cpuslocked(key);
346 void __static_key_slow_dec_deferred(struct static_key *key,
350 STATIC_KEY_CHECK_USE(key);
352 if (static_key_dec_not_one(key))
359 void __static_key_deferred_flush(void *key, struct delayed_work *work)
361 STATIC_KEY_CHECK_USE(key);
366 void jump_label_rate_limit(struct static_key_deferred *key,
369 STATIC_KEY_CHECK_USE(key);
370 key->timeout = rl;
371 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);
409 static inline struct jump_entry *static_key_entries(struct static_key *key)
411 WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);
412 return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);
415 static inline bool static_key_type(struct static_key *key)
417 return key->type & JUMP_TYPE_TRUE;
420 static inline bool static_key_linked(struct static_key *key)
422 return key->type & JUMP_TYPE_LINKED;
425 static inline void static_key_clear_linked(struct static_key *key)
427 key->type &= ~JUMP_TYPE_LINKED;
430 static inline void static_key_set_linked(struct static_key *key)
432 key->type |= JUMP_TYPE_LINKED;
444 static void static_key_set_entries(struct static_key *key,
450 type = key->type & JUMP_TYPE_MASK;
451 key->entries = entries;
452 key->type |= type;
457 struct static_key *key = jump_entry_key(entry);
458 bool enabled = static_key_enabled(key);
492 static void __jump_label_update(struct static_key *key,
497 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
503 static void __jump_label_update(struct static_key *key,
508 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {
529 struct static_key *key = NULL;
560 if (iterk == key)
563 key = iterk;
564 static_key_set_entries(key, iter);
571 static inline bool static_key_sealed(struct static_key *key)
573 return (key->type & JUMP_TYPE_LINKED) && !(key->type & ~JUMP_TYPE_MASK);
576 static inline void static_key_seal(struct static_key *key)
578 unsigned long type = key->type & JUMP_TYPE_TRUE;
579 key->type = JUMP_TYPE_LINKED | type;
614 struct static_key *key = jump_entry_key(entry);
615 bool type = static_key_type(key);
628 static inline struct static_key_mod *static_key_mod(struct static_key *key)
630 WARN_ON_ONCE(!static_key_linked(key));
631 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);
635 * key->type and key->next are the same via union.
636 * This sets key->next and preserves the type bits.
640 static void static_key_set_mod(struct static_key *key,
646 type = key->type & JUMP_TYPE_MASK;
647 key->next = mod;
648 key->type |= type;
674 static void __jump_label_mod_update(struct static_key *key)
678 for (mod = static_key_mod(key); mod; mod = mod->next) {
694 __jump_label_update(key, mod->entries, stop,
704 struct static_key *key = NULL;
721 if (iterk == key)
724 key = iterk;
725 if (within_module((unsigned long)key, mod)) {
726 static_key_set_entries(key, iter);
731 * If the key was sealed at init, then there's no need to keep a
735 if (static_key_sealed(key))
741 if (!static_key_linked(key)) {
749 jlm2->mod = __module_address((unsigned long)key);
751 jlm2->entries = static_key_entries(key);
753 static_key_set_mod(key, jlm2);
754 static_key_set_linked(key);
758 jlm->next = static_key_mod(key);
759 static_key_set_mod(key, jlm);
760 static_key_set_linked(key);
765 __jump_label_update(key, iter, iter_stop, true);
776 struct static_key *key = NULL;
780 if (jump_entry_key(iter) == key)
783 key = jump_entry_key(iter);
785 if (within_module((unsigned long)key, mod))
788 /* No @jlm allocated because key was sealed at init. */
789 if (static_key_sealed(key))
793 if (WARN_ON(!static_key_linked(key)))
796 prev = &key->next;
797 jlm = static_key_mod(key);
808 if (prev == &key->next)
809 static_key_set_mod(key, jlm->next);
815 jlm = static_key_mod(key);
818 static_key_set_entries(key, jlm->entries);
819 static_key_clear_linked(key);
895 static void jump_label_update(struct static_key *key)
903 if (static_key_linked(key)) {
904 __jump_label_mod_update(key);
909 mod = __module_address((unsigned long)key);
916 entry = static_key_entries(key);
919 __jump_label_update(key, entry, stop, init);
931 WARN_ON(static_key_enabled(&sk_true.key) != true);
932 WARN_ON(static_key_enabled(&sk_false.key) != false);
942 WARN_ON(static_key_enabled(&sk_true.key) == true);
943 WARN_ON(static_key_enabled(&sk_false.key) == false);