1 /* SPDX-License-Identifier: GPL-2.0 */
8  * Copyright (C) 2009-2012 Jason Baron <jbaron@redhat.com>
9  * Copyright (C) 2011-2012 Red Hat, Inc., Peter Zijlstra
31  * self-modifying code. Assuming toolchain and architecture support, if we
34  * (which defaults to false - and the true block is placed out of line).
38  * unconditional branch to the out-of-line true branch. Keys that are
45  * these calls then we run-time modify the branch target via a
46  * no-op -> jump or jump -> no-op conversion. For example, for an
49  * to the out-of-line of true branch.
61  * to the out-of-line block.
71  * Additional babbling in: Documentation/staging/static-keys.rst
124 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
126 	return (unsigned long)&entry->code + entry->code;  in jump_entry_code()
129 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
131 	return (unsigned long)&entry->target + entry->target;  in jump_entry_target()
134 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
136 	long offset = entry->key & ~3L;  in jump_entry_key()
138 	return (struct static_key *)((unsigned long)&entry->key + offset);  in jump_entry_key()
143 static inline unsigned long jump_entry_code(const struct jump_entry *entry)  in jump_entry_code()  argument
145 	return entry->code;  in jump_entry_code()
148 static inline unsigned long jump_entry_target(const struct jump_entry *entry)  in jump_entry_target()  argument
150 	return entry->target;  in jump_entry_target()
153 static inline struct static_key *jump_entry_key(const struct jump_entry *entry)  in jump_entry_key()  argument
155 	return (struct static_key *)((unsigned long)entry->key & ~3UL);  in jump_entry_key()
160 static inline bool jump_entry_is_branch(const struct jump_entry *entry)  in jump_entry_is_branch()  argument
162 	return (unsigned long)entry->key & 1UL;  in jump_entry_is_branch()
165 static inline bool jump_entry_is_init(const struct jump_entry *entry)  in jump_entry_is_init()  argument
167 	return (unsigned long)entry->key & 2UL;  in jump_entry_is_init()
170 static inline void jump_entry_set_init(struct jump_entry *entry, bool set)  in jump_entry_set_init()  argument
173 		entry->key |= 2;  in jump_entry_set_init()
175 		entry->key &= ~2;  in jump_entry_set_init()
178 static inline int jump_entry_size(struct jump_entry *entry)  in jump_entry_size()  argument
183 	return arch_jump_entry_size(entry);  in jump_entry_size()
223 extern void arch_jump_label_transform(struct jump_entry *entry,
225 extern bool arch_jump_label_transform_queue(struct jump_entry *entry,
239 extern enum jump_label_type jump_label_init_type(struct jump_entry *entry);
244  * in 'low-level' headers. Thus, we are initializing .enabled with a
262 	return raw_atomic_read(&key->enabled);  in static_key_count()
292 	 * Prevent key->enabled getting negative to follow the same semantics  in static_key_fast_inc_not_disabled()
295 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
299 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
307 	atomic_dec(&key->enabled);  in static_key_slow_dec()
325 	if (atomic_read(&key->enabled) != 0) {  in static_key_enable()
326 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable()
329 	atomic_set(&key->enabled, 1);  in static_key_enable()
336 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable()
337 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable()
340 	atomic_set(&key->enabled, 0);  in static_key_disable()
356 /* -------------------------------------------------------------------------- */
376 #define DEFINE_STATIC_KEY_TRUE(name)	\  argument
377 	struct static_key_true name = STATIC_KEY_TRUE_INIT
379 #define DEFINE_STATIC_KEY_TRUE_RO(name)	\  argument
380 	struct static_key_true name __ro_after_init = STATIC_KEY_TRUE_INIT
382 #define DECLARE_STATIC_KEY_TRUE(name)	\  argument
383 	extern struct static_key_true name
385 #define DEFINE_STATIC_KEY_FALSE(name)	\  argument
386 	struct static_key_false name = STATIC_KEY_FALSE_INIT
388 #define DEFINE_STATIC_KEY_FALSE_RO(name)	\  argument
389 	struct static_key_false name __ro_after_init = STATIC_KEY_FALSE_INIT
391 #define DECLARE_STATIC_KEY_FALSE(name)	\  argument
392 	extern struct static_key_false name
394 #define DEFINE_STATIC_KEY_ARRAY_TRUE(name, count)		\  argument
395 	struct static_key_true name[count] = {			\
396 		[0 ... (count) - 1] = STATIC_KEY_TRUE_INIT,	\
399 #define DEFINE_STATIC_KEY_ARRAY_FALSE(name, count)		\  argument
400 	struct static_key_false name[count] = {			\
401 		[0 ... (count) - 1] = STATIC_KEY_FALSE_INIT,	\
404 #define _DEFINE_STATIC_KEY_1(name)	DEFINE_STATIC_KEY_TRUE(name)  argument
405 #define _DEFINE_STATIC_KEY_0(name)	DEFINE_STATIC_KEY_FALSE(name)  argument
406 #define DEFINE_STATIC_KEY_MAYBE(cfg, name)			\  argument
407 	__PASTE(_DEFINE_STATIC_KEY_, IS_ENABLED(cfg))(name)
409 #define _DEFINE_STATIC_KEY_RO_1(name)	DEFINE_STATIC_KEY_TRUE_RO(name)  argument
410 #define _DEFINE_STATIC_KEY_RO_0(name)	DEFINE_STATIC_KEY_FALSE_RO(name)  argument
411 #define DEFINE_STATIC_KEY_MAYBE_RO(cfg, name)			\  argument
412 	__PASTE(_DEFINE_STATIC_KEY_RO_, IS_ENABLED(cfg))(name)
414 #define _DECLARE_STATIC_KEY_1(name)	DECLARE_STATIC_KEY_TRUE(name)  argument
415 #define _DECLARE_STATIC_KEY_0(name)	DECLARE_STATIC_KEY_FALSE(name)  argument
416 #define DECLARE_STATIC_KEY_MAYBE(cfg, name)			\  argument
417 	__PASTE(_DECLARE_STATIC_KEY_, IS_ENABLED(cfg))(name)
438  * -----------+-----------------------+------------------
442  *            |    <br-stmts>	      |	1: ...
445  *            |			      |	L: <br-stmts>
448  * -----------+-----------------------+------------------
452  *            |    <br-stmts>	      |	1: ...
455  *            |			      |	L: <br-stmts>
458  * -----------+-----------------------+------------------
469  * -----------------------------+-----------
492 		branch = !arch_static_branch(&(x)->key, true);			\
494 		branch = !arch_static_branch_jump(&(x)->key, true);		\
504 		branch = arch_static_branch_jump(&(x)->key, false);		\
506 		branch = arch_static_branch(&(x)->key, false);			\
514 #define static_branch_likely(x)		likely_notrace(static_key_enabled(&(x)->key))
515 #define static_branch_unlikely(x)	unlikely_notrace(static_key_enabled(&(x)->key))
527 #define static_branch_inc(x)		static_key_slow_inc(&(x)->key)
528 #define static_branch_dec(x)		static_key_slow_dec(&(x)->key)
529 #define static_branch_inc_cpuslocked(x)	static_key_slow_inc_cpuslocked(&(x)->key)
530 #define static_branch_dec_cpuslocked(x)	static_key_slow_dec_cpuslocked(&(x)->key)
536 #define static_branch_enable(x)			static_key_enable(&(x)->key)
537 #define static_branch_disable(x)		static_key_disable(&(x)->key)
538 #define static_branch_enable_cpuslocked(x)	static_key_enable_cpuslocked(&(x)->key)
539 #define static_branch_disable_cpuslocked(x)	static_key_disable_cpuslocked(&(x)->key)