xref: /linux/kernel/kprobes.c (revision c55722233c041865d1477a58adbc52540bfad017)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Kernel Probes (KProbes)
4  *
5  * Copyright (C) IBM Corporation, 2002, 2004
6  *
7  * 2002-Oct	Created by Vamsi Krishna S <vamsi_krishna@in.ibm.com> Kernel
8  *		Probes initial implementation (includes suggestions from
9  *		Rusty Russell).
10  * 2004-Aug	Updated by Prasanna S Panchamukhi <prasanna@in.ibm.com> with
11  *		hlists and exceptions notifier as suggested by Andi Kleen.
12  * 2004-July	Suparna Bhattacharya <suparna@in.ibm.com> added jumper probes
13  *		interface to access function arguments.
14  * 2004-Sep	Prasanna S Panchamukhi <prasanna@in.ibm.com> Changed Kprobes
15  *		exceptions notifier to be first on the priority list.
16  * 2005-May	Hien Nguyen <hien@us.ibm.com>, Jim Keniston
17  *		<jkenisto@us.ibm.com> and Prasanna S Panchamukhi
18  *		<prasanna@in.ibm.com> added function-return probes.
19  */
20 
21 #define pr_fmt(fmt) "kprobes: " fmt
22 
23 #include <linux/kprobes.h>
24 #include <linux/hash.h>
25 #include <linux/init.h>
26 #include <linux/slab.h>
27 #include <linux/stddef.h>
28 #include <linux/export.h>
29 #include <linux/kallsyms.h>
30 #include <linux/freezer.h>
31 #include <linux/seq_file.h>
32 #include <linux/debugfs.h>
33 #include <linux/sysctl.h>
34 #include <linux/kdebug.h>
35 #include <linux/kthread.h>
36 #include <linux/memory.h>
37 #include <linux/ftrace.h>
38 #include <linux/cpu.h>
39 #include <linux/jump_label.h>
40 #include <linux/static_call.h>
41 #include <linux/perf_event.h>
42 #include <linux/execmem.h>
43 #include <linux/cleanup.h>
44 #include <linux/wait.h>
45 #include <linux/wait_bit.h>
46 #include <linux/rcupdate.h>
47 
48 #include <asm/sections.h>
49 #include <asm/cacheflush.h>
50 #include <asm/errno.h>
51 #include <linux/uaccess.h>
52 
53 #define KPROBE_HASH_BITS 6
54 #define KPROBE_TABLE_SIZE (1 << KPROBE_HASH_BITS)
55 
56 #if !defined(CONFIG_OPTPROBES) || !defined(CONFIG_SYSCTL)
57 #define kprobe_sysctls_init() do { } while (0)
58 #endif
59 
60 static int kprobes_initialized;
61 /* kprobe_table can be accessed by
62  * - Normal hlist traversal and RCU add/del under 'kprobe_mutex' is held.
63  * Or
64  * - RCU hlist traversal under disabling preempt (breakpoint handlers)
65  */
66 static struct hlist_head kprobe_table[KPROBE_TABLE_SIZE];
67 
68 /* NOTE: change this value only with 'kprobe_mutex' held */
69 static bool kprobes_all_disarmed;
70 
71 /* This protects 'kprobe_table' and 'optprobe_gens' */
72 static DEFINE_MUTEX(kprobe_mutex);
73 static DEFINE_PER_CPU(struct kprobe *, kprobe_instance);
74 
75 kprobe_opcode_t * __weak kprobe_lookup_name(const char *name,
76 					unsigned int __unused)
77 {
78 	return ((kprobe_opcode_t *)(kallsyms_lookup_name(name)));
79 }
80 
81 /*
82  * Blacklist -- list of 'struct kprobe_blacklist_entry' to store info where
83  * kprobes can not probe.
84  */
85 static LIST_HEAD(kprobe_blacklist);
86 
87 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
88 /*
89  * 'kprobe::ainsn.insn' points to the copy of the instruction to be
90  * single-stepped. x86_64, POWER4 and above have no-exec support and
91  * stepping on the instruction on a vmalloced/kmalloced/data page
92  * is a recipe for disaster
93  */
94 struct kprobe_insn_page {
95 	struct list_head list;
96 	kprobe_opcode_t *insns;		/* Page of instruction slots */
97 	struct kprobe_insn_cache *cache;
98 	int nused;
99 	int ngarbage;
100 	char slot_used[];
101 };
102 
103 static int slots_per_page(struct kprobe_insn_cache *c)
104 {
105 	return PAGE_SIZE/(c->insn_size * sizeof(kprobe_opcode_t));
106 }
107 
108 enum kprobe_slot_state {
109 	SLOT_CLEAN = 0,
110 	SLOT_DIRTY = 1,
111 	SLOT_USED = 2,
112 };
113 
114 void __weak *alloc_insn_page(void)
115 {
116 	/*
117 	 * Use execmem_alloc() so this page is within +/- 2GB of where the
118 	 * kernel image and loaded module images reside. This is required
119 	 * for most of the architectures.
120 	 * (e.g. x86-64 needs this to handle the %rip-relative fixups.)
121 	 */
122 	return execmem_alloc(EXECMEM_KPROBES, PAGE_SIZE);
123 }
124 
125 static void free_insn_page(void *page)
126 {
127 	execmem_free(page);
128 }
129 
130 struct kprobe_insn_cache kprobe_insn_slots = {
131 	.mutex = __MUTEX_INITIALIZER(kprobe_insn_slots.mutex),
132 	.alloc = alloc_insn_page,
133 	.free = free_insn_page,
134 	.sym = KPROBE_INSN_PAGE_SYM,
135 	.pages = LIST_HEAD_INIT(kprobe_insn_slots.pages),
136 	.insn_size = MAX_INSN_SIZE,
137 	.nr_garbage = 0,
138 };
139 static int collect_garbage_slots(struct kprobe_insn_cache *c);
140 
141 /**
142  * __get_insn_slot - Find a slot on an executable page for an instruction.
143  * @c: Pointer to kprobe instruction cache
144  *
145  * Description: Locates available slot on existing executable pages,
146  *              allocates an executable page if there's no room on existing ones.
147  * Return: Pointer to instruction slot on success, NULL on failure.
148  */
149 kprobe_opcode_t *__get_insn_slot(struct kprobe_insn_cache *c)
150 {
151 	struct kprobe_insn_page *kip;
152 
153 	/* Since the slot array is not protected by rcu, we need a mutex */
154 	guard(mutex)(&c->mutex);
155 	do {
156 		guard(rcu)();
157 		list_for_each_entry_rcu(kip, &c->pages, list) {
158 			if (kip->nused < slots_per_page(c)) {
159 				int i;
160 
161 				for (i = 0; i < slots_per_page(c); i++) {
162 					if (kip->slot_used[i] == SLOT_CLEAN) {
163 						kip->slot_used[i] = SLOT_USED;
164 						kip->nused++;
165 						return kip->insns + (i * c->insn_size);
166 					}
167 				}
168 				/* kip->nused is broken. Fix it. */
169 				kip->nused = slots_per_page(c);
170 				WARN_ON(1);
171 			}
172 		}
173 	/* If there are any garbage slots, collect it and try again. */
174 	} while (c->nr_garbage && collect_garbage_slots(c) == 0);
175 
176 	/* All out of space.  Need to allocate a new page. */
177 	kip = kmalloc_flex(*kip, slot_used, slots_per_page(c));
178 	if (!kip)
179 		return NULL;
180 
181 	kip->insns = c->alloc();
182 	if (!kip->insns) {
183 		kfree(kip);
184 		return NULL;
185 	}
186 	INIT_LIST_HEAD(&kip->list);
187 	memset(kip->slot_used, SLOT_CLEAN, slots_per_page(c));
188 	kip->slot_used[0] = SLOT_USED;
189 	kip->nused = 1;
190 	kip->ngarbage = 0;
191 	kip->cache = c;
192 	list_add_rcu(&kip->list, &c->pages);
193 
194 	/* Record the perf ksymbol register event after adding the page */
195 	perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL, (unsigned long)kip->insns,
196 			   PAGE_SIZE, false, c->sym);
197 
198 	return kip->insns;
199 }
200 
201 /* Return true if all garbages are collected, otherwise false. */
202 static bool collect_one_slot(struct kprobe_insn_page *kip, int idx)
203 {
204 	kip->slot_used[idx] = SLOT_CLEAN;
205 	kip->nused--;
206 	if (kip->nused != 0)
207 		return false;
208 
209 	/*
210 	 * Page is no longer in use.  Free it unless
211 	 * it's the last one.  We keep the last one
212 	 * so as not to have to set it up again the
213 	 * next time somebody inserts a probe.
214 	 */
215 	if (!list_is_singular(&kip->list)) {
216 		/*
217 		 * Record perf ksymbol unregister event before removing
218 		 * the page.
219 		 */
220 		perf_event_ksymbol(PERF_RECORD_KSYMBOL_TYPE_OOL,
221 				   (unsigned long)kip->insns, PAGE_SIZE, true,
222 				   kip->cache->sym);
223 		list_del_rcu(&kip->list);
224 		synchronize_rcu();
225 		kip->cache->free(kip->insns);
226 		kfree(kip);
227 	}
228 	return true;
229 }
230 
231 static int collect_garbage_slots(struct kprobe_insn_cache *c)
232 {
233 	struct kprobe_insn_page *kip, *next;
234 
235 	/* Ensure no-one is interrupted on the garbages */
236 	synchronize_rcu();
237 
238 	list_for_each_entry_safe(kip, next, &c->pages, list) {
239 		int i;
240 
241 		if (kip->ngarbage == 0)
242 			continue;
243 		kip->ngarbage = 0;	/* we will collect all garbages */
244 		for (i = 0; i < slots_per_page(c); i++) {
245 			if (kip->slot_used[i] == SLOT_DIRTY && collect_one_slot(kip, i))
246 				break;
247 		}
248 	}
249 	c->nr_garbage = 0;
250 	return 0;
251 }
252 
253 static long __find_insn_page(struct kprobe_insn_cache *c,
254 	kprobe_opcode_t *slot, struct kprobe_insn_page **pkip)
255 {
256 	struct kprobe_insn_page *kip = NULL;
257 	long idx;
258 
259 	guard(rcu)();
260 	list_for_each_entry_rcu(kip, &c->pages, list) {
261 		idx = ((long)slot - (long)kip->insns) /
262 			(c->insn_size * sizeof(kprobe_opcode_t));
263 		if (idx >= 0 && idx < slots_per_page(c)) {
264 			*pkip = kip;
265 			return idx;
266 		}
267 	}
268 	/* Could not find this slot. */
269 	WARN_ON(1);
270 	*pkip = NULL;
271 	return -1;
272 }
273 
274 void __free_insn_slot(struct kprobe_insn_cache *c,
275 		      kprobe_opcode_t *slot, int dirty)
276 {
277 	struct kprobe_insn_page *kip = NULL;
278 	long idx;
279 
280 	guard(mutex)(&c->mutex);
281 	idx = __find_insn_page(c, slot, &kip);
282 	/* Mark and sweep: this may sleep */
283 	if (kip) {
284 		/* Check double free */
285 		WARN_ON(kip->slot_used[idx] != SLOT_USED);
286 		if (dirty) {
287 			kip->slot_used[idx] = SLOT_DIRTY;
288 			kip->ngarbage++;
289 			if (++c->nr_garbage > slots_per_page(c))
290 				collect_garbage_slots(c);
291 		} else {
292 			collect_one_slot(kip, idx);
293 		}
294 	}
295 }
296 
297 /*
298  * Check given address is on the page of kprobe instruction slots.
299  * This will be used for checking whether the address on a stack
300  * is on a text area or not.
301  */
302 bool __is_insn_slot_addr(struct kprobe_insn_cache *c, unsigned long addr)
303 {
304 	struct kprobe_insn_page *kip;
305 	bool ret = false;
306 
307 	rcu_read_lock();
308 	list_for_each_entry_rcu(kip, &c->pages, list) {
309 		if (addr >= (unsigned long)kip->insns &&
310 		    addr < (unsigned long)kip->insns + PAGE_SIZE) {
311 			ret = true;
312 			break;
313 		}
314 	}
315 	rcu_read_unlock();
316 
317 	return ret;
318 }
319 
320 int kprobe_cache_get_kallsym(struct kprobe_insn_cache *c, unsigned int *symnum,
321 			     unsigned long *value, char *type, char *sym)
322 {
323 	struct kprobe_insn_page *kip;
324 	int ret = -ERANGE;
325 
326 	rcu_read_lock();
327 	list_for_each_entry_rcu(kip, &c->pages, list) {
328 		if ((*symnum)--)
329 			continue;
330 		strscpy(sym, c->sym, KSYM_NAME_LEN);
331 		*type = 't';
332 		*value = (unsigned long)kip->insns;
333 		ret = 0;
334 		break;
335 	}
336 	rcu_read_unlock();
337 
338 	return ret;
339 }
340 
341 #ifdef CONFIG_OPTPROBES
342 void __weak *alloc_optinsn_page(void)
343 {
344 	return alloc_insn_page();
345 }
346 
347 void __weak free_optinsn_page(void *page)
348 {
349 	free_insn_page(page);
350 }
351 
352 /* For optimized_kprobe buffer */
353 struct kprobe_insn_cache kprobe_optinsn_slots = {
354 	.mutex = __MUTEX_INITIALIZER(kprobe_optinsn_slots.mutex),
355 	.alloc = alloc_optinsn_page,
356 	.free = free_optinsn_page,
357 	.sym = KPROBE_OPTINSN_PAGE_SYM,
358 	.pages = LIST_HEAD_INIT(kprobe_optinsn_slots.pages),
359 	/* .insn_size is initialized later */
360 	.nr_garbage = 0,
361 };
362 #endif /* CONFIG_OPTPROBES */
363 #endif /* __ARCH_WANT_KPROBES_INSN_SLOT */
364 
365 /* We have preemption disabled.. so it is safe to use __ versions */
366 static inline void set_kprobe_instance(struct kprobe *kp)
367 {
368 	__this_cpu_write(kprobe_instance, kp);
369 }
370 
371 static inline void reset_kprobe_instance(void)
372 {
373 	__this_cpu_write(kprobe_instance, NULL);
374 }
375 
376 /*
377  * This routine is called either:
378  *	- under the 'kprobe_mutex' - during kprobe_[un]register().
379  *				OR
380  *	- with preemption disabled - from architecture specific code.
381  */
382 struct kprobe *get_kprobe(void *addr)
383 {
384 	struct hlist_head *head;
385 	struct kprobe *p;
386 
387 	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
388 	hlist_for_each_entry_rcu(p, head, hlist,
389 				 lockdep_is_held(&kprobe_mutex)) {
390 		if (p->addr == addr)
391 			return p;
392 	}
393 
394 	return NULL;
395 }
396 NOKPROBE_SYMBOL(get_kprobe);
397 
398 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs);
399 
400 /* Return true if 'p' is an aggregator */
401 static inline bool kprobe_aggrprobe(struct kprobe *p)
402 {
403 	return p->pre_handler == aggr_pre_handler;
404 }
405 
406 /* Return true if 'p' is unused */
407 static inline bool kprobe_unused(struct kprobe *p)
408 {
409 	return kprobe_aggrprobe(p) && kprobe_disabled(p) &&
410 	       list_empty(&p->list);
411 }
412 
413 /* Keep all fields in the kprobe consistent. */
414 static inline void copy_kprobe(struct kprobe *ap, struct kprobe *p)
415 {
416 	memcpy(&p->opcode, &ap->opcode, sizeof(kprobe_opcode_t));
417 	memcpy(&p->ainsn, &ap->ainsn, sizeof(struct arch_specific_insn));
418 }
419 
420 #ifdef CONFIG_OPTPROBES
421 /* NOTE: This is protected by 'kprobe_mutex'. */
422 static bool kprobes_allow_optimization;
423 
424 /*
425  * Call all 'kprobe::pre_handler' on the list, but ignores its return value.
426  * This must be called from arch-dep optimized caller.
427  */
428 void opt_pre_handler(struct kprobe *p, struct pt_regs *regs)
429 {
430 	struct kprobe *kp;
431 
432 	list_for_each_entry_rcu(kp, &p->list, list) {
433 		if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
434 			set_kprobe_instance(kp);
435 			kp->pre_handler(kp, regs);
436 		}
437 		reset_kprobe_instance();
438 	}
439 }
440 NOKPROBE_SYMBOL(opt_pre_handler);
441 
442 /* Free optimized instructions and optimized_kprobe */
443 static void free_aggr_kprobe(struct kprobe *p)
444 {
445 	struct optimized_kprobe *op;
446 
447 	op = container_of(p, struct optimized_kprobe, kp);
448 	arch_remove_optimized_kprobe(op);
449 	arch_remove_kprobe(p);
450 	kfree(op);
451 }
452 
453 /* Return true if the kprobe is ready for optimization. */
454 static inline int kprobe_optready(struct kprobe *p)
455 {
456 	struct optimized_kprobe *op;
457 
458 	if (kprobe_aggrprobe(p)) {
459 		op = container_of(p, struct optimized_kprobe, kp);
460 		return arch_prepared_optinsn(&op->optinsn);
461 	}
462 
463 	return 0;
464 }
465 
466 /* Return true if the kprobe is disarmed. Note: p must be on hash list */
467 bool kprobe_disarmed(struct kprobe *p)
468 {
469 	struct optimized_kprobe *op;
470 
471 	/* If kprobe is not aggr/opt probe, just return kprobe is disabled */
472 	if (!kprobe_aggrprobe(p))
473 		return kprobe_disabled(p);
474 
475 	op = container_of(p, struct optimized_kprobe, kp);
476 
477 	return kprobe_disabled(p) && list_empty(&op->list);
478 }
479 
480 
481 /*
482  * Return an optimized kprobe whose optimizing code replaces
483  * instructions including 'addr' (exclude breakpoint).
484  */
485 static struct kprobe *get_optimized_kprobe(kprobe_opcode_t *addr)
486 {
487 	int i;
488 	struct kprobe *p;
489 	struct optimized_kprobe *op;
490 
491 	/* Don't check i == 0, since that is a breakpoint case. */
492 	for (i = 1; i < MAX_OPTIMIZED_LENGTH / sizeof(kprobe_opcode_t); i++) {
493 		p = get_kprobe(addr - i);
494 		/* A disabled probe can have prepared, but inactive, optinsns. */
495 		if (!p || !kprobe_optready(p) || kprobe_disarmed(p))
496 			continue;
497 
498 		op = container_of(p, struct optimized_kprobe, kp);
499 		if (arch_within_optimized_kprobe(op, addr))
500 			return p;
501 	}
502 
503 	return NULL;
504 }
505 
506 #define OPTPROBE_GEN_MAX 2
507 
508 struct optprobe_generation {
509 	/* Probes waiting for grace period to drain tasks from original text */
510 	struct list_head optimizing_list;
511 	/* Probes queued for batch unoptimization upon dispatch */
512 	struct list_head unoptimizing_list;
513 	/* In-use probes waiting for grace period to drain optinsn slot */
514 	struct list_head cooling_list;
515 	/* Unused probes unhashed and waiting for batch release */
516 	struct list_head freeing_list;
517 	/* Tasks RCU callback head for asynchronous waiting */
518 	struct rcu_head rcu;
519 	/* True if dispatched and awaiting Tasks RCU callback */
520 	bool in_flight;
521 	/* True when Tasks RCU callback has fired and ready to finalize */
522 	bool ready;
523 };
524 
525 /*
526  * Generational ring of optprobes.
527  *
528  * Incoming probe requests are queued into the waiting room generation
529  * (optprobe_gens[optprobe_cur_gen]). When dispatched, the generation
530  * unoptimizes its probes, invokes call_rcu_tasks(), and optprobe_cur_gen
531  * advances to the next slot.
532  *
533  * To ensure an idle generation is always available to collect incoming
534  * requests without dynamic allocation, the last available generation slot
535  * is never dispatched until another generation has finished.
536  */
537 static struct optprobe_generation optprobe_gens[OPTPROBE_GEN_MAX];
538 static int optprobe_cur_gen;
539 
540 static void optimize_kprobe(struct kprobe *p);
541 static struct task_struct *kprobe_optimizer_task;
542 static wait_queue_head_t kprobe_optimizer_wait;
543 static atomic_t optimizer_state;
544 enum {
545 	OPTIMIZER_ST_IDLE = 0,
546 	OPTIMIZER_ST_KICKED = 1,
547 	OPTIMIZER_ST_FLUSHING = 2,
548 };
549 
550 /* Bumped at the end of each kprobe_optimizer() pass, under 'kprobe_mutex' */
551 static unsigned long optimizer_passes;
552 
553 #define OPTIMIZE_DELAY 5
554 
555 static bool optprobe_has_queued_probes(void)
556 {
557 	struct optprobe_generation *gen = &optprobe_gens[optprobe_cur_gen];
558 
559 	return !list_empty(&gen->optimizing_list) ||
560 	       !list_empty(&gen->unoptimizing_list) ||
561 	       !list_empty(&gen->cooling_list) ||
562 	       !list_empty(&gen->freeing_list);
563 }
564 
565 static int optprobe_active_gens_count(void)
566 {
567 	int count = 0;
568 	int i;
569 
570 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
571 		if (optprobe_gens[i].in_flight || optprobe_gens[i].ready)
572 			count++;
573 	}
574 	return count;
575 }
576 
577 /*
578  * The last generation must not be fired until another generation is done.
579  * (Thus the last generation acts as the waiting room.)
580  */
581 static bool optprobe_can_fire(void)
582 {
583 	return optprobe_active_gens_count() < OPTPROBE_GEN_MAX - 1;
584 }
585 
586 static bool optprobe_has_ready_gens(void)
587 {
588 	int i;
589 
590 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
591 		if (READ_ONCE(optprobe_gens[i].ready))
592 			return true;
593 	}
594 	return false;
595 }
596 
597 static bool optprobe_optimizer_busy(void)
598 {
599 	return optprobe_has_queued_probes() || (optprobe_active_gens_count() > 0);
600 }
601 
602 /*
603  * Unoptimize (replace a jump with a breakpoint and remove the breakpoint
604  * if need) kprobes listed on 'unopt_list'.
605  */
606 static void do_unoptimize_kprobes(struct list_head *unopt_list,
607 				  struct list_head *free_list,
608 				  struct list_head *cooling_list)
609 {
610 	struct optimized_kprobe *op, *tmp;
611 	LIST_HEAD(unoptimized);
612 
613 	lockdep_assert_held(&text_mutex);
614 	/*
615 	 * The optimization/unoptimization refers 'online_cpus' via
616 	 * stop_machine() and cpu-hotplug modifies the 'online_cpus'.
617 	 * And same time, 'text_mutex' will be held in cpu-hotplug and here.
618 	 * This combination can cause a deadlock (cpu-hotplug tries to lock
619 	 * 'text_mutex' but stop_machine() can not be done because
620 	 * the 'online_cpus' has been changed)
621 	 * To avoid this deadlock, caller must have locked cpu-hotplug
622 	 * for preventing cpu-hotplug outside of 'text_mutex' locking.
623 	 */
624 	lockdep_assert_cpus_held();
625 
626 	if (list_empty(unopt_list))
627 		return;
628 
629 	arch_unoptimize_kprobes(unopt_list, &unoptimized);
630 
631 	/* Loop on newly unoptimized probes for disarming and hash list removal */
632 	list_for_each_entry_safe(op, tmp, &unoptimized, list) {
633 		/* Switching from detour code to origin */
634 		op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
635 		/*
636 		 * Disarm probes if marked disabled and not gone. Unhashed probes
637 		 * were disarmed before they were unhashed.
638 		 */
639 		if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp) &&
640 		    !hlist_unhashed(&op->kp.hlist))
641 			arch_disarm_kprobe(&op->kp);
642 		if (kprobe_unused(&op->kp)) {
643 			/*
644 			 * Remove unused probes from hash list. After waiting
645 			 * for synchronization, these probes are reclaimed.
646 			 * (reclaiming is done by do_free_cleaned_kprobes().)
647 			 */
648 			hlist_del_init_rcu(&op->kp.hlist);
649 			list_move(&op->list, free_list);
650 		} else {
651 			/*
652 			 * Keep on cooling_list until the quiescence period
653 			 * completes so that kprobe_disarmed() remains false and
654 			 * unregister_kprobes() does not prematurely free it.
655 			 */
656 			list_move(&op->list, cooling_list);
657 		}
658 	}
659 }
660 
661 /* Reclaim all kprobes on the 'free_list' */
662 static void do_free_cleaned_kprobes(struct list_head *free_list)
663 {
664 	struct optimized_kprobe *op, *tmp;
665 
666 	list_for_each_entry_safe(op, tmp, free_list, list) {
667 		struct kprobe *_p;
668 
669 		list_del_init(&op->list);
670 		if (WARN_ON_ONCE(!kprobe_unused(&op->kp))) {
671 			/*
672 			 * This must not happen, but if there is a kprobe
673 			 * still in use, keep it on kprobes hash list.
674 			 */
675 			continue;
676 		}
677 
678 		/*
679 		 * The aggregator was holding back another probe while it sat on the
680 		 * unoptimizing/freeing lists. Now that the aggregator has been fully
681 		 * reverted we can safely retry the optimization of that sibling.
682 		 */
683 		_p = get_optimized_kprobe(op->kp.addr);
684 		if (unlikely(_p))
685 			optimize_kprobe(_p);
686 
687 		free_aggr_kprobe(&op->kp);
688 	}
689 }
690 
691 static void kick_kprobe_optimizer(void);
692 
693 static void optprobe_generation_rcu_cb(struct rcu_head *rcu)
694 {
695 	struct optprobe_generation *gen;
696 
697 	gen = container_of(rcu, struct optprobe_generation, rcu);
698 	WRITE_ONCE(gen->ready, true);
699 	wake_up(&kprobe_optimizer_wait);
700 }
701 
702 static void optprobe_finalize_generation(struct optprobe_generation *gen)
703 {
704 	struct optimized_kprobe *op, *tmp;
705 
706 	lockdep_assert_held(&kprobe_mutex);
707 
708 	scoped_guard(cpus_read_lock) {
709 		guard(mutex)(&text_mutex);
710 
711 		/* Optimization never be done when disarmed */
712 		if (!kprobes_all_disarmed && kprobes_allow_optimization &&
713 		    !list_empty(&gen->optimizing_list))
714 			arch_optimize_kprobes(&gen->optimizing_list);
715 	}
716 
717 	/* Free cleaned kprobes after quiescence period */
718 	do_free_cleaned_kprobes(&gen->freeing_list);
719 
720 	/* Finalize unoptimized kprobes whose quiescence period completed */
721 	list_for_each_entry_safe(op, tmp, &gen->cooling_list, list) {
722 		if (kprobe_unused(&op->kp)) {
723 			/*
724 			 * Unregistered while quiescence period was in flight.
725 			 * Unhash it now and move to cur_gen's freeing list so
726 			 * it will be reclaimed after the next quiescence period.
727 			 */
728 			hlist_del_init_rcu(&op->kp.hlist);
729 			list_move(&op->list, &optprobe_gens[optprobe_cur_gen].freeing_list);
730 			kick_kprobe_optimizer();
731 		} else {
732 			struct kprobe *_p;
733 
734 			/* Still in use; now safely disarmed */
735 			list_del_init(&op->list);
736 			if (!kprobe_disabled(&op->kp)) {
737 				optimize_kprobe(&op->kp);
738 			} else {
739 				_p = get_optimized_kprobe(op->kp.addr);
740 				if (unlikely(_p))
741 					optimize_kprobe(_p);
742 			}
743 		}
744 	}
745 
746 	gen->in_flight = false;
747 	WRITE_ONCE(gen->ready, false);
748 }
749 
750 static bool optprobe_dispatch_generation(void)
751 {
752 	struct optprobe_generation *gen;
753 
754 	lockdep_assert_held(&kprobe_mutex);
755 
756 	if (!optprobe_can_fire() || !optprobe_has_queued_probes())
757 		return false;
758 
759 	gen = &optprobe_gens[optprobe_cur_gen];
760 
761 	scoped_guard(cpus_read_lock) {
762 		guard(mutex)(&text_mutex);
763 
764 		/*
765 		 * Unoptimize kprobes and collect cleaned (unused and disarmed)
766 		 * kprobes before waiting for quiescence period.
767 		 */
768 		do_unoptimize_kprobes(&gen->unoptimizing_list, &gen->freeing_list,
769 				      &gen->cooling_list);
770 	}
771 
772 	/* Advance cur_gen to the next generation slot */
773 	optprobe_cur_gen = (optprobe_cur_gen + 1) % OPTPROBE_GEN_MAX;
774 
775 	gen->in_flight = true;
776 	WRITE_ONCE(gen->ready, false);
777 
778 	call_rcu_tasks(&gen->rcu, optprobe_generation_rcu_cb);
779 	return true;
780 }
781 
782 /* Kprobe jump optimizer */
783 static void kprobe_optimizer(void)
784 {
785 	bool progress = false;
786 	int i;
787 
788 	guard(mutex)(&kprobe_mutex);
789 
790 	/* Step 1: Finalize any generation whose Tasks RCU grace period completed */
791 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
792 		if (READ_ONCE(optprobe_gens[i].ready)) {
793 			optprobe_finalize_generation(&optprobe_gens[i]);
794 			progress = true;
795 		}
796 	}
797 
798 	/* Step 2: Dispatch waiting room generation if allowed */
799 	if (optprobe_dispatch_generation())
800 		progress = true;
801 
802 	/* Step 3: Wake up flushers if progress was made */
803 	if (progress) {
804 		optimizer_passes++;
805 		wake_up_var_locked(&optimizer_passes, &kprobe_mutex);
806 	}
807 
808 	if (optprobe_has_queued_probes() && optprobe_can_fire()) {
809 		/* Probes remain and can be fired immediately (e.g. retried siblings) */
810 		kick_kprobe_optimizer();
811 	}
812 }
813 
814 static int kprobe_optimizer_thread(void *data)
815 {
816 	while (!kthread_should_stop()) {
817 		/* Wait until there is work to do or a generation is ready */
818 		wait_event_interruptible(kprobe_optimizer_wait,
819 			atomic_read(&optimizer_state) != OPTIMIZER_ST_IDLE ||
820 			optprobe_has_ready_gens() ||
821 			kthread_should_stop());
822 
823 		if (kthread_should_stop())
824 			break;
825 
826 		/*
827 		 * If it was a normal kick and no generation is ready to finalize,
828 		 * wait for OPTIMIZE_DELAY to batch incoming requests.
829 		 * This wait can be interrupted by a flush request or a ready generation.
830 		 */
831 		if (atomic_read(&optimizer_state) == OPTIMIZER_ST_KICKED &&
832 		    !optprobe_has_ready_gens()) {
833 			wait_event_interruptible_timeout(
834 				kprobe_optimizer_wait,
835 				atomic_read(&optimizer_state) == OPTIMIZER_ST_FLUSHING ||
836 				optprobe_has_ready_gens() ||
837 				kthread_should_stop(),
838 				OPTIMIZE_DELAY);
839 		}
840 
841 		if (kthread_should_stop())
842 			break;
843 
844 		atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
845 
846 		kprobe_optimizer();
847 	}
848 	return 0;
849 }
850 
851 /* Start optimizer after OPTIMIZE_DELAY passed */
852 static void kick_kprobe_optimizer(void)
853 {
854 	lockdep_assert_held(&kprobe_mutex);
855 	if (atomic_cmpxchg(&optimizer_state,
856 		OPTIMIZER_ST_IDLE, OPTIMIZER_ST_KICKED) == OPTIMIZER_ST_IDLE)
857 		wake_up(&kprobe_optimizer_wait);
858 }
859 
860 static void wait_for_kprobe_optimizer_locked(void)
861 {
862 	lockdep_assert_held(&kprobe_mutex);
863 
864 	while (optprobe_optimizer_busy()) {
865 		unsigned long passes = optimizer_passes;
866 
867 		/* Wake up optimizer thread if it can make progress */
868 		if ((optprobe_can_fire() && optprobe_has_queued_probes()) ||
869 		    optprobe_has_ready_gens()) {
870 			if (atomic_xchg_acquire(&optimizer_state,
871 				OPTIMIZER_ST_FLUSHING) != OPTIMIZER_ST_FLUSHING)
872 				wake_up(&kprobe_optimizer_wait);
873 		}
874 
875 		/*
876 		 * kprobe_optimizer() holds 'kprobe_mutex' for a whole pass, which
877 		 * this drops while sleeping, so a new count means progress was made.
878 		 */
879 		wait_var_event_mutex(&optimizer_passes,
880 				     optimizer_passes != passes, &kprobe_mutex);
881 	}
882 }
883 
884 /* Wait for completing optimization and unoptimization */
885 void wait_for_kprobe_optimizer(void)
886 {
887 	guard(mutex)(&kprobe_mutex);
888 
889 	wait_for_kprobe_optimizer_locked();
890 }
891 
892 bool optprobe_queued_unopt(struct optimized_kprobe *op)
893 {
894 	struct optimized_kprobe *_op;
895 	int i;
896 
897 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
898 		list_for_each_entry(_op, &optprobe_gens[i].unoptimizing_list, list) {
899 			if (op == _op)
900 				return true;
901 		}
902 	}
903 
904 	return false;
905 }
906 
907 static bool optprobe_queued_freeing(struct optimized_kprobe *op)
908 {
909 	struct optimized_kprobe *_op;
910 	int i;
911 
912 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
913 		list_for_each_entry(_op, &optprobe_gens[i].freeing_list, list) {
914 			if (op == _op)
915 				return true;
916 		}
917 	}
918 
919 	return false;
920 }
921 
922 static bool optprobe_queued_cooling(struct optimized_kprobe *op)
923 {
924 	struct optimized_kprobe *_op;
925 	int i;
926 
927 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
928 		list_for_each_entry(_op, &optprobe_gens[i].cooling_list, list) {
929 			if (op == _op)
930 				return true;
931 		}
932 	}
933 
934 	return false;
935 }
936 
937 /* Optimize kprobe if p is ready to be optimized */
938 static void optimize_kprobe(struct kprobe *p)
939 {
940 	struct optimized_kprobe *op;
941 
942 	/* Check if the kprobe is disabled or not ready for optimization. */
943 	if (!kprobe_optready(p) || !kprobes_allow_optimization ||
944 	    (kprobe_disabled(p) || kprobes_all_disarmed))
945 		return;
946 
947 	/* kprobes with 'post_handler' can not be optimized */
948 	if (p->post_handler)
949 		return;
950 
951 	op = container_of(p, struct optimized_kprobe, kp);
952 
953 	/* Check there is no other kprobes at the optimized instructions */
954 	if (arch_check_optimized_kprobe(op) < 0)
955 		return;
956 
957 	/* Check if it is already optimized. */
958 	if (op->kp.flags & KPROBE_FLAG_OPTIMIZED) {
959 		if (optprobe_queued_unopt(op)) {
960 			/* This is under unoptimizing. Just dequeue the probe */
961 			list_del_init(&op->list);
962 		}
963 		return;
964 	}
965 
966 	if (optprobe_queued_cooling(op)) {
967 		/* Under in-flight unoptimization. It will be re-optimized upon finalize */
968 		return;
969 	}
970 
971 	if (optprobe_queued_freeing(op))
972 		list_del_init(&op->list);
973 
974 	op->kp.flags |= KPROBE_FLAG_OPTIMIZED;
975 
976 	/*
977 	 * On the 'unoptimizing_list' and 'optimizing_list',
978 	 * 'op' must have OPTIMIZED flag
979 	 */
980 	if (WARN_ON_ONCE(!list_empty(&op->list)))
981 		return;
982 
983 	list_add(&op->list, &optprobe_gens[optprobe_cur_gen].optimizing_list);
984 	kick_kprobe_optimizer();
985 }
986 
987 /* Short cut to direct unoptimizing */
988 static void force_unoptimize_kprobe(struct optimized_kprobe *op)
989 {
990 	lockdep_assert_cpus_held();
991 	arch_unoptimize_kprobe(op);
992 	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
993 }
994 
995 /* Unoptimize a kprobe if p is optimized */
996 static void unoptimize_kprobe(struct kprobe *p, bool force)
997 {
998 	struct optprobe_generation *cur;
999 	struct optimized_kprobe *op;
1000 
1001 	if (!kprobe_aggrprobe(p) || kprobe_disarmed(p))
1002 		return; /* This is not an optprobe nor optimized */
1003 
1004 	op = container_of(p, struct optimized_kprobe, kp);
1005 	cur = &optprobe_gens[optprobe_cur_gen];
1006 
1007 	if (!list_empty(&op->list)) {
1008 		if (optprobe_queued_unopt(op)) {
1009 			/* Queued in unoptimizing queue */
1010 			if (force) {
1011 				/*
1012 				 * Forcibly unoptimize the kprobe here, and queue it
1013 				 * for cooling down (or freeing if unused).
1014 				 */
1015 				force_unoptimize_kprobe(op);
1016 				/* Disarm probes if marked disabled and not gone */
1017 				if (kprobe_disabled(&op->kp) && !kprobe_gone(&op->kp))
1018 					arch_disarm_kprobe(&op->kp);
1019 				if (kprobe_unused(&op->kp)) {
1020 					hlist_del_init_rcu(&op->kp.hlist);
1021 					list_move(&op->list, &cur->freeing_list);
1022 				} else {
1023 					list_move(&op->list, &cur->cooling_list);
1024 				}
1025 				kick_kprobe_optimizer();
1026 			}
1027 		} else if (optprobe_queued_cooling(op)) {
1028 			if (force && kprobe_unused(&op->kp)) {
1029 				/*
1030 				 * Already unoptimized, move to freeing list for
1031 				 * release afterwards.
1032 				 */
1033 				if (!kprobe_gone(&op->kp))
1034 					arch_disarm_kprobe(&op->kp);
1035 				hlist_del_init_rcu(&op->kp.hlist);
1036 				list_move(&op->list, &cur->freeing_list);
1037 				kick_kprobe_optimizer();
1038 			}
1039 		} else {
1040 			/* Dequeue from the optimizing queue */
1041 			list_del_init(&op->list);
1042 			op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
1043 		}
1044 		return;
1045 	}
1046 
1047 	if (!kprobe_optimized(p))
1048 		return;
1049 
1050 	/* Optimized kprobe case */
1051 	if (force) {
1052 		/* Forcibly update the code: this is a special case */
1053 		force_unoptimize_kprobe(op);
1054 		list_add(&op->list, &cur->cooling_list);
1055 	} else {
1056 		list_add(&op->list, &cur->unoptimizing_list);
1057 	}
1058 	kick_kprobe_optimizer();
1059 }
1060 
1061 /* Cancel unoptimizing for reusing */
1062 static int reuse_unused_kprobe(struct kprobe *ap)
1063 {
1064 	struct optimized_kprobe *op;
1065 
1066 	/*
1067 	 * Unused kprobe MUST be on the way of delayed unoptimizing (means
1068 	 * there is still a relative jump) and disabled.
1069 	 */
1070 	op = container_of(ap, struct optimized_kprobe, kp);
1071 	WARN_ON_ONCE(list_empty(&op->list));
1072 	/* Already disarmed, the caller arms it only for an enabled probe */
1073 	if (optprobe_queued_cooling(op))
1074 		return 0;
1075 
1076 	/* Enable the probe again */
1077 	ap->flags &= ~KPROBE_FLAG_DISABLED;
1078 	/* Optimize it again. (remove from 'op->list') */
1079 	if (!kprobe_optready(ap))
1080 		return -EINVAL;
1081 
1082 	optimize_kprobe(ap);
1083 	return 0;
1084 }
1085 
1086 /* Remove optimized instructions */
1087 static void kill_optimized_kprobe(struct kprobe *p)
1088 {
1089 	struct optimized_kprobe *op;
1090 
1091 	op = container_of(p, struct optimized_kprobe, kp);
1092 	if (!list_empty(&op->list)) {
1093 		if (kprobe_unused(p)) {
1094 			if (optprobe_queued_unopt(op) || optprobe_queued_cooling(op)) {
1095 				hlist_del_init_rcu(&op->kp.hlist);
1096 				list_move(&op->list,
1097 					  &optprobe_gens[optprobe_cur_gen].freeing_list);
1098 				kick_kprobe_optimizer();
1099 			} else if (!optprobe_queued_freeing(op)) {
1100 				list_del_init(&op->list);
1101 			}
1102 		} else {
1103 			list_del_init(&op->list);
1104 		}
1105 	}
1106 	op->kp.flags &= ~KPROBE_FLAG_OPTIMIZED;
1107 
1108 	/*
1109 	 * Don't remove the slot if it is queued for freeing or unoptimization;
1110 	 * the optimizer will reclaim it after the quiescence period.
1111 	 */
1112 	if (!optprobe_queued_freeing(op) && !optprobe_queued_unopt(op) &&
1113 	    !optprobe_queued_cooling(op))
1114 		arch_remove_optimized_kprobe(op);
1115 }
1116 
1117 static inline
1118 void __prepare_optimized_kprobe(struct optimized_kprobe *op, struct kprobe *p)
1119 {
1120 	if (!kprobe_ftrace(p))
1121 		arch_prepare_optimized_kprobe(op, p);
1122 }
1123 
1124 /* Try to prepare optimized instructions */
1125 static void prepare_optimized_kprobe(struct kprobe *p)
1126 {
1127 	struct optimized_kprobe *op;
1128 
1129 	op = container_of(p, struct optimized_kprobe, kp);
1130 	__prepare_optimized_kprobe(op, p);
1131 }
1132 
1133 /* Allocate new optimized_kprobe and try to prepare optimized instructions. */
1134 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1135 {
1136 	struct optimized_kprobe *op;
1137 
1138 	op = kzalloc_obj(struct optimized_kprobe);
1139 	if (!op)
1140 		return NULL;
1141 
1142 	INIT_LIST_HEAD(&op->list);
1143 	op->kp.addr = p->addr;
1144 	__prepare_optimized_kprobe(op, p);
1145 
1146 	return &op->kp;
1147 }
1148 
1149 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p);
1150 
1151 /*
1152  * Prepare an optimized_kprobe and optimize it.
1153  * NOTE: 'p' must be a normal registered kprobe.
1154  */
1155 static void try_to_optimize_kprobe(struct kprobe *p)
1156 {
1157 	struct kprobe *ap;
1158 	struct optimized_kprobe *op;
1159 
1160 	/* Impossible to optimize ftrace-based kprobe. */
1161 	if (kprobe_ftrace(p))
1162 		return;
1163 
1164 	/* For preparing optimization, jump_label_text_reserved() is called. */
1165 	guard(cpus_read_lock)();
1166 	guard(jump_label_lock)();
1167 	guard(mutex)(&text_mutex);
1168 
1169 	ap = alloc_aggr_kprobe(p);
1170 	if (!ap)
1171 		return;
1172 
1173 	op = container_of(ap, struct optimized_kprobe, kp);
1174 	if (!arch_prepared_optinsn(&op->optinsn)) {
1175 		/* If failed to setup optimizing, fallback to kprobe. */
1176 		arch_remove_optimized_kprobe(op);
1177 		kfree(op);
1178 		return;
1179 	}
1180 
1181 	init_aggr_kprobe(ap, p);
1182 	optimize_kprobe(ap);	/* This just kicks optimizer thread. */
1183 }
1184 
1185 static void optimize_all_kprobes(void)
1186 {
1187 	struct hlist_head *head;
1188 	struct kprobe *p;
1189 	unsigned int i;
1190 
1191 	guard(mutex)(&kprobe_mutex);
1192 	/* If optimization is already allowed, just return. */
1193 	if (kprobes_allow_optimization)
1194 		return;
1195 
1196 	cpus_read_lock();
1197 	kprobes_allow_optimization = true;
1198 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1199 		head = &kprobe_table[i];
1200 		hlist_for_each_entry(p, head, hlist)
1201 			if (!kprobe_disabled(p))
1202 				optimize_kprobe(p);
1203 	}
1204 	cpus_read_unlock();
1205 	pr_info("kprobe jump-optimization is enabled. All kprobes are optimized if possible.\n");
1206 }
1207 
1208 #ifdef CONFIG_SYSCTL
1209 static void unoptimize_all_kprobes(void)
1210 {
1211 	struct hlist_head *head;
1212 	struct kprobe *p;
1213 	unsigned int i;
1214 
1215 	guard(mutex)(&kprobe_mutex);
1216 	/* If optimization is already prohibited, just return. */
1217 	if (!kprobes_allow_optimization)
1218 		return;
1219 
1220 	cpus_read_lock();
1221 	kprobes_allow_optimization = false;
1222 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
1223 		head = &kprobe_table[i];
1224 		hlist_for_each_entry(p, head, hlist) {
1225 			if (!kprobe_disabled(p))
1226 				unoptimize_kprobe(p, false);
1227 		}
1228 	}
1229 	cpus_read_unlock();
1230 	/* Wait for unoptimizing completion. */
1231 	wait_for_kprobe_optimizer_locked();
1232 	pr_info("kprobe jump-optimization is disabled. All kprobes are based on software breakpoint.\n");
1233 }
1234 
1235 static DEFINE_MUTEX(kprobe_sysctl_mutex);
1236 static int sysctl_kprobes_optimization;
1237 static int proc_kprobes_optimization_handler(const struct ctl_table *table,
1238 					     int write, void *buffer,
1239 					     size_t *length, loff_t *ppos)
1240 {
1241 	int ret;
1242 
1243 	guard(mutex)(&kprobe_sysctl_mutex);
1244 	sysctl_kprobes_optimization = kprobes_allow_optimization ? 1 : 0;
1245 	ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
1246 
1247 	if (sysctl_kprobes_optimization)
1248 		optimize_all_kprobes();
1249 	else
1250 		unoptimize_all_kprobes();
1251 
1252 	return ret;
1253 }
1254 
1255 static const struct ctl_table kprobe_sysctls[] = {
1256 	{
1257 		.procname	= "kprobes-optimization",
1258 		.data		= &sysctl_kprobes_optimization,
1259 		.maxlen		= sizeof(int),
1260 		.mode		= 0644,
1261 		.proc_handler	= proc_kprobes_optimization_handler,
1262 		.extra1		= SYSCTL_ZERO,
1263 		.extra2		= SYSCTL_ONE,
1264 	},
1265 };
1266 
1267 static void __init kprobe_sysctls_init(void)
1268 {
1269 	register_sysctl_init("debug", kprobe_sysctls);
1270 }
1271 #endif /* CONFIG_SYSCTL */
1272 
1273 /* Return true if the probe is queued on unoptimizing lists */
1274 static bool kprobe_queued_unopt(struct kprobe *p)
1275 {
1276 	struct optimized_kprobe *op;
1277 
1278 	if (kprobe_aggrprobe(p)) {
1279 		op = container_of(p, struct optimized_kprobe, kp);
1280 		if (optprobe_queued_unopt(op))
1281 			return true;
1282 	}
1283 	return false;
1284 }
1285 
1286 /* Put a breakpoint for a probe. */
1287 static void __arm_kprobe(struct kprobe *p)
1288 {
1289 	struct kprobe *_p;
1290 
1291 	lockdep_assert_held(&text_mutex);
1292 
1293 	/* Find the overlapping optimized kprobes. */
1294 	_p = get_optimized_kprobe(p->addr);
1295 	if (unlikely(_p))
1296 		/* Fallback to unoptimized kprobe */
1297 		unoptimize_kprobe(_p, true);
1298 
1299 	if (!kprobe_queued_unopt(p))
1300 		arch_arm_kprobe(p);
1301 	optimize_kprobe(p);	/* Try to optimize (add kprobe to a list) */
1302 	if (unlikely(kprobe_queued_unopt(p)))
1303 		unoptimize_kprobe(p, true);
1304 }
1305 
1306 /* Remove the breakpoint of a probe. */
1307 static void __disarm_kprobe(struct kprobe *p, bool reopt)
1308 {
1309 	struct kprobe *_p;
1310 
1311 	lockdep_assert_held(&text_mutex);
1312 
1313 	/* Try to unoptimize */
1314 	unoptimize_kprobe(p, kprobes_all_disarmed);
1315 
1316 	if (!kprobe_queued_unopt(p)) {
1317 		arch_disarm_kprobe(p);
1318 		/* If another kprobe was blocked, re-optimize it. */
1319 		_p = get_optimized_kprobe(p->addr);
1320 		if (unlikely(_p) && reopt)
1321 			optimize_kprobe(_p);
1322 	}
1323 }
1324 
1325 static void __init init_optprobe(void)
1326 {
1327 	int i;
1328 
1329 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
1330 	/* Init 'kprobe_optinsn_slots' for allocation */
1331 	kprobe_optinsn_slots.insn_size = MAX_OPTINSN_SIZE;
1332 #endif
1333 
1334 	for (i = 0; i < OPTPROBE_GEN_MAX; i++) {
1335 		INIT_LIST_HEAD(&optprobe_gens[i].optimizing_list);
1336 		INIT_LIST_HEAD(&optprobe_gens[i].unoptimizing_list);
1337 		INIT_LIST_HEAD(&optprobe_gens[i].cooling_list);
1338 		INIT_LIST_HEAD(&optprobe_gens[i].freeing_list);
1339 		optprobe_gens[i].in_flight = false;
1340 		optprobe_gens[i].ready = false;
1341 	}
1342 	optprobe_cur_gen = 0;
1343 
1344 	init_waitqueue_head(&kprobe_optimizer_wait);
1345 	atomic_set(&optimizer_state, OPTIMIZER_ST_IDLE);
1346 	kprobe_optimizer_task = kthread_run(kprobe_optimizer_thread, NULL,
1347 					    "kprobe-optimizer");
1348 }
1349 #else /* !CONFIG_OPTPROBES */
1350 
1351 #define init_optprobe()				do {} while (0)
1352 #define optimize_kprobe(p)			do {} while (0)
1353 #define unoptimize_kprobe(p, f)			do {} while (0)
1354 #define kill_optimized_kprobe(p)		do {} while (0)
1355 #define prepare_optimized_kprobe(p)		do {} while (0)
1356 #define try_to_optimize_kprobe(p)		do {} while (0)
1357 #define __arm_kprobe(p)				arch_arm_kprobe(p)
1358 #define __disarm_kprobe(p, o)			arch_disarm_kprobe(p)
1359 #define kprobe_disarmed(p)			kprobe_disabled(p)
1360 #define wait_for_kprobe_optimizer_locked()			\
1361 	lockdep_assert_held(&kprobe_mutex)
1362 
1363 static int reuse_unused_kprobe(struct kprobe *ap)
1364 {
1365 	/*
1366 	 * If the optimized kprobe is NOT supported, the aggr kprobe is
1367 	 * released at the same time that the last aggregated kprobe is
1368 	 * unregistered.
1369 	 * Thus there should be no chance to reuse unused kprobe.
1370 	 */
1371 	WARN_ON_ONCE(1);
1372 	return -EINVAL;
1373 }
1374 
1375 static void free_aggr_kprobe(struct kprobe *p)
1376 {
1377 	arch_remove_kprobe(p);
1378 	kfree(p);
1379 }
1380 
1381 static struct kprobe *alloc_aggr_kprobe(struct kprobe *p)
1382 {
1383 	return kzalloc_obj(struct kprobe);
1384 }
1385 #endif /* CONFIG_OPTPROBES */
1386 
1387 #ifdef CONFIG_KPROBES_ON_FTRACE
1388 static struct ftrace_ops kprobe_ftrace_ops __read_mostly = {
1389 	.func = kprobe_ftrace_handler,
1390 	.flags = FTRACE_OPS_FL_SAVE_REGS,
1391 };
1392 
1393 static struct ftrace_ops kprobe_ipmodify_ops __read_mostly = {
1394 	.func = kprobe_ftrace_handler,
1395 	.flags = FTRACE_OPS_FL_SAVE_REGS | FTRACE_OPS_FL_IPMODIFY,
1396 };
1397 
1398 static int kprobe_ipmodify_enabled;
1399 static int kprobe_ftrace_enabled;
1400 bool kprobe_ftrace_disabled;
1401 
1402 static int __arm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1403 			       int *cnt)
1404 {
1405 	int ret;
1406 
1407 	lockdep_assert_held(&kprobe_mutex);
1408 
1409 	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 0, 0);
1410 	if (ret < 0)
1411 		return ret;
1412 
1413 	if (*cnt == 0) {
1414 		ret = register_ftrace_function(ops);
1415 		if (ret < 0) {
1416 			/*
1417 			 * At this point, sinec ops is not registered, we should be sefe from
1418 			 * registering empty filter.
1419 			 */
1420 			ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1421 			return ret;
1422 		}
1423 	}
1424 
1425 	(*cnt)++;
1426 	return ret;
1427 }
1428 
1429 static int arm_kprobe_ftrace(struct kprobe *p)
1430 {
1431 	bool ipmodify = (p->post_handler != NULL);
1432 
1433 	return __arm_kprobe_ftrace(p,
1434 		ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1435 		ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1436 }
1437 
1438 static int __disarm_kprobe_ftrace(struct kprobe *p, struct ftrace_ops *ops,
1439 				  int *cnt)
1440 {
1441 	int ret;
1442 
1443 	lockdep_assert_held(&kprobe_mutex);
1444 	if (unlikely(kprobe_ftrace_disabled)) {
1445 		/* Now ftrace is disabled forever, disarm is already done. */
1446 		return 0;
1447 	}
1448 
1449 	if (*cnt == 1) {
1450 		ret = unregister_ftrace_function(ops);
1451 		if (WARN(ret < 0, "Failed to unregister kprobe-ftrace (error %d)\n", ret))
1452 			return ret;
1453 	}
1454 
1455 	(*cnt)--;
1456 
1457 	ret = ftrace_set_filter_ip(ops, (unsigned long)p->addr, 1, 0);
1458 	if (WARN_ONCE(ret < 0, "Failed to disarm kprobe-ftrace at %pS (error %d)\n",
1459 		      p->addr, ret)) {
1460 		if (*cnt == 0)
1461 			register_ftrace_function(ops);
1462 		(*cnt)++;
1463 		return ret;
1464 	}
1465 	return 0;
1466 }
1467 
1468 static int disarm_kprobe_ftrace(struct kprobe *p)
1469 {
1470 	bool ipmodify = (p->post_handler != NULL);
1471 
1472 	return __disarm_kprobe_ftrace(p,
1473 		ipmodify ? &kprobe_ipmodify_ops : &kprobe_ftrace_ops,
1474 		ipmodify ? &kprobe_ipmodify_enabled : &kprobe_ftrace_enabled);
1475 }
1476 
1477 void kprobe_ftrace_kill(void)
1478 {
1479 	kprobe_ftrace_disabled = true;
1480 }
1481 #else	/* !CONFIG_KPROBES_ON_FTRACE */
1482 static inline int arm_kprobe_ftrace(struct kprobe *p)
1483 {
1484 	return -ENODEV;
1485 }
1486 
1487 static inline int disarm_kprobe_ftrace(struct kprobe *p)
1488 {
1489 	return -ENODEV;
1490 }
1491 #endif
1492 
1493 static int prepare_kprobe(struct kprobe *p)
1494 {
1495 	/* Must ensure p->addr is really on ftrace */
1496 	if (kprobe_ftrace(p))
1497 		return arch_prepare_kprobe_ftrace(p);
1498 
1499 	return arch_prepare_kprobe(p);
1500 }
1501 
1502 static int arm_kprobe(struct kprobe *kp)
1503 {
1504 	if (unlikely(kprobe_ftrace(kp)))
1505 		return arm_kprobe_ftrace(kp);
1506 
1507 	guard(cpus_read_lock)();
1508 	guard(mutex)(&text_mutex);
1509 	__arm_kprobe(kp);
1510 	return 0;
1511 }
1512 
1513 static int disarm_kprobe(struct kprobe *kp, bool reopt)
1514 {
1515 	if (unlikely(kprobe_ftrace(kp)))
1516 		return disarm_kprobe_ftrace(kp);
1517 
1518 	guard(cpus_read_lock)();
1519 	guard(mutex)(&text_mutex);
1520 	__disarm_kprobe(kp, reopt);
1521 	return 0;
1522 }
1523 
1524 /*
1525  * Aggregate handlers for multiple kprobes support - these handlers
1526  * take care of invoking the individual kprobe handlers on p->list
1527  */
1528 static int aggr_pre_handler(struct kprobe *p, struct pt_regs *regs)
1529 {
1530 	struct kprobe *kp;
1531 
1532 	list_for_each_entry_rcu(kp, &p->list, list) {
1533 		if (kp->pre_handler && likely(!kprobe_disabled(kp))) {
1534 			set_kprobe_instance(kp);
1535 			if (kp->pre_handler(kp, regs))
1536 				return 1;
1537 		}
1538 		reset_kprobe_instance();
1539 	}
1540 	return 0;
1541 }
1542 NOKPROBE_SYMBOL(aggr_pre_handler);
1543 
1544 static void aggr_post_handler(struct kprobe *p, struct pt_regs *regs,
1545 			      unsigned long flags)
1546 {
1547 	struct kprobe *kp;
1548 
1549 	list_for_each_entry_rcu(kp, &p->list, list) {
1550 		if (kp->post_handler && likely(!kprobe_disabled(kp))) {
1551 			set_kprobe_instance(kp);
1552 			kp->post_handler(kp, regs, flags);
1553 			reset_kprobe_instance();
1554 		}
1555 	}
1556 }
1557 NOKPROBE_SYMBOL(aggr_post_handler);
1558 
1559 /* Walks the list and increments 'nmissed' if 'p' has child probes. */
1560 void kprobes_inc_nmissed_count(struct kprobe *p)
1561 {
1562 	struct kprobe *kp;
1563 
1564 	if (!kprobe_aggrprobe(p)) {
1565 		p->nmissed++;
1566 	} else {
1567 		list_for_each_entry_rcu(kp, &p->list, list)
1568 			kp->nmissed++;
1569 	}
1570 }
1571 NOKPROBE_SYMBOL(kprobes_inc_nmissed_count);
1572 
1573 static struct kprobe kprobe_busy = {
1574 	.addr = (void *) get_kprobe,
1575 };
1576 
1577 void kprobe_busy_begin(void)
1578 {
1579 	struct kprobe_ctlblk *kcb;
1580 
1581 	preempt_disable();
1582 	__this_cpu_write(current_kprobe, &kprobe_busy);
1583 	kcb = get_kprobe_ctlblk();
1584 	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
1585 }
1586 
1587 void kprobe_busy_end(void)
1588 {
1589 	__this_cpu_write(current_kprobe, NULL);
1590 	preempt_enable();
1591 }
1592 
1593 /* Add the new probe to 'ap->list'. */
1594 static void add_new_kprobe(struct kprobe *ap, struct kprobe *p)
1595 {
1596 	/* A reused cooling probe is already unoptimized */
1597 	if (p->post_handler && !kprobe_unused(ap))
1598 		unoptimize_kprobe(ap, true);	/* Fall back to normal kprobe */
1599 
1600 	list_add_rcu(&p->list, &ap->list);
1601 }
1602 
1603 /*
1604  * Fill in the required fields of the aggregator kprobe. Replace the
1605  * earlier kprobe in the hlist with the aggregator kprobe.
1606  */
1607 static void init_aggr_kprobe(struct kprobe *ap, struct kprobe *p)
1608 {
1609 	/* Copy the insn slot of 'p' to 'ap'. */
1610 	copy_kprobe(p, ap);
1611 	flush_insn_slot(ap);
1612 	ap->addr = p->addr;
1613 	ap->flags = p->flags & ~KPROBE_FLAG_OPTIMIZED;
1614 	ap->pre_handler = aggr_pre_handler;
1615 	/* We don't care the kprobe which has gone. */
1616 	if (p->post_handler && !kprobe_gone(p))
1617 		ap->post_handler = aggr_post_handler;
1618 
1619 	INIT_LIST_HEAD(&ap->list);
1620 	INIT_HLIST_NODE(&ap->hlist);
1621 
1622 	list_add_rcu(&p->list, &ap->list);
1623 	hlist_replace_rcu(&p->hlist, &ap->hlist);
1624 }
1625 
1626 /*
1627  * This registers the second or subsequent kprobe at the same address.
1628  */
1629 static int register_aggr_kprobe(struct kprobe *orig_p, struct kprobe *p)
1630 {
1631 	int ret = 0;
1632 	struct kprobe *ap = orig_p;
1633 	bool reset_post_handler = false;
1634 	bool ftrace_switch_ops = false;
1635 
1636 	scoped_guard(cpus_read_lock) {
1637 		/* For preparing optimization, jump_label_text_reserved() is called */
1638 		guard(jump_label_lock)();
1639 		guard(mutex)(&text_mutex);
1640 
1641 		if (!kprobe_aggrprobe(orig_p)) {
1642 			/* If 'orig_p' is not an 'aggr_kprobe', create new one. */
1643 			ap = alloc_aggr_kprobe(orig_p);
1644 			if (!ap)
1645 				return -ENOMEM;
1646 			init_aggr_kprobe(ap, orig_p);
1647 		} else if (kprobe_unused(ap)) {
1648 			/* This probe is going to die. Rescue it */
1649 			ret = reuse_unused_kprobe(ap);
1650 			if (ret)
1651 				return ret;
1652 		}
1653 
1654 		if (kprobe_gone(ap)) {
1655 			/*
1656 			 * Attempting to insert new probe at the same location that
1657 			 * had a probe in the module vaddr area which already
1658 			 * freed. So, the instruction slot has already been
1659 			 * released. We need a new slot for the new probe.
1660 			 */
1661 			ret = arch_prepare_kprobe(ap);
1662 			if (ret)
1663 				/*
1664 				 * Even if fail to allocate new slot, don't need to
1665 				 * free the 'ap'. It will be used next time, or
1666 				 * freed by unregister_kprobe().
1667 				 */
1668 				return ret;
1669 
1670 			/* Prepare optimized instructions if possible. */
1671 			prepare_optimized_kprobe(ap);
1672 
1673 			/*
1674 			 * Clear gone flag to prevent allocating new slot again, and
1675 			 * set disabled flag because it is not armed yet.
1676 			 */
1677 			ap->flags = (ap->flags & ~KPROBE_FLAG_GONE)
1678 					| KPROBE_FLAG_DISABLED;
1679 		}
1680 
1681 		if (p->post_handler && !ap->post_handler) {
1682 			reset_post_handler = true;
1683 			if (kprobe_ftrace(ap) && !kprobes_all_disarmed &&
1684 			    !kprobe_disabled(ap))
1685 				ftrace_switch_ops = true;
1686 			else
1687 				ap->post_handler = aggr_post_handler;
1688 		}
1689 
1690 		/* Copy the insn slot of 'p' to 'ap'. */
1691 		copy_kprobe(ap, p);
1692 		if (!ftrace_switch_ops)
1693 			add_new_kprobe(ap, p);
1694 	}
1695 
1696 	if (ftrace_switch_ops) {
1697 		/*
1698 		 * Must switch ftrace ops outside of text_mutex and cpus_read_lock
1699 		 * to prevent deadlocks when ftrace code modification acquires text_mutex.
1700 		 */
1701 		ret = disarm_kprobe_ftrace(ap);
1702 		if (ret)
1703 			return ret;
1704 
1705 		ap->post_handler = aggr_post_handler;
1706 		ret = arm_kprobe_ftrace(ap);
1707 		if (ret) {
1708 			ap->post_handler = NULL;
1709 			if (arm_kprobe_ftrace(ap))
1710 				ap->flags |= KPROBE_FLAG_DISABLED;
1711 			return ret;
1712 		}
1713 		add_new_kprobe(ap, p);
1714 	}
1715 
1716 	if (kprobe_disabled(ap) && !kprobe_disabled(p)) {
1717 		ap->flags &= ~KPROBE_FLAG_DISABLED;
1718 		if (!kprobes_all_disarmed) {
1719 			/* Arm the breakpoint again. */
1720 			ret = arm_kprobe(ap);
1721 			if (ret) {
1722 				ap->flags |= KPROBE_FLAG_DISABLED;
1723 				if (reset_post_handler)
1724 					ap->post_handler = NULL;
1725 				list_del_rcu(&p->list);
1726 				synchronize_rcu();
1727 			}
1728 		}
1729 	}
1730 	return ret;
1731 }
1732 
1733 bool __weak arch_within_kprobe_blacklist(unsigned long addr)
1734 {
1735 	/* The '__kprobes' functions and entry code must not be probed. */
1736 	return addr >= (unsigned long)__kprobes_text_start &&
1737 	       addr < (unsigned long)__kprobes_text_end;
1738 }
1739 
1740 static bool __within_kprobe_blacklist(unsigned long addr)
1741 {
1742 	struct kprobe_blacklist_entry *ent;
1743 
1744 	if (arch_within_kprobe_blacklist(addr))
1745 		return true;
1746 	/*
1747 	 * If 'kprobe_blacklist' is defined, check the address and
1748 	 * reject any probe registration in the prohibited area.
1749 	 * Note: this can return true during transition period where
1750 	 * (start_addr, end_addr) in the black list is shrinking
1751 	 * but old entry has not been removed yet. This is acceptable
1752 	 * because the worst case is that we reject more probes than
1753 	 * we should.
1754 	 */
1755 	guard(rcu)();
1756 	list_for_each_entry_rcu(ent, &kprobe_blacklist, list) {
1757 		if (addr >= ent->start_addr && addr < ent->end_addr)
1758 			return true;
1759 	}
1760 	return false;
1761 }
1762 
1763 bool within_kprobe_blacklist(unsigned long addr)
1764 {
1765 	char symname[KSYM_NAME_LEN], *p;
1766 
1767 	if (__within_kprobe_blacklist(addr))
1768 		return true;
1769 
1770 	/* Check if the address is on a suffixed-symbol */
1771 	if (!lookup_symbol_name(addr, symname)) {
1772 		p = strchr(symname, '.');
1773 		if (!p)
1774 			return false;
1775 		*p = '\0';
1776 		addr = (unsigned long)kprobe_lookup_name(symname, 0);
1777 		if (addr)
1778 			return __within_kprobe_blacklist(addr);
1779 	}
1780 	return false;
1781 }
1782 
1783 /*
1784  * arch_adjust_kprobe_addr - adjust the address
1785  * @addr: symbol base address
1786  * @offset: offset within the symbol
1787  * @on_func_entry: was this @addr+@offset on the function entry
1788  *
1789  * Typically returns @addr + @offset, except for special cases where the
1790  * function might be prefixed by a CFI landing pad, in that case any offset
1791  * inside the landing pad is mapped to the first 'real' instruction of the
1792  * symbol.
1793  *
1794  * Specifically, for things like IBT/BTI, skip the resp. ENDBR/BTI.C
1795  * instruction at +0.
1796  */
1797 kprobe_opcode_t *__weak arch_adjust_kprobe_addr(unsigned long addr,
1798 						unsigned long offset,
1799 						bool *on_func_entry)
1800 {
1801 	*on_func_entry = !offset;
1802 	return (kprobe_opcode_t *)(addr + offset);
1803 }
1804 
1805 /*
1806  * If 'symbol_name' is specified, look it up and add the 'offset'
1807  * to it. This way, we can specify a relative address to a symbol.
1808  * This returns encoded errors if it fails to look up symbol or invalid
1809  * combination of parameters.
1810  */
1811 static kprobe_opcode_t *
1812 _kprobe_addr(kprobe_opcode_t *addr, const char *symbol_name,
1813 	     unsigned long offset, bool *on_func_entry)
1814 {
1815 	if ((symbol_name && addr) || (!symbol_name && !addr))
1816 		return ERR_PTR(-EINVAL);
1817 
1818 	if (symbol_name) {
1819 		/*
1820 		 * Input: @sym + @offset
1821 		 * Output: @addr + @offset
1822 		 *
1823 		 * NOTE: kprobe_lookup_name() does *NOT* fold the offset
1824 		 *       argument into it's output!
1825 		 */
1826 		addr = kprobe_lookup_name(symbol_name, offset);
1827 		if (!addr)
1828 			return ERR_PTR(-ENOENT);
1829 	}
1830 
1831 	/*
1832 	 * So here we have @addr + @offset, displace it into a new
1833 	 * @addr' + @offset' where @addr' is the symbol start address.
1834 	 */
1835 	addr = (void *)addr + offset;
1836 	if (!kallsyms_lookup_size_offset((unsigned long)addr, NULL, &offset))
1837 		return ERR_PTR(-ENOENT);
1838 	addr = (void *)addr - offset;
1839 
1840 	/*
1841 	 * Then ask the architecture to re-combine them, taking care of
1842 	 * magical function entry details while telling us if this was indeed
1843 	 * at the start of the function.
1844 	 */
1845 	addr = arch_adjust_kprobe_addr((unsigned long)addr, offset, on_func_entry);
1846 	if (!addr)
1847 		return ERR_PTR(-EINVAL);
1848 
1849 	return addr;
1850 }
1851 
1852 /*
1853  * Check the 'p' is valid and return the aggregator kprobe
1854  * at the same address.
1855  */
1856 static struct kprobe *__get_valid_kprobe_at(struct kprobe *p, kprobe_opcode_t *addr)
1857 {
1858 	struct hlist_head *head;
1859 	struct kprobe *ap, *list_p;
1860 
1861 	lockdep_assert_held(&kprobe_mutex);
1862 
1863 	if (unlikely(!addr))
1864 		return NULL;
1865 
1866 	head = &kprobe_table[hash_ptr(addr, KPROBE_HASH_BITS)];
1867 	hlist_for_each_entry_rcu(ap, head, hlist,
1868 				 lockdep_is_held(&kprobe_mutex)) {
1869 		if (ap == p) {
1870 			p->addr = addr;
1871 			return ap;
1872 		}
1873 		if (ap->addr == addr && kprobe_aggrprobe(ap)) {
1874 			list_for_each_entry(list_p, &ap->list, list) {
1875 				if (list_p == p) {
1876 					p->addr = addr;
1877 					return ap;
1878 				}
1879 			}
1880 		}
1881 	}
1882 
1883 	return NULL;
1884 }
1885 
1886 static inline struct kprobe *__get_valid_kprobe(struct kprobe *p)
1887 {
1888 	return __get_valid_kprobe_at(p, p->addr);
1889 }
1890 
1891 /* Return error if the kprobe is being re-registered */
1892 static inline int check_kprobe_rereg_at(struct kprobe *p, kprobe_opcode_t *addr)
1893 {
1894 	guard(mutex)(&kprobe_mutex);
1895 
1896 	if (__get_valid_kprobe_at(p, addr))
1897 		return -EINVAL;
1898 
1899 	return 0;
1900 }
1901 
1902 static inline int check_kprobe_rereg(struct kprobe *p)
1903 {
1904 	return check_kprobe_rereg_at(p, p->addr);
1905 }
1906 
1907 static int check_ftrace_location(struct kprobe *p)
1908 {
1909 	unsigned long addr = (unsigned long)p->addr;
1910 
1911 	if (ftrace_location(addr) == addr) {
1912 #ifdef CONFIG_KPROBES_ON_FTRACE
1913 		p->flags |= KPROBE_FLAG_FTRACE;
1914 #else
1915 		return -EINVAL;
1916 #endif
1917 	}
1918 	return 0;
1919 }
1920 
1921 static bool is_cfi_preamble_symbol(unsigned long addr)
1922 {
1923 	char symbuf[KSYM_NAME_LEN];
1924 
1925 	if (lookup_symbol_name(addr, symbuf))
1926 		return false;
1927 
1928 	return str_has_prefix(symbuf, "__cfi_") ||
1929 		str_has_prefix(symbuf, "__pfx_");
1930 }
1931 
1932 static int check_kprobe_address_safe(struct kprobe *p,
1933 				     struct module **probed_mod)
1934 {
1935 	int ret;
1936 
1937 	ret = check_ftrace_location(p);
1938 	if (ret)
1939 		return ret;
1940 
1941 	guard(jump_label_lock)();
1942 
1943 	/* Ensure the address is in a text area, and find a module if exists. */
1944 	*probed_mod = NULL;
1945 	if (!core_kernel_text((unsigned long) p->addr)) {
1946 		guard(rcu)();
1947 		*probed_mod = __module_text_address((unsigned long) p->addr);
1948 		if (!(*probed_mod))
1949 			return -EINVAL;
1950 
1951 		/*
1952 		 * We must hold a refcount of the probed module while updating
1953 		 * its code to prohibit unexpected unloading.
1954 		 */
1955 		if (unlikely(!try_module_get(*probed_mod)))
1956 			return -ENOENT;
1957 	}
1958 	/* Ensure it is not in reserved area. */
1959 	if (in_gate_area_no_mm((unsigned long) p->addr) ||
1960 	    within_kprobe_blacklist((unsigned long) p->addr) ||
1961 	    jump_label_text_reserved(p->addr, p->addr) ||
1962 	    static_call_text_reserved(p->addr, p->addr) ||
1963 	    find_bug((unsigned long)p->addr) ||
1964 	    is_cfi_preamble_symbol((unsigned long)p->addr)) {
1965 		module_put(*probed_mod);
1966 		return -EINVAL;
1967 	}
1968 
1969 	/* Get module refcount and reject __init functions for loaded modules. */
1970 	if (IS_ENABLED(CONFIG_MODULES) && *probed_mod) {
1971 		/*
1972 		 * If the module freed '.init.text', we couldn't insert
1973 		 * kprobes in there.
1974 		 */
1975 		if (within_module_init((unsigned long)p->addr, *probed_mod) &&
1976 		    !module_is_coming(*probed_mod)) {
1977 			module_put(*probed_mod);
1978 			return -ENOENT;
1979 		}
1980 	}
1981 
1982 	return 0;
1983 }
1984 
1985 static int __register_kprobe(struct kprobe *p)
1986 {
1987 	int ret;
1988 	struct kprobe *old_p;
1989 
1990 	guard(mutex)(&kprobe_mutex);
1991 
1992 	old_p = get_kprobe(p->addr);
1993 	if (old_p)
1994 		/* Since this may unoptimize 'old_p', locking 'text_mutex'. */
1995 		return register_aggr_kprobe(old_p, p);
1996 
1997 	scoped_guard(cpus_read_lock) {
1998 		/* Prevent text modification */
1999 		guard(mutex)(&text_mutex);
2000 		ret = prepare_kprobe(p);
2001 		if (ret)
2002 			return ret;
2003 	}
2004 
2005 	INIT_HLIST_NODE(&p->hlist);
2006 	hlist_add_head_rcu(&p->hlist,
2007 		       &kprobe_table[hash_ptr(p->addr, KPROBE_HASH_BITS)]);
2008 
2009 	if (!kprobes_all_disarmed && !kprobe_disabled(p)) {
2010 		ret = arm_kprobe(p);
2011 		if (ret) {
2012 			hlist_del_rcu(&p->hlist);
2013 			synchronize_rcu();
2014 			arch_remove_kprobe(p);
2015 			return ret;
2016 		}
2017 	}
2018 
2019 	/* Try to optimize kprobe */
2020 	try_to_optimize_kprobe(p);
2021 	return 0;
2022 }
2023 
2024 int register_kprobe(struct kprobe *p)
2025 {
2026 	int ret;
2027 	struct module *probed_mod;
2028 	kprobe_opcode_t *addr;
2029 	bool on_func_entry;
2030 
2031 	/* Canonicalize probe address from symbol */
2032 	addr = _kprobe_addr(p->addr, p->symbol_name, p->offset, &on_func_entry);
2033 	if (IS_ERR(addr))
2034 		return PTR_ERR(addr);
2035 	p->addr = addr;
2036 
2037 	ret = check_kprobe_rereg(p);
2038 	if (ret)
2039 		return ret;
2040 
2041 	/* User can pass only KPROBE_FLAG_DISABLED to register_kprobe */
2042 	p->flags &= KPROBE_FLAG_DISABLED;
2043 	if (on_func_entry)
2044 		p->flags |= KPROBE_FLAG_ON_FUNC_ENTRY;
2045 	p->nmissed = 0;
2046 	INIT_LIST_HEAD(&p->list);
2047 
2048 	ret = check_kprobe_address_safe(p, &probed_mod);
2049 	if (ret)
2050 		return ret;
2051 
2052 	ret = __register_kprobe(p);
2053 
2054 	if (probed_mod)
2055 		module_put(probed_mod);
2056 
2057 	return ret;
2058 }
2059 EXPORT_SYMBOL_GPL(register_kprobe);
2060 
2061 /* Check if all probes on the 'ap' are disabled. */
2062 static bool aggr_kprobe_disabled(struct kprobe *ap)
2063 {
2064 	struct kprobe *kp;
2065 
2066 	lockdep_assert_held(&kprobe_mutex);
2067 
2068 	list_for_each_entry(kp, &ap->list, list)
2069 		if (!kprobe_disabled(kp))
2070 			/*
2071 			 * Since there is an active probe on the list,
2072 			 * we can't disable this 'ap'.
2073 			 */
2074 			return false;
2075 
2076 	return true;
2077 }
2078 
2079 static struct kprobe *__disable_kprobe(struct kprobe *p)
2080 {
2081 	struct kprobe *orig_p;
2082 	int ret;
2083 
2084 	lockdep_assert_held(&kprobe_mutex);
2085 
2086 	/* Get an original kprobe for return */
2087 	orig_p = __get_valid_kprobe(p);
2088 	if (unlikely(orig_p == NULL))
2089 		return ERR_PTR(-EINVAL);
2090 
2091 	if (kprobe_disabled(p))
2092 		return orig_p;
2093 
2094 	/* Disable probe if it is a child probe */
2095 	if (p != orig_p)
2096 		p->flags |= KPROBE_FLAG_DISABLED;
2097 
2098 	/* Try to disarm and disable this/parent probe */
2099 	if (p == orig_p || aggr_kprobe_disabled(orig_p)) {
2100 		/*
2101 		 * Don't be lazy here.  Even if 'kprobes_all_disarmed'
2102 		 * is false, 'orig_p' might not have been armed yet.
2103 		 * Note arm_all_kprobes() __tries__ to arm all kprobes
2104 		 * on the best effort basis.
2105 		 */
2106 		if (!kprobes_all_disarmed && !kprobe_disabled(orig_p)) {
2107 			ret = disarm_kprobe(orig_p, true);
2108 			if (ret) {
2109 				p->flags &= ~KPROBE_FLAG_DISABLED;
2110 				return ERR_PTR(ret);
2111 			}
2112 		}
2113 		orig_p->flags |= KPROBE_FLAG_DISABLED;
2114 	}
2115 
2116 	return orig_p;
2117 }
2118 
2119 /*
2120  * Unregister a kprobe without a scheduler synchronization.
2121  */
2122 static int __unregister_kprobe_top(struct kprobe *p)
2123 {
2124 	struct kprobe *ap, *list_p;
2125 
2126 	/* Disable kprobe. This will disarm it if needed. */
2127 	ap = __disable_kprobe(p);
2128 	if (IS_ERR(ap))
2129 		return PTR_ERR(ap);
2130 
2131 	WARN_ON(ap != p && !kprobe_aggrprobe(ap));
2132 
2133 	/*
2134 	 * If the probe is an independent(and non-optimized) kprobe
2135 	 * (not an aggrprobe), the last kprobe on the aggrprobe, or
2136 	 * kprobe is already disarmed, just remove from the hash list.
2137 	 */
2138 	if (ap == p ||
2139 		(list_is_singular(&ap->list) && kprobe_disarmed(ap))) {
2140 		/*
2141 		 * !disarmed could be happen if the probe is under delayed
2142 		 * unoptimizing.
2143 		 */
2144 		hlist_del_rcu(&ap->hlist);
2145 		return 0;
2146 	}
2147 
2148 	/* If disabling probe has special handlers, update aggrprobe */
2149 	if (p->post_handler && !kprobe_gone(p)) {
2150 		list_for_each_entry(list_p, &ap->list, list) {
2151 			if ((list_p != p) && (list_p->post_handler))
2152 				break;
2153 		}
2154 		/* No other probe has post_handler */
2155 		if (list_entry_is_head(list_p, &ap->list, list)) {
2156 			/*
2157 			 * For the kprobe-on-ftrace case, we keep the
2158 			 * post_handler setting to identify this aggrprobe
2159 			 * armed with kprobe_ipmodify_ops.
2160 			 */
2161 			if (!kprobe_ftrace(ap))
2162 				ap->post_handler = NULL;
2163 		}
2164 	}
2165 
2166 	/*
2167 	 * Remove from the aggrprobe: this path will do nothing in
2168 	 * __unregister_kprobe_bottom().
2169 	 */
2170 	list_del_rcu(&p->list);
2171 	if (!kprobe_disabled(ap) && !kprobes_all_disarmed)
2172 		/*
2173 		 * Try to optimize this probe again, because post
2174 		 * handler may have been changed.
2175 		 */
2176 		optimize_kprobe(ap);
2177 	return 0;
2178 
2179 }
2180 
2181 static void __unregister_kprobe_bottom(struct kprobe *p)
2182 {
2183 	struct kprobe *ap;
2184 
2185 	if (list_empty(&p->list))
2186 		/* This is an independent kprobe */
2187 		arch_remove_kprobe(p);
2188 	else if (list_is_singular(&p->list)) {
2189 		/* This is the last child of an aggrprobe */
2190 		ap = list_entry(p->list.next, struct kprobe, list);
2191 		list_del(&p->list);
2192 		free_aggr_kprobe(ap);
2193 	}
2194 	/* Otherwise, do nothing. */
2195 }
2196 
2197 int register_kprobes(struct kprobe **kps, int num)
2198 {
2199 	int i, ret = 0;
2200 
2201 	if (num <= 0)
2202 		return -EINVAL;
2203 	for (i = 0; i < num; i++) {
2204 		ret = register_kprobe(kps[i]);
2205 		if (ret < 0) {
2206 			if (i > 0)
2207 				unregister_kprobes(kps, i);
2208 			break;
2209 		}
2210 	}
2211 	return ret;
2212 }
2213 EXPORT_SYMBOL_GPL(register_kprobes);
2214 
2215 void unregister_kprobe(struct kprobe *p)
2216 {
2217 	unregister_kprobes(&p, 1);
2218 }
2219 EXPORT_SYMBOL_GPL(unregister_kprobe);
2220 
2221 void unregister_kprobes(struct kprobe **kps, int num)
2222 {
2223 	int i;
2224 
2225 	if (num <= 0)
2226 		return;
2227 	scoped_guard(mutex, &kprobe_mutex) {
2228 		for (i = 0; i < num; i++)
2229 			if (__unregister_kprobe_top(kps[i]) < 0)
2230 				kps[i]->addr = NULL;
2231 	}
2232 	synchronize_rcu();
2233 	for (i = 0; i < num; i++)
2234 		if (kps[i]->addr)
2235 			__unregister_kprobe_bottom(kps[i]);
2236 }
2237 EXPORT_SYMBOL_GPL(unregister_kprobes);
2238 
2239 int __weak kprobe_exceptions_notify(struct notifier_block *self,
2240 					unsigned long val, void *data)
2241 {
2242 	return NOTIFY_DONE;
2243 }
2244 NOKPROBE_SYMBOL(kprobe_exceptions_notify);
2245 
2246 static struct notifier_block kprobe_exceptions_nb = {
2247 	.notifier_call = kprobe_exceptions_notify,
2248 	.priority = 0x7fffffff /* we need to be notified first */
2249 };
2250 
2251 #ifdef CONFIG_KRETPROBES
2252 
2253 #if !defined(CONFIG_KRETPROBE_ON_RETHOOK)
2254 
2255 /* callbacks for objpool of kretprobe instances */
2256 static int kretprobe_init_inst(void *nod, void *context)
2257 {
2258 	struct kretprobe_instance *ri = nod;
2259 
2260 	ri->rph = context;
2261 	return 0;
2262 }
2263 static int kretprobe_fini_pool(struct objpool_head *head, void *context)
2264 {
2265 	kfree(context);
2266 	return 0;
2267 }
2268 
2269 static void free_rp_inst_rcu(struct rcu_head *head)
2270 {
2271 	struct kretprobe_instance *ri = container_of(head, struct kretprobe_instance, rcu);
2272 	struct kretprobe_holder *rph = ri->rph;
2273 
2274 	objpool_drop(ri, &rph->pool);
2275 }
2276 NOKPROBE_SYMBOL(free_rp_inst_rcu);
2277 
2278 static void recycle_rp_inst(struct kretprobe_instance *ri)
2279 {
2280 	struct kretprobe *rp = get_kretprobe(ri);
2281 
2282 	if (likely(rp))
2283 		objpool_push(ri, &rp->rph->pool);
2284 	else
2285 		call_rcu(&ri->rcu, free_rp_inst_rcu);
2286 }
2287 NOKPROBE_SYMBOL(recycle_rp_inst);
2288 
2289 /*
2290  * This function is called from delayed_put_task_struct() when a task is
2291  * dead and cleaned up to recycle any kretprobe instances associated with
2292  * this task. These left over instances represent probed functions that
2293  * have been called but will never return.
2294  */
2295 void kprobe_flush_task(struct task_struct *tk)
2296 {
2297 	struct kretprobe_instance *ri;
2298 	struct llist_node *node;
2299 
2300 	/* Early boot, not yet initialized. */
2301 	if (unlikely(!kprobes_initialized))
2302 		return;
2303 
2304 	kprobe_busy_begin();
2305 
2306 	node = __llist_del_all(&tk->kretprobe_instances);
2307 	while (node) {
2308 		ri = container_of(node, struct kretprobe_instance, llist);
2309 		node = node->next;
2310 
2311 		recycle_rp_inst(ri);
2312 	}
2313 
2314 	kprobe_busy_end();
2315 }
2316 NOKPROBE_SYMBOL(kprobe_flush_task);
2317 
2318 static inline void free_rp_inst(struct kretprobe *rp)
2319 {
2320 	struct kretprobe_holder *rph = rp->rph;
2321 
2322 	if (!rph)
2323 		return;
2324 	rp->rph = NULL;
2325 	objpool_fini(&rph->pool);
2326 }
2327 
2328 /* This assumes the 'tsk' is the current task or the is not running. */
2329 static kprobe_opcode_t *__kretprobe_find_ret_addr(struct task_struct *tsk,
2330 						  struct llist_node **cur)
2331 {
2332 	struct kretprobe_instance *ri = NULL;
2333 	struct llist_node *node = *cur;
2334 
2335 	if (!node)
2336 		node = tsk->kretprobe_instances.first;
2337 	else
2338 		node = node->next;
2339 
2340 	while (node) {
2341 		ri = container_of(node, struct kretprobe_instance, llist);
2342 		if (ri->ret_addr != kretprobe_trampoline_addr()) {
2343 			*cur = node;
2344 			return ri->ret_addr;
2345 		}
2346 		node = node->next;
2347 	}
2348 	return NULL;
2349 }
2350 NOKPROBE_SYMBOL(__kretprobe_find_ret_addr);
2351 
2352 /**
2353  * kretprobe_find_ret_addr -- Find correct return address modified by kretprobe
2354  * @tsk: Target task
2355  * @fp: A frame pointer
2356  * @cur: a storage of the loop cursor llist_node pointer for next call
2357  *
2358  * Find the correct return address modified by a kretprobe on @tsk in unsigned
2359  * long type. If it finds the return address, this returns that address value,
2360  * or this returns 0.
2361  * The @tsk must be 'current' or a task which is not running. @fp is a hint
2362  * to get the currect return address - which is compared with the
2363  * kretprobe_instance::fp field. The @cur is a loop cursor for searching the
2364  * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the
2365  * first call, but '@cur' itself must NOT NULL.
2366  */
2367 unsigned long kretprobe_find_ret_addr(struct task_struct *tsk, void *fp,
2368 				      struct llist_node **cur)
2369 {
2370 	struct kretprobe_instance *ri;
2371 	kprobe_opcode_t *ret;
2372 
2373 	if (WARN_ON_ONCE(!cur))
2374 		return 0;
2375 
2376 	do {
2377 		ret = __kretprobe_find_ret_addr(tsk, cur);
2378 		if (!ret)
2379 			break;
2380 		ri = container_of(*cur, struct kretprobe_instance, llist);
2381 	} while (ri->fp != fp);
2382 
2383 	return (unsigned long)ret;
2384 }
2385 NOKPROBE_SYMBOL(kretprobe_find_ret_addr);
2386 
2387 void __weak arch_kretprobe_fixup_return(struct pt_regs *regs,
2388 					kprobe_opcode_t *correct_ret_addr)
2389 {
2390 	/*
2391 	 * Do nothing by default. Please fill this to update the fake return
2392 	 * address on the stack with the correct one on each arch if possible.
2393 	 */
2394 }
2395 
2396 unsigned long __kretprobe_trampoline_handler(struct pt_regs *regs,
2397 					     void *frame_pointer)
2398 {
2399 	struct kretprobe_instance *ri = NULL;
2400 	struct llist_node *first, *node = NULL;
2401 	kprobe_opcode_t *correct_ret_addr;
2402 	struct kretprobe *rp;
2403 
2404 	/* Find correct address and all nodes for this frame. */
2405 	correct_ret_addr = __kretprobe_find_ret_addr(current, &node);
2406 	if (!correct_ret_addr) {
2407 		pr_err("kretprobe: Return address not found, not execute handler. Maybe there is a bug in the kernel.\n");
2408 		BUG_ON(1);
2409 	}
2410 
2411 	/*
2412 	 * Set the return address as the instruction pointer, because if the
2413 	 * user handler calls stack_trace_save_regs() with this 'regs',
2414 	 * the stack trace will start from the instruction pointer.
2415 	 */
2416 	instruction_pointer_set(regs, (unsigned long)correct_ret_addr);
2417 
2418 	/* Run the user handler of the nodes. */
2419 	first = current->kretprobe_instances.first;
2420 	while (first) {
2421 		ri = container_of(first, struct kretprobe_instance, llist);
2422 
2423 		if (WARN_ON_ONCE(ri->fp != frame_pointer))
2424 			break;
2425 
2426 		rp = get_kretprobe(ri);
2427 		if (rp && rp->handler) {
2428 			struct kprobe *prev = kprobe_running();
2429 
2430 			__this_cpu_write(current_kprobe, &rp->kp);
2431 			ri->ret_addr = correct_ret_addr;
2432 			rp->handler(ri, regs);
2433 			__this_cpu_write(current_kprobe, prev);
2434 		}
2435 		if (first == node)
2436 			break;
2437 
2438 		first = first->next;
2439 	}
2440 
2441 	arch_kretprobe_fixup_return(regs, correct_ret_addr);
2442 
2443 	/* Unlink all nodes for this frame. */
2444 	first = current->kretprobe_instances.first;
2445 	current->kretprobe_instances.first = node->next;
2446 	node->next = NULL;
2447 
2448 	/* Recycle free instances. */
2449 	while (first) {
2450 		ri = container_of(first, struct kretprobe_instance, llist);
2451 		first = first->next;
2452 
2453 		recycle_rp_inst(ri);
2454 	}
2455 
2456 	return (unsigned long)correct_ret_addr;
2457 }
2458 NOKPROBE_SYMBOL(__kretprobe_trampoline_handler)
2459 
2460 /*
2461  * This kprobe pre_handler is registered with every kretprobe. When probe
2462  * hits it will set up the return probe.
2463  */
2464 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2465 {
2466 	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2467 	struct kretprobe_holder *rph = rp->rph;
2468 	struct kretprobe_instance *ri;
2469 
2470 	ri = objpool_pop(&rph->pool);
2471 	if (!ri) {
2472 		rp->nmissed++;
2473 		return 0;
2474 	}
2475 
2476 	if (rp->entry_handler && rp->entry_handler(ri, regs)) {
2477 		objpool_push(ri, &rph->pool);
2478 		return 0;
2479 	}
2480 
2481 	arch_prepare_kretprobe(ri, regs);
2482 
2483 	__llist_add(&ri->llist, &current->kretprobe_instances);
2484 
2485 	return 0;
2486 }
2487 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2488 #else /* CONFIG_KRETPROBE_ON_RETHOOK */
2489 /*
2490  * This kprobe pre_handler is registered with every kretprobe. When probe
2491  * hits it will set up the return probe.
2492  */
2493 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2494 {
2495 	struct kretprobe *rp = container_of(p, struct kretprobe, kp);
2496 	struct kretprobe_instance *ri;
2497 	struct rethook_node *rhn;
2498 
2499 	rhn = rethook_try_get(rp->rh);
2500 	if (!rhn) {
2501 		rp->nmissed++;
2502 		return 0;
2503 	}
2504 
2505 	ri = container_of(rhn, struct kretprobe_instance, node);
2506 
2507 	if (rp->entry_handler && rp->entry_handler(ri, regs))
2508 		rethook_recycle(rhn);
2509 	else
2510 		rethook_hook(rhn, regs, kprobe_ftrace(p));
2511 
2512 	return 0;
2513 }
2514 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2515 
2516 static void kretprobe_rethook_handler(struct rethook_node *rh, void *data,
2517 				      unsigned long ret_addr,
2518 				      struct pt_regs *regs)
2519 {
2520 	struct kretprobe *rp = (struct kretprobe *)data;
2521 	struct kretprobe_instance *ri;
2522 	struct kprobe_ctlblk *kcb;
2523 
2524 	/* The data must NOT be null. This means rethook data structure is broken. */
2525 	if (WARN_ON_ONCE(!data) || !rp->handler)
2526 		return;
2527 
2528 	__this_cpu_write(current_kprobe, &rp->kp);
2529 	kcb = get_kprobe_ctlblk();
2530 	kcb->kprobe_status = KPROBE_HIT_ACTIVE;
2531 
2532 	ri = container_of(rh, struct kretprobe_instance, node);
2533 	rp->handler(ri, regs);
2534 
2535 	__this_cpu_write(current_kprobe, NULL);
2536 }
2537 NOKPROBE_SYMBOL(kretprobe_rethook_handler);
2538 
2539 #endif /* !CONFIG_KRETPROBE_ON_RETHOOK */
2540 
2541 /**
2542  * kprobe_on_func_entry() -- check whether given address is function entry
2543  * @addr: Target address
2544  * @sym:  Target symbol name
2545  * @offset: The offset from the symbol or the address
2546  *
2547  * This checks whether the given @addr+@offset or @sym+@offset is on the
2548  * function entry address or not.
2549  * This returns 0 if it is the function entry, or -EINVAL if it is not.
2550  * And also it returns -ENOENT if it fails the symbol or address lookup.
2551  * Caller must pass @addr or @sym (either one must be NULL), or this
2552  * returns -EINVAL.
2553  */
2554 int kprobe_on_func_entry(kprobe_opcode_t *addr, const char *sym, unsigned long offset)
2555 {
2556 	bool on_func_entry;
2557 	kprobe_opcode_t *kp_addr = _kprobe_addr(addr, sym, offset, &on_func_entry);
2558 
2559 	if (IS_ERR(kp_addr))
2560 		return PTR_ERR(kp_addr);
2561 
2562 	if (!on_func_entry)
2563 		return -EINVAL;
2564 
2565 	return 0;
2566 }
2567 
2568 int register_kretprobe(struct kretprobe *rp)
2569 {
2570 	int ret;
2571 	int i;
2572 	void *addr;
2573 	bool on_func_entry;
2574 
2575 	addr = _kprobe_addr(rp->kp.addr, rp->kp.symbol_name, rp->kp.offset, &on_func_entry);
2576 	if (IS_ERR(addr))
2577 		return PTR_ERR(addr);
2578 
2579 	if (!on_func_entry)
2580 		return -EINVAL;
2581 
2582 	if (check_kprobe_rereg_at(&rp->kp, addr))
2583 		return -EINVAL;
2584 
2585 	if (kretprobe_blacklist_size) {
2586 		for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
2587 			if (kretprobe_blacklist[i].addr == addr)
2588 				return -EINVAL;
2589 		}
2590 	}
2591 
2592 	if (rp->data_size > KRETPROBE_MAX_DATA_SIZE)
2593 		return -E2BIG;
2594 
2595 	rp->kp.pre_handler = pre_handler_kretprobe;
2596 	rp->kp.post_handler = NULL;
2597 
2598 	/* Pre-allocate memory for max kretprobe instances */
2599 	if (rp->maxactive <= 0)
2600 		rp->maxactive = max_t(unsigned int, 10, 2*num_possible_cpus());
2601 
2602 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
2603 	rp->rh = rethook_alloc((void *)rp, kretprobe_rethook_handler,
2604 				sizeof(struct kretprobe_instance) +
2605 				rp->data_size, rp->maxactive);
2606 	if (IS_ERR(rp->rh))
2607 		return PTR_ERR(rp->rh);
2608 
2609 	rp->nmissed = 0;
2610 	/* Establish function entry probe point */
2611 	ret = register_kprobe(&rp->kp);
2612 	if (ret != 0) {
2613 		rethook_free(rp->rh);
2614 		rp->rh = NULL;
2615 	}
2616 #else	/* !CONFIG_KRETPROBE_ON_RETHOOK */
2617 	rp->rph = kzalloc_obj(struct kretprobe_holder);
2618 	if (!rp->rph)
2619 		return -ENOMEM;
2620 
2621 	if (objpool_init(&rp->rph->pool, rp->maxactive, rp->data_size +
2622 			sizeof(struct kretprobe_instance), GFP_KERNEL,
2623 			rp->rph, kretprobe_init_inst, kretprobe_fini_pool)) {
2624 		kfree(rp->rph);
2625 		rp->rph = NULL;
2626 		return -ENOMEM;
2627 	}
2628 	rcu_assign_pointer(rp->rph->rp, rp);
2629 	rp->nmissed = 0;
2630 	/* Establish function entry probe point */
2631 	ret = register_kprobe(&rp->kp);
2632 	if (ret != 0)
2633 		free_rp_inst(rp);
2634 #endif
2635 	return ret;
2636 }
2637 EXPORT_SYMBOL_GPL(register_kretprobe);
2638 
2639 int register_kretprobes(struct kretprobe **rps, int num)
2640 {
2641 	int ret = 0, i;
2642 
2643 	if (num <= 0)
2644 		return -EINVAL;
2645 	for (i = 0; i < num; i++) {
2646 		ret = register_kretprobe(rps[i]);
2647 		if (ret < 0) {
2648 			if (i > 0)
2649 				unregister_kretprobes(rps, i);
2650 			break;
2651 		}
2652 	}
2653 	return ret;
2654 }
2655 EXPORT_SYMBOL_GPL(register_kretprobes);
2656 
2657 void unregister_kretprobe(struct kretprobe *rp)
2658 {
2659 	unregister_kretprobes(&rp, 1);
2660 }
2661 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2662 
2663 void unregister_kretprobes(struct kretprobe **rps, int num)
2664 {
2665 	int i;
2666 
2667 	if (num <= 0)
2668 		return;
2669 	for (i = 0; i < num; i++) {
2670 		guard(mutex)(&kprobe_mutex);
2671 
2672 		if (__unregister_kprobe_top(&rps[i]->kp) < 0) {
2673 			rps[i]->kp.addr = NULL;
2674 			continue;
2675 		}
2676 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
2677 		rethook_free(rps[i]->rh);
2678 #else
2679 		rcu_assign_pointer(rps[i]->rph->rp, NULL);
2680 #endif
2681 	}
2682 
2683 	synchronize_rcu();
2684 	for (i = 0; i < num; i++) {
2685 		if (rps[i]->kp.addr) {
2686 			__unregister_kprobe_bottom(&rps[i]->kp);
2687 #ifdef CONFIG_KRETPROBE_ON_RETHOOK
2688 			rps[i]->rh = NULL;
2689 #else
2690 			free_rp_inst(rps[i]);
2691 #endif
2692 		}
2693 	}
2694 }
2695 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2696 
2697 #else /* CONFIG_KRETPROBES */
2698 int register_kretprobe(struct kretprobe *rp)
2699 {
2700 	return -EOPNOTSUPP;
2701 }
2702 EXPORT_SYMBOL_GPL(register_kretprobe);
2703 
2704 int register_kretprobes(struct kretprobe **rps, int num)
2705 {
2706 	return -EOPNOTSUPP;
2707 }
2708 EXPORT_SYMBOL_GPL(register_kretprobes);
2709 
2710 void unregister_kretprobe(struct kretprobe *rp)
2711 {
2712 }
2713 EXPORT_SYMBOL_GPL(unregister_kretprobe);
2714 
2715 void unregister_kretprobes(struct kretprobe **rps, int num)
2716 {
2717 }
2718 EXPORT_SYMBOL_GPL(unregister_kretprobes);
2719 
2720 static int pre_handler_kretprobe(struct kprobe *p, struct pt_regs *regs)
2721 {
2722 	return 0;
2723 }
2724 NOKPROBE_SYMBOL(pre_handler_kretprobe);
2725 
2726 #endif /* CONFIG_KRETPROBES */
2727 
2728 /* Set the kprobe gone and remove its instruction buffer. */
2729 static void kill_kprobe(struct kprobe *p)
2730 {
2731 	struct kprobe *kp;
2732 
2733 	lockdep_assert_held(&kprobe_mutex);
2734 
2735 	/*
2736 	 * The module is going away. We should disarm the kprobe which
2737 	 * is using ftrace, because ftrace framework is still available at
2738 	 * 'MODULE_STATE_GOING' notification.
2739 	 */
2740 	if (kprobe_ftrace(p) && !kprobe_disabled(p) && !kprobes_all_disarmed)
2741 		disarm_kprobe_ftrace(p);
2742 
2743 	p->flags |= KPROBE_FLAG_GONE;
2744 	if (kprobe_aggrprobe(p)) {
2745 		/*
2746 		 * If this is an aggr_kprobe, we have to list all the
2747 		 * chained probes and mark them GONE.
2748 		 */
2749 		list_for_each_entry(kp, &p->list, list)
2750 			kp->flags |= KPROBE_FLAG_GONE;
2751 		p->post_handler = NULL;
2752 		kill_optimized_kprobe(p);
2753 	}
2754 	/*
2755 	 * Here, we can remove insn_slot safely, because no thread calls
2756 	 * the original probed function (which will be freed soon) any more.
2757 	 */
2758 	arch_remove_kprobe(p);
2759 }
2760 
2761 /* Disable one kprobe */
2762 int disable_kprobe(struct kprobe *kp)
2763 {
2764 	struct kprobe *p;
2765 
2766 	guard(mutex)(&kprobe_mutex);
2767 
2768 	/* Disable this kprobe */
2769 	p = __disable_kprobe(kp);
2770 
2771 	return IS_ERR(p) ? PTR_ERR(p) : 0;
2772 }
2773 EXPORT_SYMBOL_GPL(disable_kprobe);
2774 
2775 /* Enable one kprobe */
2776 int enable_kprobe(struct kprobe *kp)
2777 {
2778 	int ret = 0;
2779 	struct kprobe *p;
2780 
2781 	guard(mutex)(&kprobe_mutex);
2782 
2783 	/* Check whether specified probe is valid. */
2784 	p = __get_valid_kprobe(kp);
2785 	if (unlikely(p == NULL))
2786 		return -EINVAL;
2787 
2788 	if (kprobe_gone(kp))
2789 		/* This kprobe has gone, we couldn't enable it. */
2790 		return -EINVAL;
2791 
2792 	if (p != kp)
2793 		kp->flags &= ~KPROBE_FLAG_DISABLED;
2794 
2795 	if (!kprobes_all_disarmed && kprobe_disabled(p)) {
2796 		p->flags &= ~KPROBE_FLAG_DISABLED;
2797 		ret = arm_kprobe(p);
2798 		if (ret) {
2799 			p->flags |= KPROBE_FLAG_DISABLED;
2800 			if (p != kp)
2801 				kp->flags |= KPROBE_FLAG_DISABLED;
2802 		}
2803 	}
2804 	return ret;
2805 }
2806 EXPORT_SYMBOL_GPL(enable_kprobe);
2807 
2808 /* Caller must NOT call this in usual path. This is only for critical case */
2809 void dump_kprobe(struct kprobe *kp)
2810 {
2811 	pr_err("Dump kprobe:\n.symbol_name = %s, .offset = %x, .addr = %pS\n",
2812 	       kp->symbol_name, kp->offset, kp->addr);
2813 }
2814 NOKPROBE_SYMBOL(dump_kprobe);
2815 
2816 int kprobe_add_ksym_blacklist(unsigned long entry)
2817 {
2818 	struct kprobe_blacklist_entry *ent;
2819 	unsigned long offset = 0, size = 0;
2820 
2821 	if (!kernel_text_address(entry) ||
2822 	    !kallsyms_lookup_size_offset(entry, &size, &offset))
2823 		return -EINVAL;
2824 
2825 	ent = kmalloc_obj(*ent);
2826 	if (!ent)
2827 		return -ENOMEM;
2828 	ent->start_addr = entry;
2829 	ent->end_addr = entry + size;
2830 	INIT_LIST_HEAD(&ent->list);
2831 	list_add_tail_rcu(&ent->list, &kprobe_blacklist);
2832 
2833 	return (int)size;
2834 }
2835 
2836 /* Add all symbols in given area into kprobe blacklist */
2837 int kprobe_add_area_blacklist(unsigned long start, unsigned long end)
2838 {
2839 	unsigned long entry;
2840 	int ret = 0;
2841 
2842 	for (entry = start; entry < end; entry += ret) {
2843 		ret = kprobe_add_ksym_blacklist(entry);
2844 		if (ret < 0)
2845 			return ret;
2846 		if (ret == 0)	/* In case of alias symbol */
2847 			ret = 1;
2848 	}
2849 	return 0;
2850 }
2851 
2852 int __weak arch_kprobe_get_kallsym(unsigned int *symnum, unsigned long *value,
2853 				   char *type, char *sym)
2854 {
2855 	return -ERANGE;
2856 }
2857 
2858 int kprobe_get_kallsym(unsigned int symnum, unsigned long *value, char *type,
2859 		       char *sym)
2860 {
2861 #ifdef __ARCH_WANT_KPROBES_INSN_SLOT
2862 	if (!kprobe_cache_get_kallsym(&kprobe_insn_slots, &symnum, value, type, sym))
2863 		return 0;
2864 #ifdef CONFIG_OPTPROBES
2865 	if (!kprobe_cache_get_kallsym(&kprobe_optinsn_slots, &symnum, value, type, sym))
2866 		return 0;
2867 #endif
2868 #endif
2869 	if (!arch_kprobe_get_kallsym(&symnum, value, type, sym))
2870 		return 0;
2871 	return -ERANGE;
2872 }
2873 
2874 int __init __weak arch_populate_kprobe_blacklist(void)
2875 {
2876 	return 0;
2877 }
2878 
2879 /*
2880  * Lookup and populate the kprobe_blacklist.
2881  *
2882  * Unlike the kretprobe blacklist, we'll need to determine
2883  * the range of addresses that belong to the said functions,
2884  * since a kprobe need not necessarily be at the beginning
2885  * of a function.
2886  */
2887 static int __init populate_kprobe_blacklist(unsigned long *start,
2888 					     unsigned long *end)
2889 {
2890 	unsigned long entry;
2891 	unsigned long *iter;
2892 	int ret;
2893 
2894 	for (iter = start; iter < end; iter++) {
2895 		entry = (unsigned long)dereference_symbol_descriptor((void *)*iter);
2896 		ret = kprobe_add_ksym_blacklist(entry);
2897 		if (ret == -EINVAL)
2898 			continue;
2899 		if (ret < 0)
2900 			return ret;
2901 	}
2902 
2903 	/* Symbols in '__kprobes_text' are blacklisted */
2904 	ret = kprobe_add_area_blacklist((unsigned long)__kprobes_text_start,
2905 					(unsigned long)__kprobes_text_end);
2906 	if (ret)
2907 		return ret;
2908 
2909 	/* Symbols in 'noinstr' section are blacklisted */
2910 	ret = kprobe_add_area_blacklist((unsigned long)__noinstr_text_start,
2911 					(unsigned long)__noinstr_text_end);
2912 
2913 	return ret ? : arch_populate_kprobe_blacklist();
2914 }
2915 
2916 #ifdef CONFIG_MODULES
2917 /* Remove all symbols in given area from kprobe blacklist */
2918 static void kprobe_remove_area_blacklist(unsigned long start, unsigned long end)
2919 {
2920 	struct kprobe_blacklist_entry *ent, *n;
2921 
2922 	list_for_each_entry_safe(ent, n, &kprobe_blacklist, list) {
2923 		if (ent->start_addr < start || ent->start_addr >= end)
2924 			continue;
2925 		list_del_rcu(&ent->list);
2926 		kfree_rcu(ent, rcu);
2927 	}
2928 }
2929 
2930 static void kprobe_remove_ksym_blacklist(unsigned long entry)
2931 {
2932 	kprobe_remove_area_blacklist(entry, entry + 1);
2933 }
2934 
2935 static void add_module_kprobe_blacklist(struct module *mod)
2936 {
2937 	unsigned long start, end;
2938 	int i;
2939 
2940 	if (mod->kprobe_blacklist) {
2941 		for (i = 0; i < mod->num_kprobe_blacklist; i++)
2942 			kprobe_add_ksym_blacklist(mod->kprobe_blacklist[i]);
2943 	}
2944 
2945 	start = (unsigned long)mod->kprobes_text_start;
2946 	if (start) {
2947 		end = start + mod->kprobes_text_size;
2948 		kprobe_add_area_blacklist(start, end);
2949 	}
2950 
2951 	start = (unsigned long)mod->noinstr_text_start;
2952 	if (start) {
2953 		end = start + mod->noinstr_text_size;
2954 		kprobe_add_area_blacklist(start, end);
2955 	}
2956 }
2957 
2958 static void remove_module_kprobe_blacklist(struct module *mod)
2959 {
2960 	unsigned long start, end;
2961 	int i;
2962 
2963 	if (mod->kprobe_blacklist) {
2964 		for (i = 0; i < mod->num_kprobe_blacklist; i++)
2965 			kprobe_remove_ksym_blacklist(mod->kprobe_blacklist[i]);
2966 	}
2967 
2968 	start = (unsigned long)mod->kprobes_text_start;
2969 	if (start) {
2970 		end = start + mod->kprobes_text_size;
2971 		kprobe_remove_area_blacklist(start, end);
2972 	}
2973 
2974 	start = (unsigned long)mod->noinstr_text_start;
2975 	if (start) {
2976 		end = start + mod->noinstr_text_size;
2977 		kprobe_remove_area_blacklist(start, end);
2978 	}
2979 }
2980 
2981 /* Module notifier call back, checking kprobes on the module */
2982 static int kprobes_module_callback(struct notifier_block *nb,
2983 				   unsigned long val, void *data)
2984 {
2985 	struct module *mod = data;
2986 	struct hlist_head *head;
2987 	struct kprobe *p;
2988 	unsigned int i;
2989 	int checkcore = (val == MODULE_STATE_GOING);
2990 
2991 	guard(mutex)(&kprobe_mutex);
2992 
2993 	if (val == MODULE_STATE_COMING)
2994 		add_module_kprobe_blacklist(mod);
2995 
2996 	if (val != MODULE_STATE_GOING && val != MODULE_STATE_LIVE)
2997 		return NOTIFY_DONE;
2998 
2999 	/*
3000 	 * When 'MODULE_STATE_GOING' was notified, both of module '.text' and
3001 	 * '.init.text' sections would be freed. When 'MODULE_STATE_LIVE' was
3002 	 * notified, only '.init.text' section would be freed. We need to
3003 	 * disable kprobes which have been inserted in the sections.
3004 	 */
3005 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
3006 		head = &kprobe_table[i];
3007 		hlist_for_each_entry(p, head, hlist)
3008 			if (within_module_init((unsigned long)p->addr, mod) ||
3009 			    (checkcore &&
3010 			     within_module_core((unsigned long)p->addr, mod))) {
3011 				/*
3012 				 * The vaddr this probe is installed will soon
3013 				 * be vfreed buy not synced to disk. Hence,
3014 				 * disarming the breakpoint isn't needed.
3015 				 *
3016 				 * Note, this will also move any optimized probes
3017 				 * that are pending to be removed from their
3018 				 * corresponding lists to the 'freeing_list' and
3019 				 * will not be touched by the delayed
3020 				 * kprobe_optimizer() work handler.
3021 				 */
3022 				kill_kprobe(p);
3023 			}
3024 	}
3025 	if (val == MODULE_STATE_GOING)
3026 		remove_module_kprobe_blacklist(mod);
3027 	return NOTIFY_DONE;
3028 }
3029 
3030 static struct notifier_block kprobe_module_nb = {
3031 	.notifier_call = kprobes_module_callback,
3032 	.priority = 0
3033 };
3034 
3035 static int kprobe_register_module_notifier(void)
3036 {
3037 	return register_module_notifier(&kprobe_module_nb);
3038 }
3039 #else
3040 static int kprobe_register_module_notifier(void)
3041 {
3042 	return 0;
3043 }
3044 #endif /* CONFIG_MODULES */
3045 
3046 void kprobe_free_init_mem(void)
3047 {
3048 	void *start = (void *)(&__init_begin);
3049 	void *end = (void *)(&__init_end);
3050 	struct hlist_head *head;
3051 	struct kprobe *p;
3052 	int i;
3053 
3054 	guard(mutex)(&kprobe_mutex);
3055 
3056 	/* Kill all kprobes on initmem because the target code has been freed. */
3057 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
3058 		head = &kprobe_table[i];
3059 		hlist_for_each_entry(p, head, hlist) {
3060 			if (start <= (void *)p->addr && (void *)p->addr < end)
3061 				kill_kprobe(p);
3062 		}
3063 	}
3064 }
3065 
3066 static int __init init_kprobes(void)
3067 {
3068 	int i, err;
3069 
3070 	/* FIXME allocate the probe table, currently defined statically */
3071 	/* initialize all list heads */
3072 	for (i = 0; i < KPROBE_TABLE_SIZE; i++)
3073 		INIT_HLIST_HEAD(&kprobe_table[i]);
3074 
3075 	err = populate_kprobe_blacklist(__start_kprobe_blacklist,
3076 					__stop_kprobe_blacklist);
3077 	if (err)
3078 		pr_err("Failed to populate blacklist (error %d), kprobes not restricted, be careful using them!\n", err);
3079 
3080 	if (kretprobe_blacklist_size) {
3081 		/* lookup the function address from its name */
3082 		for (i = 0; kretprobe_blacklist[i].name != NULL; i++) {
3083 			kretprobe_blacklist[i].addr =
3084 				kprobe_lookup_name(kretprobe_blacklist[i].name, 0);
3085 			if (!kretprobe_blacklist[i].addr)
3086 				pr_err("Failed to lookup symbol '%s' for kretprobe blacklist. Maybe the target function is removed or renamed.\n",
3087 				       kretprobe_blacklist[i].name);
3088 		}
3089 	}
3090 
3091 	/* By default, kprobes are armed */
3092 	kprobes_all_disarmed = false;
3093 
3094 	/* Initialize the optimization infrastructure */
3095 	init_optprobe();
3096 
3097 	err = arch_init_kprobes();
3098 	if (!err)
3099 		err = register_die_notifier(&kprobe_exceptions_nb);
3100 	if (!err)
3101 		err = kprobe_register_module_notifier();
3102 
3103 	kprobes_initialized = (err == 0);
3104 	kprobe_sysctls_init();
3105 	return err;
3106 }
3107 early_initcall(init_kprobes);
3108 
3109 #if defined(CONFIG_OPTPROBES)
3110 static int __init init_optprobes(void)
3111 {
3112 	/*
3113 	 * Enable kprobe optimization - this kicks the optimizer which
3114 	 * depends on synchronize_rcu_tasks() and ksoftirqd, that is
3115 	 * not spawned in early initcall. So delay the optimization.
3116 	 */
3117 	optimize_all_kprobes();
3118 
3119 	return 0;
3120 }
3121 subsys_initcall(init_optprobes);
3122 #endif
3123 
3124 #ifdef CONFIG_DEBUG_FS
3125 static void report_probe(struct seq_file *pi, struct kprobe *p,
3126 		const char *sym, int offset, char *modname, struct kprobe *pp)
3127 {
3128 	char *kprobe_type;
3129 	void *addr = p->addr;
3130 
3131 	if (p->pre_handler == pre_handler_kretprobe)
3132 		kprobe_type = "r";
3133 	else
3134 		kprobe_type = "k";
3135 
3136 	if (!kallsyms_show_value(pi->file->f_cred))
3137 		addr = NULL;
3138 
3139 	if (sym)
3140 		seq_printf(pi, "%px  %s  %s+0x%x  %s ",
3141 			addr, kprobe_type, sym, offset,
3142 			(modname ? modname : " "));
3143 	else	/* try to use %pS */
3144 		seq_printf(pi, "%px  %s  %pS ",
3145 			addr, kprobe_type, p->addr);
3146 
3147 	if (!pp)
3148 		pp = p;
3149 	seq_printf(pi, "%s%s%s%s\n",
3150 		(kprobe_gone(p) ? "[GONE]" : ""),
3151 		((kprobe_disabled(p) && !kprobe_gone(p)) ?  "[DISABLED]" : ""),
3152 		(kprobe_optimized(pp) ? "[OPTIMIZED]" : ""),
3153 		(kprobe_ftrace(pp) ? "[FTRACE]" : ""));
3154 }
3155 
3156 static void *kprobe_seq_start(struct seq_file *f, loff_t *pos)
3157 {
3158 	return (*pos < KPROBE_TABLE_SIZE) ? pos : NULL;
3159 }
3160 
3161 static void *kprobe_seq_next(struct seq_file *f, void *v, loff_t *pos)
3162 {
3163 	(*pos)++;
3164 	if (*pos >= KPROBE_TABLE_SIZE)
3165 		return NULL;
3166 	return pos;
3167 }
3168 
3169 static void kprobe_seq_stop(struct seq_file *f, void *v)
3170 {
3171 	/* Nothing to do */
3172 }
3173 
3174 static int show_kprobe_addr(struct seq_file *pi, void *v)
3175 {
3176 	struct hlist_head *head;
3177 	struct kprobe *p, *kp;
3178 	const char *sym;
3179 	unsigned int i = *(loff_t *) v;
3180 	unsigned long offset = 0;
3181 	char *modname, namebuf[KSYM_NAME_LEN];
3182 
3183 	head = &kprobe_table[i];
3184 	preempt_disable();
3185 	hlist_for_each_entry_rcu(p, head, hlist) {
3186 		sym = kallsyms_lookup((unsigned long)p->addr, NULL,
3187 					&offset, &modname, namebuf);
3188 		if (kprobe_aggrprobe(p)) {
3189 			list_for_each_entry_rcu(kp, &p->list, list)
3190 				report_probe(pi, kp, sym, offset, modname, p);
3191 		} else
3192 			report_probe(pi, p, sym, offset, modname, NULL);
3193 	}
3194 	preempt_enable();
3195 	return 0;
3196 }
3197 
3198 static const struct seq_operations kprobes_sops = {
3199 	.start = kprobe_seq_start,
3200 	.next  = kprobe_seq_next,
3201 	.stop  = kprobe_seq_stop,
3202 	.show  = show_kprobe_addr
3203 };
3204 
3205 DEFINE_SEQ_ATTRIBUTE(kprobes);
3206 
3207 /* kprobes/blacklist -- shows which functions can not be probed */
3208 static void *kprobe_blacklist_seq_start(struct seq_file *m, loff_t *pos)
3209 {
3210 	mutex_lock(&kprobe_mutex);
3211 	return seq_list_start(&kprobe_blacklist, *pos);
3212 }
3213 
3214 static void *kprobe_blacklist_seq_next(struct seq_file *m, void *v, loff_t *pos)
3215 {
3216 	return seq_list_next(v, &kprobe_blacklist, pos);
3217 }
3218 
3219 static int kprobe_blacklist_seq_show(struct seq_file *m, void *v)
3220 {
3221 	struct kprobe_blacklist_entry *ent =
3222 		list_entry(v, struct kprobe_blacklist_entry, list);
3223 
3224 	/*
3225 	 * If '/proc/kallsyms' is not showing kernel address, we won't
3226 	 * show them here either.
3227 	 */
3228 	if (!kallsyms_show_value(m->file->f_cred))
3229 		seq_printf(m, "0x%px-0x%px\t%ps\n", NULL, NULL,
3230 			   (void *)ent->start_addr);
3231 	else
3232 		seq_printf(m, "0x%px-0x%px\t%ps\n", (void *)ent->start_addr,
3233 			   (void *)ent->end_addr, (void *)ent->start_addr);
3234 	return 0;
3235 }
3236 
3237 static void kprobe_blacklist_seq_stop(struct seq_file *f, void *v)
3238 {
3239 	mutex_unlock(&kprobe_mutex);
3240 }
3241 
3242 static const struct seq_operations kprobe_blacklist_sops = {
3243 	.start = kprobe_blacklist_seq_start,
3244 	.next  = kprobe_blacklist_seq_next,
3245 	.stop  = kprobe_blacklist_seq_stop,
3246 	.show  = kprobe_blacklist_seq_show,
3247 };
3248 DEFINE_SEQ_ATTRIBUTE(kprobe_blacklist);
3249 
3250 static int arm_all_kprobes(void)
3251 {
3252 	struct hlist_head *head;
3253 	struct kprobe *p;
3254 	unsigned int i, total = 0, errors = 0;
3255 	int err, ret = 0;
3256 
3257 	guard(mutex)(&kprobe_mutex);
3258 
3259 	/* If kprobes are armed, just return */
3260 	if (!kprobes_all_disarmed)
3261 		return 0;
3262 
3263 	/*
3264 	 * optimize_kprobe() called by arm_kprobe() checks
3265 	 * kprobes_all_disarmed, so set kprobes_all_disarmed before
3266 	 * arm_kprobe.
3267 	 */
3268 	kprobes_all_disarmed = false;
3269 	/* Arming kprobes doesn't optimize kprobe itself */
3270 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
3271 		head = &kprobe_table[i];
3272 		/* Arm all kprobes on a best-effort basis */
3273 		hlist_for_each_entry(p, head, hlist) {
3274 			if (!kprobe_disabled(p)) {
3275 				err = arm_kprobe(p);
3276 				if (err)  {
3277 					errors++;
3278 					ret = err;
3279 				}
3280 				total++;
3281 			}
3282 		}
3283 	}
3284 
3285 	if (errors)
3286 		pr_warn("Kprobes globally enabled, but failed to enable %d out of %d probes. Please check which kprobes are kept disabled via debugfs.\n",
3287 			errors, total);
3288 	else
3289 		pr_info("Kprobes globally enabled\n");
3290 
3291 	return ret;
3292 }
3293 
3294 static int disarm_all_kprobes(void)
3295 {
3296 	struct hlist_head *head;
3297 	struct kprobe *p;
3298 	unsigned int i, total = 0, errors = 0;
3299 	int err, ret = 0;
3300 
3301 	guard(mutex)(&kprobe_mutex);
3302 
3303 	/* If kprobes are already disarmed, just return */
3304 	if (kprobes_all_disarmed)
3305 		return 0;
3306 
3307 	kprobes_all_disarmed = true;
3308 
3309 	for (i = 0; i < KPROBE_TABLE_SIZE; i++) {
3310 		head = &kprobe_table[i];
3311 		/* Disarm all kprobes on a best-effort basis */
3312 		hlist_for_each_entry(p, head, hlist) {
3313 			if (!arch_trampoline_kprobe(p) && !kprobe_disabled(p)) {
3314 				err = disarm_kprobe(p, false);
3315 				if (err) {
3316 					errors++;
3317 					ret = err;
3318 				}
3319 				total++;
3320 			}
3321 		}
3322 	}
3323 
3324 	if (errors)
3325 		pr_warn("Kprobes globally disabled, but failed to disable %d out of %d probes. Please check which kprobes are kept enabled via debugfs.\n",
3326 			errors, total);
3327 	else
3328 		pr_info("Kprobes globally disabled\n");
3329 
3330 	/* Wait for disarming all kprobes by optimizer */
3331 	wait_for_kprobe_optimizer_locked();
3332 	return ret;
3333 }
3334 
3335 static int kprobes_enabled_set(void *data, u64 val)
3336 {
3337 	if (val)
3338 		return arm_all_kprobes();
3339 
3340 	return disarm_all_kprobes();
3341 }
3342 
3343 static int kprobes_enabled_get(void *data, u64 *val)
3344 {
3345 	*val = !kprobes_all_disarmed;
3346 	return 0;
3347 }
3348 
3349 DEFINE_DEBUGFS_ATTRIBUTE(fops_kp, kprobes_enabled_get,
3350 			 kprobes_enabled_set, "%llu\n");
3351 
3352 static int __init debugfs_kprobe_init(void)
3353 {
3354 	struct dentry *dir;
3355 
3356 	dir = debugfs_create_dir("kprobes", NULL);
3357 
3358 	debugfs_create_file("list", 0400, dir, NULL, &kprobes_fops);
3359 
3360 	debugfs_create_file("enabled", 0600, dir, NULL, &fops_kp);
3361 
3362 	debugfs_create_file("blacklist", 0400, dir, NULL,
3363 			    &kprobe_blacklist_fops);
3364 
3365 	return 0;
3366 }
3367 
3368 late_initcall(debugfs_kprobe_init);
3369 #endif /* CONFIG_DEBUG_FS */
3370