xref: /linux/kernel/trace/trace_probe.c (revision 0941144ebeaf3ce46c80711f00e935921eb28b21)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Common code for probe-based Dynamic events.
4  *
5  * This code was copied from kernel/trace/trace_kprobe.c written by
6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
7  *
8  * Updates to make this generic:
9  * Copyright (C) IBM Corporation, 2010-2011
10  * Author:     Srikar Dronamraju
11  */
12 #define pr_fmt(fmt)	"trace_probe: " fmt
13 
14 #include <linux/bpf.h>
15 #include <linux/fs.h>
16 
17 #include "trace_btf.h"
18 #include "trace_probe.h"
19 
20 #undef C
21 #define C(a, b)		b
22 
23 static const char *trace_probe_err_text[] = { ERRORS };
24 
25 static const char *reserved_field_names[] = {
26 	"common_type",
27 	"common_flags",
28 	"common_preempt_count",
29 	"common_pid",
30 	"common_tgid",
31 	FIELD_STRING_IP,
32 	FIELD_STRING_RETIP,
33 	FIELD_STRING_FUNC,
34 };
35 
36 /* Printing  in basic type function template */
37 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)			\
38 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
39 {									\
40 	trace_seq_printf(s, fmt, *(type *)data);			\
41 	return !trace_seq_has_overflowed(s);				\
42 }									\
43 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
44 
45 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
46 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
47 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
48 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
49 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
50 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
51 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
52 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
53 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
54 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
55 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
56 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
57 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
58 
59 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
60 {
61 	trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
62 	return !trace_seq_has_overflowed(s);
63 }
64 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
65 
66 /* Print type function for string type */
67 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
68 {
69 	int len = *(u32 *)data >> 16;
70 
71 	if (!len)
72 		trace_seq_puts(s, FAULT_STRING);
73 	else
74 		trace_seq_printf(s, "\"%s\"",
75 				 (const char *)get_loc_data(data, ent));
76 	return !trace_seq_has_overflowed(s);
77 }
78 
79 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
80 
81 /* Fetch type information table */
82 static const struct fetch_type probe_fetch_types[] = {
83 	/* Special types */
84 	__ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
85 			    "__data_loc char[]"),
86 	__ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
87 			    "__data_loc char[]"),
88 	__ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
89 			    "__data_loc char[]"),
90 	/* Basic types */
91 	ASSIGN_FETCH_TYPE(u8,  u8,  0),
92 	ASSIGN_FETCH_TYPE(u16, u16, 0),
93 	ASSIGN_FETCH_TYPE(u32, u32, 0),
94 	ASSIGN_FETCH_TYPE(u64, u64, 0),
95 	ASSIGN_FETCH_TYPE(s8,  u8,  1),
96 	ASSIGN_FETCH_TYPE(s16, u16, 1),
97 	ASSIGN_FETCH_TYPE(s32, u32, 1),
98 	ASSIGN_FETCH_TYPE(s64, u64, 1),
99 	ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
100 	ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
101 	ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
102 	ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
103 	ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
104 	ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
105 
106 	ASSIGN_FETCH_TYPE_END
107 };
108 
109 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
110 {
111 	int i;
112 
113 	/* Reject the symbol/symstr for uprobes */
114 	if (type && (flags & TPARG_FL_USER) &&
115 	    (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
116 		return NULL;
117 
118 	if (!type)
119 		type = DEFAULT_FETCH_TYPE_STR;
120 
121 	/* Special case: bitfield */
122 	if (*type == 'b') {
123 		unsigned long bs;
124 
125 		type = strchr(type, '/');
126 		if (!type)
127 			goto fail;
128 
129 		type++;
130 		if (kstrtoul(type, 0, &bs))
131 			goto fail;
132 
133 		switch (bs) {
134 		case 8:
135 			return find_fetch_type("u8", flags);
136 		case 16:
137 			return find_fetch_type("u16", flags);
138 		case 32:
139 			return find_fetch_type("u32", flags);
140 		case 64:
141 			return find_fetch_type("u64", flags);
142 		default:
143 			goto fail;
144 		}
145 	}
146 
147 	for (i = 0; probe_fetch_types[i].name; i++) {
148 		if (strcmp(type, probe_fetch_types[i].name) == 0)
149 			return &probe_fetch_types[i];
150 	}
151 
152 fail:
153 	return NULL;
154 }
155 
156 static struct trace_probe_log trace_probe_log;
157 extern struct mutex dyn_event_ops_mutex;
158 
159 const char *trace_probe_log_init(const char *subsystem, int argc, const char **argv)
160 {
161 	lockdep_assert_held(&dyn_event_ops_mutex);
162 
163 	trace_probe_log.subsystem = subsystem;
164 	trace_probe_log.argc = argc;
165 	trace_probe_log.argv = argv;
166 	trace_probe_log.index = 0;
167 	return subsystem;
168 }
169 
170 void trace_probe_log_clear(void)
171 {
172 	lockdep_assert_held(&dyn_event_ops_mutex);
173 
174 	memset(&trace_probe_log, 0, sizeof(trace_probe_log));
175 }
176 
177 void trace_probe_log_set_index(int index)
178 {
179 	lockdep_assert_held(&dyn_event_ops_mutex);
180 
181 	trace_probe_log.index = index;
182 }
183 
184 void __trace_probe_log_err(int offset, int err_type)
185 {
186 	char *command, *p;
187 	int i, len = 0, pos = 0;
188 
189 	lockdep_assert_held(&dyn_event_ops_mutex);
190 
191 	if (!trace_probe_log.argv || !trace_probe_log.argc)
192 		return;
193 
194 	/* Recalculate the length and allocate buffer */
195 	for (i = 0; i < trace_probe_log.argc; i++) {
196 		if (i == trace_probe_log.index)
197 			pos = len;
198 		len += strlen(trace_probe_log.argv[i]) + 1;
199 	}
200 	command = kzalloc(len, GFP_KERNEL);
201 	if (!command)
202 		return;
203 
204 	if (trace_probe_log.index >= trace_probe_log.argc) {
205 		/**
206 		 * Set the error position is next to the last arg + space.
207 		 * Note that len includes the terminal null and the cursor
208 		 * appears at pos + 1.
209 		 */
210 		pos = len;
211 		offset = 0;
212 	}
213 
214 	/* And make a command string from argv array */
215 	p = command;
216 	for (i = 0; i < trace_probe_log.argc; i++) {
217 		len = strlen(trace_probe_log.argv[i]);
218 		memcpy(p, trace_probe_log.argv[i], len);
219 		p[len] = ' ';
220 		p += len + 1;
221 	}
222 	*(p - 1) = '\0';
223 
224 	tracing_log_err(NULL, trace_probe_log.subsystem, command,
225 			trace_probe_err_text, err_type, pos + offset);
226 
227 	kfree(command);
228 }
229 
230 /* Split symbol and offset. */
231 int traceprobe_split_symbol_offset(char *symbol, long *offset)
232 {
233 	char *tmp;
234 	int ret;
235 
236 	if (!offset)
237 		return -EINVAL;
238 
239 	tmp = strpbrk(symbol, "+-");
240 	if (tmp) {
241 		ret = kstrtol(tmp, 0, offset);
242 		if (ret)
243 			return ret;
244 		*tmp = '\0';
245 	} else
246 		*offset = 0;
247 
248 	return 0;
249 }
250 
251 /**
252  * traceprobe_parse_event_name() - Parse a string into group and event names
253  * @pevent: A pointer to the string to be parsed.
254  * @pgroup: A pointer to the group name.
255  * @buf:    A buffer to store the parsed group name.
256  * @offset: The offset of the string in the original user command, for logging.
257  *
258  * This parses a string with the format `[GROUP/][EVENT]` or `[GROUP.][EVENT]`
259  * (either GROUP or EVENT or both must be specified).
260  * Since the parsed group name is stored in @buf, the caller must ensure @buf
261  * is at least MAX_EVENT_NAME_LEN bytes.
262  *
263  * Return: 0 on success, or -EINVAL on failure.
264  *
265  * If success, *@pevent is updated to point to the event name part of the
266  * original string, or NULL if there is no event name.
267  * Also, *@pgroup is updated to point to the parsed group which is stored
268  * in @buf, or NULL if there is no group name.
269  */
270 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
271 				char *buf, int offset)
272 {
273 	const char *slash, *event = *pevent;
274 	int len;
275 
276 	slash = strchr(event, '/');
277 	if (!slash)
278 		slash = strchr(event, '.');
279 
280 	if (slash) {
281 		if (slash == event) {
282 			trace_probe_log_err(offset, NO_GROUP_NAME);
283 			return -EINVAL;
284 		}
285 		if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
286 			trace_probe_log_err(offset, GROUP_TOO_LONG);
287 			return -EINVAL;
288 		}
289 		strscpy(buf, event, slash - event + 1);
290 		if (!is_good_system_name(buf)) {
291 			trace_probe_log_err(offset, BAD_GROUP_NAME);
292 			return -EINVAL;
293 		}
294 		*pgroup = buf;
295 		*pevent = slash + 1;
296 		offset += slash - event + 1;
297 		event = *pevent;
298 	}
299 	len = strlen(event);
300 	if (len == 0) {
301 		if (slash) {
302 			*pevent = NULL;
303 			return 0;
304 		}
305 		trace_probe_log_err(offset, NO_EVENT_NAME);
306 		return -EINVAL;
307 	} else if (len >= MAX_EVENT_NAME_LEN) {
308 		trace_probe_log_err(offset, EVENT_TOO_LONG);
309 		return -EINVAL;
310 	}
311 	if (!is_good_name(event)) {
312 		trace_probe_log_err(offset, BAD_EVENT_NAME);
313 		return -EINVAL;
314 	}
315 	return 0;
316 }
317 
318 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
319 				 struct traceprobe_parse_context *ctx)
320 {
321 	struct ftrace_event_field *field;
322 	struct list_head *head;
323 
324 	head = trace_get_fields(ctx->event);
325 	list_for_each_entry(field, head, link) {
326 		if (!strcmp(arg, field->name)) {
327 			code->op = FETCH_OP_TP_ARG;
328 			code->data = field;
329 			return 0;
330 		}
331 	}
332 	return -ENOENT;
333 }
334 
335 static int parse_trace_event(char *arg, struct fetch_insn *code,
336 			     struct traceprobe_parse_context *ctx)
337 {
338 	int ret;
339 
340 	if (code->data)
341 		return -EFAULT;
342 	ret = parse_trace_event_arg(arg, code, ctx);
343 	if (!ret)
344 		return 0;
345 	return -EINVAL;
346 }
347 
348 /* this_cpu_* parser */
349 #define THIS_CPU_PTR_PREFIX "this_cpu_ptr("
350 #define THIS_CPU_READ_PREFIX "this_cpu_read("
351 #define THIS_CPU_PTR_LEN (sizeof(THIS_CPU_PTR_PREFIX) - 1)
352 #define THIS_CPU_READ_LEN (sizeof(THIS_CPU_READ_PREFIX) - 1)
353 
354 static int
355 parse_probe_arg(char *arg, const struct fetch_type *type,
356 		struct fetch_insn **pcode, struct fetch_insn *end,
357 		struct traceprobe_parse_context *ctx);
358 
359 static int parse_this_cpu(char *arg, struct traceprobe_parse_context *ctx)
360 {
361 	bool is_read = false;
362 	char *tmp;
363 
364 	/*
365 	 * This is only for kernel probes, excluding eprobe, because per-cpu
366 	 * pointer should not be recorded by events.
367 	 */
368 	if (!(ctx->flags & TPARG_FL_KERNEL) ||
369 	    (ctx->flags & TPARG_FL_TEVENT)) {
370 		trace_probe_log_err(ctx->offset, NOSUP_PERCPU);
371 		return -EINVAL;
372 	}
373 	if (str_has_prefix(arg, THIS_CPU_PTR_PREFIX)) {
374 		arg += THIS_CPU_PTR_LEN;
375 		ctx->offset += THIS_CPU_PTR_LEN;
376 	} else if (str_has_prefix(arg, THIS_CPU_READ_PREFIX)) {
377 		arg += THIS_CPU_READ_LEN;
378 		ctx->offset += THIS_CPU_READ_LEN;
379 		is_read = true;
380 	} else {
381 		trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
382 		return -EINVAL;
383 	}
384 
385 	tmp = strrchr(arg, ')');
386 	if (!tmp) {
387 		trace_probe_log_err(ctx->offset + strlen(arg),
388 					DEREF_OPEN_BRACE);
389 		return -EINVAL;
390 	}
391 	*tmp = '\0';
392 
393 	ctx->stack[ctx->depth].type = STATE_DEREF;
394 	ctx->stack[ctx->depth].deref.deref = FETCH_OP_CPU_PTR;
395 	ctx->stack[ctx->depth].deref.offset = 0;
396 	ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
397 	ctx->stack[ctx->depth].deref.inner_arg = arg;
398 	ctx->stack[ctx->depth].deref.is_cpu_read = is_read;
399 	return 0;
400 }
401 
402 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
403 
404 static u32 btf_type_int(const struct btf_type *t)
405 {
406 	return *(u32 *)(t + 1);
407 }
408 
409 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
410 {
411 	const struct btf_type *real_type;
412 	u32 intdata;
413 
414 	if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR)
415 		return false;
416 
417 	real_type = btf_type_skip_modifiers(btf, type->type, NULL);
418 	if (!real_type)
419 		return false;
420 
421 	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
422 		return false;
423 
424 	intdata = btf_type_int(real_type);
425 	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
426 		&& BTF_INT_BITS(intdata) == 8;
427 }
428 
429 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
430 {
431 	const struct btf_type *real_type;
432 	const struct btf_array *array;
433 	u32 intdata;
434 
435 	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
436 		return false;
437 
438 	array = (const struct btf_array *)(type + 1);
439 
440 	real_type = btf_type_skip_modifiers(btf, array->type, NULL);
441 	if (!real_type)
442 		return false;
443 
444 	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
445 		return false;
446 
447 	intdata = btf_type_int(real_type);
448 	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
449 		&& BTF_INT_BITS(intdata) == 8;
450 }
451 
452 static struct btf *ctx_btf(struct traceprobe_parse_context *ctx)
453 {
454 	return ctx->struct_btf ? : ctx->btf;
455 }
456 
457 static int check_prepare_btf_string_fetch(char *typename,
458 				struct fetch_insn **pcode,
459 				struct traceprobe_parse_context *ctx)
460 {
461 	struct btf *btf = ctx_btf(ctx);
462 
463 	if (!btf || !ctx->last_type)
464 		return 0;
465 
466 	/* char [] does not need any change. */
467 	if (btf_type_is_char_array(btf, ctx->last_type))
468 		return 0;
469 
470 	/* char * requires dereference the pointer. */
471 	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
472 		struct fetch_insn *code = *pcode + 1;
473 
474 		if (code->op == FETCH_OP_END) {
475 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
476 			return -E2BIG;
477 		}
478 		if (typename[0] == 'u')
479 			code->op = FETCH_OP_UDEREF;
480 		else
481 			code->op = FETCH_OP_DEREF;
482 		code->offset = 0;
483 		*pcode = code;
484 		return 0;
485 	}
486 	/* Other types are not available for string */
487 	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
488 	return -EINVAL;
489 }
490 
491 static const char *fetch_type_from_btf_type(struct btf *btf,
492 					const struct btf_type *type,
493 					struct traceprobe_parse_context *ctx)
494 {
495 	u32 intdata;
496 
497 	if (ctx->last_bitsize) {
498 		u32 bits = ctx->last_bitsize + ctx->last_bitoffs;
499 
500 		if (bits <= 8)
501 			return "u8";
502 		if (bits <= 16)
503 			return "u16";
504 		if (bits <= 32)
505 			return "u32";
506 		return "u64";
507 	}
508 
509 	/* TODO: const char * could be converted as a string */
510 	switch (BTF_INFO_KIND(type->info)) {
511 	case BTF_KIND_ENUM:
512 		/* enum is "int", so convert to "s32" */
513 		return "s32";
514 	case BTF_KIND_ENUM64:
515 		return "s64";
516 	case BTF_KIND_PTR:
517 		/* pointer will be converted to "x??" */
518 		return IS_ENABLED(CONFIG_64BIT) ? "x64" : "x32";
519 	case BTF_KIND_INT:
520 		intdata = btf_type_int(type);
521 		if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
522 			switch (BTF_INT_BITS(intdata)) {
523 			case 8:
524 				return "s8";
525 			case 16:
526 				return "s16";
527 			case 32:
528 				return "s32";
529 			case 64:
530 				return "s64";
531 			}
532 		} else {	/* unsigned */
533 			switch (BTF_INT_BITS(intdata)) {
534 			case 8:
535 				return "u8";
536 			case 16:
537 				return "u16";
538 			case 32:
539 				return "u32";
540 			case 64:
541 				return "u64";
542 			}
543 		}
544 	}
545 	/* TODO: support other types */
546 
547 	return NULL;
548 }
549 
550 static int query_btf_context(struct traceprobe_parse_context *ctx)
551 {
552 	const struct btf_param *param;
553 	const struct btf_type *type;
554 	struct btf *btf;
555 	s32 nr;
556 
557 	if (ctx->btf)
558 		return 0;
559 
560 	if (!ctx->funcname)
561 		return -EINVAL;
562 
563 	type = btf_find_func_proto(ctx->funcname, &btf);
564 	if (!type)
565 		return -ENOENT;
566 
567 	ctx->btf = btf;
568 	ctx->proto = type;
569 
570 	/* ctx->params is optional, since func(void) will not have params. */
571 	nr = 0;
572 	param = btf_get_func_param(type, &nr);
573 	if (!IS_ERR_OR_NULL(param)) {
574 		/* Hide the first 'data' argument of tracepoint */
575 		if (ctx->flags & TPARG_FL_TPOINT) {
576 			nr--;
577 			param++;
578 		}
579 	}
580 
581 	if (nr > 0) {
582 		ctx->nr_params = nr;
583 		ctx->params = param;
584 	} else {
585 		ctx->nr_params = 0;
586 		ctx->params = NULL;
587 	}
588 
589 	return 0;
590 }
591 
592 static void clear_struct_btf(struct traceprobe_parse_context *ctx)
593 {
594 	if (ctx->struct_btf) {
595 		btf_put(ctx->struct_btf);
596 		ctx->struct_btf = NULL;
597 		ctx->last_struct = NULL;
598 	}
599 }
600 
601 static void clear_btf_context(struct traceprobe_parse_context *ctx)
602 {
603 	if (ctx->btf) {
604 		btf_put(ctx->btf);
605 		ctx->btf = NULL;
606 		ctx->proto = NULL;
607 		ctx->params = NULL;
608 		ctx->nr_params = 0;
609 	}
610 }
611 
612 /* Return 1 if the field separator is arrow operator ('->') */
613 static int split_next_field(char *varname, char **next_field,
614 			    struct traceprobe_parse_context *ctx)
615 {
616 	char *field;
617 	int ret = 0;
618 
619 	field = strpbrk(varname, ".-");
620 	if (field) {
621 		if (field[0] == '-' && field[1] == '>') {
622 			field[0] = '\0';
623 			field += 2;
624 			ret = 1;
625 		} else if (field[0] == '.') {
626 			field[0] = '\0';
627 			field += 1;
628 		} else {
629 			trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
630 			return -EINVAL;
631 		}
632 		*next_field = field;
633 	}
634 
635 	return ret;
636 }
637 
638 /* Inner loop for solving dot operator ('.'). Return bit-offset of the given field */
639 static int get_bitoffset_of_field(char **pfieldname, const struct btf_type **ptype,
640 				  struct traceprobe_parse_context *ctx)
641 {
642 	const struct btf_type *type = *ptype;
643 	const struct btf_member *field;
644 	const struct btf_type *container_type;
645 	struct btf *btf = ctx_btf(ctx);
646 	char *fieldname = *pfieldname;
647 	int bitoffs = 0;
648 	u32 anon_offs;
649 	char *next;
650 	int is_ptr;
651 
652 	do {
653 		next = NULL;
654 		is_ptr = split_next_field(fieldname, &next, ctx);
655 		if (is_ptr < 0)
656 			return is_ptr;
657 
658 		anon_offs = 0;
659 		field = btf_find_struct_member(btf, type, fieldname,
660 						&anon_offs, &container_type);
661 		if (IS_ERR(field)) {
662 			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
663 			return PTR_ERR(field);
664 		}
665 		if (!field) {
666 			trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
667 			return -ENOENT;
668 		}
669 		/* Add anonymous structure/union offset */
670 		bitoffs += anon_offs;
671 
672 		type = btf_type_skip_modifiers(btf, field->type, NULL);
673 		if (!type) {
674 			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
675 			return -EINVAL;
676 		}
677 
678 		/* Accumulate the bit-offsets of the dot-connected fields */
679 		if (btf_type_kflag(container_type)) {
680 			bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
681 			ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
682 		} else {
683 			bitoffs += field->offset;
684 			ctx->last_bitsize = 0;
685 			if (BTF_INFO_KIND(type->info) == BTF_KIND_INT) {
686 				u32 intdata = btf_type_int(type);
687 
688 				bitoffs += BTF_INT_OFFSET(intdata);
689 				if ((bitoffs % 8) ||
690 				    BTF_INT_BITS(intdata) != BYTES_TO_BITS(type->size) ||
691 				    !is_power_of_2(type->size))
692 					ctx->last_bitsize = BTF_INT_BITS(intdata);
693 			}
694 		}
695 
696 		if (next)
697 			ctx->offset += next - fieldname;
698 		fieldname = next;
699 	} while (!is_ptr && fieldname);
700 
701 	*pfieldname = fieldname;
702 	*ptype = type;
703 
704 	return bitoffs;
705 }
706 /*
707  * Parse the field of data structure. The @type must be a pointer type
708  * pointing the target data structure type.
709  */
710 static int parse_btf_field(char *fieldname, const struct btf_type *type,
711 			   struct fetch_insn **pcode, struct fetch_insn *end,
712 			   struct traceprobe_parse_context *ctx)
713 {
714 	struct fetch_insn *code = *pcode;
715 	struct btf *btf = ctx_btf(ctx);
716 	bool is_first_field = true;
717 	int bitoffs;
718 
719 	do {
720 		/* For the first field of typecast, @type will be the target structure type. */
721 		if (!(is_first_field && ctx->struct_btf)) {
722 			/* Outer loop for solving arrow operator ('->') */
723 			if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
724 				trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
725 				return -EINVAL;
726 			}
727 
728 			/* Convert a struct pointer type to a struct type */
729 			type = btf_type_skip_modifiers(btf, type->type, NULL);
730 			if (!type) {
731 				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
732 				return -EINVAL;
733 			}
734 		}
735 
736 		bitoffs = get_bitoffset_of_field(&fieldname, &type, ctx);
737 		if (bitoffs < 0)
738 			return bitoffs;
739 		if (++code == end) {
740 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
741 			return -EINVAL;
742 		}
743 		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
744 		code->offset = bitoffs / 8;
745 		if (is_first_field && ctx->struct_btf) {
746 			/* The first field can be typecasted with field option. */
747 			code->offset -= ctx->prefix_byteoffs;
748 		}
749 		*pcode = code;
750 
751 		ctx->last_bitoffs = bitoffs % 8;
752 		ctx->last_type = type;
753 		is_first_field = false;
754 	} while (fieldname);
755 
756 	return 0;
757 }
758 
759 static int __store_entry_arg(struct trace_probe *tp, int argnum);
760 
761 static int parse_btf_arg(char *varname,
762 			 struct fetch_insn **pcode, struct fetch_insn *end,
763 			 struct traceprobe_parse_context *ctx)
764 {
765 	struct fetch_insn *code = *pcode;
766 	const struct btf_param *params;
767 	const struct btf_type *type;
768 	char *field = NULL;
769 	int i, is_ptr, ret;
770 	u32 tid;
771 
772 	/* Note: field is not separated at this point, so check prefix. */
773 	if (!str_has_prefix(varname, "$current") &&
774 	    !ctx->funcname && !(ctx->flags & TPARG_FL_TEVENT))
775 		return -EINVAL;
776 
777 	is_ptr = split_next_field(varname, &field, ctx);
778 	if (is_ptr < 0)
779 		return is_ptr;
780 	if (!is_ptr && field) {
781 		/* dot-connected field on an argument is not supported. */
782 		trace_probe_log_err(ctx->offset + field - varname,
783 				    NOSUP_DAT_ARG);
784 		return -EOPNOTSUPP;
785 	}
786 
787 	if (!strcmp(varname, "$current")) {
788 		code->op = FETCH_OP_CURRENT;
789 		/* If no typecast is specified for $current, use task_struct by default */
790 		ret = bpf_find_btf_id("task_struct", BTF_KIND_STRUCT, &ctx->struct_btf);
791 		if (ret < 0) {
792 			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
793 			return -ENOENT;
794 		}
795 		tid = (u32)ret;
796 		type = ctx->last_struct =
797 			btf_type_skip_modifiers(ctx->struct_btf, tid, NULL);
798 		goto found_type;
799 	}
800 
801 	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
802 		code->op = FETCH_OP_RETVAL;
803 		/* Check whether the function return type is not void, even with typecast. */
804 		if (query_btf_context(ctx) == 0) {
805 			if (ctx->proto->type == 0) {
806 				trace_probe_log_err(ctx->offset, NO_RETVAL);
807 				return -ENOENT;
808 			}
809 			tid = ctx->proto->type;
810 			goto found;
811 		}
812 		if (field) {
813 			trace_probe_log_err(ctx->offset + field - varname,
814 					    NO_BTF_ENTRY);
815 			return -ENOENT;
816 		}
817 		return 0;
818 	}
819 
820 	if (!ctx->btf) {
821 		ret = query_btf_context(ctx);
822 		if (ret < 0 || ctx->nr_params == 0) {
823 			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
824 			return -ENOENT;
825 		}
826 	}
827 	params = ctx->params;
828 
829 	for (i = 0; i < ctx->nr_params; i++) {
830 		const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
831 
832 		if (name && !strcmp(name, varname)) {
833 			if (tparg_is_function_entry(ctx->flags)) {
834 				code->op = FETCH_OP_ARG;
835 				if (ctx->flags & TPARG_FL_TPOINT)
836 					code->param = i + 1;
837 				else
838 					code->param = i;
839 			} else if (tparg_is_function_return(ctx->flags)) {
840 				code->op = FETCH_OP_EDATA;
841 				ret = __store_entry_arg(ctx->tp, i);
842 				if (ret < 0) {
843 					/* internal error */
844 					return ret;
845 				}
846 				code->offset = ret;
847 			}
848 			tid = params[i].type;
849 			goto found;
850 		}
851 	}
852 	trace_probe_log_err(ctx->offset, NO_BTFARG);
853 	return -ENOENT;
854 
855 found:
856 	type = btf_type_skip_modifiers(ctx->btf, tid, NULL);
857 found_type:
858 	if (!type) {
859 		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
860 		return -EINVAL;
861 	}
862 	/* Initialize the last type information */
863 	ctx->last_type = type;
864 	ctx->last_bitoffs = 0;
865 	ctx->last_bitsize = 0;
866 	if (field) {
867 		ctx->offset += field - varname;
868 		return parse_btf_field(field, type, pcode, end, ctx);
869 	}
870 	return 0;
871 }
872 
873 static const struct fetch_type *find_fetch_type_from_btf_type(
874 					struct traceprobe_parse_context *ctx)
875 {
876 	struct btf *btf = ctx_btf(ctx);
877 	const char *typestr = NULL;
878 
879 	if (btf && ctx->last_type)
880 		typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
881 
882 	return find_fetch_type(typestr, ctx->flags);
883 }
884 
885 static int parse_btf_bitfield(struct fetch_insn **pcode,
886 			      const struct fetch_type *t,
887 			      struct traceprobe_parse_context *ctx)
888 {
889 	struct fetch_insn *code = *pcode;
890 
891 	if ((ctx->last_bitsize == 0 ||
892 	     ctx->last_bitsize == BYTES_TO_BITS(t->size)) &&
893 	    ctx->last_bitoffs == 0)
894 		return 0;
895 
896 	if (ctx->last_bitsize == 0 ||
897 	    BYTES_TO_BITS(t->size) < (ctx->last_bitsize + ctx->last_bitoffs)) {
898 		trace_probe_log_err(ctx->offset, BAD_BITFIELD);
899 		return -EINVAL;
900 	}
901 
902 	code++;
903 	if (code->op != FETCH_OP_NOP) {
904 		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
905 		return -EINVAL;
906 	}
907 	*pcode = code;
908 
909 	code->op = FETCH_OP_MOD_BF;
910 #ifdef __BIG_ENDIAN_BITFIELD
911 	code->lshift = ctx->last_bitoffs;
912 #else
913 	code->lshift = BYTES_TO_BITS(t->size) - (ctx->last_bitsize + ctx->last_bitoffs);
914 #endif
915 	code->rshift = BYTES_TO_BITS(t->size) - ctx->last_bitsize;
916 	code->basesize = t->size;
917 	return 0;
918 }
919 
920 static int query_btf_struct(const char *sname, struct traceprobe_parse_context *ctx)
921 {
922 	struct btf *btf = NULL;
923 	int id;
924 
925 	/* A struct_btf should only be used by a single argument */
926 	if (WARN_ON_ONCE(ctx->struct_btf)) {
927 		btf_put(ctx->struct_btf);
928 		ctx->struct_btf = NULL;
929 	}
930 
931 	id = bpf_find_btf_id(sname, BTF_KIND_STRUCT, &btf);
932 	if (id < 0)
933 		return id;
934 	ctx->struct_btf = btf;
935 	ctx->last_struct = btf_type_by_id(ctx->struct_btf, id);
936 	return 0;
937 }
938 
939 static int parse_btf_casttype(char *casttype, struct traceprobe_parse_context *ctx)
940 {
941 	char *field;
942 	int ret;
943 
944 	/* Field option - evaluated later. */
945 	field = strchr(casttype, ',');
946 	if (field)
947 		*field++ = '\0';
948 
949 	ret = query_btf_struct(casttype, ctx);
950 	if (ret < 0) {
951 		trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
952 		return -EINVAL;
953 	}
954 
955 	if (field) {
956 		struct btf_type *type = (struct btf_type *)ctx->last_struct;
957 
958 		ctx->offset += field - casttype;
959 		ret = get_bitoffset_of_field(&field, &ctx->last_struct, ctx);
960 		if (ret < 0)
961 			return ret;
962 		if (ret % 8) {
963 			trace_probe_log_err(ctx->offset, TYPECAST_NOT_ALIGNED);
964 			return -EINVAL;
965 		}
966 		if (field != NULL) {
967 			/* this means @field skips an arrow operator ("->"). */
968 			trace_probe_log_err(ctx->offset - 2, TYPECAST_BAD_ARROW);
969 			return -EINVAL;
970 		}
971 		ctx->prefix_byteoffs = ret / 8;
972 		/* Restore the original struct type (overwritten by get_bitoffset_of_field) */
973 		ctx->last_struct = type;
974 	}
975 
976 	return ret;
977 }
978 
979 /* Find the matching closing parenthesis for a given opening parenthesis. */
980 static char *find_matched_close_paren(char *s)
981 {
982 	char *p = s;
983 	int count = 0;
984 
985 	while (*p) {
986 		if (*p == '(')
987 			count++;
988 		else if (*p == ')') {
989 			if (--count == 0)
990 				return p;
991 		}
992 		p++;
993 	}
994 	return NULL;
995 }
996 
997 static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
998 {
999 	int orig_offset = ctx->offset;
1000 	char *close;
1001 	char *tmp;
1002 	char *fieldname;
1003 
1004 	if (!(tparg_is_event_probe(ctx->flags) ||
1005 	      tparg_is_function_entry(ctx->flags) ||
1006 	      tparg_is_function_return(ctx->flags))) {
1007 		trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1008 		return -EOPNOTSUPP;
1009 	}
1010 
1011 	/*
1012 	 * Always consider the token after typecast as a nested call
1013 	 * For example: (STRUCT)VAR->FIELD and (STRUCT)(VAR)->FIELD are same.
1014 	 * VAR is solved in the nested call.
1015 	 */
1016 	tmp = strchr(arg, ')');
1017 	if (!tmp) {
1018 		trace_probe_log_err(ctx->offset + strlen(arg),
1019 				    DEREF_OPEN_BRACE);
1020 		return -EINVAL;
1021 	}
1022 	*tmp++ = '\0';
1023 
1024 	ctx->offset += tmp - arg;
1025 	if (*tmp == '(') {
1026 		close = find_matched_close_paren(tmp);
1027 
1028 		if (!close) {
1029 			trace_probe_log_err(ctx->offset, DEREF_OPEN_BRACE);
1030 			return -EINVAL;
1031 		}
1032 		/* We expect a field access for typecast */
1033 		if (close[1] != '-' || close[2] != '>') {
1034 			trace_probe_log_err(ctx->offset + close - tmp + 1,
1035 					    TYPECAST_REQ_FIELD);
1036 			return -EINVAL;
1037 		}
1038 		/* Skip '(' */
1039 		ctx->offset += 1;
1040 		tmp++;
1041 	} else if (*tmp == '+' || *tmp == '-' ||
1042 		   str_has_prefix(tmp, THIS_CPU_PTR_PREFIX) ||
1043 		   str_has_prefix(tmp, THIS_CPU_READ_PREFIX)) {
1044 		/* Dereference can have another field access inside it. */
1045 		char *open = strchr(tmp + 1, '(');
1046 
1047 		if (!open) {
1048 			trace_probe_log_err(ctx->offset,
1049 					    DEREF_NEED_BRACE);
1050 			return -EINVAL;
1051 		}
1052 		close = find_matched_close_paren(open);
1053 		if (!close) {
1054 			trace_probe_log_err(ctx->offset + strlen(tmp),
1055 					    DEREF_OPEN_BRACE);
1056 			return -EINVAL;
1057 		}
1058 		close++;
1059 		/* We expect a field access for typecast */
1060 		if (close[0] != '-' || close[1] != '>') {
1061 			trace_probe_log_err(ctx->offset + close - tmp,
1062 					    TYPECAST_REQ_FIELD);
1063 			return -EINVAL;
1064 		}
1065 	} else {
1066 		if (tmp[0] == '@') {
1067 			/* @sym+offset is not allowed without parenthesized */
1068 			close = strpbrk(tmp, "+-");
1069 			if (close && isdigit(close[1])) {
1070 				trace_probe_log_err(ctx->offset,
1071 						    TYPECAST_SYM_OFFSET);
1072 				return -EINVAL;
1073 			}
1074 		}
1075 		/* Inner variable name */
1076 		close = strchr(tmp, '-');
1077 		if (!close || close[1] != '>') {
1078 			trace_probe_log_err(ctx->offset + strlen(tmp),
1079 					    TYPECAST_REQ_FIELD);
1080 			return -EINVAL;
1081 		}
1082 	}
1083 	*close = '\0';
1084 
1085 	/* Let fieldname point the field name. */
1086 	if (close[1] == '-')
1087 		fieldname = close + 3; /* Skip "->" after closing parenthesis */
1088 	else
1089 		fieldname = close + 2; /* Skip ">" after inner variable name */
1090 
1091 	ctx->stack[ctx->depth].type = STATE_TYPECAST;
1092 	ctx->stack[ctx->depth].typecast.casttype = arg + 1;
1093 	ctx->stack[ctx->depth].typecast.fieldname = fieldname;
1094 	ctx->stack[ctx->depth].typecast.orig_offset = orig_offset;
1095 	ctx->stack[ctx->depth].typecast.field_offset_diff = fieldname - arg;
1096 	ctx->stack[ctx->depth].typecast.inner_arg = tmp;
1097 	return 0;
1098 }
1099 
1100 #else /* !CONFIG_PROBE_EVENTS_BTF_ARGS */
1101 
1102 static void clear_struct_btf(struct traceprobe_parse_context *ctx)
1103 {
1104 	ctx->struct_btf = NULL;
1105 }
1106 
1107 static void clear_btf_context(struct traceprobe_parse_context *ctx)
1108 {
1109 	ctx->btf = NULL;
1110 }
1111 
1112 static int query_btf_context(struct traceprobe_parse_context *ctx)
1113 {
1114 	return -EOPNOTSUPP;
1115 }
1116 
1117 static int parse_btf_arg(char *varname,
1118 			 struct fetch_insn **pcode, struct fetch_insn *end,
1119 			 struct traceprobe_parse_context *ctx)
1120 {
1121 	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1122 	return -EOPNOTSUPP;
1123 }
1124 
1125 static int parse_btf_bitfield(struct fetch_insn **pcode,
1126 			      const struct fetch_type *t,
1127 			      struct traceprobe_parse_context *ctx)
1128 {
1129 	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1130 	return -EOPNOTSUPP;
1131 }
1132 
1133 #define find_fetch_type_from_btf_type(ctx)		\
1134 	find_fetch_type(NULL, ctx->flags)
1135 
1136 static int check_prepare_btf_string_fetch(char *typename,
1137 				struct fetch_insn **pcode,
1138 				struct traceprobe_parse_context *ctx)
1139 {
1140 	return 0;
1141 }
1142 
1143 static int parse_btf_casttype(char *casttype,
1144 			      struct traceprobe_parse_context *ctx)
1145 {
1146 	return -EOPNOTSUPP;
1147 }
1148 
1149 static int parse_btf_field(char *fieldname, const struct btf_type *type,
1150 			   struct fetch_insn **pcode, struct fetch_insn *end,
1151 			   struct traceprobe_parse_context *ctx)
1152 {
1153 	return -EOPNOTSUPP;
1154 }
1155 
1156 static int handle_typecast(char *arg, struct traceprobe_parse_context *ctx)
1157 {
1158 	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1159 	return -EOPNOTSUPP;
1160 }
1161 
1162 #endif /* CONFIG_PROBE_EVENTS_BTF_ARGS */
1163 
1164 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1165 
1166 static void store_entry_arg_at(struct fetch_insn *code, int argnum, int offset)
1167 {
1168 	code[0].op = FETCH_OP_ARG;
1169 	code[0].param = argnum;
1170 	code[1].op = FETCH_OP_ST_EDATA;
1171 	code[1].offset = offset;
1172 }
1173 
1174 static int get_entry_arg_max_offset(struct probe_entry_arg *earg)
1175 {
1176 	int i, max_offset = 0;
1177 
1178 	/*
1179 	 * earg->code[] array has an operation sequence which is run in
1180 	 * the entry handler.
1181 	 * The sequence stopped by FETCH_OP_END and each data stored in
1182 	 * the entry data buffer by FETCH_OP_ST_EDATA. The FETCH_OP_ST_EDATA
1183 	 * stores the data at the data buffer + its offset, and all data are
1184 	 * "unsigned long" size. The offset must be increased when a data is
1185 	 * stored. Thus we need to find the last FETCH_OP_ST_EDATA in the
1186 	 * code array.
1187 	 */
1188 	for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i++) {
1189 		if (earg->code[i].op == FETCH_OP_ST_EDATA)
1190 			if (earg->code[i].offset > max_offset)
1191 				max_offset = earg->code[i].offset;
1192 	}
1193 	return max_offset;
1194 }
1195 
1196 /*
1197  * Add the entry code to store the 'argnum'th parameter and return the offset
1198  * in the entry data buffer where the data will be stored.
1199  */
1200 static int __store_entry_arg(struct trace_probe *tp, int argnum)
1201 {
1202 	struct probe_entry_arg *earg = tp->entry_arg;
1203 	int i, offset, last_offset = 0;
1204 
1205 	if (!earg) {
1206 		earg = kzalloc_flex(*earg, code, 2 * tp->nr_args + 1);
1207 		if (!earg)
1208 			return -ENOMEM;
1209 		earg->size = 2 * tp->nr_args + 1;
1210 		/* Fill the code buffer with 'end' to simplify it */
1211 		for (i = 0; i < earg->size; i++)
1212 			earg->code[i].op = FETCH_OP_END;
1213 		tp->entry_arg = earg;
1214 		store_entry_arg_at(earg->code, argnum, 0);
1215 		return 0;
1216 	}
1217 
1218 	/*
1219 	 * NOTE: if anyone change the following rule, please rewrite this.
1220 	 * The entry code array is filled with the pair of
1221 	 *
1222 	 * [FETCH_OP_ARG(argnum)]
1223 	 * [FETCH_OP_ST_EDATA(offset of entry data buffer)]
1224 	 *
1225 	 * and the rest of entries are filled with [FETCH_OP_END].
1226 	 * The offset should be incremented, thus the last pair should
1227 	 * have the largest offset.
1228 	 */
1229 
1230 	/* Search the offset for the sprcified argnum. */
1231 	for (i = 0; i < earg->size - 1 && earg->code[i].op != FETCH_OP_END; i += 2) {
1232 		if (WARN_ON_ONCE(earg->code[i].op != FETCH_OP_ARG))
1233 			return -EINVAL;
1234 
1235 		if (earg->code[i].param != argnum)
1236 			continue;
1237 
1238 		if (WARN_ON_ONCE(earg->code[i + 1].op != FETCH_OP_ST_EDATA))
1239 			return -EINVAL;
1240 
1241 		return earg->code[i + 1].offset;
1242 	}
1243 	/* Not found, append new entry if possible. */
1244 	if (i >= earg->size - 1)
1245 		return -ENOSPC;
1246 
1247 	/* The last entry must have the largest offset. */
1248 	if (i != 0) {
1249 		if (WARN_ON_ONCE(earg->code[i - 1].op != FETCH_OP_ST_EDATA))
1250 			return -EINVAL;
1251 		last_offset = earg->code[i - 1].offset;
1252 	}
1253 
1254 	offset = last_offset + sizeof(unsigned long);
1255 	store_entry_arg_at(&earg->code[i], argnum, offset);
1256 	return offset;
1257 }
1258 
1259 int traceprobe_get_entry_data_size(struct trace_probe *tp)
1260 {
1261 	struct probe_entry_arg *earg = tp->entry_arg;
1262 
1263 	if (!earg)
1264 		return 0;
1265 
1266 	return get_entry_arg_max_offset(earg) + sizeof(unsigned long);
1267 }
1268 
1269 void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
1270 {
1271 	struct probe_entry_arg *earg = tp->entry_arg;
1272 	unsigned long val = 0;
1273 	int i;
1274 
1275 	if (!earg)
1276 		return;
1277 
1278 	for (i = 0; i < earg->size; i++) {
1279 		struct fetch_insn *code = &earg->code[i];
1280 
1281 		switch (code->op) {
1282 		case FETCH_OP_ARG:
1283 			val = regs_get_kernel_argument(regs, code->param);
1284 			break;
1285 		case FETCH_OP_ST_EDATA:
1286 			*(unsigned long *)((u8 *)edata + code->offset) = val;
1287 			break;
1288 		case FETCH_OP_END:
1289 			goto end;
1290 		default:
1291 			break;
1292 		}
1293 	}
1294 end:
1295 	return;
1296 }
1297 NOKPROBE_SYMBOL(store_trace_entry_data)
1298 #endif
1299 
1300 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
1301 
1302 static int parse_probe_var_retval(char *orig_arg,
1303 				  struct fetch_insn **pcode,
1304 				  struct fetch_insn *end,
1305 				  struct traceprobe_parse_context *ctx)
1306 {
1307 	struct fetch_insn *code = *pcode;
1308 
1309 	if (!(ctx->flags & TPARG_FL_RETURN)) {
1310 		trace_probe_log_err(ctx->offset, RETVAL_ON_PROBE);
1311 		return -EINVAL;
1312 	}
1313 	if (!(ctx->flags & TPARG_FL_KERNEL) ||
1314 	    !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1315 		code->op = FETCH_OP_RETVAL;
1316 		return 0;
1317 	}
1318 	return parse_btf_arg(orig_arg, pcode, end, ctx);
1319 }
1320 
1321 static int parse_probe_var_stack(char *arg, int len, struct fetch_insn *code,
1322 				 struct traceprobe_parse_context *ctx)
1323 {
1324 	unsigned long param;
1325 	int ret;
1326 
1327 	if (arg[len] == '\0') {
1328 		code->op = FETCH_OP_STACKP;
1329 		return 0;
1330 	}
1331 
1332 	if (isdigit(arg[len])) {
1333 		ret = kstrtoul(arg + len, 10, &param);
1334 		if (ret) {
1335 			trace_probe_log_err(ctx->offset, BAD_VAR);
1336 			return ret;
1337 		}
1338 
1339 		if ((ctx->flags & TPARG_FL_KERNEL) &&
1340 		    param > PARAM_MAX_STACK) {
1341 			trace_probe_log_err(ctx->offset, BAD_STACK_NUM);
1342 			return -EINVAL;
1343 		}
1344 		code->op = FETCH_OP_STACK;
1345 		code->param = (unsigned int)param;
1346 		return 0;
1347 	}
1348 
1349 	trace_probe_log_err(ctx->offset, BAD_VAR);
1350 	return -EINVAL;
1351 }
1352 
1353 static int parse_probe_var_current(char *orig_arg, char *arg,
1354 				   struct fetch_insn **pcode,
1355 				   struct fetch_insn *end,
1356 				   struct traceprobe_parse_context *ctx)
1357 {
1358 	struct fetch_insn *code = *pcode;
1359 
1360 	/* $current is only supported by kernel probe. */
1361 	if (!(ctx->flags & TPARG_FL_KERNEL)) {
1362 		trace_probe_log_err(ctx->offset, BAD_VAR);
1363 		return -EINVAL;
1364 	}
1365 	arg += strlen("current");
1366 	if (*arg == '-' && IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS))
1367 		return parse_btf_arg(orig_arg, pcode, end, ctx);
1368 
1369 	if (*arg != '\0') {
1370 		trace_probe_log_err(ctx->offset, BAD_VAR);
1371 		return -EINVAL;
1372 	}
1373 
1374 	code->op = FETCH_OP_CURRENT;
1375 	return 0;
1376 }
1377 
1378 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
1379 static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
1380 			       struct traceprobe_parse_context *ctx)
1381 {
1382 	unsigned long param;
1383 	int ret;
1384 
1385 	ret = kstrtoul(arg + len, 10, &param);
1386 	if (ret) {
1387 		trace_probe_log_err(ctx->offset, BAD_VAR);
1388 		return ret;
1389 	}
1390 
1391 	if (!param || param > PARAM_MAX_STACK) {
1392 		trace_probe_log_err(ctx->offset, BAD_ARG_NUM);
1393 		return -EINVAL;
1394 	}
1395 	param--; /* argN starts from 1, but internal arg[N] starts from 0 */
1396 
1397 	if (tparg_is_function_entry(ctx->flags)) {
1398 		code->op = FETCH_OP_ARG;
1399 		code->param = (unsigned int)param;
1400 		/*
1401 		 * The tracepoint probe will probe a stub function, and the
1402 		 * first parameter of the stub is a dummy and should be ignored.
1403 		 */
1404 		if (ctx->flags & TPARG_FL_TPOINT)
1405 			code->param++;
1406 	} else if (tparg_is_function_return(ctx->flags)) {
1407 		/* function entry argument access from return probe */
1408 		ret = __store_entry_arg(ctx->tp, param);
1409 		if (ret < 0)	/* This error should be an internal error */
1410 			return ret;
1411 
1412 		code->op = FETCH_OP_EDATA;
1413 		code->offset = ret;
1414 	} else {
1415 		trace_probe_log_err(ctx->offset, NOFENTRY_ARGS);
1416 		return -EINVAL;
1417 	}
1418 	return 0;
1419 }
1420 #else
1421 static int parse_probe_var_arg(char *arg, int len, struct fetch_insn *code,
1422 			       struct traceprobe_parse_context *ctx)
1423 {
1424 	trace_probe_log_err(ctx->offset, BAD_VAR);
1425 	return -EINVAL;
1426 }
1427 #endif
1428 
1429 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
1430 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
1431 			    struct fetch_insn **pcode,
1432 			    struct fetch_insn *end,
1433 			    struct traceprobe_parse_context *ctx)
1434 {
1435 	struct fetch_insn *code = *pcode;
1436 	char *arg = orig_arg + 1;
1437 	int len, ret;
1438 
1439 	if (ctx->flags & TPARG_FL_TEVENT) {
1440 		ret = parse_trace_event(arg, code, ctx);
1441 		if (!ret)
1442 			return 0;
1443 	}
1444 
1445 	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
1446 		code->op = FETCH_OP_COMM;
1447 		return 0;
1448 	}
1449 
1450 	/* eprobe only support event fields or '$comm'. */
1451 	if (ctx->flags & TPARG_FL_TEVENT)
1452 		goto inval;
1453 
1454 	if (str_has_prefix(arg, "retval"))
1455 		return parse_probe_var_retval(orig_arg, pcode, end, ctx);
1456 
1457 	len = str_has_prefix(arg, "stack");
1458 	if (len)
1459 		return parse_probe_var_stack(arg, len, code, ctx);
1460 
1461 	/* $current returns the address of the current task_struct. */
1462 	if (str_has_prefix(arg, "current"))
1463 		return parse_probe_var_current(orig_arg, arg, pcode, end, ctx);
1464 
1465 	len = str_has_prefix(arg, "arg");
1466 	if (len)
1467 		return parse_probe_var_arg(arg, len, code, ctx);
1468 
1469 inval:
1470 	trace_probe_log_err(ctx->offset, BAD_VAR);
1471 	return -EINVAL;
1472 }
1473 
1474 static int str_to_immediate(char *str, unsigned long *imm)
1475 {
1476 	if (isdigit(str[0]))
1477 		return kstrtoul(str, 0, imm);
1478 	else if (str[0] == '-')
1479 		return kstrtol(str, 0, (long *)imm);
1480 	else if (str[0] == '+')
1481 		return kstrtol(str + 1, 0, (long *)imm);
1482 	return -EINVAL;
1483 }
1484 
1485 static int __parse_imm_string(char *str, char **pbuf, int offs)
1486 {
1487 	size_t len = strlen(str);
1488 
1489 	if (!len || str[len - 1] != '"') {
1490 		trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1491 		return -EINVAL;
1492 	}
1493 	*pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1494 	if (!*pbuf)
1495 		return -ENOMEM;
1496 	return 0;
1497 }
1498 
1499 static int parse_probe_arg_register(char *arg, struct fetch_insn *code,
1500 				    struct traceprobe_parse_context *ctx)
1501 {
1502 	int ret;
1503 
1504 	if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1505 		/* eprobe and fprobe do not handle registers */
1506 		trace_probe_log_err(ctx->offset, BAD_VAR);
1507 		return -EINVAL;
1508 	}
1509 	ret = regs_query_register_offset(arg + 1);
1510 	if (ret >= 0) {
1511 		code->op = FETCH_OP_REG;
1512 		code->param = (unsigned int)ret;
1513 		return 0;
1514 	}
1515 	trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1516 	return -EINVAL;
1517 }
1518 
1519 static int parse_probe_arg_mem_symbol(char *arg, struct fetch_insn **pcode,
1520 				      struct fetch_insn *end,
1521 				      struct traceprobe_parse_context *ctx)
1522 {
1523 	struct fetch_insn *code = *pcode;
1524 	unsigned long param;
1525 	long offset = 0;
1526 	int ret;
1527 
1528 	if (isdigit(arg[1])) {
1529 		ret = kstrtoul(arg + 1, 0, &param);
1530 		if (ret) {
1531 			trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1532 			return ret;
1533 		}
1534 		/* load address */
1535 		code->op = FETCH_OP_IMM;
1536 		code->immediate = param;
1537 	} else if (arg[1] == '+') {
1538 		/* Kernel probes do not support file offsets */
1539 		if (ctx->flags & TPARG_FL_KERNEL) {
1540 			trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1541 			return -EINVAL;
1542 		}
1543 		ret = kstrtol(arg + 2, 0, &offset);
1544 		if (ret) {
1545 			trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1546 			return ret;
1547 		}
1548 
1549 		code->op = FETCH_OP_FOFFS;
1550 		code->immediate = (unsigned long)offset;
1551 		offset = 0;
1552 	} else {
1553 		/* uprobes don't support symbols */
1554 		if (!(ctx->flags & TPARG_FL_KERNEL)) {
1555 			trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1556 			return -EINVAL;
1557 		}
1558 		/* Preserve symbol for updating */
1559 		code->op = FETCH_NOP_SYMBOL;
1560 		code->data = kstrdup(arg + 1, GFP_KERNEL);
1561 		if (!code->data)
1562 			return -ENOMEM;
1563 		if (++code == end) {
1564 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1565 			return -EINVAL;
1566 		}
1567 		code->op = FETCH_OP_IMM;
1568 		code->immediate = 0;
1569 	}
1570 	/* These are fetching from memory */
1571 	if (++code == end) {
1572 		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1573 		return -EINVAL;
1574 	}
1575 	*pcode = code;
1576 	code->op = FETCH_OP_DEREF;
1577 	code->offset = offset;
1578 	return 0;
1579 }
1580 
1581 static int parse_probe_arg_deref(char *arg, struct traceprobe_parse_context *ctx)
1582 {
1583 	int deref = FETCH_OP_DEREF;
1584 	long offset = 0;
1585 	char *tmp;
1586 	int ret;
1587 
1588 	if (arg[1] == 'u') {
1589 		deref = FETCH_OP_UDEREF;
1590 		arg[1] = arg[0];
1591 		arg++;
1592 	}
1593 	if (arg[0] == '+')
1594 		arg++;	/* Skip '+', because kstrtol() rejects it. */
1595 	tmp = strchr(arg, '(');
1596 	if (!tmp) {
1597 		trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1598 		return -EINVAL;
1599 	}
1600 	*tmp = '\0';
1601 	ret = kstrtol(arg, 0, &offset);
1602 	if (ret) {
1603 		trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1604 		return ret;
1605 	}
1606 	ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1607 	arg = tmp + 1;
1608 
1609 	tmp = strrchr(arg, ')');
1610 	if (!tmp) {
1611 		trace_probe_log_err(ctx->offset + strlen(arg),
1612 					DEREF_OPEN_BRACE);
1613 		return -EINVAL;
1614 	}
1615 	*tmp = '\0';
1616 
1617 	ctx->stack[ctx->depth].type = STATE_DEREF;
1618 	ctx->stack[ctx->depth].deref.deref = deref;
1619 	ctx->stack[ctx->depth].deref.offset = offset;
1620 	ctx->stack[ctx->depth].deref.cur_offs = ctx->offset;
1621 	ctx->stack[ctx->depth].deref.inner_arg = arg;
1622 	ctx->stack[ctx->depth].deref.is_cpu_read = false;
1623 	return 0;
1624 }
1625 
1626 static int parse_probe_arg_imm(char *arg, struct fetch_insn *code,
1627 			       struct traceprobe_parse_context *ctx)
1628 {
1629 	char *tmp;
1630 	int ret;
1631 
1632 	if (arg[1] == '"') {	/* Immediate string */
1633 		ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1634 		if (ret)
1635 			return ret;
1636 		code->op = FETCH_OP_IMMSTR;
1637 		code->data = tmp;
1638 	} else {
1639 		ret = str_to_immediate(arg + 1, &code->immediate);
1640 		if (ret) {
1641 			trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1642 			return ret;
1643 		}
1644 		code->op = FETCH_OP_IMM;
1645 	}
1646 	return 0;
1647 }
1648 
1649 static int parse_probe_arg_default(char *arg, struct fetch_insn **pcode,
1650 				   struct fetch_insn *end,
1651 				   struct traceprobe_parse_context *ctx)
1652 {
1653 	int ret;
1654 
1655 	if (isalpha(arg[0]) || arg[0] == '_') {
1656 		/* BTF variable or event field */
1657 		if (ctx->flags & TPARG_FL_TEVENT) {
1658 			ret = parse_trace_event(arg, *pcode, ctx);
1659 			if (ret < 0) {
1660 				trace_probe_log_err(ctx->offset, NO_EVENT_FIELD);
1661 				return -EINVAL;
1662 			}
1663 			return 0;
1664 		}
1665 		if (!tparg_is_function_entry(ctx->flags) &&
1666 		    !tparg_is_function_return(ctx->flags)) {
1667 			trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1668 			return -EINVAL;
1669 		}
1670 		return parse_btf_arg(arg, pcode, end, ctx);
1671 	}
1672 
1673 	return 0;
1674 }
1675 
1676 static int parse_probe_arg_nested(char **parg, struct traceprobe_parse_context *ctx)
1677 {
1678 	char *arg = *parg;
1679 	int ret;
1680 
1681 	while (true) {
1682 		/* Determine if this is a nested argument */
1683 		if (arg[0] != '+' && arg[0] != '-' && arg[0] != '(' &&
1684 		    !str_has_prefix(arg, THIS_CPU_PTR_PREFIX) &&
1685 		    !str_has_prefix(arg, THIS_CPU_READ_PREFIX))
1686 			break;
1687 
1688 		/* If nested, check the maximum depth limit */
1689 		if (ctx->depth >= TRACEPROBE_MAX_NESTED_LEVEL) {
1690 			trace_probe_log_err(ctx->offset, TOO_MANY_NESTED);
1691 			return -E2BIG;
1692 		}
1693 
1694 		/* Perform the actual parsing subroutine calls */
1695 		switch (arg[0]) {
1696 		case '+':
1697 		case '-':
1698 			ret = parse_probe_arg_deref(arg, ctx);
1699 			if (ret)
1700 				return ret;
1701 			arg = ctx->stack[ctx->depth].deref.inner_arg;
1702 			break;
1703 		case '(':
1704 			ret = handle_typecast(arg, ctx);
1705 			if (ret)
1706 				return ret;
1707 			arg = ctx->stack[ctx->depth].typecast.inner_arg;
1708 			break;
1709 		default:
1710 			ret = parse_this_cpu(arg, ctx);
1711 			if (ret)
1712 				return ret;
1713 			arg = ctx->stack[ctx->depth].deref.inner_arg;
1714 			break;
1715 		}
1716 		ctx->depth++;
1717 	}
1718 
1719 	*parg = arg;
1720 	return 0;
1721 }
1722 
1723 static int parse_probe_arg_leaf(char *arg, const struct fetch_type *type,
1724 				struct fetch_insn **pcode, struct fetch_insn *end,
1725 				struct traceprobe_parse_context *ctx)
1726 {
1727 	struct fetch_insn *code = *pcode;
1728 	int ret;
1729 
1730 	switch (arg[0]) {
1731 	case '$':
1732 		ret = parse_probe_vars(arg, type, pcode, end, ctx);
1733 		break;
1734 	case '%':	/* named register */
1735 		ret = parse_probe_arg_register(arg, code, ctx);
1736 		break;
1737 	case '@':	/* memory, file-offset or symbol */
1738 		ret = parse_probe_arg_mem_symbol(arg, pcode, end, ctx);
1739 		break;
1740 	case '\\':	/* Immediate value */
1741 		ret = parse_probe_arg_imm(arg, code, ctx);
1742 		break;
1743 	default:
1744 		ret = parse_probe_arg_default(arg, pcode, end, ctx);
1745 		break;
1746 	}
1747 
1748 	if (ret)
1749 		return ret;
1750 
1751 	if (code->op == FETCH_OP_NOP) {
1752 		/* Parsed, but do not find fetch method */
1753 		trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1754 		return -EINVAL;
1755 	}
1756 
1757 	return 0;
1758 }
1759 
1760 static int unwind_parse_states(struct fetch_insn **pcode, struct fetch_insn *end,
1761 			       struct traceprobe_parse_context *ctx)
1762 {
1763 	struct parse_state *state;
1764 	struct fetch_insn *code;
1765 	int ret;
1766 
1767 	while (ctx->depth > 0) {
1768 		ctx->depth--;
1769 		state = &ctx->stack[ctx->depth];
1770 
1771 		if (state->type == STATE_DEREF) {
1772 			code = *pcode;
1773 			ctx->offset = state->deref.cur_offs;
1774 			if (code->op == FETCH_OP_COMM || code->op == FETCH_OP_IMMSTR) {
1775 				trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1776 				return -EINVAL;
1777 			}
1778 
1779 			if (!(state->deref.deref == FETCH_OP_CPU_PTR &&
1780 			      *state->deref.inner_arg == '@')) {
1781 				code++;
1782 				if (code == end) {
1783 					trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1784 					return -EINVAL;
1785 				}
1786 			}
1787 			*pcode = code;
1788 
1789 			code->op = state->deref.deref;
1790 			code->offset = state->deref.offset;
1791 			ctx->last_type = NULL;
1792 
1793 			if (state->deref.is_cpu_read) {
1794 				code = *pcode;
1795 				code++;
1796 				if (code == end) {
1797 					trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1798 					return -EINVAL;
1799 				}
1800 				code->op = FETCH_OP_DEREF;
1801 				code->offset = 0;
1802 				*pcode = code;
1803 			}
1804 		} else if (state->type == STATE_TYPECAST) {
1805 			clear_struct_btf(ctx);
1806 
1807 			/* resolve the typecast struct name */
1808 			ctx->offset = state->typecast.orig_offset + 1; /* for the '(' */
1809 			ret = parse_btf_casttype(state->typecast.casttype, ctx);
1810 			if (ret < 0)
1811 				return ret;
1812 
1813 			ctx->offset = state->typecast.orig_offset +
1814 				      state->typecast.field_offset_diff;
1815 			ret = parse_btf_field(state->typecast.fieldname,
1816 					      ctx->last_struct, pcode,
1817 					      end, ctx);
1818 			ctx->prefix_byteoffs = 0;
1819 			if (ret < 0)
1820 				return ret;
1821 		}
1822 	}
1823 
1824 	return 0;
1825 }
1826 
1827 /* Loop-based (non-recursive) argument parser */
1828 static int
1829 parse_probe_arg(char *arg, const struct fetch_type *type,
1830 		struct fetch_insn **pcode, struct fetch_insn *end,
1831 		struct traceprobe_parse_context *ctx)
1832 {
1833 	int ret;
1834 
1835 	ctx->depth = 0;
1836 
1837 	ret = parse_probe_arg_nested(&arg, ctx);
1838 	if (ret)
1839 		return ret;
1840 
1841 	ret = parse_probe_arg_leaf(arg, type, pcode, end, ctx);
1842 	if (ret)
1843 		return ret;
1844 
1845 	return unwind_parse_states(pcode, end, ctx);
1846 }
1847 
1848 /* Bitfield type needs to be parsed into a fetch function */
1849 static int __parse_bitfield_probe_arg(const char *bf,
1850 				      const struct fetch_type *t,
1851 				      struct fetch_insn **pcode)
1852 {
1853 	struct fetch_insn *code = *pcode;
1854 	unsigned long bw, bo;
1855 	char *tail;
1856 
1857 	if (*bf != 'b')
1858 		return 0;
1859 
1860 	bw = simple_strtoul(bf + 1, &tail, 0);	/* Use simple one */
1861 
1862 	if (bw == 0 || *tail != '@')
1863 		return -EINVAL;
1864 
1865 	bf = tail + 1;
1866 	bo = simple_strtoul(bf, &tail, 0);
1867 
1868 	if (tail == bf || *tail != '/')
1869 		return -EINVAL;
1870 	code++;
1871 	if (code->op != FETCH_OP_NOP)
1872 		return -EINVAL;
1873 	*pcode = code;
1874 
1875 	code->op = FETCH_OP_MOD_BF;
1876 	code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1877 	code->rshift = BYTES_TO_BITS(t->size) - bw;
1878 	code->basesize = t->size;
1879 
1880 	return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1881 }
1882 
1883 /* Split type part from @arg and return it. */
1884 static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1885 				  struct traceprobe_parse_context *ctx)
1886 {
1887 	char *t = NULL, *t2, *t3;
1888 	int offs;
1889 
1890 	t = strchr(arg, ':');
1891 	if (t) {
1892 		*t++ = '\0';
1893 		t2 = strchr(t, '[');
1894 		if (t2) {
1895 			*t2++ = '\0';
1896 			t3 = strchr(t2, ']');
1897 			if (!t3) {
1898 				offs = t2 + strlen(t2) - arg;
1899 
1900 				trace_probe_log_err(ctx->offset + offs,
1901 						    ARRAY_NO_CLOSE);
1902 				return ERR_PTR(-EINVAL);
1903 			} else if (t3[1] != '\0') {
1904 				trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1905 						    BAD_ARRAY_SUFFIX);
1906 				return ERR_PTR(-EINVAL);
1907 			}
1908 			*t3 = '\0';
1909 			if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1910 				trace_probe_log_err(ctx->offset + t2 - arg,
1911 						    BAD_ARRAY_NUM);
1912 				return ERR_PTR(-EINVAL);
1913 			}
1914 			if (parg->count > MAX_ARRAY_LEN) {
1915 				trace_probe_log_err(ctx->offset + t2 - arg,
1916 						    ARRAY_TOO_BIG);
1917 				return ERR_PTR(-EINVAL);
1918 			}
1919 		}
1920 	}
1921 	offs = t ? t - arg : 0;
1922 
1923 	/*
1924 	 * Since $comm and immediate string can not be dereferenced,
1925 	 * we can find those by strcmp. But ignore for eprobes.
1926 	 */
1927 	if (!(ctx->flags & TPARG_FL_TEVENT) &&
1928 	    (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1929 	     strncmp(arg, "\\\"", 2) == 0)) {
1930 		/* The type of $comm must be "string", and not an array type. */
1931 		if (parg->count || (t && strcmp(t, "string"))) {
1932 			trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1933 			return ERR_PTR(-EINVAL);
1934 		}
1935 		parg->type = find_fetch_type("string", ctx->flags);
1936 	} else
1937 		parg->type = find_fetch_type(t, ctx->flags);
1938 
1939 	if (!parg->type) {
1940 		trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1941 		return ERR_PTR(-EINVAL);
1942 	}
1943 
1944 	return t;
1945 }
1946 
1947 /* After parsing, adjust the fetch_insn according to the probe_arg */
1948 static int finalize_fetch_insn(struct fetch_insn *code,
1949 			       struct probe_arg *parg,
1950 			       char *type,
1951 			       int type_offset,
1952 			       struct traceprobe_parse_context *ctx)
1953 {
1954 	struct fetch_insn *scode;
1955 	int ret;
1956 
1957 	/* Store operation */
1958 	if (parg->type->is_string) {
1959 		bool is_uderef;
1960 
1961 		/* Check bad combination of the type and the last fetch_insn. */
1962 		if (!strcmp(parg->type->name, "symstr")) {
1963 			if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1964 			    code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1965 			    code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1966 				trace_probe_log_err(ctx->offset + type_offset,
1967 						    BAD_SYMSTRING);
1968 				return -EINVAL;
1969 			}
1970 		} else {
1971 			if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1972 			    code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1973 			    code->op != FETCH_OP_IMMSTR && code->op != FETCH_OP_TP_ARG) {
1974 				trace_probe_log_err(ctx->offset + type_offset,
1975 						    BAD_STRING);
1976 				return -EINVAL;
1977 			}
1978 		}
1979 
1980 		is_uderef = (code->op == FETCH_OP_UDEREF);
1981 		if (!strcmp(parg->type->name, "symstr") ||
1982 		    (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1983 		     code->op == FETCH_OP_IMMSTR) || code->op == FETCH_OP_TP_ARG ||
1984 		     parg->count) {
1985 			/*
1986 			 * IMM, DATA and COMM is pointing actual address, those
1987 			 * must be kept, and if parg->count != 0, this is an
1988 			 * array of string pointers instead of string address
1989 			 * itself.
1990 			 * For the symstr, it doesn't need to dereference, thus
1991 			 * it just get the value.
1992 			 */
1993 			if (parg->count && (code->op != FETCH_OP_DEREF &&
1994 					    code->op != FETCH_OP_UDEREF)) {
1995 				trace_probe_log_err(ctx->offset + type_offset,
1996 						    BAD_STRING);
1997 				return -EINVAL;
1998 			}
1999 			code++;
2000 			if (code->op != FETCH_OP_NOP) {
2001 				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
2002 				return -EINVAL;
2003 			}
2004 		}
2005 
2006 		/* If op == DEREF, replace it with STRING */
2007 		if (!strcmp(parg->type->name, "ustring") || is_uderef)
2008 			code->op = FETCH_OP_ST_USTRING;
2009 		else if (!strcmp(parg->type->name, "symstr"))
2010 			code->op = FETCH_OP_ST_SYMSTR;
2011 		else
2012 			code->op = FETCH_OP_ST_STRING;
2013 		code->size = parg->type->size;
2014 		parg->dynamic = true;
2015 	} else if (code->op == FETCH_OP_DEREF) {
2016 		code->op = FETCH_OP_ST_MEM;
2017 		code->size = parg->type->size;
2018 	} else if (code->op == FETCH_OP_UDEREF) {
2019 		code->op = FETCH_OP_ST_UMEM;
2020 		code->size = parg->type->size;
2021 	} else {
2022 		code++;
2023 		if (code->op != FETCH_OP_NOP) {
2024 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
2025 			return -E2BIG;
2026 		}
2027 		code->op = FETCH_OP_ST_RAW;
2028 		code->size = parg->type->size;
2029 	}
2030 
2031 	/* Save storing fetch_insn. */
2032 	scode = code;
2033 
2034 	/* Modify operation */
2035 	if (type != NULL) {
2036 		/* Bitfield needs a special fetch_insn. */
2037 		ret = __parse_bitfield_probe_arg(type, parg->type, &code);
2038 		if (ret) {
2039 			trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
2040 			return ret;
2041 		}
2042 	} else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
2043 		   ctx->last_type) {
2044 		/* If user not specified the type, try parsing BTF bitfield. */
2045 		ret = parse_btf_bitfield(&code, parg->type, ctx);
2046 		if (ret)
2047 			return ret;
2048 	}
2049 
2050 	/* Loop(Array) operation */
2051 	if (parg->count) {
2052 		if (scode->op != FETCH_OP_ST_MEM &&
2053 		    scode->op != FETCH_OP_ST_STRING &&
2054 		    scode->op != FETCH_OP_ST_USTRING) {
2055 			trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
2056 			return -EINVAL;
2057 		}
2058 		code++;
2059 		if (code->op != FETCH_OP_NOP) {
2060 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
2061 			return -E2BIG;
2062 		}
2063 		code->op = FETCH_OP_LP_ARRAY;
2064 		code->param = parg->count;
2065 	}
2066 
2067 	/* Finalize the fetch_insn array. */
2068 	code++;
2069 	code->op = FETCH_OP_END;
2070 
2071 	return 0;
2072 }
2073 
2074 /* String length checking wrapper */
2075 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
2076 					   struct probe_arg *parg,
2077 					   struct traceprobe_parse_context *ctx)
2078 {
2079 	struct fetch_insn *code, *tmp = NULL;
2080 	char *type, *arg __free(kfree) = NULL;
2081 	int ret, len;
2082 
2083 	len = strlen(argv);
2084 	if (len > MAX_ARGSTR_LEN) {
2085 		trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
2086 		return -E2BIG;
2087 	} else if (len == 0) {
2088 		trace_probe_log_err(ctx->offset, NO_ARG_BODY);
2089 		return -EINVAL;
2090 	}
2091 
2092 	arg = kstrdup(argv, GFP_KERNEL);
2093 	if (!arg)
2094 		return -ENOMEM;
2095 
2096 	parg->comm = kstrdup(arg, GFP_KERNEL);
2097 	if (!parg->comm)
2098 		return -ENOMEM;
2099 
2100 	type = parse_probe_arg_type(arg, parg, ctx);
2101 	if (IS_ERR(type))
2102 		return PTR_ERR(type);
2103 
2104 	code = tmp = kzalloc_objs(*code, FETCH_INSN_MAX);
2105 	if (!code)
2106 		return -ENOMEM;
2107 	code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
2108 
2109 	ctx->last_type = NULL;
2110 	ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
2111 			      ctx);
2112 	if (ret < 0)
2113 		goto fail;
2114 	/* Update storing type if BTF is available */
2115 	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
2116 	    ctx->last_type) {
2117 		if (!type) {
2118 			parg->type = find_fetch_type_from_btf_type(ctx);
2119 		} else if (strstr(type, "string")) {
2120 			ret = check_prepare_btf_string_fetch(type, &code, ctx);
2121 			if (ret)
2122 				goto fail;
2123 		}
2124 	}
2125 	parg->offset = *size;
2126 	*size += parg->type->size * (parg->count ?: 1);
2127 
2128 	if (*size > MAX_PROBE_EVENT_SIZE) {
2129 		ret = -E2BIG;
2130 		trace_probe_log_err(ctx->offset, EVENT_TOO_BIG);
2131 		goto fail;
2132 	}
2133 
2134 	if (parg->count) {
2135 		len = strlen(parg->type->fmttype) + 6;
2136 		parg->fmt = kmalloc(len, GFP_KERNEL);
2137 		if (!parg->fmt) {
2138 			ret = -ENOMEM;
2139 			goto fail;
2140 		}
2141 		snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
2142 			 parg->count);
2143 	}
2144 
2145 	ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
2146 	if (ret < 0)
2147 		goto fail;
2148 
2149 	for (; code < tmp + FETCH_INSN_MAX; code++)
2150 		if (code->op == FETCH_OP_END)
2151 			break;
2152 	/* Shrink down the code buffer */
2153 	parg->code = kzalloc_objs(*code, code - tmp + 1);
2154 	if (!parg->code)
2155 		ret = -ENOMEM;
2156 	else
2157 		memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
2158 
2159 fail:
2160 	if (ret < 0) {
2161 		for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
2162 			if (code->op == FETCH_NOP_SYMBOL ||
2163 			    code->op == FETCH_OP_IMMSTR)
2164 				kfree(code->data);
2165 	}
2166 	kfree(tmp);
2167 
2168 	/* struct_btf should not be passed to other arguments */
2169 	clear_struct_btf(ctx);
2170 
2171 	return ret;
2172 }
2173 
2174 /* Return 1 if name is reserved or already used by another argument */
2175 static int traceprobe_conflict_field_name(const char *name,
2176 					  struct probe_arg *args, int narg)
2177 {
2178 	int i;
2179 
2180 	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
2181 		if (strcmp(reserved_field_names[i], name) == 0)
2182 			return 1;
2183 
2184 	for (i = 0; i < narg; i++)
2185 		if (strcmp(args[i].name, name) == 0)
2186 			return 1;
2187 
2188 	return 0;
2189 }
2190 
2191 static char *generate_probe_arg_name(const char *arg, int idx)
2192 {
2193 	char *name = NULL;
2194 	const char *end;
2195 
2196 	/*
2197 	 * If argument name is omitted, try arg as a name (BTF variable)
2198 	 * or "argN".
2199 	 */
2200 	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
2201 		end = strchr(arg, ':');
2202 		if (!end)
2203 			end = arg + strlen(arg);
2204 
2205 		name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
2206 		if (!name || !is_good_name(name)) {
2207 			kfree(name);
2208 			name = NULL;
2209 		}
2210 	}
2211 
2212 	if (!name)
2213 		name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
2214 
2215 	return name;
2216 }
2217 
2218 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
2219 			       struct traceprobe_parse_context *ctx)
2220 {
2221 	struct probe_arg *parg = &tp->args[i];
2222 	const char *body;
2223 
2224 	ctx->tp = tp;
2225 	body = strchr(arg, '=');
2226 	if (body) {
2227 		if (body - arg > MAX_ARG_NAME_LEN) {
2228 			trace_probe_log_err(0, ARG_NAME_TOO_LONG);
2229 			return -EINVAL;
2230 		} else if (body == arg) {
2231 			trace_probe_log_err(0, NO_ARG_NAME);
2232 			return -EINVAL;
2233 		}
2234 		parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
2235 		body++;
2236 	} else {
2237 		parg->name = generate_probe_arg_name(arg, i);
2238 		body = arg;
2239 	}
2240 	if (!parg->name)
2241 		return -ENOMEM;
2242 
2243 	if (!is_good_name(parg->name)) {
2244 		trace_probe_log_err(0, BAD_ARG_NAME);
2245 		return -EINVAL;
2246 	}
2247 	if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
2248 		trace_probe_log_err(0, USED_ARG_NAME);
2249 		return -EINVAL;
2250 	}
2251 	ctx->offset = body - arg;
2252 	/* Parse fetch argument */
2253 	return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
2254 }
2255 
2256 void traceprobe_free_probe_arg(struct probe_arg *arg)
2257 {
2258 	struct fetch_insn *code = arg->code;
2259 
2260 	while (code && code->op != FETCH_OP_END) {
2261 		if (code->op == FETCH_NOP_SYMBOL ||
2262 		    code->op == FETCH_OP_IMMSTR)
2263 			kfree(code->data);
2264 		code++;
2265 	}
2266 	kfree(arg->code);
2267 	kfree(arg->name);
2268 	kfree(arg->comm);
2269 	kfree(arg->fmt);
2270 }
2271 
2272 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
2273 			    struct traceprobe_parse_context *ctx)
2274 {
2275 	int i, found = 0;
2276 
2277 	for (i = 0; i < argc; i++)
2278 		if (str_has_prefix(argv[i], "$arg")) {
2279 			trace_probe_log_set_index(i + 2);
2280 
2281 			if (!tparg_is_function_entry(ctx->flags) &&
2282 			    !tparg_is_function_return(ctx->flags)) {
2283 				trace_probe_log_err(0, NOFENTRY_ARGS);
2284 				return -EINVAL;
2285 			}
2286 
2287 			if (isdigit(argv[i][4])) {
2288 				found = 1;
2289 				continue;
2290 			}
2291 
2292 			if (argv[i][4] != '*') {
2293 				trace_probe_log_err(0, BAD_VAR);
2294 				return -EINVAL;
2295 			}
2296 
2297 			if (*args_idx >= 0 && *args_idx < argc) {
2298 				trace_probe_log_err(0, DOUBLE_ARGS);
2299 				return -EINVAL;
2300 			}
2301 			found = 1;
2302 			*args_idx = i;
2303 		}
2304 
2305 	return found;
2306 }
2307 
2308 static int sprint_nth_btf_arg(int idx, const char *type,
2309 			      char *buf, int bufsize,
2310 			      struct traceprobe_parse_context *ctx)
2311 {
2312 	const char *name;
2313 	int ret;
2314 
2315 	if (idx >= ctx->nr_params) {
2316 		trace_probe_log_err(0, NO_BTFARG);
2317 		return -ENOENT;
2318 	}
2319 	name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
2320 	if (!name) {
2321 		trace_probe_log_err(0, NO_BTF_ENTRY);
2322 		return -ENOENT;
2323 	}
2324 	ret = snprintf(buf, bufsize, "%s%s", name, type);
2325 	if (ret >= bufsize) {
2326 		trace_probe_log_err(0, ARGS_2LONG);
2327 		return -E2BIG;
2328 	}
2329 	return ret;
2330 }
2331 
2332 /* Return new_argv which must be freed after use */
2333 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
2334 					 int *new_argc, char *buf, int bufsize,
2335 					 struct traceprobe_parse_context *ctx)
2336 {
2337 	int i, j, n, used, ret, args_idx = -1;
2338 	const char **new_argv __free(kfree) = NULL;
2339 
2340 	ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
2341 	if (ret < 0)
2342 		return ERR_PTR(ret);
2343 
2344 	if (!ret) {
2345 		*new_argc = argc;
2346 		return NULL;
2347 	}
2348 
2349 	ret = query_btf_context(ctx);
2350 	if (ret < 0 || ctx->nr_params == 0) {
2351 		if (args_idx != -1) {
2352 			/* $arg* requires BTF info */
2353 			trace_probe_log_err(0, NOSUP_BTFARG);
2354 			return ERR_PTR(-EOPNOTSUPP);
2355 		}
2356 		*new_argc = argc;
2357 		return NULL;
2358 	}
2359 
2360 	if (args_idx >= 0)
2361 		*new_argc = argc + ctx->nr_params - 1;
2362 	else
2363 		*new_argc = argc;
2364 
2365 	new_argv = kzalloc_objs(*new_argv, *new_argc);
2366 	if (!new_argv)
2367 		return ERR_PTR(-ENOMEM);
2368 
2369 	used = 0;
2370 	for (i = 0, j = 0; i < argc; i++) {
2371 		trace_probe_log_set_index(i + 2);
2372 		if (i == args_idx) {
2373 			for (n = 0; n < ctx->nr_params; n++) {
2374 				ret = sprint_nth_btf_arg(n, "", buf + used,
2375 							 bufsize - used, ctx);
2376 				if (ret < 0)
2377 					return ERR_PTR(ret);
2378 
2379 				new_argv[j++] = buf + used;
2380 				used += ret + 1;
2381 			}
2382 			continue;
2383 		}
2384 
2385 		if (str_has_prefix(argv[i], "$arg")) {
2386 			char *type = NULL;
2387 
2388 			n = simple_strtoul(argv[i] + 4, &type, 10);
2389 			if (type && !(*type == ':' || *type == '\0')) {
2390 				trace_probe_log_err(0, BAD_VAR);
2391 				return ERR_PTR(-ENOENT);
2392 			}
2393 			/* Note: $argN starts from $arg1, so $arg0 is invalid. */
2394 			if (n == 0) {
2395 				trace_probe_log_err(0, BAD_ARG_NUM);
2396 				return ERR_PTR(-EINVAL);
2397 			}
2398 			ret = sprint_nth_btf_arg(n - 1, type, buf + used,
2399 						 bufsize - used, ctx);
2400 			if (ret < 0)
2401 				return ERR_PTR(ret);
2402 			new_argv[j++] = buf + used;
2403 			used += ret + 1;
2404 		} else
2405 			new_argv[j++] = argv[i];
2406 	}
2407 
2408 	return_ptr(new_argv);
2409 }
2410 
2411 /* @buf: *buf must be equal to NULL. Caller must to free *buf */
2412 int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
2413 {
2414 	int i, used, ret;
2415 	const int bufsize = MAX_DENTRY_ARGS_LEN;
2416 	char *tmpbuf __free(kfree) = NULL;
2417 
2418 	if (*buf)
2419 		return -EINVAL;
2420 
2421 	used = 0;
2422 	for (i = 0; i < argc; i++) {
2423 		char *tmp __free(kfree) = NULL;
2424 		char *equal;
2425 		size_t arg_len;
2426 
2427 		if (!glob_match("*:%p[dD]", argv[i]))
2428 			continue;
2429 
2430 		if (!tmpbuf) {
2431 			tmpbuf = kmalloc(bufsize, GFP_KERNEL);
2432 			if (!tmpbuf)
2433 				return -ENOMEM;
2434 		}
2435 
2436 		tmp = kstrdup(argv[i], GFP_KERNEL);
2437 		if (!tmp)
2438 			return -ENOMEM;
2439 
2440 		equal = strchr(tmp, '=');
2441 		if (equal)
2442 			*equal = '\0';
2443 		arg_len = strlen(argv[i]);
2444 		tmp[arg_len - 4] = '\0';
2445 		if (argv[i][arg_len - 1] == 'd')
2446 			ret = snprintf(tmpbuf + used, bufsize - used,
2447 				       "%s%s+0x0(+0x%zx(%s)):string",
2448 				       equal ? tmp : "", equal ? "=" : "",
2449 				       offsetof(struct dentry, d_name.name),
2450 				       equal ? equal + 1 : tmp);
2451 		else
2452 			ret = snprintf(tmpbuf + used, bufsize - used,
2453 				       "%s%s+0x0(+0x%zx(+0x%zx(%s))):string",
2454 				       equal ? tmp : "", equal ? "=" : "",
2455 				       offsetof(struct dentry, d_name.name),
2456 				       offsetof(struct file, f_path.dentry),
2457 				       equal ? equal + 1 : tmp);
2458 
2459 		if (ret >= bufsize - used)
2460 			return -ENOMEM;
2461 		argv[i] = tmpbuf + used;
2462 		used += ret + 1;
2463 	}
2464 
2465 	*buf = no_free_ptr(tmpbuf);
2466 	return 0;
2467 }
2468 
2469 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
2470 {
2471 	clear_btf_context(ctx);
2472 }
2473 
2474 int traceprobe_update_arg(struct probe_arg *arg)
2475 {
2476 	struct fetch_insn *code = arg->code;
2477 	long offset;
2478 	char *tmp;
2479 	char c;
2480 	int ret = 0;
2481 
2482 	while (code && code->op != FETCH_OP_END) {
2483 		if (code->op == FETCH_NOP_SYMBOL) {
2484 			if (code[1].op != FETCH_OP_IMM)
2485 				return -EINVAL;
2486 
2487 			tmp = strpbrk(code->data, "+-");
2488 			if (tmp)
2489 				c = *tmp;
2490 			ret = traceprobe_split_symbol_offset(code->data,
2491 							     &offset);
2492 			if (ret)
2493 				return ret;
2494 
2495 			code[1].immediate =
2496 				(unsigned long)kallsyms_lookup_name(code->data);
2497 			if (tmp)
2498 				*tmp = c;
2499 			if (!code[1].immediate)
2500 				return -ENOENT;
2501 			code[1].immediate += offset;
2502 		}
2503 		code++;
2504 	}
2505 	return 0;
2506 }
2507 
2508 /* When len=0, we just calculate the needed length */
2509 #define LEN_OR_ZERO (len > pos ? len - pos : 0)
2510 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
2511 			   enum probe_print_type ptype)
2512 {
2513 	struct probe_arg *parg;
2514 	int i, j;
2515 	int pos = 0;
2516 	const char *fmt, *arg;
2517 
2518 	switch (ptype) {
2519 	case PROBE_PRINT_NORMAL:
2520 		fmt = "(%lx)";
2521 		arg = ", REC->" FIELD_STRING_IP;
2522 		break;
2523 	case PROBE_PRINT_RETURN:
2524 		fmt = "(%lx <- %lx)";
2525 		arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
2526 		break;
2527 	case PROBE_PRINT_EVENT:
2528 		fmt = "";
2529 		arg = "";
2530 		break;
2531 	default:
2532 		WARN_ON_ONCE(1);
2533 		return 0;
2534 	}
2535 
2536 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
2537 
2538 	for (i = 0; i < tp->nr_args; i++) {
2539 		parg = tp->args + i;
2540 		pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
2541 		if (parg->count) {
2542 			pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
2543 					parg->type->fmt);
2544 			for (j = 1; j < parg->count; j++)
2545 				pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
2546 						parg->type->fmt);
2547 			pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
2548 		} else
2549 			pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
2550 					parg->type->fmt);
2551 	}
2552 
2553 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
2554 
2555 	for (i = 0; i < tp->nr_args; i++) {
2556 		parg = tp->args + i;
2557 		if (parg->count) {
2558 			if (parg->type->is_string)
2559 				fmt = ", __get_str(%s[%d])";
2560 			else
2561 				fmt = ", REC->%s[%d]";
2562 			for (j = 0; j < parg->count; j++)
2563 				pos += snprintf(buf + pos, LEN_OR_ZERO,
2564 						fmt, parg->name, j);
2565 		} else {
2566 			if (parg->type->is_string)
2567 				fmt = ", __get_str(%s)";
2568 			else
2569 				fmt = ", REC->%s";
2570 			pos += snprintf(buf + pos, LEN_OR_ZERO,
2571 					fmt, parg->name);
2572 		}
2573 	}
2574 
2575 	/* return the length of print_fmt */
2576 	return pos;
2577 }
2578 #undef LEN_OR_ZERO
2579 
2580 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
2581 {
2582 	struct trace_event_call *call = trace_probe_event_call(tp);
2583 	int len;
2584 	char *print_fmt;
2585 
2586 	/* First: called with 0 length to calculate the needed length */
2587 	len = __set_print_fmt(tp, NULL, 0, ptype);
2588 	print_fmt = kmalloc(len + 1, GFP_KERNEL);
2589 	if (!print_fmt)
2590 		return -ENOMEM;
2591 
2592 	/* Second: actually write the @print_fmt */
2593 	__set_print_fmt(tp, print_fmt, len + 1, ptype);
2594 	call->print_fmt = print_fmt;
2595 
2596 	return 0;
2597 }
2598 
2599 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
2600 				 size_t offset, struct trace_probe *tp)
2601 {
2602 	struct trace_probe_event *tpe = trace_probe_event_from_call(event_call);
2603 	int ret, i;
2604 
2605 	/*
2606 	 * A field created by trace_define_field() only stores the name and
2607 	 * type pointers, it does not copy the strings. Here they point into
2608 	 * the probe_arg of @tp, which is freed when @tp is removed. For an
2609 	 * event with multiple probes attached, the field list is defined
2610 	 * once by the first probe but kept alive by the surviving siblings,
2611 	 * so removing that first probe would leave the fields referencing
2612 	 * freed memory. Duplicate the strings and anchor the copies on the
2613 	 * trace_probe_event, which lives as long as the field list itself.
2614 	 *
2615 	 * event_define_fields() ignores the return value of this hook, so
2616 	 * if a previous attempt failed before creating any field, it may
2617 	 * call here again. Release duplicates left behind by such an
2618 	 * attempt before starting over.
2619 	 */
2620 	for (i = 0; i < tpe->nr_field_strings; i++)
2621 		kfree(tpe->field_strings[i]);
2622 	kfree(tpe->field_strings);
2623 	tpe->field_strings = NULL;
2624 	tpe->nr_field_strings = 0;
2625 
2626 	if (tp->nr_args) {
2627 		tpe->field_strings = kcalloc(tp->nr_args * 2, sizeof(char *),
2628 					     GFP_KERNEL);
2629 		if (!tpe->field_strings)
2630 			return -ENOMEM;
2631 	}
2632 
2633 	/* Set argument names as fields */
2634 	for (i = 0; i < tp->nr_args; i++) {
2635 		struct probe_arg *parg = &tp->args[i];
2636 		const char *fmt = parg->type->fmttype;
2637 		int size = parg->type->size;
2638 		char *name, *type;
2639 
2640 		if (parg->fmt)
2641 			fmt = parg->fmt;
2642 		if (parg->count)
2643 			size *= parg->count;
2644 
2645 		name = kstrdup(parg->name, GFP_KERNEL);
2646 		type = kstrdup(fmt, GFP_KERNEL);
2647 		if (!name || !type) {
2648 			kfree(name);
2649 			kfree(type);
2650 			return -ENOMEM;
2651 		}
2652 		tpe->field_strings[tpe->nr_field_strings++] = name;
2653 		tpe->field_strings[tpe->nr_field_strings++] = type;
2654 
2655 		ret = trace_define_field(event_call, type, name,
2656 					 offset + parg->offset, size,
2657 					 parg->type->is_signed,
2658 					 FILTER_OTHER);
2659 		if (ret)
2660 			return ret;
2661 	}
2662 	return 0;
2663 }
2664 
2665 static void trace_probe_event_free(struct trace_probe_event *tpe)
2666 {
2667 	int i;
2668 
2669 	for (i = 0; i < tpe->nr_field_strings; i++)
2670 		kfree(tpe->field_strings[i]);
2671 	kfree(tpe->field_strings);
2672 	kfree(tpe->class.system);
2673 	kfree(tpe->call.name);
2674 	kfree(tpe->call.print_fmt);
2675 	kfree(tpe);
2676 }
2677 
2678 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
2679 {
2680 	if (trace_probe_has_sibling(tp))
2681 		return -EBUSY;
2682 
2683 	list_del_init(&tp->list);
2684 	trace_probe_event_free(tp->event);
2685 
2686 	tp->event = to->event;
2687 	list_add_tail(&tp->list, trace_probe_probe_list(to));
2688 
2689 	return 0;
2690 }
2691 
2692 void trace_probe_unlink(struct trace_probe *tp)
2693 {
2694 	list_del_init(&tp->list);
2695 	if (list_empty(trace_probe_probe_list(tp)))
2696 		trace_probe_event_free(tp->event);
2697 	tp->event = NULL;
2698 }
2699 
2700 void trace_probe_cleanup(struct trace_probe *tp)
2701 {
2702 	int i;
2703 
2704 	for (i = 0; i < tp->nr_args; i++)
2705 		traceprobe_free_probe_arg(&tp->args[i]);
2706 
2707 	if (tp->entry_arg) {
2708 		kfree(tp->entry_arg);
2709 		tp->entry_arg = NULL;
2710 	}
2711 
2712 	if (tp->event)
2713 		trace_probe_unlink(tp);
2714 }
2715 
2716 int trace_probe_init(struct trace_probe *tp, const char *event,
2717 		     const char *group, bool alloc_filter, int nargs)
2718 {
2719 	struct trace_event_call *call;
2720 	size_t size = sizeof(struct trace_probe_event);
2721 	int ret = 0;
2722 
2723 	if (!event || !group)
2724 		return -EINVAL;
2725 
2726 	if (alloc_filter)
2727 		size += sizeof(struct trace_uprobe_filter);
2728 
2729 	tp->event = kzalloc(size, GFP_KERNEL);
2730 	if (!tp->event)
2731 		return -ENOMEM;
2732 
2733 	INIT_LIST_HEAD(&tp->event->files);
2734 	INIT_LIST_HEAD(&tp->event->class.fields);
2735 	INIT_LIST_HEAD(&tp->event->probes);
2736 	INIT_LIST_HEAD(&tp->list);
2737 	list_add(&tp->list, &tp->event->probes);
2738 
2739 	call = trace_probe_event_call(tp);
2740 	call->class = &tp->event->class;
2741 	call->name = kstrdup(event, GFP_KERNEL);
2742 	if (!call->name) {
2743 		ret = -ENOMEM;
2744 		goto error;
2745 	}
2746 
2747 	tp->event->class.system = kstrdup(group, GFP_KERNEL);
2748 	if (!tp->event->class.system) {
2749 		ret = -ENOMEM;
2750 		goto error;
2751 	}
2752 
2753 	tp->nr_args = nargs;
2754 	/* Make sure pointers in args[] are NULL */
2755 	if (nargs)
2756 		memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
2757 
2758 	return 0;
2759 
2760 error:
2761 	trace_probe_cleanup(tp);
2762 	return ret;
2763 }
2764 
2765 static struct trace_event_call *
2766 find_trace_event_call(const char *system, const char *event_name)
2767 {
2768 	struct trace_event_call *tp_event;
2769 	const char *name;
2770 
2771 	list_for_each_entry(tp_event, &ftrace_events, list) {
2772 		if (!tp_event->class->system ||
2773 		    strcmp(system, tp_event->class->system))
2774 			continue;
2775 		name = trace_event_name(tp_event);
2776 		if (!name || strcmp(event_name, name))
2777 			continue;
2778 		return tp_event;
2779 	}
2780 
2781 	return NULL;
2782 }
2783 
2784 int trace_probe_register_event_call(struct trace_probe *tp)
2785 {
2786 	struct trace_event_call *call = trace_probe_event_call(tp);
2787 	int ret;
2788 
2789 	lockdep_assert_held(&event_mutex);
2790 
2791 	if (find_trace_event_call(trace_probe_group_name(tp),
2792 				  trace_probe_name(tp)))
2793 		return -EEXIST;
2794 
2795 	ret = register_trace_event(&call->event);
2796 	if (!ret)
2797 		return -ENODEV;
2798 
2799 	ret = trace_add_event_call(call);
2800 	if (ret)
2801 		unregister_trace_event(&call->event);
2802 
2803 	return ret;
2804 }
2805 
2806 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2807 {
2808 	struct event_file_link *link;
2809 
2810 	link = kmalloc_obj(*link);
2811 	if (!link)
2812 		return -ENOMEM;
2813 
2814 	link->file = file;
2815 	INIT_LIST_HEAD(&link->list);
2816 	list_add_tail_rcu(&link->list, &tp->event->files);
2817 	trace_probe_set_flag(tp, TP_FLAG_TRACE);
2818 	return 0;
2819 }
2820 
2821 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2822 						  struct trace_event_file *file)
2823 {
2824 	struct event_file_link *link;
2825 
2826 	trace_probe_for_each_link(link, tp) {
2827 		if (link->file == file)
2828 			return link;
2829 	}
2830 
2831 	return NULL;
2832 }
2833 
2834 int trace_probe_remove_file(struct trace_probe *tp,
2835 			    struct trace_event_file *file)
2836 {
2837 	struct event_file_link *link;
2838 
2839 	link = trace_probe_get_file_link(tp, file);
2840 	if (!link)
2841 		return -ENOENT;
2842 
2843 	list_del_rcu(&link->list);
2844 	kvfree_rcu_mightsleep(link);
2845 
2846 	if (list_empty(&tp->event->files))
2847 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2848 
2849 	return 0;
2850 }
2851 
2852 /*
2853  * Return the smallest index of different type argument (start from 1).
2854  * If all argument types and name are same, return 0.
2855  */
2856 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2857 {
2858 	int i;
2859 
2860 	/* In case of more arguments */
2861 	if (a->nr_args < b->nr_args)
2862 		return a->nr_args + 1;
2863 	if (a->nr_args > b->nr_args)
2864 		return b->nr_args + 1;
2865 
2866 	for (i = 0; i < a->nr_args; i++) {
2867 		if ((a->args[i].type != b->args[i].type) ||
2868 		    (a->args[i].count != b->args[i].count) ||
2869 		    strcmp(a->args[i].name, b->args[i].name))
2870 			return i + 1;
2871 	}
2872 
2873 	return 0;
2874 }
2875 
2876 bool trace_probe_match_command_args(struct trace_probe *tp,
2877 				    int argc, const char **argv)
2878 {
2879 	int i;
2880 
2881 	if (tp->nr_args < argc)
2882 		return false;
2883 
2884 	for (i = 0; i < argc; i++) {
2885 		int len = strlen(tp->args[i].name);
2886 
2887 		if (strncmp(argv[i], tp->args[i].name, len) ||
2888 		    argv[i][len] != '=' ||
2889 		    strcmp(argv[i] + len + 1, tp->args[i].comm))
2890 			return false;
2891 	}
2892 	return true;
2893 }
2894 
2895 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2896 {
2897 	int argc = 0, ret = 0;
2898 	char **argv;
2899 
2900 	argv = argv_split(GFP_KERNEL, raw_command, &argc);
2901 	if (!argv)
2902 		return -ENOMEM;
2903 
2904 	if (argc)
2905 		ret = createfn(argc, (const char **)argv);
2906 
2907 	argv_free(argv);
2908 
2909 	return ret;
2910 }
2911 
2912 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2913 		 u8 *data, void *field)
2914 {
2915 	void *p;
2916 	int i, j;
2917 
2918 	for (i = 0; i < nr_args; i++) {
2919 		struct probe_arg *a = args + i;
2920 
2921 		trace_seq_printf(s, " %s=", a->name);
2922 		if (likely(!a->count)) {
2923 			if (!a->type->print(s, data + a->offset, field))
2924 				return -ENOMEM;
2925 			continue;
2926 		}
2927 		trace_seq_putc(s, '{');
2928 		p = data + a->offset;
2929 		for (j = 0; j < a->count; j++) {
2930 			if (!a->type->print(s, p, field))
2931 				return -ENOMEM;
2932 			trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2933 			p += a->type->size;
2934 		}
2935 	}
2936 	return 0;
2937 }
2938 
2939 #ifdef CONFIG_PROBE_EVENTS_DUMP_FETCHARG
2940 
2941 struct fetch_op_decode {
2942 	const char *name;
2943 	void (*decode)(struct seq_file *m, struct fetch_insn *insn);
2944 };
2945 
2946 static const struct fetch_op_decode fetch_op_decode[];
2947 
2948 static void fetcharg_decode_none(struct seq_file *m, struct fetch_insn *insn)
2949 {
2950 	seq_puts(m, fetch_op_decode[insn->op].name);
2951 }
2952 
2953 static void fetcharg_decode_param(struct seq_file *m, struct fetch_insn *insn)
2954 {
2955 	seq_printf(m, "%s(%u)", fetch_op_decode[insn->op].name, insn->param);
2956 }
2957 
2958 static void fetcharg_decode_imm(struct seq_file *m, struct fetch_insn *insn)
2959 {
2960 	seq_printf(m, "%s(0x%lx)", fetch_op_decode[insn->op].name, insn->immediate);
2961 }
2962 
2963 static void fetcharg_decode_string(struct seq_file *m, struct fetch_insn *insn)
2964 {
2965 	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
2966 }
2967 
2968 static void fetcharg_decode_symbol(struct seq_file *m, struct fetch_insn *insn)
2969 {
2970 	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, (char *)insn->data);
2971 }
2972 
2973 static void fetcharg_decode_offset(struct seq_file *m, struct fetch_insn *insn)
2974 {
2975 	seq_printf(m, "%s(offset=%d)", fetch_op_decode[insn->op].name, insn->offset);
2976 }
2977 
2978 static void fetcharg_decode_store(struct seq_file *m, struct fetch_insn *insn)
2979 {
2980 	if (insn->op == FETCH_OP_ST_RAW)
2981 		seq_printf(m, "%s(size=%u)", fetch_op_decode[insn->op].name, insn->size);
2982 	else
2983 		seq_printf(m, "%s(offset=%d,size=%u)", fetch_op_decode[insn->op].name,
2984 			  insn->offset, insn->size);
2985 }
2986 
2987 static void fetcharg_decode_bf(struct seq_file *m, struct fetch_insn *insn)
2988 {
2989 	seq_printf(m, "%s(basesize=%u,lshift=%u,rshift=%u)",
2990 		   fetch_op_decode[insn->op].name, insn->basesize, insn->lshift, insn->rshift);
2991 }
2992 
2993 static void fetcharg_decode_tp_arg(struct seq_file *m, struct fetch_insn *insn)
2994 {
2995 	struct ftrace_event_field *field = insn->data;
2996 
2997 	seq_printf(m, "%s(%s)", fetch_op_decode[insn->op].name, field->name);
2998 }
2999 
3000 #define FETCH_OP(opname, decode_fn) \
3001 	[FETCH_OP_##opname] = { .name = #opname, .decode = fetcharg_decode_##decode_fn }
3002 
3003 static const struct fetch_op_decode fetch_op_decode[] = FETCH_OP_LIST;
3004 #undef FETCH_OP
3005 
3006 static void trace_probe_dump_arg(struct seq_file *m, struct probe_arg *parg)
3007 {
3008 	int i;
3009 
3010 	seq_printf(m, "#  %s: ", parg->name);
3011 	for (i = 0; i < FETCH_INSN_MAX; i++) {
3012 		struct fetch_insn *insn = parg->code + i;
3013 
3014 		if (insn->op >= ARRAY_SIZE(fetch_op_decode) || !fetch_op_decode[insn->op].decode)
3015 			seq_printf(m, "unknown(%d)", insn->op);
3016 		else
3017 			fetch_op_decode[insn->op].decode(m, insn);
3018 
3019 		if (insn->op == FETCH_OP_END)
3020 			break;
3021 		seq_puts(m, " -> ");
3022 	}
3023 	seq_putc(m, '\n');
3024 }
3025 
3026 void trace_probe_dump_args(struct seq_file *m, struct trace_probe *tp)
3027 {
3028 	int i;
3029 
3030 	for (i = 0; i < tp->nr_args; i++)
3031 		trace_probe_dump_arg(m, &tp->args[i]);
3032 }
3033 #endif /* CONFIG_PROBE_EVENTS_DUMP_FETCHARG */
3034