xref: /linux/kernel/trace/trace_probe.c (revision 405e6c37c89ef0df2bfc7a988820a3df22dacb1b)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Common code for probe-based Dynamic events.
4  *
5  * This code was copied from kernel/trace/trace_kprobe.c written by
6  * Masami Hiramatsu <masami.hiramatsu.pt@hitachi.com>
7  *
8  * Updates to make this generic:
9  * Copyright (C) IBM Corporation, 2010-2011
10  * Author:     Srikar Dronamraju
11  */
12 #define pr_fmt(fmt)	"trace_probe: " fmt
13 
14 #include <linux/bpf.h>
15 #include <linux/fs.h>
16 #include "trace_btf.h"
17 
18 #include "trace_probe.h"
19 
20 #undef C
21 #define C(a, b)		b
22 
23 static const char *trace_probe_err_text[] = { ERRORS };
24 
25 static const char *reserved_field_names[] = {
26 	"common_type",
27 	"common_flags",
28 	"common_preempt_count",
29 	"common_pid",
30 	"common_tgid",
31 	FIELD_STRING_IP,
32 	FIELD_STRING_RETIP,
33 	FIELD_STRING_FUNC,
34 };
35 
36 /* Printing  in basic type function template */
37 #define DEFINE_BASIC_PRINT_TYPE_FUNC(tname, type, fmt)			\
38 int PRINT_TYPE_FUNC_NAME(tname)(struct trace_seq *s, void *data, void *ent)\
39 {									\
40 	trace_seq_printf(s, fmt, *(type *)data);			\
41 	return !trace_seq_has_overflowed(s);				\
42 }									\
43 const char PRINT_TYPE_FMT_NAME(tname)[] = fmt;
44 
45 DEFINE_BASIC_PRINT_TYPE_FUNC(u8,  u8,  "%u")
46 DEFINE_BASIC_PRINT_TYPE_FUNC(u16, u16, "%u")
47 DEFINE_BASIC_PRINT_TYPE_FUNC(u32, u32, "%u")
48 DEFINE_BASIC_PRINT_TYPE_FUNC(u64, u64, "%Lu")
49 DEFINE_BASIC_PRINT_TYPE_FUNC(s8,  s8,  "%d")
50 DEFINE_BASIC_PRINT_TYPE_FUNC(s16, s16, "%d")
51 DEFINE_BASIC_PRINT_TYPE_FUNC(s32, s32, "%d")
52 DEFINE_BASIC_PRINT_TYPE_FUNC(s64, s64, "%Ld")
53 DEFINE_BASIC_PRINT_TYPE_FUNC(x8,  u8,  "0x%x")
54 DEFINE_BASIC_PRINT_TYPE_FUNC(x16, u16, "0x%x")
55 DEFINE_BASIC_PRINT_TYPE_FUNC(x32, u32, "0x%x")
56 DEFINE_BASIC_PRINT_TYPE_FUNC(x64, u64, "0x%Lx")
57 DEFINE_BASIC_PRINT_TYPE_FUNC(char, u8, "'%c'")
58 
PRINT_TYPE_FUNC_NAME(symbol)59 int PRINT_TYPE_FUNC_NAME(symbol)(struct trace_seq *s, void *data, void *ent)
60 {
61 	trace_seq_printf(s, "%pS", (void *)*(unsigned long *)data);
62 	return !trace_seq_has_overflowed(s);
63 }
64 const char PRINT_TYPE_FMT_NAME(symbol)[] = "%pS";
65 
66 /* Print type function for string type */
PRINT_TYPE_FUNC_NAME(string)67 int PRINT_TYPE_FUNC_NAME(string)(struct trace_seq *s, void *data, void *ent)
68 {
69 	int len = *(u32 *)data >> 16;
70 
71 	if (!len)
72 		trace_seq_puts(s, FAULT_STRING);
73 	else
74 		trace_seq_printf(s, "\"%s\"",
75 				 (const char *)get_loc_data(data, ent));
76 	return !trace_seq_has_overflowed(s);
77 }
78 
79 const char PRINT_TYPE_FMT_NAME(string)[] = "\\\"%s\\\"";
80 
81 /* Fetch type information table */
82 static const struct fetch_type probe_fetch_types[] = {
83 	/* Special types */
84 	__ASSIGN_FETCH_TYPE("string", string, string, sizeof(u32), 1, 1,
85 			    "__data_loc char[]"),
86 	__ASSIGN_FETCH_TYPE("ustring", string, string, sizeof(u32), 1, 1,
87 			    "__data_loc char[]"),
88 	__ASSIGN_FETCH_TYPE("symstr", string, string, sizeof(u32), 1, 1,
89 			    "__data_loc char[]"),
90 	/* Basic types */
91 	ASSIGN_FETCH_TYPE(u8,  u8,  0),
92 	ASSIGN_FETCH_TYPE(u16, u16, 0),
93 	ASSIGN_FETCH_TYPE(u32, u32, 0),
94 	ASSIGN_FETCH_TYPE(u64, u64, 0),
95 	ASSIGN_FETCH_TYPE(s8,  u8,  1),
96 	ASSIGN_FETCH_TYPE(s16, u16, 1),
97 	ASSIGN_FETCH_TYPE(s32, u32, 1),
98 	ASSIGN_FETCH_TYPE(s64, u64, 1),
99 	ASSIGN_FETCH_TYPE_ALIAS(x8,  u8,  u8,  0),
100 	ASSIGN_FETCH_TYPE_ALIAS(x16, u16, u16, 0),
101 	ASSIGN_FETCH_TYPE_ALIAS(x32, u32, u32, 0),
102 	ASSIGN_FETCH_TYPE_ALIAS(x64, u64, u64, 0),
103 	ASSIGN_FETCH_TYPE_ALIAS(char, u8, u8,  0),
104 	ASSIGN_FETCH_TYPE_ALIAS(symbol, ADDR_FETCH_TYPE, ADDR_FETCH_TYPE, 0),
105 
106 	ASSIGN_FETCH_TYPE_END
107 };
108 
find_fetch_type(const char * type,unsigned long flags)109 static const struct fetch_type *find_fetch_type(const char *type, unsigned long flags)
110 {
111 	int i;
112 
113 	/* Reject the symbol/symstr for uprobes */
114 	if (type && (flags & TPARG_FL_USER) &&
115 	    (!strcmp(type, "symbol") || !strcmp(type, "symstr")))
116 		return NULL;
117 
118 	if (!type)
119 		type = DEFAULT_FETCH_TYPE_STR;
120 
121 	/* Special case: bitfield */
122 	if (*type == 'b') {
123 		unsigned long bs;
124 
125 		type = strchr(type, '/');
126 		if (!type)
127 			goto fail;
128 
129 		type++;
130 		if (kstrtoul(type, 0, &bs))
131 			goto fail;
132 
133 		switch (bs) {
134 		case 8:
135 			return find_fetch_type("u8", flags);
136 		case 16:
137 			return find_fetch_type("u16", flags);
138 		case 32:
139 			return find_fetch_type("u32", flags);
140 		case 64:
141 			return find_fetch_type("u64", flags);
142 		default:
143 			goto fail;
144 		}
145 	}
146 
147 	for (i = 0; probe_fetch_types[i].name; i++) {
148 		if (strcmp(type, probe_fetch_types[i].name) == 0)
149 			return &probe_fetch_types[i];
150 	}
151 
152 fail:
153 	return NULL;
154 }
155 
156 static struct trace_probe_log trace_probe_log;
157 extern struct mutex dyn_event_ops_mutex;
158 
trace_probe_log_init(const char * subsystem,int argc,const char ** argv)159 void trace_probe_log_init(const char *subsystem, int argc, const char **argv)
160 {
161 	lockdep_assert_held(&dyn_event_ops_mutex);
162 
163 	trace_probe_log.subsystem = subsystem;
164 	trace_probe_log.argc = argc;
165 	trace_probe_log.argv = argv;
166 	trace_probe_log.index = 0;
167 }
168 
trace_probe_log_clear(void)169 void trace_probe_log_clear(void)
170 {
171 	lockdep_assert_held(&dyn_event_ops_mutex);
172 
173 	memset(&trace_probe_log, 0, sizeof(trace_probe_log));
174 }
175 
trace_probe_log_set_index(int index)176 void trace_probe_log_set_index(int index)
177 {
178 	lockdep_assert_held(&dyn_event_ops_mutex);
179 
180 	trace_probe_log.index = index;
181 }
182 
__trace_probe_log_err(int offset,int err_type)183 void __trace_probe_log_err(int offset, int err_type)
184 {
185 	char *command, *p;
186 	int i, len = 0, pos = 0;
187 
188 	lockdep_assert_held(&dyn_event_ops_mutex);
189 
190 	if (!trace_probe_log.argv)
191 		return;
192 
193 	/* Recalculate the length and allocate buffer */
194 	for (i = 0; i < trace_probe_log.argc; i++) {
195 		if (i == trace_probe_log.index)
196 			pos = len;
197 		len += strlen(trace_probe_log.argv[i]) + 1;
198 	}
199 	command = kzalloc(len, GFP_KERNEL);
200 	if (!command)
201 		return;
202 
203 	if (trace_probe_log.index >= trace_probe_log.argc) {
204 		/**
205 		 * Set the error position is next to the last arg + space.
206 		 * Note that len includes the terminal null and the cursor
207 		 * appears at pos + 1.
208 		 */
209 		pos = len;
210 		offset = 0;
211 	}
212 
213 	/* And make a command string from argv array */
214 	p = command;
215 	for (i = 0; i < trace_probe_log.argc; i++) {
216 		len = strlen(trace_probe_log.argv[i]);
217 		strcpy(p, trace_probe_log.argv[i]);
218 		p[len] = ' ';
219 		p += len + 1;
220 	}
221 	*(p - 1) = '\0';
222 
223 	tracing_log_err(NULL, trace_probe_log.subsystem, command,
224 			trace_probe_err_text, err_type, pos + offset);
225 
226 	kfree(command);
227 }
228 
229 /* Split symbol and offset. */
traceprobe_split_symbol_offset(char * symbol,long * offset)230 int traceprobe_split_symbol_offset(char *symbol, long *offset)
231 {
232 	char *tmp;
233 	int ret;
234 
235 	if (!offset)
236 		return -EINVAL;
237 
238 	tmp = strpbrk(symbol, "+-");
239 	if (tmp) {
240 		ret = kstrtol(tmp, 0, offset);
241 		if (ret)
242 			return ret;
243 		*tmp = '\0';
244 	} else
245 		*offset = 0;
246 
247 	return 0;
248 }
249 
250 /* @buf must has MAX_EVENT_NAME_LEN size */
traceprobe_parse_event_name(const char ** pevent,const char ** pgroup,char * buf,int offset)251 int traceprobe_parse_event_name(const char **pevent, const char **pgroup,
252 				char *buf, int offset)
253 {
254 	const char *slash, *event = *pevent;
255 	int len;
256 
257 	slash = strchr(event, '/');
258 	if (!slash)
259 		slash = strchr(event, '.');
260 
261 	if (slash) {
262 		if (slash == event) {
263 			trace_probe_log_err(offset, NO_GROUP_NAME);
264 			return -EINVAL;
265 		}
266 		if (slash - event + 1 > MAX_EVENT_NAME_LEN) {
267 			trace_probe_log_err(offset, GROUP_TOO_LONG);
268 			return -EINVAL;
269 		}
270 		strscpy(buf, event, slash - event + 1);
271 		if (!is_good_system_name(buf)) {
272 			trace_probe_log_err(offset, BAD_GROUP_NAME);
273 			return -EINVAL;
274 		}
275 		*pgroup = buf;
276 		*pevent = slash + 1;
277 		offset += slash - event + 1;
278 		event = *pevent;
279 	}
280 	len = strlen(event);
281 	if (len == 0) {
282 		if (slash) {
283 			*pevent = NULL;
284 			return 0;
285 		}
286 		trace_probe_log_err(offset, NO_EVENT_NAME);
287 		return -EINVAL;
288 	} else if (len >= MAX_EVENT_NAME_LEN) {
289 		trace_probe_log_err(offset, EVENT_TOO_LONG);
290 		return -EINVAL;
291 	}
292 	if (!is_good_name(event)) {
293 		trace_probe_log_err(offset, BAD_EVENT_NAME);
294 		return -EINVAL;
295 	}
296 	return 0;
297 }
298 
parse_trace_event_arg(char * arg,struct fetch_insn * code,struct traceprobe_parse_context * ctx)299 static int parse_trace_event_arg(char *arg, struct fetch_insn *code,
300 				 struct traceprobe_parse_context *ctx)
301 {
302 	struct ftrace_event_field *field;
303 	struct list_head *head;
304 
305 	head = trace_get_fields(ctx->event);
306 	list_for_each_entry(field, head, link) {
307 		if (!strcmp(arg, field->name)) {
308 			code->op = FETCH_OP_TP_ARG;
309 			code->data = field;
310 			return 0;
311 		}
312 	}
313 	return -ENOENT;
314 }
315 
316 #ifdef CONFIG_PROBE_EVENTS_BTF_ARGS
317 
btf_type_int(const struct btf_type * t)318 static u32 btf_type_int(const struct btf_type *t)
319 {
320 	return *(u32 *)(t + 1);
321 }
322 
btf_type_is_char_ptr(struct btf * btf,const struct btf_type * type)323 static bool btf_type_is_char_ptr(struct btf *btf, const struct btf_type *type)
324 {
325 	const struct btf_type *real_type;
326 	u32 intdata;
327 	s32 tid;
328 
329 	real_type = btf_type_skip_modifiers(btf, type->type, &tid);
330 	if (!real_type)
331 		return false;
332 
333 	if (BTF_INFO_KIND(real_type->info) != BTF_KIND_INT)
334 		return false;
335 
336 	intdata = btf_type_int(real_type);
337 	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
338 		&& BTF_INT_BITS(intdata) == 8;
339 }
340 
btf_type_is_char_array(struct btf * btf,const struct btf_type * type)341 static bool btf_type_is_char_array(struct btf *btf, const struct btf_type *type)
342 {
343 	const struct btf_type *real_type;
344 	const struct btf_array *array;
345 	u32 intdata;
346 	s32 tid;
347 
348 	if (BTF_INFO_KIND(type->info) != BTF_KIND_ARRAY)
349 		return false;
350 
351 	array = (const struct btf_array *)(type + 1);
352 
353 	real_type = btf_type_skip_modifiers(btf, array->type, &tid);
354 
355 	intdata = btf_type_int(real_type);
356 	return !(BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED)
357 		&& BTF_INT_BITS(intdata) == 8;
358 }
359 
check_prepare_btf_string_fetch(char * typename,struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)360 static int check_prepare_btf_string_fetch(char *typename,
361 				struct fetch_insn **pcode,
362 				struct traceprobe_parse_context *ctx)
363 {
364 	struct btf *btf = ctx->btf;
365 
366 	if (!btf || !ctx->last_type)
367 		return 0;
368 
369 	/* char [] does not need any change. */
370 	if (btf_type_is_char_array(btf, ctx->last_type))
371 		return 0;
372 
373 	/* char * requires dereference the pointer. */
374 	if (btf_type_is_char_ptr(btf, ctx->last_type)) {
375 		struct fetch_insn *code = *pcode + 1;
376 
377 		if (code->op == FETCH_OP_END) {
378 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
379 			return -E2BIG;
380 		}
381 		if (typename[0] == 'u')
382 			code->op = FETCH_OP_UDEREF;
383 		else
384 			code->op = FETCH_OP_DEREF;
385 		code->offset = 0;
386 		*pcode = code;
387 		return 0;
388 	}
389 	/* Other types are not available for string */
390 	trace_probe_log_err(ctx->offset, BAD_TYPE4STR);
391 	return -EINVAL;
392 }
393 
fetch_type_from_btf_type(struct btf * btf,const struct btf_type * type,struct traceprobe_parse_context * ctx)394 static const char *fetch_type_from_btf_type(struct btf *btf,
395 					const struct btf_type *type,
396 					struct traceprobe_parse_context *ctx)
397 {
398 	u32 intdata;
399 
400 	/* TODO: const char * could be converted as a string */
401 	switch (BTF_INFO_KIND(type->info)) {
402 	case BTF_KIND_ENUM:
403 		/* enum is "int", so convert to "s32" */
404 		return "s32";
405 	case BTF_KIND_ENUM64:
406 		return "s64";
407 	case BTF_KIND_PTR:
408 		/* pointer will be converted to "x??" */
409 		if (IS_ENABLED(CONFIG_64BIT))
410 			return "x64";
411 		else
412 			return "x32";
413 	case BTF_KIND_INT:
414 		intdata = btf_type_int(type);
415 		if (BTF_INT_ENCODING(intdata) & BTF_INT_SIGNED) {
416 			switch (BTF_INT_BITS(intdata)) {
417 			case 8:
418 				return "s8";
419 			case 16:
420 				return "s16";
421 			case 32:
422 				return "s32";
423 			case 64:
424 				return "s64";
425 			}
426 		} else {	/* unsigned */
427 			switch (BTF_INT_BITS(intdata)) {
428 			case 8:
429 				return "u8";
430 			case 16:
431 				return "u16";
432 			case 32:
433 				return "u32";
434 			case 64:
435 				return "u64";
436 			}
437 			/* bitfield, size is encoded in the type */
438 			ctx->last_bitsize = BTF_INT_BITS(intdata);
439 			ctx->last_bitoffs += BTF_INT_OFFSET(intdata);
440 			return "u64";
441 		}
442 	}
443 	/* TODO: support other types */
444 
445 	return NULL;
446 }
447 
query_btf_context(struct traceprobe_parse_context * ctx)448 static int query_btf_context(struct traceprobe_parse_context *ctx)
449 {
450 	const struct btf_param *param;
451 	const struct btf_type *type;
452 	struct btf *btf;
453 	s32 nr;
454 
455 	if (ctx->btf)
456 		return 0;
457 
458 	if (!ctx->funcname)
459 		return -EINVAL;
460 
461 	type = btf_find_func_proto(ctx->funcname, &btf);
462 	if (!type)
463 		return -ENOENT;
464 
465 	ctx->btf = btf;
466 	ctx->proto = type;
467 
468 	/* ctx->params is optional, since func(void) will not have params. */
469 	nr = 0;
470 	param = btf_get_func_param(type, &nr);
471 	if (!IS_ERR_OR_NULL(param)) {
472 		/* Hide the first 'data' argument of tracepoint */
473 		if (ctx->flags & TPARG_FL_TPOINT) {
474 			nr--;
475 			param++;
476 		}
477 	}
478 
479 	if (nr > 0) {
480 		ctx->nr_params = nr;
481 		ctx->params = param;
482 	} else {
483 		ctx->nr_params = 0;
484 		ctx->params = NULL;
485 	}
486 
487 	return 0;
488 }
489 
clear_btf_context(struct traceprobe_parse_context * ctx)490 static void clear_btf_context(struct traceprobe_parse_context *ctx)
491 {
492 	if (ctx->btf) {
493 		btf_put(ctx->btf);
494 		ctx->btf = NULL;
495 		ctx->proto = NULL;
496 		ctx->params = NULL;
497 		ctx->nr_params = 0;
498 	}
499 }
500 
501 /* Return 1 if the field separater is arrow operator ('->') */
split_next_field(char * varname,char ** next_field,struct traceprobe_parse_context * ctx)502 static int split_next_field(char *varname, char **next_field,
503 			    struct traceprobe_parse_context *ctx)
504 {
505 	char *field;
506 	int ret = 0;
507 
508 	field = strpbrk(varname, ".-");
509 	if (field) {
510 		if (field[0] == '-' && field[1] == '>') {
511 			field[0] = '\0';
512 			field += 2;
513 			ret = 1;
514 		} else if (field[0] == '.') {
515 			field[0] = '\0';
516 			field += 1;
517 		} else {
518 			trace_probe_log_err(ctx->offset + field - varname, BAD_HYPHEN);
519 			return -EINVAL;
520 		}
521 		*next_field = field;
522 	}
523 
524 	return ret;
525 }
526 
527 /*
528  * Parse the field of data structure. The @type must be a pointer type
529  * pointing the target data structure type.
530  */
parse_btf_field(char * fieldname,const struct btf_type * type,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)531 static int parse_btf_field(char *fieldname, const struct btf_type *type,
532 			   struct fetch_insn **pcode, struct fetch_insn *end,
533 			   struct traceprobe_parse_context *ctx)
534 {
535 	struct fetch_insn *code = *pcode;
536 	const struct btf_member *field;
537 	u32 bitoffs, anon_offs;
538 	char *next;
539 	int is_ptr;
540 	s32 tid;
541 
542 	do {
543 		/* Outer loop for solving arrow operator ('->') */
544 		if (BTF_INFO_KIND(type->info) != BTF_KIND_PTR) {
545 			trace_probe_log_err(ctx->offset, NO_PTR_STRCT);
546 			return -EINVAL;
547 		}
548 		/* Convert a struct pointer type to a struct type */
549 		type = btf_type_skip_modifiers(ctx->btf, type->type, &tid);
550 		if (!type) {
551 			trace_probe_log_err(ctx->offset, BAD_BTF_TID);
552 			return -EINVAL;
553 		}
554 
555 		bitoffs = 0;
556 		do {
557 			/* Inner loop for solving dot operator ('.') */
558 			next = NULL;
559 			is_ptr = split_next_field(fieldname, &next, ctx);
560 			if (is_ptr < 0)
561 				return is_ptr;
562 
563 			anon_offs = 0;
564 			field = btf_find_struct_member(ctx->btf, type, fieldname,
565 						       &anon_offs);
566 			if (IS_ERR(field)) {
567 				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
568 				return PTR_ERR(field);
569 			}
570 			if (!field) {
571 				trace_probe_log_err(ctx->offset, NO_BTF_FIELD);
572 				return -ENOENT;
573 			}
574 			/* Add anonymous structure/union offset */
575 			bitoffs += anon_offs;
576 
577 			/* Accumulate the bit-offsets of the dot-connected fields */
578 			if (btf_type_kflag(type)) {
579 				bitoffs += BTF_MEMBER_BIT_OFFSET(field->offset);
580 				ctx->last_bitsize = BTF_MEMBER_BITFIELD_SIZE(field->offset);
581 			} else {
582 				bitoffs += field->offset;
583 				ctx->last_bitsize = 0;
584 			}
585 
586 			type = btf_type_skip_modifiers(ctx->btf, field->type, &tid);
587 			if (!type) {
588 				trace_probe_log_err(ctx->offset, BAD_BTF_TID);
589 				return -EINVAL;
590 			}
591 
592 			ctx->offset += next - fieldname;
593 			fieldname = next;
594 		} while (!is_ptr && fieldname);
595 
596 		if (++code == end) {
597 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
598 			return -EINVAL;
599 		}
600 		code->op = FETCH_OP_DEREF;	/* TODO: user deref support */
601 		code->offset = bitoffs / 8;
602 		*pcode = code;
603 
604 		ctx->last_bitoffs = bitoffs % 8;
605 		ctx->last_type = type;
606 	} while (fieldname);
607 
608 	return 0;
609 }
610 
611 static int __store_entry_arg(struct trace_probe *tp, int argnum);
612 
parse_btf_arg(char * varname,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)613 static int parse_btf_arg(char *varname,
614 			 struct fetch_insn **pcode, struct fetch_insn *end,
615 			 struct traceprobe_parse_context *ctx)
616 {
617 	struct fetch_insn *code = *pcode;
618 	const struct btf_param *params;
619 	const struct btf_type *type;
620 	char *field = NULL;
621 	int i, is_ptr, ret;
622 	u32 tid;
623 
624 	if (WARN_ON_ONCE(!ctx->funcname))
625 		return -EINVAL;
626 
627 	is_ptr = split_next_field(varname, &field, ctx);
628 	if (is_ptr < 0)
629 		return is_ptr;
630 	if (!is_ptr && field) {
631 		/* dot-connected field on an argument is not supported. */
632 		trace_probe_log_err(ctx->offset + field - varname,
633 				    NOSUP_DAT_ARG);
634 		return -EOPNOTSUPP;
635 	}
636 
637 	if (ctx->flags & TPARG_FL_RETURN && !strcmp(varname, "$retval")) {
638 		code->op = FETCH_OP_RETVAL;
639 		/* Check whether the function return type is not void */
640 		if (query_btf_context(ctx) == 0) {
641 			if (ctx->proto->type == 0) {
642 				trace_probe_log_err(ctx->offset, NO_RETVAL);
643 				return -ENOENT;
644 			}
645 			tid = ctx->proto->type;
646 			goto found;
647 		}
648 		if (field) {
649 			trace_probe_log_err(ctx->offset + field - varname,
650 					    NO_BTF_ENTRY);
651 			return -ENOENT;
652 		}
653 		return 0;
654 	}
655 
656 	if (!ctx->btf) {
657 		ret = query_btf_context(ctx);
658 		if (ret < 0 || ctx->nr_params == 0) {
659 			trace_probe_log_err(ctx->offset, NO_BTF_ENTRY);
660 			return PTR_ERR(params);
661 		}
662 	}
663 	params = ctx->params;
664 
665 	for (i = 0; i < ctx->nr_params; i++) {
666 		const char *name = btf_name_by_offset(ctx->btf, params[i].name_off);
667 
668 		if (name && !strcmp(name, varname)) {
669 			if (tparg_is_function_entry(ctx->flags)) {
670 				code->op = FETCH_OP_ARG;
671 				if (ctx->flags & TPARG_FL_TPOINT)
672 					code->param = i + 1;
673 				else
674 					code->param = i;
675 			} else if (tparg_is_function_return(ctx->flags)) {
676 				code->op = FETCH_OP_EDATA;
677 				ret = __store_entry_arg(ctx->tp, i);
678 				if (ret < 0) {
679 					/* internal error */
680 					return ret;
681 				}
682 				code->offset = ret;
683 			}
684 			tid = params[i].type;
685 			goto found;
686 		}
687 	}
688 	trace_probe_log_err(ctx->offset, NO_BTFARG);
689 	return -ENOENT;
690 
691 found:
692 	type = btf_type_skip_modifiers(ctx->btf, tid, &tid);
693 	if (!type) {
694 		trace_probe_log_err(ctx->offset, BAD_BTF_TID);
695 		return -EINVAL;
696 	}
697 	/* Initialize the last type information */
698 	ctx->last_type = type;
699 	ctx->last_bitoffs = 0;
700 	ctx->last_bitsize = 0;
701 	if (field) {
702 		ctx->offset += field - varname;
703 		return parse_btf_field(field, type, pcode, end, ctx);
704 	}
705 	return 0;
706 }
707 
find_fetch_type_from_btf_type(struct traceprobe_parse_context * ctx)708 static const struct fetch_type *find_fetch_type_from_btf_type(
709 					struct traceprobe_parse_context *ctx)
710 {
711 	struct btf *btf = ctx->btf;
712 	const char *typestr = NULL;
713 
714 	if (btf && ctx->last_type)
715 		typestr = fetch_type_from_btf_type(btf, ctx->last_type, ctx);
716 
717 	return find_fetch_type(typestr, ctx->flags);
718 }
719 
parse_btf_bitfield(struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)720 static int parse_btf_bitfield(struct fetch_insn **pcode,
721 			      struct traceprobe_parse_context *ctx)
722 {
723 	struct fetch_insn *code = *pcode;
724 
725 	if ((ctx->last_bitsize % 8 == 0) && ctx->last_bitoffs == 0)
726 		return 0;
727 
728 	code++;
729 	if (code->op != FETCH_OP_NOP) {
730 		trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
731 		return -EINVAL;
732 	}
733 	*pcode = code;
734 
735 	code->op = FETCH_OP_MOD_BF;
736 	code->lshift = 64 - (ctx->last_bitsize + ctx->last_bitoffs);
737 	code->rshift = 64 - ctx->last_bitsize;
738 	code->basesize = 64 / 8;
739 	return 0;
740 }
741 
742 #else
clear_btf_context(struct traceprobe_parse_context * ctx)743 static void clear_btf_context(struct traceprobe_parse_context *ctx)
744 {
745 	ctx->btf = NULL;
746 }
747 
query_btf_context(struct traceprobe_parse_context * ctx)748 static int query_btf_context(struct traceprobe_parse_context *ctx)
749 {
750 	return -EOPNOTSUPP;
751 }
752 
parse_btf_arg(char * varname,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)753 static int parse_btf_arg(char *varname,
754 			 struct fetch_insn **pcode, struct fetch_insn *end,
755 			 struct traceprobe_parse_context *ctx)
756 {
757 	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
758 	return -EOPNOTSUPP;
759 }
760 
parse_btf_bitfield(struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)761 static int parse_btf_bitfield(struct fetch_insn **pcode,
762 			      struct traceprobe_parse_context *ctx)
763 {
764 	trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
765 	return -EOPNOTSUPP;
766 }
767 
768 #define find_fetch_type_from_btf_type(ctx)		\
769 	find_fetch_type(NULL, ctx->flags)
770 
check_prepare_btf_string_fetch(char * typename,struct fetch_insn ** pcode,struct traceprobe_parse_context * ctx)771 static int check_prepare_btf_string_fetch(char *typename,
772 				struct fetch_insn **pcode,
773 				struct traceprobe_parse_context *ctx)
774 {
775 	return 0;
776 }
777 
778 #endif
779 
780 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
781 
782 /*
783  * Add the entry code to store the 'argnum'th parameter and return the offset
784  * in the entry data buffer where the data will be stored.
785  */
__store_entry_arg(struct trace_probe * tp,int argnum)786 static int __store_entry_arg(struct trace_probe *tp, int argnum)
787 {
788 	struct probe_entry_arg *earg = tp->entry_arg;
789 	bool match = false;
790 	int i, offset;
791 
792 	if (!earg) {
793 		earg = kzalloc(sizeof(*tp->entry_arg), GFP_KERNEL);
794 		if (!earg)
795 			return -ENOMEM;
796 		earg->size = 2 * tp->nr_args + 1;
797 		earg->code = kcalloc(earg->size, sizeof(struct fetch_insn),
798 				     GFP_KERNEL);
799 		if (!earg->code) {
800 			kfree(earg);
801 			return -ENOMEM;
802 		}
803 		/* Fill the code buffer with 'end' to simplify it */
804 		for (i = 0; i < earg->size; i++)
805 			earg->code[i].op = FETCH_OP_END;
806 		tp->entry_arg = earg;
807 	}
808 
809 	/*
810 	 * The entry code array is repeating the pair of
811 	 * [FETCH_OP_ARG(argnum)][FETCH_OP_ST_EDATA(offset of entry data buffer)]
812 	 * and the rest of entries are filled with [FETCH_OP_END].
813 	 *
814 	 * To reduce the redundant function parameter fetching, we scan the entry
815 	 * code array to find the FETCH_OP_ARG which already fetches the 'argnum'
816 	 * parameter. If it doesn't match, update 'offset' to find the last
817 	 * offset.
818 	 * If we find the FETCH_OP_END without matching FETCH_OP_ARG entry, we
819 	 * will save the entry with FETCH_OP_ARG and FETCH_OP_ST_EDATA, and
820 	 * return data offset so that caller can find the data offset in the entry
821 	 * data buffer.
822 	 */
823 	offset = 0;
824 	for (i = 0; i < earg->size - 1; i++) {
825 		switch (earg->code[i].op) {
826 		case FETCH_OP_END:
827 			earg->code[i].op = FETCH_OP_ARG;
828 			earg->code[i].param = argnum;
829 			earg->code[i + 1].op = FETCH_OP_ST_EDATA;
830 			earg->code[i + 1].offset = offset;
831 			return offset;
832 		case FETCH_OP_ARG:
833 			match = (earg->code[i].param == argnum);
834 			break;
835 		case FETCH_OP_ST_EDATA:
836 			offset = earg->code[i].offset;
837 			if (match)
838 				return offset;
839 			offset += sizeof(unsigned long);
840 			break;
841 		default:
842 			break;
843 		}
844 	}
845 	return -ENOSPC;
846 }
847 
traceprobe_get_entry_data_size(struct trace_probe * tp)848 int traceprobe_get_entry_data_size(struct trace_probe *tp)
849 {
850 	struct probe_entry_arg *earg = tp->entry_arg;
851 	int i, size = 0;
852 
853 	if (!earg)
854 		return 0;
855 
856 	/*
857 	 * earg->code[] array has an operation sequence which is run in
858 	 * the entry handler.
859 	 * The sequence stopped by FETCH_OP_END and each data stored in
860 	 * the entry data buffer by FETCH_OP_ST_EDATA. The FETCH_OP_ST_EDATA
861 	 * stores the data at the data buffer + its offset, and all data are
862 	 * "unsigned long" size. The offset must be increased when a data is
863 	 * stored. Thus we need to find the last FETCH_OP_ST_EDATA in the
864 	 * code array.
865 	 */
866 	for (i = 0; i < earg->size; i++) {
867 		switch (earg->code[i].op) {
868 		case FETCH_OP_END:
869 			goto out;
870 		case FETCH_OP_ST_EDATA:
871 			size = earg->code[i].offset + sizeof(unsigned long);
872 			break;
873 		default:
874 			break;
875 		}
876 	}
877 out:
878 	return size;
879 }
880 
store_trace_entry_data(void * edata,struct trace_probe * tp,struct pt_regs * regs)881 void store_trace_entry_data(void *edata, struct trace_probe *tp, struct pt_regs *regs)
882 {
883 	struct probe_entry_arg *earg = tp->entry_arg;
884 	unsigned long val = 0;
885 	int i;
886 
887 	if (!earg)
888 		return;
889 
890 	for (i = 0; i < earg->size; i++) {
891 		struct fetch_insn *code = &earg->code[i];
892 
893 		switch (code->op) {
894 		case FETCH_OP_ARG:
895 			val = regs_get_kernel_argument(regs, code->param);
896 			break;
897 		case FETCH_OP_ST_EDATA:
898 			*(unsigned long *)((unsigned long)edata + code->offset) = val;
899 			break;
900 		case FETCH_OP_END:
901 			goto end;
902 		default:
903 			break;
904 		}
905 	}
906 end:
907 	return;
908 }
NOKPROBE_SYMBOL(store_trace_entry_data)909 NOKPROBE_SYMBOL(store_trace_entry_data)
910 #endif
911 
912 #define PARAM_MAX_STACK (THREAD_SIZE / sizeof(unsigned long))
913 
914 /* Parse $vars. @orig_arg points '$', which syncs to @ctx->offset */
915 static int parse_probe_vars(char *orig_arg, const struct fetch_type *t,
916 			    struct fetch_insn **pcode,
917 			    struct fetch_insn *end,
918 			    struct traceprobe_parse_context *ctx)
919 {
920 	struct fetch_insn *code = *pcode;
921 	int err = TP_ERR_BAD_VAR;
922 	char *arg = orig_arg + 1;
923 	unsigned long param;
924 	int ret = 0;
925 	int len;
926 
927 	if (ctx->flags & TPARG_FL_TEVENT) {
928 		if (code->data)
929 			return -EFAULT;
930 		ret = parse_trace_event_arg(arg, code, ctx);
931 		if (!ret)
932 			return 0;
933 		if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
934 			code->op = FETCH_OP_COMM;
935 			return 0;
936 		}
937 		/* backward compatibility */
938 		ctx->offset = 0;
939 		goto inval;
940 	}
941 
942 	if (str_has_prefix(arg, "retval")) {
943 		if (!(ctx->flags & TPARG_FL_RETURN)) {
944 			err = TP_ERR_RETVAL_ON_PROBE;
945 			goto inval;
946 		}
947 		if (!(ctx->flags & TPARG_FL_KERNEL) ||
948 		    !IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
949 			code->op = FETCH_OP_RETVAL;
950 			return 0;
951 		}
952 		return parse_btf_arg(orig_arg, pcode, end, ctx);
953 	}
954 
955 	len = str_has_prefix(arg, "stack");
956 	if (len) {
957 
958 		if (arg[len] == '\0') {
959 			code->op = FETCH_OP_STACKP;
960 			return 0;
961 		}
962 
963 		if (isdigit(arg[len])) {
964 			ret = kstrtoul(arg + len, 10, &param);
965 			if (ret)
966 				goto inval;
967 
968 			if ((ctx->flags & TPARG_FL_KERNEL) &&
969 			    param > PARAM_MAX_STACK) {
970 				err = TP_ERR_BAD_STACK_NUM;
971 				goto inval;
972 			}
973 			code->op = FETCH_OP_STACK;
974 			code->param = (unsigned int)param;
975 			return 0;
976 		}
977 		goto inval;
978 	}
979 
980 	if (strcmp(arg, "comm") == 0 || strcmp(arg, "COMM") == 0) {
981 		code->op = FETCH_OP_COMM;
982 		return 0;
983 	}
984 
985 #ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API
986 	len = str_has_prefix(arg, "arg");
987 	if (len) {
988 		ret = kstrtoul(arg + len, 10, &param);
989 		if (ret)
990 			goto inval;
991 
992 		if (!param || param > PARAM_MAX_STACK) {
993 			err = TP_ERR_BAD_ARG_NUM;
994 			goto inval;
995 		}
996 		param--; /* argN starts from 1, but internal arg[N] starts from 0 */
997 
998 		if (tparg_is_function_entry(ctx->flags)) {
999 			code->op = FETCH_OP_ARG;
1000 			code->param = (unsigned int)param;
1001 			/*
1002 			 * The tracepoint probe will probe a stub function, and the
1003 			 * first parameter of the stub is a dummy and should be ignored.
1004 			 */
1005 			if (ctx->flags & TPARG_FL_TPOINT)
1006 				code->param++;
1007 		} else if (tparg_is_function_return(ctx->flags)) {
1008 			/* function entry argument access from return probe */
1009 			ret = __store_entry_arg(ctx->tp, param);
1010 			if (ret < 0)	/* This error should be an internal error */
1011 				return ret;
1012 
1013 			code->op = FETCH_OP_EDATA;
1014 			code->offset = ret;
1015 		} else {
1016 			err = TP_ERR_NOFENTRY_ARGS;
1017 			goto inval;
1018 		}
1019 		return 0;
1020 	}
1021 #endif
1022 
1023 inval:
1024 	__trace_probe_log_err(ctx->offset, err);
1025 	return -EINVAL;
1026 }
1027 
str_to_immediate(char * str,unsigned long * imm)1028 static int str_to_immediate(char *str, unsigned long *imm)
1029 {
1030 	if (isdigit(str[0]))
1031 		return kstrtoul(str, 0, imm);
1032 	else if (str[0] == '-')
1033 		return kstrtol(str, 0, (long *)imm);
1034 	else if (str[0] == '+')
1035 		return kstrtol(str + 1, 0, (long *)imm);
1036 	return -EINVAL;
1037 }
1038 
__parse_imm_string(char * str,char ** pbuf,int offs)1039 static int __parse_imm_string(char *str, char **pbuf, int offs)
1040 {
1041 	size_t len = strlen(str);
1042 
1043 	if (str[len - 1] != '"') {
1044 		trace_probe_log_err(offs + len, IMMSTR_NO_CLOSE);
1045 		return -EINVAL;
1046 	}
1047 	*pbuf = kstrndup(str, len - 1, GFP_KERNEL);
1048 	if (!*pbuf)
1049 		return -ENOMEM;
1050 	return 0;
1051 }
1052 
1053 /* Recursive argument parser */
1054 static int
parse_probe_arg(char * arg,const struct fetch_type * type,struct fetch_insn ** pcode,struct fetch_insn * end,struct traceprobe_parse_context * ctx)1055 parse_probe_arg(char *arg, const struct fetch_type *type,
1056 		struct fetch_insn **pcode, struct fetch_insn *end,
1057 		struct traceprobe_parse_context *ctx)
1058 {
1059 	struct fetch_insn *code = *pcode;
1060 	unsigned long param;
1061 	int deref = FETCH_OP_DEREF;
1062 	long offset = 0;
1063 	char *tmp;
1064 	int ret = 0;
1065 
1066 	switch (arg[0]) {
1067 	case '$':
1068 		ret = parse_probe_vars(arg, type, pcode, end, ctx);
1069 		break;
1070 
1071 	case '%':	/* named register */
1072 		if (ctx->flags & (TPARG_FL_TEVENT | TPARG_FL_FPROBE)) {
1073 			/* eprobe and fprobe do not handle registers */
1074 			trace_probe_log_err(ctx->offset, BAD_VAR);
1075 			break;
1076 		}
1077 		ret = regs_query_register_offset(arg + 1);
1078 		if (ret >= 0) {
1079 			code->op = FETCH_OP_REG;
1080 			code->param = (unsigned int)ret;
1081 			ret = 0;
1082 		} else
1083 			trace_probe_log_err(ctx->offset, BAD_REG_NAME);
1084 		break;
1085 
1086 	case '@':	/* memory, file-offset or symbol */
1087 		if (isdigit(arg[1])) {
1088 			ret = kstrtoul(arg + 1, 0, &param);
1089 			if (ret) {
1090 				trace_probe_log_err(ctx->offset, BAD_MEM_ADDR);
1091 				break;
1092 			}
1093 			/* load address */
1094 			code->op = FETCH_OP_IMM;
1095 			code->immediate = param;
1096 		} else if (arg[1] == '+') {
1097 			/* kprobes don't support file offsets */
1098 			if (ctx->flags & TPARG_FL_KERNEL) {
1099 				trace_probe_log_err(ctx->offset, FILE_ON_KPROBE);
1100 				return -EINVAL;
1101 			}
1102 			ret = kstrtol(arg + 2, 0, &offset);
1103 			if (ret) {
1104 				trace_probe_log_err(ctx->offset, BAD_FILE_OFFS);
1105 				break;
1106 			}
1107 
1108 			code->op = FETCH_OP_FOFFS;
1109 			code->immediate = (unsigned long)offset;  // imm64?
1110 		} else {
1111 			/* uprobes don't support symbols */
1112 			if (!(ctx->flags & TPARG_FL_KERNEL)) {
1113 				trace_probe_log_err(ctx->offset, SYM_ON_UPROBE);
1114 				return -EINVAL;
1115 			}
1116 			/* Preserve symbol for updating */
1117 			code->op = FETCH_NOP_SYMBOL;
1118 			code->data = kstrdup(arg + 1, GFP_KERNEL);
1119 			if (!code->data)
1120 				return -ENOMEM;
1121 			if (++code == end) {
1122 				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1123 				return -EINVAL;
1124 			}
1125 			code->op = FETCH_OP_IMM;
1126 			code->immediate = 0;
1127 		}
1128 		/* These are fetching from memory */
1129 		if (++code == end) {
1130 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1131 			return -EINVAL;
1132 		}
1133 		*pcode = code;
1134 		code->op = FETCH_OP_DEREF;
1135 		code->offset = offset;
1136 		break;
1137 
1138 	case '+':	/* deref memory */
1139 	case '-':
1140 		if (arg[1] == 'u') {
1141 			deref = FETCH_OP_UDEREF;
1142 			arg[1] = arg[0];
1143 			arg++;
1144 		}
1145 		if (arg[0] == '+')
1146 			arg++;	/* Skip '+', because kstrtol() rejects it. */
1147 		tmp = strchr(arg, '(');
1148 		if (!tmp) {
1149 			trace_probe_log_err(ctx->offset, DEREF_NEED_BRACE);
1150 			return -EINVAL;
1151 		}
1152 		*tmp = '\0';
1153 		ret = kstrtol(arg, 0, &offset);
1154 		if (ret) {
1155 			trace_probe_log_err(ctx->offset, BAD_DEREF_OFFS);
1156 			break;
1157 		}
1158 		ctx->offset += (tmp + 1 - arg) + (arg[0] != '-' ? 1 : 0);
1159 		arg = tmp + 1;
1160 		tmp = strrchr(arg, ')');
1161 		if (!tmp) {
1162 			trace_probe_log_err(ctx->offset + strlen(arg),
1163 					    DEREF_OPEN_BRACE);
1164 			return -EINVAL;
1165 		} else {
1166 			const struct fetch_type *t2 = find_fetch_type(NULL, ctx->flags);
1167 			int cur_offs = ctx->offset;
1168 
1169 			*tmp = '\0';
1170 			ret = parse_probe_arg(arg, t2, &code, end, ctx);
1171 			if (ret)
1172 				break;
1173 			ctx->offset = cur_offs;
1174 			if (code->op == FETCH_OP_COMM ||
1175 			    code->op == FETCH_OP_DATA) {
1176 				trace_probe_log_err(ctx->offset, COMM_CANT_DEREF);
1177 				return -EINVAL;
1178 			}
1179 			if (++code == end) {
1180 				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1181 				return -EINVAL;
1182 			}
1183 			*pcode = code;
1184 
1185 			code->op = deref;
1186 			code->offset = offset;
1187 			/* Reset the last type if used */
1188 			ctx->last_type = NULL;
1189 		}
1190 		break;
1191 	case '\\':	/* Immediate value */
1192 		if (arg[1] == '"') {	/* Immediate string */
1193 			ret = __parse_imm_string(arg + 2, &tmp, ctx->offset + 2);
1194 			if (ret)
1195 				break;
1196 			code->op = FETCH_OP_DATA;
1197 			code->data = tmp;
1198 		} else {
1199 			ret = str_to_immediate(arg + 1, &code->immediate);
1200 			if (ret)
1201 				trace_probe_log_err(ctx->offset + 1, BAD_IMM);
1202 			else
1203 				code->op = FETCH_OP_IMM;
1204 		}
1205 		break;
1206 	default:
1207 		if (isalpha(arg[0]) || arg[0] == '_') {	/* BTF variable */
1208 			if (!tparg_is_function_entry(ctx->flags) &&
1209 			    !tparg_is_function_return(ctx->flags)) {
1210 				trace_probe_log_err(ctx->offset, NOSUP_BTFARG);
1211 				return -EINVAL;
1212 			}
1213 			ret = parse_btf_arg(arg, pcode, end, ctx);
1214 			break;
1215 		}
1216 	}
1217 	if (!ret && code->op == FETCH_OP_NOP) {
1218 		/* Parsed, but do not find fetch method */
1219 		trace_probe_log_err(ctx->offset, BAD_FETCH_ARG);
1220 		ret = -EINVAL;
1221 	}
1222 	return ret;
1223 }
1224 
1225 /* Bitfield type needs to be parsed into a fetch function */
__parse_bitfield_probe_arg(const char * bf,const struct fetch_type * t,struct fetch_insn ** pcode)1226 static int __parse_bitfield_probe_arg(const char *bf,
1227 				      const struct fetch_type *t,
1228 				      struct fetch_insn **pcode)
1229 {
1230 	struct fetch_insn *code = *pcode;
1231 	unsigned long bw, bo;
1232 	char *tail;
1233 
1234 	if (*bf != 'b')
1235 		return 0;
1236 
1237 	bw = simple_strtoul(bf + 1, &tail, 0);	/* Use simple one */
1238 
1239 	if (bw == 0 || *tail != '@')
1240 		return -EINVAL;
1241 
1242 	bf = tail + 1;
1243 	bo = simple_strtoul(bf, &tail, 0);
1244 
1245 	if (tail == bf || *tail != '/')
1246 		return -EINVAL;
1247 	code++;
1248 	if (code->op != FETCH_OP_NOP)
1249 		return -EINVAL;
1250 	*pcode = code;
1251 
1252 	code->op = FETCH_OP_MOD_BF;
1253 	code->lshift = BYTES_TO_BITS(t->size) - (bw + bo);
1254 	code->rshift = BYTES_TO_BITS(t->size) - bw;
1255 	code->basesize = t->size;
1256 
1257 	return (BYTES_TO_BITS(t->size) < (bw + bo)) ? -EINVAL : 0;
1258 }
1259 
1260 /* Split type part from @arg and return it. */
parse_probe_arg_type(char * arg,struct probe_arg * parg,struct traceprobe_parse_context * ctx)1261 static char *parse_probe_arg_type(char *arg, struct probe_arg *parg,
1262 				  struct traceprobe_parse_context *ctx)
1263 {
1264 	char *t = NULL, *t2, *t3;
1265 	int offs;
1266 
1267 	t = strchr(arg, ':');
1268 	if (t) {
1269 		*t++ = '\0';
1270 		t2 = strchr(t, '[');
1271 		if (t2) {
1272 			*t2++ = '\0';
1273 			t3 = strchr(t2, ']');
1274 			if (!t3) {
1275 				offs = t2 + strlen(t2) - arg;
1276 
1277 				trace_probe_log_err(ctx->offset + offs,
1278 						    ARRAY_NO_CLOSE);
1279 				return ERR_PTR(-EINVAL);
1280 			} else if (t3[1] != '\0') {
1281 				trace_probe_log_err(ctx->offset + t3 + 1 - arg,
1282 						    BAD_ARRAY_SUFFIX);
1283 				return ERR_PTR(-EINVAL);
1284 			}
1285 			*t3 = '\0';
1286 			if (kstrtouint(t2, 0, &parg->count) || !parg->count) {
1287 				trace_probe_log_err(ctx->offset + t2 - arg,
1288 						    BAD_ARRAY_NUM);
1289 				return ERR_PTR(-EINVAL);
1290 			}
1291 			if (parg->count > MAX_ARRAY_LEN) {
1292 				trace_probe_log_err(ctx->offset + t2 - arg,
1293 						    ARRAY_TOO_BIG);
1294 				return ERR_PTR(-EINVAL);
1295 			}
1296 		}
1297 	}
1298 	offs = t ? t - arg : 0;
1299 
1300 	/*
1301 	 * Since $comm and immediate string can not be dereferenced,
1302 	 * we can find those by strcmp. But ignore for eprobes.
1303 	 */
1304 	if (!(ctx->flags & TPARG_FL_TEVENT) &&
1305 	    (strcmp(arg, "$comm") == 0 || strcmp(arg, "$COMM") == 0 ||
1306 	     strncmp(arg, "\\\"", 2) == 0)) {
1307 		/* The type of $comm must be "string", and not an array type. */
1308 		if (parg->count || (t && strcmp(t, "string"))) {
1309 			trace_probe_log_err(ctx->offset + offs, NEED_STRING_TYPE);
1310 			return ERR_PTR(-EINVAL);
1311 		}
1312 		parg->type = find_fetch_type("string", ctx->flags);
1313 	} else
1314 		parg->type = find_fetch_type(t, ctx->flags);
1315 
1316 	if (!parg->type) {
1317 		trace_probe_log_err(ctx->offset + offs, BAD_TYPE);
1318 		return ERR_PTR(-EINVAL);
1319 	}
1320 
1321 	return t;
1322 }
1323 
1324 /* After parsing, adjust the fetch_insn according to the probe_arg */
finalize_fetch_insn(struct fetch_insn * code,struct probe_arg * parg,char * type,int type_offset,struct traceprobe_parse_context * ctx)1325 static int finalize_fetch_insn(struct fetch_insn *code,
1326 			       struct probe_arg *parg,
1327 			       char *type,
1328 			       int type_offset,
1329 			       struct traceprobe_parse_context *ctx)
1330 {
1331 	struct fetch_insn *scode;
1332 	int ret;
1333 
1334 	/* Store operation */
1335 	if (parg->type->is_string) {
1336 		/* Check bad combination of the type and the last fetch_insn. */
1337 		if (!strcmp(parg->type->name, "symstr")) {
1338 			if (code->op != FETCH_OP_REG && code->op != FETCH_OP_STACK &&
1339 			    code->op != FETCH_OP_RETVAL && code->op != FETCH_OP_ARG &&
1340 			    code->op != FETCH_OP_DEREF && code->op != FETCH_OP_TP_ARG) {
1341 				trace_probe_log_err(ctx->offset + type_offset,
1342 						    BAD_SYMSTRING);
1343 				return -EINVAL;
1344 			}
1345 		} else {
1346 			if (code->op != FETCH_OP_DEREF && code->op != FETCH_OP_UDEREF &&
1347 			    code->op != FETCH_OP_IMM && code->op != FETCH_OP_COMM &&
1348 			    code->op != FETCH_OP_DATA && code->op != FETCH_OP_TP_ARG) {
1349 				trace_probe_log_err(ctx->offset + type_offset,
1350 						    BAD_STRING);
1351 				return -EINVAL;
1352 			}
1353 		}
1354 
1355 		if (!strcmp(parg->type->name, "symstr") ||
1356 		    (code->op == FETCH_OP_IMM || code->op == FETCH_OP_COMM ||
1357 		     code->op == FETCH_OP_DATA) || code->op == FETCH_OP_TP_ARG ||
1358 		     parg->count) {
1359 			/*
1360 			 * IMM, DATA and COMM is pointing actual address, those
1361 			 * must be kept, and if parg->count != 0, this is an
1362 			 * array of string pointers instead of string address
1363 			 * itself.
1364 			 * For the symstr, it doesn't need to dereference, thus
1365 			 * it just get the value.
1366 			 */
1367 			code++;
1368 			if (code->op != FETCH_OP_NOP) {
1369 				trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1370 				return -EINVAL;
1371 			}
1372 		}
1373 
1374 		/* If op == DEREF, replace it with STRING */
1375 		if (!strcmp(parg->type->name, "ustring") ||
1376 		    code->op == FETCH_OP_UDEREF)
1377 			code->op = FETCH_OP_ST_USTRING;
1378 		else if (!strcmp(parg->type->name, "symstr"))
1379 			code->op = FETCH_OP_ST_SYMSTR;
1380 		else
1381 			code->op = FETCH_OP_ST_STRING;
1382 		code->size = parg->type->size;
1383 		parg->dynamic = true;
1384 	} else if (code->op == FETCH_OP_DEREF) {
1385 		code->op = FETCH_OP_ST_MEM;
1386 		code->size = parg->type->size;
1387 	} else if (code->op == FETCH_OP_UDEREF) {
1388 		code->op = FETCH_OP_ST_UMEM;
1389 		code->size = parg->type->size;
1390 	} else {
1391 		code++;
1392 		if (code->op != FETCH_OP_NOP) {
1393 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1394 			return -E2BIG;
1395 		}
1396 		code->op = FETCH_OP_ST_RAW;
1397 		code->size = parg->type->size;
1398 	}
1399 
1400 	/* Save storing fetch_insn. */
1401 	scode = code;
1402 
1403 	/* Modify operation */
1404 	if (type != NULL) {
1405 		/* Bitfield needs a special fetch_insn. */
1406 		ret = __parse_bitfield_probe_arg(type, parg->type, &code);
1407 		if (ret) {
1408 			trace_probe_log_err(ctx->offset + type_offset, BAD_BITFIELD);
1409 			return ret;
1410 		}
1411 	} else if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1412 		   ctx->last_type) {
1413 		/* If user not specified the type, try parsing BTF bitfield. */
1414 		ret = parse_btf_bitfield(&code, ctx);
1415 		if (ret)
1416 			return ret;
1417 	}
1418 
1419 	/* Loop(Array) operation */
1420 	if (parg->count) {
1421 		if (scode->op != FETCH_OP_ST_MEM &&
1422 		    scode->op != FETCH_OP_ST_STRING &&
1423 		    scode->op != FETCH_OP_ST_USTRING) {
1424 			trace_probe_log_err(ctx->offset + type_offset, BAD_STRING);
1425 			return -EINVAL;
1426 		}
1427 		code++;
1428 		if (code->op != FETCH_OP_NOP) {
1429 			trace_probe_log_err(ctx->offset, TOO_MANY_OPS);
1430 			return -E2BIG;
1431 		}
1432 		code->op = FETCH_OP_LP_ARRAY;
1433 		code->param = parg->count;
1434 	}
1435 
1436 	/* Finalize the fetch_insn array. */
1437 	code++;
1438 	code->op = FETCH_OP_END;
1439 
1440 	return 0;
1441 }
1442 
1443 /* String length checking wrapper */
traceprobe_parse_probe_arg_body(const char * argv,ssize_t * size,struct probe_arg * parg,struct traceprobe_parse_context * ctx)1444 static int traceprobe_parse_probe_arg_body(const char *argv, ssize_t *size,
1445 					   struct probe_arg *parg,
1446 					   struct traceprobe_parse_context *ctx)
1447 {
1448 	struct fetch_insn *code, *tmp = NULL;
1449 	char *type, *arg __free(kfree) = NULL;
1450 	int ret, len;
1451 
1452 	len = strlen(argv);
1453 	if (len > MAX_ARGSTR_LEN) {
1454 		trace_probe_log_err(ctx->offset, ARG_TOO_LONG);
1455 		return -E2BIG;
1456 	} else if (len == 0) {
1457 		trace_probe_log_err(ctx->offset, NO_ARG_BODY);
1458 		return -EINVAL;
1459 	}
1460 
1461 	arg = kstrdup(argv, GFP_KERNEL);
1462 	if (!arg)
1463 		return -ENOMEM;
1464 
1465 	parg->comm = kstrdup(arg, GFP_KERNEL);
1466 	if (!parg->comm)
1467 		return -ENOMEM;
1468 
1469 	type = parse_probe_arg_type(arg, parg, ctx);
1470 	if (IS_ERR(type))
1471 		return PTR_ERR(type);
1472 
1473 	code = tmp = kcalloc(FETCH_INSN_MAX, sizeof(*code), GFP_KERNEL);
1474 	if (!code)
1475 		return -ENOMEM;
1476 	code[FETCH_INSN_MAX - 1].op = FETCH_OP_END;
1477 
1478 	ctx->last_type = NULL;
1479 	ret = parse_probe_arg(arg, parg->type, &code, &code[FETCH_INSN_MAX - 1],
1480 			      ctx);
1481 	if (ret < 0)
1482 		goto fail;
1483 
1484 	/* Update storing type if BTF is available */
1485 	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS) &&
1486 	    ctx->last_type) {
1487 		if (!type) {
1488 			parg->type = find_fetch_type_from_btf_type(ctx);
1489 		} else if (strstr(type, "string")) {
1490 			ret = check_prepare_btf_string_fetch(type, &code, ctx);
1491 			if (ret)
1492 				goto fail;
1493 		}
1494 	}
1495 	parg->offset = *size;
1496 	*size += parg->type->size * (parg->count ?: 1);
1497 
1498 	if (parg->count) {
1499 		len = strlen(parg->type->fmttype) + 6;
1500 		parg->fmt = kmalloc(len, GFP_KERNEL);
1501 		if (!parg->fmt) {
1502 			ret = -ENOMEM;
1503 			goto fail;
1504 		}
1505 		snprintf(parg->fmt, len, "%s[%d]", parg->type->fmttype,
1506 			 parg->count);
1507 	}
1508 
1509 	ret = finalize_fetch_insn(code, parg, type, type ? type - arg : 0, ctx);
1510 	if (ret < 0)
1511 		goto fail;
1512 
1513 	for (; code < tmp + FETCH_INSN_MAX; code++)
1514 		if (code->op == FETCH_OP_END)
1515 			break;
1516 	/* Shrink down the code buffer */
1517 	parg->code = kcalloc(code - tmp + 1, sizeof(*code), GFP_KERNEL);
1518 	if (!parg->code)
1519 		ret = -ENOMEM;
1520 	else
1521 		memcpy(parg->code, tmp, sizeof(*code) * (code - tmp + 1));
1522 
1523 fail:
1524 	if (ret < 0) {
1525 		for (code = tmp; code < tmp + FETCH_INSN_MAX; code++)
1526 			if (code->op == FETCH_NOP_SYMBOL ||
1527 			    code->op == FETCH_OP_DATA)
1528 				kfree(code->data);
1529 	}
1530 	kfree(tmp);
1531 
1532 	return ret;
1533 }
1534 
1535 /* Return 1 if name is reserved or already used by another argument */
traceprobe_conflict_field_name(const char * name,struct probe_arg * args,int narg)1536 static int traceprobe_conflict_field_name(const char *name,
1537 					  struct probe_arg *args, int narg)
1538 {
1539 	int i;
1540 
1541 	for (i = 0; i < ARRAY_SIZE(reserved_field_names); i++)
1542 		if (strcmp(reserved_field_names[i], name) == 0)
1543 			return 1;
1544 
1545 	for (i = 0; i < narg; i++)
1546 		if (strcmp(args[i].name, name) == 0)
1547 			return 1;
1548 
1549 	return 0;
1550 }
1551 
generate_probe_arg_name(const char * arg,int idx)1552 static char *generate_probe_arg_name(const char *arg, int idx)
1553 {
1554 	char *name = NULL;
1555 	const char *end;
1556 
1557 	/*
1558 	 * If argument name is omitted, try arg as a name (BTF variable)
1559 	 * or "argN".
1560 	 */
1561 	if (IS_ENABLED(CONFIG_PROBE_EVENTS_BTF_ARGS)) {
1562 		end = strchr(arg, ':');
1563 		if (!end)
1564 			end = arg + strlen(arg);
1565 
1566 		name = kmemdup_nul(arg, end - arg, GFP_KERNEL);
1567 		if (!name || !is_good_name(name)) {
1568 			kfree(name);
1569 			name = NULL;
1570 		}
1571 	}
1572 
1573 	if (!name)
1574 		name = kasprintf(GFP_KERNEL, "arg%d", idx + 1);
1575 
1576 	return name;
1577 }
1578 
traceprobe_parse_probe_arg(struct trace_probe * tp,int i,const char * arg,struct traceprobe_parse_context * ctx)1579 int traceprobe_parse_probe_arg(struct trace_probe *tp, int i, const char *arg,
1580 			       struct traceprobe_parse_context *ctx)
1581 {
1582 	struct probe_arg *parg = &tp->args[i];
1583 	const char *body;
1584 
1585 	ctx->tp = tp;
1586 	body = strchr(arg, '=');
1587 	if (body) {
1588 		if (body - arg > MAX_ARG_NAME_LEN) {
1589 			trace_probe_log_err(0, ARG_NAME_TOO_LONG);
1590 			return -EINVAL;
1591 		} else if (body == arg) {
1592 			trace_probe_log_err(0, NO_ARG_NAME);
1593 			return -EINVAL;
1594 		}
1595 		parg->name = kmemdup_nul(arg, body - arg, GFP_KERNEL);
1596 		body++;
1597 	} else {
1598 		parg->name = generate_probe_arg_name(arg, i);
1599 		body = arg;
1600 	}
1601 	if (!parg->name)
1602 		return -ENOMEM;
1603 
1604 	if (!is_good_name(parg->name)) {
1605 		trace_probe_log_err(0, BAD_ARG_NAME);
1606 		return -EINVAL;
1607 	}
1608 	if (traceprobe_conflict_field_name(parg->name, tp->args, i)) {
1609 		trace_probe_log_err(0, USED_ARG_NAME);
1610 		return -EINVAL;
1611 	}
1612 	ctx->offset = body - arg;
1613 	/* Parse fetch argument */
1614 	return traceprobe_parse_probe_arg_body(body, &tp->size, parg, ctx);
1615 }
1616 
traceprobe_free_probe_arg(struct probe_arg * arg)1617 void traceprobe_free_probe_arg(struct probe_arg *arg)
1618 {
1619 	struct fetch_insn *code = arg->code;
1620 
1621 	while (code && code->op != FETCH_OP_END) {
1622 		if (code->op == FETCH_NOP_SYMBOL ||
1623 		    code->op == FETCH_OP_DATA)
1624 			kfree(code->data);
1625 		code++;
1626 	}
1627 	kfree(arg->code);
1628 	kfree(arg->name);
1629 	kfree(arg->comm);
1630 	kfree(arg->fmt);
1631 }
1632 
argv_has_var_arg(int argc,const char * argv[],int * args_idx,struct traceprobe_parse_context * ctx)1633 static int argv_has_var_arg(int argc, const char *argv[], int *args_idx,
1634 			    struct traceprobe_parse_context *ctx)
1635 {
1636 	int i, found = 0;
1637 
1638 	for (i = 0; i < argc; i++)
1639 		if (str_has_prefix(argv[i], "$arg")) {
1640 			trace_probe_log_set_index(i + 2);
1641 
1642 			if (!tparg_is_function_entry(ctx->flags) &&
1643 			    !tparg_is_function_return(ctx->flags)) {
1644 				trace_probe_log_err(0, NOFENTRY_ARGS);
1645 				return -EINVAL;
1646 			}
1647 
1648 			if (isdigit(argv[i][4])) {
1649 				found = 1;
1650 				continue;
1651 			}
1652 
1653 			if (argv[i][4] != '*') {
1654 				trace_probe_log_err(0, BAD_VAR);
1655 				return -EINVAL;
1656 			}
1657 
1658 			if (*args_idx >= 0 && *args_idx < argc) {
1659 				trace_probe_log_err(0, DOUBLE_ARGS);
1660 				return -EINVAL;
1661 			}
1662 			found = 1;
1663 			*args_idx = i;
1664 		}
1665 
1666 	return found;
1667 }
1668 
sprint_nth_btf_arg(int idx,const char * type,char * buf,int bufsize,struct traceprobe_parse_context * ctx)1669 static int sprint_nth_btf_arg(int idx, const char *type,
1670 			      char *buf, int bufsize,
1671 			      struct traceprobe_parse_context *ctx)
1672 {
1673 	const char *name;
1674 	int ret;
1675 
1676 	if (idx >= ctx->nr_params) {
1677 		trace_probe_log_err(0, NO_BTFARG);
1678 		return -ENOENT;
1679 	}
1680 	name = btf_name_by_offset(ctx->btf, ctx->params[idx].name_off);
1681 	if (!name) {
1682 		trace_probe_log_err(0, NO_BTF_ENTRY);
1683 		return -ENOENT;
1684 	}
1685 	ret = snprintf(buf, bufsize, "%s%s", name, type);
1686 	if (ret >= bufsize) {
1687 		trace_probe_log_err(0, ARGS_2LONG);
1688 		return -E2BIG;
1689 	}
1690 	return ret;
1691 }
1692 
1693 /* Return new_argv which must be freed after use */
traceprobe_expand_meta_args(int argc,const char * argv[],int * new_argc,char * buf,int bufsize,struct traceprobe_parse_context * ctx)1694 const char **traceprobe_expand_meta_args(int argc, const char *argv[],
1695 					 int *new_argc, char *buf, int bufsize,
1696 					 struct traceprobe_parse_context *ctx)
1697 {
1698 	const struct btf_param *params = NULL;
1699 	int i, j, n, used, ret, args_idx = -1;
1700 	const char **new_argv __free(kfree) = NULL;
1701 
1702 	ret = argv_has_var_arg(argc, argv, &args_idx, ctx);
1703 	if (ret < 0)
1704 		return ERR_PTR(ret);
1705 
1706 	if (!ret) {
1707 		*new_argc = argc;
1708 		return NULL;
1709 	}
1710 
1711 	ret = query_btf_context(ctx);
1712 	if (ret < 0 || ctx->nr_params == 0) {
1713 		if (args_idx != -1) {
1714 			/* $arg* requires BTF info */
1715 			trace_probe_log_err(0, NOSUP_BTFARG);
1716 			return (const char **)params;
1717 		}
1718 		*new_argc = argc;
1719 		return NULL;
1720 	}
1721 
1722 	if (args_idx >= 0)
1723 		*new_argc = argc + ctx->nr_params - 1;
1724 	else
1725 		*new_argc = argc;
1726 
1727 	new_argv = kcalloc(*new_argc, sizeof(char *), GFP_KERNEL);
1728 	if (!new_argv)
1729 		return ERR_PTR(-ENOMEM);
1730 
1731 	used = 0;
1732 	for (i = 0, j = 0; i < argc; i++) {
1733 		trace_probe_log_set_index(i + 2);
1734 		if (i == args_idx) {
1735 			for (n = 0; n < ctx->nr_params; n++) {
1736 				ret = sprint_nth_btf_arg(n, "", buf + used,
1737 							 bufsize - used, ctx);
1738 				if (ret < 0)
1739 					return ERR_PTR(ret);
1740 
1741 				new_argv[j++] = buf + used;
1742 				used += ret + 1;
1743 			}
1744 			continue;
1745 		}
1746 
1747 		if (str_has_prefix(argv[i], "$arg")) {
1748 			char *type = NULL;
1749 
1750 			n = simple_strtoul(argv[i] + 4, &type, 10);
1751 			if (type && !(*type == ':' || *type == '\0')) {
1752 				trace_probe_log_err(0, BAD_VAR);
1753 				return ERR_PTR(-ENOENT);
1754 			}
1755 			/* Note: $argN starts from $arg1 */
1756 			ret = sprint_nth_btf_arg(n - 1, type, buf + used,
1757 						 bufsize - used, ctx);
1758 			if (ret < 0)
1759 				return ERR_PTR(ret);
1760 			new_argv[j++] = buf + used;
1761 			used += ret + 1;
1762 		} else
1763 			new_argv[j++] = argv[i];
1764 	}
1765 
1766 	return_ptr(new_argv);
1767 }
1768 
1769 /* @buf: *buf must be equal to NULL. Caller must to free *buf */
traceprobe_expand_dentry_args(int argc,const char * argv[],char ** buf)1770 int traceprobe_expand_dentry_args(int argc, const char *argv[], char **buf)
1771 {
1772 	int i, used, ret;
1773 	const int bufsize = MAX_DENTRY_ARGS_LEN;
1774 	char *tmpbuf __free(kfree) = NULL;
1775 
1776 	if (*buf)
1777 		return -EINVAL;
1778 
1779 	used = 0;
1780 	for (i = 0; i < argc; i++) {
1781 		char *tmp __free(kfree) = NULL;
1782 		char *equal;
1783 		size_t arg_len;
1784 
1785 		if (!glob_match("*:%p[dD]", argv[i]))
1786 			continue;
1787 
1788 		if (!tmpbuf) {
1789 			tmpbuf = kmalloc(bufsize, GFP_KERNEL);
1790 			if (!tmpbuf)
1791 				return -ENOMEM;
1792 		}
1793 
1794 		tmp = kstrdup(argv[i], GFP_KERNEL);
1795 		if (!tmp)
1796 			return -ENOMEM;
1797 
1798 		equal = strchr(tmp, '=');
1799 		if (equal)
1800 			*equal = '\0';
1801 		arg_len = strlen(argv[i]);
1802 		tmp[arg_len - 4] = '\0';
1803 		if (argv[i][arg_len - 1] == 'd')
1804 			ret = snprintf(tmpbuf + used, bufsize - used,
1805 				       "%s%s+0x0(+0x%zx(%s)):string",
1806 				       equal ? tmp : "", equal ? "=" : "",
1807 				       offsetof(struct dentry, d_name.name),
1808 				       equal ? equal + 1 : tmp);
1809 		else
1810 			ret = snprintf(tmpbuf + used, bufsize - used,
1811 				       "%s%s+0x0(+0x%zx(+0x%zx(%s))):string",
1812 				       equal ? tmp : "", equal ? "=" : "",
1813 				       offsetof(struct dentry, d_name.name),
1814 				       offsetof(struct file, f_path.dentry),
1815 				       equal ? equal + 1 : tmp);
1816 
1817 		if (ret >= bufsize - used)
1818 			return -ENOMEM;
1819 		argv[i] = tmpbuf + used;
1820 		used += ret + 1;
1821 	}
1822 
1823 	*buf = no_free_ptr(tmpbuf);
1824 	return 0;
1825 }
1826 
traceprobe_finish_parse(struct traceprobe_parse_context * ctx)1827 void traceprobe_finish_parse(struct traceprobe_parse_context *ctx)
1828 {
1829 	clear_btf_context(ctx);
1830 }
1831 
traceprobe_update_arg(struct probe_arg * arg)1832 int traceprobe_update_arg(struct probe_arg *arg)
1833 {
1834 	struct fetch_insn *code = arg->code;
1835 	long offset;
1836 	char *tmp;
1837 	char c;
1838 	int ret = 0;
1839 
1840 	while (code && code->op != FETCH_OP_END) {
1841 		if (code->op == FETCH_NOP_SYMBOL) {
1842 			if (code[1].op != FETCH_OP_IMM)
1843 				return -EINVAL;
1844 
1845 			tmp = strpbrk(code->data, "+-");
1846 			if (tmp)
1847 				c = *tmp;
1848 			ret = traceprobe_split_symbol_offset(code->data,
1849 							     &offset);
1850 			if (ret)
1851 				return ret;
1852 
1853 			code[1].immediate =
1854 				(unsigned long)kallsyms_lookup_name(code->data);
1855 			if (tmp)
1856 				*tmp = c;
1857 			if (!code[1].immediate)
1858 				return -ENOENT;
1859 			code[1].immediate += offset;
1860 		}
1861 		code++;
1862 	}
1863 	return 0;
1864 }
1865 
1866 /* When len=0, we just calculate the needed length */
1867 #define LEN_OR_ZERO (len ? len - pos : 0)
__set_print_fmt(struct trace_probe * tp,char * buf,int len,enum probe_print_type ptype)1868 static int __set_print_fmt(struct trace_probe *tp, char *buf, int len,
1869 			   enum probe_print_type ptype)
1870 {
1871 	struct probe_arg *parg;
1872 	int i, j;
1873 	int pos = 0;
1874 	const char *fmt, *arg;
1875 
1876 	switch (ptype) {
1877 	case PROBE_PRINT_NORMAL:
1878 		fmt = "(%lx)";
1879 		arg = ", REC->" FIELD_STRING_IP;
1880 		break;
1881 	case PROBE_PRINT_RETURN:
1882 		fmt = "(%lx <- %lx)";
1883 		arg = ", REC->" FIELD_STRING_FUNC ", REC->" FIELD_STRING_RETIP;
1884 		break;
1885 	case PROBE_PRINT_EVENT:
1886 		fmt = "";
1887 		arg = "";
1888 		break;
1889 	default:
1890 		WARN_ON_ONCE(1);
1891 		return 0;
1892 	}
1893 
1894 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", fmt);
1895 
1896 	for (i = 0; i < tp->nr_args; i++) {
1897 		parg = tp->args + i;
1898 		pos += snprintf(buf + pos, LEN_OR_ZERO, " %s=", parg->name);
1899 		if (parg->count) {
1900 			pos += snprintf(buf + pos, LEN_OR_ZERO, "{%s",
1901 					parg->type->fmt);
1902 			for (j = 1; j < parg->count; j++)
1903 				pos += snprintf(buf + pos, LEN_OR_ZERO, ",%s",
1904 						parg->type->fmt);
1905 			pos += snprintf(buf + pos, LEN_OR_ZERO, "}");
1906 		} else
1907 			pos += snprintf(buf + pos, LEN_OR_ZERO, "%s",
1908 					parg->type->fmt);
1909 	}
1910 
1911 	pos += snprintf(buf + pos, LEN_OR_ZERO, "\"%s", arg);
1912 
1913 	for (i = 0; i < tp->nr_args; i++) {
1914 		parg = tp->args + i;
1915 		if (parg->count) {
1916 			if (parg->type->is_string)
1917 				fmt = ", __get_str(%s[%d])";
1918 			else
1919 				fmt = ", REC->%s[%d]";
1920 			for (j = 0; j < parg->count; j++)
1921 				pos += snprintf(buf + pos, LEN_OR_ZERO,
1922 						fmt, parg->name, j);
1923 		} else {
1924 			if (parg->type->is_string)
1925 				fmt = ", __get_str(%s)";
1926 			else
1927 				fmt = ", REC->%s";
1928 			pos += snprintf(buf + pos, LEN_OR_ZERO,
1929 					fmt, parg->name);
1930 		}
1931 	}
1932 
1933 	/* return the length of print_fmt */
1934 	return pos;
1935 }
1936 #undef LEN_OR_ZERO
1937 
traceprobe_set_print_fmt(struct trace_probe * tp,enum probe_print_type ptype)1938 int traceprobe_set_print_fmt(struct trace_probe *tp, enum probe_print_type ptype)
1939 {
1940 	struct trace_event_call *call = trace_probe_event_call(tp);
1941 	int len;
1942 	char *print_fmt;
1943 
1944 	/* First: called with 0 length to calculate the needed length */
1945 	len = __set_print_fmt(tp, NULL, 0, ptype);
1946 	print_fmt = kmalloc(len + 1, GFP_KERNEL);
1947 	if (!print_fmt)
1948 		return -ENOMEM;
1949 
1950 	/* Second: actually write the @print_fmt */
1951 	__set_print_fmt(tp, print_fmt, len + 1, ptype);
1952 	call->print_fmt = print_fmt;
1953 
1954 	return 0;
1955 }
1956 
traceprobe_define_arg_fields(struct trace_event_call * event_call,size_t offset,struct trace_probe * tp)1957 int traceprobe_define_arg_fields(struct trace_event_call *event_call,
1958 				 size_t offset, struct trace_probe *tp)
1959 {
1960 	int ret, i;
1961 
1962 	/* Set argument names as fields */
1963 	for (i = 0; i < tp->nr_args; i++) {
1964 		struct probe_arg *parg = &tp->args[i];
1965 		const char *fmt = parg->type->fmttype;
1966 		int size = parg->type->size;
1967 
1968 		if (parg->fmt)
1969 			fmt = parg->fmt;
1970 		if (parg->count)
1971 			size *= parg->count;
1972 		ret = trace_define_field(event_call, fmt, parg->name,
1973 					 offset + parg->offset, size,
1974 					 parg->type->is_signed,
1975 					 FILTER_OTHER);
1976 		if (ret)
1977 			return ret;
1978 	}
1979 	return 0;
1980 }
1981 
trace_probe_event_free(struct trace_probe_event * tpe)1982 static void trace_probe_event_free(struct trace_probe_event *tpe)
1983 {
1984 	kfree(tpe->class.system);
1985 	kfree(tpe->call.name);
1986 	kfree(tpe->call.print_fmt);
1987 	kfree(tpe);
1988 }
1989 
trace_probe_append(struct trace_probe * tp,struct trace_probe * to)1990 int trace_probe_append(struct trace_probe *tp, struct trace_probe *to)
1991 {
1992 	if (trace_probe_has_sibling(tp))
1993 		return -EBUSY;
1994 
1995 	list_del_init(&tp->list);
1996 	trace_probe_event_free(tp->event);
1997 
1998 	tp->event = to->event;
1999 	list_add_tail(&tp->list, trace_probe_probe_list(to));
2000 
2001 	return 0;
2002 }
2003 
trace_probe_unlink(struct trace_probe * tp)2004 void trace_probe_unlink(struct trace_probe *tp)
2005 {
2006 	list_del_init(&tp->list);
2007 	if (list_empty(trace_probe_probe_list(tp)))
2008 		trace_probe_event_free(tp->event);
2009 	tp->event = NULL;
2010 }
2011 
trace_probe_cleanup(struct trace_probe * tp)2012 void trace_probe_cleanup(struct trace_probe *tp)
2013 {
2014 	int i;
2015 
2016 	for (i = 0; i < tp->nr_args; i++)
2017 		traceprobe_free_probe_arg(&tp->args[i]);
2018 
2019 	if (tp->entry_arg) {
2020 		kfree(tp->entry_arg->code);
2021 		kfree(tp->entry_arg);
2022 		tp->entry_arg = NULL;
2023 	}
2024 
2025 	if (tp->event)
2026 		trace_probe_unlink(tp);
2027 }
2028 
trace_probe_init(struct trace_probe * tp,const char * event,const char * group,bool alloc_filter,int nargs)2029 int trace_probe_init(struct trace_probe *tp, const char *event,
2030 		     const char *group, bool alloc_filter, int nargs)
2031 {
2032 	struct trace_event_call *call;
2033 	size_t size = sizeof(struct trace_probe_event);
2034 	int ret = 0;
2035 
2036 	if (!event || !group)
2037 		return -EINVAL;
2038 
2039 	if (alloc_filter)
2040 		size += sizeof(struct trace_uprobe_filter);
2041 
2042 	tp->event = kzalloc(size, GFP_KERNEL);
2043 	if (!tp->event)
2044 		return -ENOMEM;
2045 
2046 	INIT_LIST_HEAD(&tp->event->files);
2047 	INIT_LIST_HEAD(&tp->event->class.fields);
2048 	INIT_LIST_HEAD(&tp->event->probes);
2049 	INIT_LIST_HEAD(&tp->list);
2050 	list_add(&tp->list, &tp->event->probes);
2051 
2052 	call = trace_probe_event_call(tp);
2053 	call->class = &tp->event->class;
2054 	call->name = kstrdup(event, GFP_KERNEL);
2055 	if (!call->name) {
2056 		ret = -ENOMEM;
2057 		goto error;
2058 	}
2059 
2060 	tp->event->class.system = kstrdup(group, GFP_KERNEL);
2061 	if (!tp->event->class.system) {
2062 		ret = -ENOMEM;
2063 		goto error;
2064 	}
2065 
2066 	tp->nr_args = nargs;
2067 	/* Make sure pointers in args[] are NULL */
2068 	if (nargs)
2069 		memset(tp->args, 0, sizeof(tp->args[0]) * nargs);
2070 
2071 	return 0;
2072 
2073 error:
2074 	trace_probe_cleanup(tp);
2075 	return ret;
2076 }
2077 
2078 static struct trace_event_call *
find_trace_event_call(const char * system,const char * event_name)2079 find_trace_event_call(const char *system, const char *event_name)
2080 {
2081 	struct trace_event_call *tp_event;
2082 	const char *name;
2083 
2084 	list_for_each_entry(tp_event, &ftrace_events, list) {
2085 		if (!tp_event->class->system ||
2086 		    strcmp(system, tp_event->class->system))
2087 			continue;
2088 		name = trace_event_name(tp_event);
2089 		if (!name || strcmp(event_name, name))
2090 			continue;
2091 		return tp_event;
2092 	}
2093 
2094 	return NULL;
2095 }
2096 
trace_probe_register_event_call(struct trace_probe * tp)2097 int trace_probe_register_event_call(struct trace_probe *tp)
2098 {
2099 	struct trace_event_call *call = trace_probe_event_call(tp);
2100 	int ret;
2101 
2102 	lockdep_assert_held(&event_mutex);
2103 
2104 	if (find_trace_event_call(trace_probe_group_name(tp),
2105 				  trace_probe_name(tp)))
2106 		return -EEXIST;
2107 
2108 	ret = register_trace_event(&call->event);
2109 	if (!ret)
2110 		return -ENODEV;
2111 
2112 	ret = trace_add_event_call(call);
2113 	if (ret)
2114 		unregister_trace_event(&call->event);
2115 
2116 	return ret;
2117 }
2118 
trace_probe_add_file(struct trace_probe * tp,struct trace_event_file * file)2119 int trace_probe_add_file(struct trace_probe *tp, struct trace_event_file *file)
2120 {
2121 	struct event_file_link *link;
2122 
2123 	link = kmalloc(sizeof(*link), GFP_KERNEL);
2124 	if (!link)
2125 		return -ENOMEM;
2126 
2127 	link->file = file;
2128 	INIT_LIST_HEAD(&link->list);
2129 	list_add_tail_rcu(&link->list, &tp->event->files);
2130 	trace_probe_set_flag(tp, TP_FLAG_TRACE);
2131 	return 0;
2132 }
2133 
trace_probe_get_file_link(struct trace_probe * tp,struct trace_event_file * file)2134 struct event_file_link *trace_probe_get_file_link(struct trace_probe *tp,
2135 						  struct trace_event_file *file)
2136 {
2137 	struct event_file_link *link;
2138 
2139 	trace_probe_for_each_link(link, tp) {
2140 		if (link->file == file)
2141 			return link;
2142 	}
2143 
2144 	return NULL;
2145 }
2146 
trace_probe_remove_file(struct trace_probe * tp,struct trace_event_file * file)2147 int trace_probe_remove_file(struct trace_probe *tp,
2148 			    struct trace_event_file *file)
2149 {
2150 	struct event_file_link *link;
2151 
2152 	link = trace_probe_get_file_link(tp, file);
2153 	if (!link)
2154 		return -ENOENT;
2155 
2156 	list_del_rcu(&link->list);
2157 	kvfree_rcu_mightsleep(link);
2158 
2159 	if (list_empty(&tp->event->files))
2160 		trace_probe_clear_flag(tp, TP_FLAG_TRACE);
2161 
2162 	return 0;
2163 }
2164 
2165 /*
2166  * Return the smallest index of different type argument (start from 1).
2167  * If all argument types and name are same, return 0.
2168  */
trace_probe_compare_arg_type(struct trace_probe * a,struct trace_probe * b)2169 int trace_probe_compare_arg_type(struct trace_probe *a, struct trace_probe *b)
2170 {
2171 	int i;
2172 
2173 	/* In case of more arguments */
2174 	if (a->nr_args < b->nr_args)
2175 		return a->nr_args + 1;
2176 	if (a->nr_args > b->nr_args)
2177 		return b->nr_args + 1;
2178 
2179 	for (i = 0; i < a->nr_args; i++) {
2180 		if ((b->nr_args <= i) ||
2181 		    ((a->args[i].type != b->args[i].type) ||
2182 		     (a->args[i].count != b->args[i].count) ||
2183 		     strcmp(a->args[i].name, b->args[i].name)))
2184 			return i + 1;
2185 	}
2186 
2187 	return 0;
2188 }
2189 
trace_probe_match_command_args(struct trace_probe * tp,int argc,const char ** argv)2190 bool trace_probe_match_command_args(struct trace_probe *tp,
2191 				    int argc, const char **argv)
2192 {
2193 	char buf[MAX_ARGSTR_LEN + 1];
2194 	int i;
2195 
2196 	if (tp->nr_args < argc)
2197 		return false;
2198 
2199 	for (i = 0; i < argc; i++) {
2200 		snprintf(buf, sizeof(buf), "%s=%s",
2201 			 tp->args[i].name, tp->args[i].comm);
2202 		if (strcmp(buf, argv[i]))
2203 			return false;
2204 	}
2205 	return true;
2206 }
2207 
trace_probe_create(const char * raw_command,int (* createfn)(int,const char **))2208 int trace_probe_create(const char *raw_command, int (*createfn)(int, const char **))
2209 {
2210 	int argc = 0, ret = 0;
2211 	char **argv;
2212 
2213 	argv = argv_split(GFP_KERNEL, raw_command, &argc);
2214 	if (!argv)
2215 		return -ENOMEM;
2216 
2217 	if (argc)
2218 		ret = createfn(argc, (const char **)argv);
2219 
2220 	argv_free(argv);
2221 
2222 	return ret;
2223 }
2224 
trace_probe_print_args(struct trace_seq * s,struct probe_arg * args,int nr_args,u8 * data,void * field)2225 int trace_probe_print_args(struct trace_seq *s, struct probe_arg *args, int nr_args,
2226 		 u8 *data, void *field)
2227 {
2228 	void *p;
2229 	int i, j;
2230 
2231 	for (i = 0; i < nr_args; i++) {
2232 		struct probe_arg *a = args + i;
2233 
2234 		trace_seq_printf(s, " %s=", a->name);
2235 		if (likely(!a->count)) {
2236 			if (!a->type->print(s, data + a->offset, field))
2237 				return -ENOMEM;
2238 			continue;
2239 		}
2240 		trace_seq_putc(s, '{');
2241 		p = data + a->offset;
2242 		for (j = 0; j < a->count; j++) {
2243 			if (!a->type->print(s, p, field))
2244 				return -ENOMEM;
2245 			trace_seq_putc(s, j == a->count - 1 ? '}' : ',');
2246 			p += a->type->size;
2247 		}
2248 	}
2249 	return 0;
2250 }
2251