1 // SPDX-License-Identifier: GPL-2.0
40 	 * 3. map: 1 always, pointer points to additional struct with number
85  * Map of C-string key/value pairs with fixed maximum capacity. Each map has
87  * way appropriate. Map is "write-only", there is no way to get data out of
88  * map. Map is intended to be used to provide metadata for profilers and is
89  * not to be used for internal in-app communication. All methods are
90  * thread-safe.
100 	/* number of used entries in map */
116 #define TLS_NOT_SET -1
128 	 * - -1 (TLS_NOT_SET) - no metavariable;
129 	 * - 0 (TLS_LOCAL_EXEC) - Local Executable mode;
130 	 * - 1 (TLS_IMM_EXEC) - Immediate Executable mode;
131 	 * - 2 (TLS_GENERAL_DYN) - General Dynamic mode;
138 	 * TLS_LOCAL_EXEC: offset from thread pointer (fs:0 for x86-64,
159 	 * cnt <0 - map value isn't set;
160 	 * 0 - map has id set, but no key/value entries
183 	/* if map_descrs[i].cnt == -1, metavar is not present/set */
186 	 * payload has compactly packed values of str and map variables in the
265  * For x86-64, this is mapped onto two entries in GOT.
282 	 * - -1 (TLS_NOT_SET), if no metavar is present;  in calc_location()
283 	 * - 0 (TLS_LOCAL_EXEC), if metavar uses Local Executable mode of TLS  in calc_location()
284 	 * (offset from fs:0 for x86-64 or tpidr_el0 for aarch64);  in calc_location()
285 	 * - 1 (TLS_IMM_EXEC), if metavar uses Immediate Executable mode of TLS;  in calc_location()
286 	 * - 2 (TLS_GENERAL_DYN), if metavar uses General Dynamic mode of TLS;  in calc_location()
289 	 * to get NULL for "no metavar" location, or correct pointer for local  in calc_location()
292 	if (loc->tls_mode <= TLS_LOCAL_EXEC) {  in calc_location()
295 		void *addr = tls_base + loc->offset;  in calc_location()
296 		/* multiply by (tls_mode + 1) to get NULL, if we have no  in calc_location()
298 		return (void *)((loc->tls_mode + 1) * (int64_t)addr);  in calc_location()
304 	 *  - loc->offset is pointing to a GOT entry containing fixed offset  in calc_location()
308 	 *  - loc->offset is pointing to a beginning of double GOT entries;  in calc_location()
309 	 *  - (for aarch64 only) second entry points to tls_index_t struct;  in calc_location()
310 	 *  - (for x86-64 only) two GOT entries are already tls_index_t;  in calc_location()
311 	 *  - tls_index_t->module is used to find start of TLS section in  in calc_location()
313 	 *  - tls_index_t->offset provides offset within that TLS section,  in calc_location()
321 			    (void *)loc->offset);  in calc_location()
324 		/* dtv = ((struct tcbhead *)tls_base)->dtv[tls_index.module] */  in calc_location()
326 				    &((struct tcbhead *)tls_base)->dtv);  in calc_location()
332 	/* if pointer has (void *)-1 value, then TLS wasn't initialized yet */  in calc_location()
333 	return tls_ptr && tls_ptr != (void *)-1  in calc_location()
348 	void *location = calc_location(&cfg->int_locs[idx], tls_base);  in read_int_var()
353 	data->int_vals[idx] = value->val;  in read_int_var()
354 	if (value->header.len)  in read_int_var()
355 		data->int_vals_set_mask |= (1 << idx);  in read_int_var()
367 	data->str_lens[idx] = 0;  in read_str_var()
368 	location = calc_location(&cfg->str_locs[idx], tls_base);  in read_str_var()
373 	len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, value->ptr);  in read_str_var()
384 	data->str_lens[idx] = len;  in read_str_var()
394 	struct strobe_map_descr* descr = &data->map_descrs[idx];  in read_map_var()
395 	struct strobe_map_raw map;  in read_map_var()  local
399 	descr->tag_len = 0; /* presume no tag is set */  in read_map_var()
400 	descr->cnt = -1; /* presume no value is set */  in read_map_var()
402 	location = calc_location(&cfg->map_locs[idx], tls_base);  in read_map_var()
407 	if (bpf_probe_read_user(&map, sizeof(struct strobe_map_raw), value->ptr))  in read_map_var()
410 	descr->id = map.id;  in read_map_var()
411 	descr->cnt = map.cnt;  in read_map_var()
412 	if (cfg->req_meta_idx == idx) {  in read_map_var()
413 		data->req_id = map.id;  in read_map_var()
414 		data->req_meta_valid = 1;  in read_map_var()
417 	len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN, map.tag);  in read_map_var()
419 		descr->tag_len = len;  in read_map_var()
429 		if (i >= map.cnt)  in read_map_var()
432 		descr->key_lens[i] = 0;  in read_map_var()
433 		len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,  in read_map_var()
434 					      map.entries[i].key);  in read_map_var()
436 			descr->key_lens[i] = len;  in read_map_var()
439 		descr->val_lens[i] = 0;  in read_map_var()
440 		len = bpf_probe_read_user_str(&data->payload[off], STROBE_MAX_STR_LEN,  in read_map_var()
441 					      map.entries[i].val);  in read_map_var()
443 			descr->val_lens[i] = len;  in read_map_var()
470 	/* lose precision info for ctx->payload_off, verifier won't track  in read_var_callback()
473 	ctx->payload_off ^= index;  in read_var_callback()
474 	barrier_var(ctx->payload_off);  in read_var_callback()
475 	ctx->payload_off ^= index;  in read_var_callback()
476 	switch (ctx->type) {  in read_var_callback()
480 		read_int_var(ctx->cfg, index, ctx->tls_base, ctx->value, ctx->data);  in read_var_callback()
485 		if (ctx->payload_off > sizeof(ctx->data->payload) - READ_MAP_VAR_PAYLOAD_CAP)  in read_var_callback()
487 		ctx->payload_off = read_map_var(ctx->cfg, index, ctx->tls_base,  in read_var_callback()
488 						ctx->value, ctx->data, ctx->payload_off);  in read_var_callback()
493 		if (ctx->payload_off > sizeof(ctx->data->payload) - STROBE_MAX_STR_LEN)  in read_var_callback()
495 		ctx->payload_off = read_str_var(ctx->cfg, index, ctx->tls_base,  in read_var_callback()
496 						ctx->value, ctx->data, ctx->payload_off);  in read_var_callback()
504  * read_strobe_meta returns NULL, if no metadata was read; otherwise returns
525 	data->int_vals_set_mask = 0;  in read_strobe_meta()
526 	data->req_meta_valid = 0;  in read_strobe_meta()
530 	 * tls_base = (void *)task->thread.fsbase;  in read_strobe_meta()
561 	if (payload_off > sizeof(data->payload))  in read_strobe_meta()
562 		payload_off = sizeof(data->payload);  in read_strobe_meta()
594 	return &data->payload[payload_off];  in read_strobe_meta()
610 	sample->pid = pid;  in on_event()
611 	bpf_get_current_comm(&sample->comm, TASK_COMM_LEN);  in on_event()
613 	sample->ktime = ktime_ns;  in on_event()
616 	sample_end = read_strobe_meta(task, &sample->metadata);  in on_event()
617 	sample->has_meta = sample_end != NULL;  in on_event()
618 	sample_end = sample_end ? : &sample->metadata;  in on_event()
621 		sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_1, 0);  in on_event()
622 		sample->user_stack_id = bpf_get_stackid(ctx, &stacks_1, BPF_F_USER_STACK);  in on_event()
624 		sample->kernel_stack_id = bpf_get_stackid(ctx, &stacks_0, 0);  in on_event()
625 		sample->user_stack_id = bpf_get_stackid(ctx, &stacks_0, BPF_F_USER_STACK);  in on_event()
628 	uint64_t sample_size = sample_end - (void *)sample;  in on_event()