1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 /* Copyright (c) 2022 Meta Platforms, Inc. and affiliates. */ 3 #ifndef __USDT_BPF_H__ 4 #define __USDT_BPF_H__ 5 6 #include <linux/errno.h> 7 #include "bpf_helpers.h" 8 #include "bpf_tracing.h" 9 10 /* Below types and maps are internal implementation details of libbpf's USDT 11 * support and are subjects to change. Also, bpf_usdt_xxx() API helpers should 12 * be considered an unstable API as well and might be adjusted based on user 13 * feedback from using libbpf's USDT support in production. 14 */ 15 16 /* User can override BPF_USDT_MAX_SPEC_CNT to change default size of internal 17 * map that keeps track of USDT argument specifications. This might be 18 * necessary if there are a lot of USDT attachments. 19 */ 20 #ifndef BPF_USDT_MAX_SPEC_CNT 21 #define BPF_USDT_MAX_SPEC_CNT 256 22 #endif 23 /* User can override BPF_USDT_MAX_IP_CNT to change default size of internal 24 * map that keeps track of IP (memory address) mapping to USDT argument 25 * specification. 26 * Note, if kernel supports BPF cookies, this map is not used and could be 27 * resized all the way to 1 to save a bit of memory. 28 */ 29 #ifndef BPF_USDT_MAX_IP_CNT 30 #define BPF_USDT_MAX_IP_CNT (4 * BPF_USDT_MAX_SPEC_CNT) 31 #endif 32 33 enum __bpf_usdt_arg_type { 34 BPF_USDT_ARG_CONST, 35 BPF_USDT_ARG_REG, 36 BPF_USDT_ARG_REG_DEREF, 37 }; 38 39 struct __bpf_usdt_arg_spec { 40 /* u64 scalar interpreted depending on arg_type, see below */ 41 __u64 val_off; 42 /* arg location case, see bpf_usdt_arg() for details */ 43 enum __bpf_usdt_arg_type arg_type; 44 /* offset of referenced register within struct pt_regs */ 45 short reg_off; 46 /* whether arg should be interpreted as signed value */ 47 bool arg_signed; 48 /* number of bits that need to be cleared and, optionally, 49 * sign-extended to cast arguments that are 1, 2, or 4 bytes 50 * long into final 8-byte u64/s64 value returned to user 51 */ 52 char arg_bitshift; 53 }; 54 55 /* should match USDT_MAX_ARG_CNT in usdt.c exactly */ 56 #define BPF_USDT_MAX_ARG_CNT 12 57 struct __bpf_usdt_spec { 58 struct __bpf_usdt_arg_spec args[BPF_USDT_MAX_ARG_CNT]; 59 __u64 usdt_cookie; 60 short arg_cnt; 61 }; 62 63 struct { 64 __uint(type, BPF_MAP_TYPE_ARRAY); 65 __uint(max_entries, BPF_USDT_MAX_SPEC_CNT); 66 __type(key, int); 67 __type(value, struct __bpf_usdt_spec); 68 } __bpf_usdt_specs SEC(".maps") __weak; 69 70 struct { 71 __uint(type, BPF_MAP_TYPE_HASH); 72 __uint(max_entries, BPF_USDT_MAX_IP_CNT); 73 __type(key, long); 74 __type(value, __u32); 75 } __bpf_usdt_ip_to_spec_id SEC(".maps") __weak; 76 77 extern const _Bool LINUX_HAS_BPF_COOKIE __kconfig; 78 79 static __always_inline 80 int __bpf_usdt_spec_id(struct pt_regs *ctx) 81 { 82 if (!LINUX_HAS_BPF_COOKIE) { 83 long ip = PT_REGS_IP(ctx); 84 int *spec_id_ptr; 85 86 spec_id_ptr = bpf_map_lookup_elem(&__bpf_usdt_ip_to_spec_id, &ip); 87 return spec_id_ptr ? *spec_id_ptr : -ESRCH; 88 } 89 90 return bpf_get_attach_cookie(ctx); 91 } 92 93 /* Return number of USDT arguments defined for currently traced USDT. */ 94 __weak __hidden 95 int bpf_usdt_arg_cnt(struct pt_regs *ctx) 96 { 97 struct __bpf_usdt_spec *spec; 98 int spec_id; 99 100 spec_id = __bpf_usdt_spec_id(ctx); 101 if (spec_id < 0) 102 return -ESRCH; 103 104 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); 105 if (!spec) 106 return -ESRCH; 107 108 return spec->arg_cnt; 109 } 110 111 /* Returns the size in bytes of the #*arg_num* (zero-indexed) USDT argument. 112 * Returns negative error if argument is not found or arg_num is invalid. 113 */ 114 static __always_inline 115 int bpf_usdt_arg_size(struct pt_regs *ctx, __u64 arg_num) 116 { 117 struct __bpf_usdt_arg_spec *arg_spec; 118 struct __bpf_usdt_spec *spec; 119 int spec_id; 120 121 spec_id = __bpf_usdt_spec_id(ctx); 122 if (spec_id < 0) 123 return -ESRCH; 124 125 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); 126 if (!spec) 127 return -ESRCH; 128 129 if (arg_num >= BPF_USDT_MAX_ARG_CNT) 130 return -ENOENT; 131 barrier_var(arg_num); 132 if (arg_num >= spec->arg_cnt) 133 return -ENOENT; 134 135 arg_spec = &spec->args[arg_num]; 136 137 /* arg_spec->arg_bitshift = 64 - arg_sz * 8 138 * so: arg_sz = (64 - arg_spec->arg_bitshift) / 8 139 */ 140 return (unsigned int)(64 - arg_spec->arg_bitshift) / 8; 141 } 142 143 /* Fetch USDT argument #*arg_num* (zero-indexed) and put its value into *res. 144 * Returns 0 on success; negative error, otherwise. 145 * On error *res is guaranteed to be set to zero. 146 */ 147 __weak __hidden 148 int bpf_usdt_arg(struct pt_regs *ctx, __u64 arg_num, long *res) 149 { 150 struct __bpf_usdt_spec *spec; 151 struct __bpf_usdt_arg_spec *arg_spec; 152 unsigned long val; 153 int err, spec_id; 154 155 *res = 0; 156 157 spec_id = __bpf_usdt_spec_id(ctx); 158 if (spec_id < 0) 159 return -ESRCH; 160 161 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); 162 if (!spec) 163 return -ESRCH; 164 165 if (arg_num >= BPF_USDT_MAX_ARG_CNT) 166 return -ENOENT; 167 barrier_var(arg_num); 168 if (arg_num >= spec->arg_cnt) 169 return -ENOENT; 170 171 arg_spec = &spec->args[arg_num]; 172 switch (arg_spec->arg_type) { 173 case BPF_USDT_ARG_CONST: 174 /* Arg is just a constant ("-4@$-9" in USDT arg spec). 175 * value is recorded in arg_spec->val_off directly. 176 */ 177 val = arg_spec->val_off; 178 break; 179 case BPF_USDT_ARG_REG: 180 /* Arg is in a register (e.g, "8@%rax" in USDT arg spec), 181 * so we read the contents of that register directly from 182 * struct pt_regs. To keep things simple user-space parts 183 * record offsetof(struct pt_regs, <regname>) in arg_spec->reg_off. 184 */ 185 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off); 186 if (err) 187 return err; 188 break; 189 case BPF_USDT_ARG_REG_DEREF: 190 /* Arg is in memory addressed by register, plus some offset 191 * (e.g., "-4@-1204(%rbp)" in USDT arg spec). Register is 192 * identified like with BPF_USDT_ARG_REG case, and the offset 193 * is in arg_spec->val_off. We first fetch register contents 194 * from pt_regs, then do another user-space probe read to 195 * fetch argument value itself. 196 */ 197 err = bpf_probe_read_kernel(&val, sizeof(val), (void *)ctx + arg_spec->reg_off); 198 if (err) 199 return err; 200 err = bpf_probe_read_user(&val, sizeof(val), (void *)val + arg_spec->val_off); 201 if (err) 202 return err; 203 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 204 val >>= arg_spec->arg_bitshift; 205 #endif 206 break; 207 default: 208 return -EINVAL; 209 } 210 211 /* cast arg from 1, 2, or 4 bytes to final 8 byte size clearing 212 * necessary upper arg_bitshift bits, with sign extension if argument 213 * is signed 214 */ 215 val <<= arg_spec->arg_bitshift; 216 if (arg_spec->arg_signed) 217 val = ((long)val) >> arg_spec->arg_bitshift; 218 else 219 val = val >> arg_spec->arg_bitshift; 220 *res = val; 221 return 0; 222 } 223 224 /* Retrieve user-specified cookie value provided during attach as 225 * bpf_usdt_opts.usdt_cookie. This serves the same purpose as BPF cookie 226 * returned by bpf_get_attach_cookie(). Libbpf's support for USDT is itself 227 * utilizing BPF cookies internally, so user can't use BPF cookie directly 228 * for USDT programs and has to use bpf_usdt_cookie() API instead. 229 */ 230 __weak __hidden 231 long bpf_usdt_cookie(struct pt_regs *ctx) 232 { 233 struct __bpf_usdt_spec *spec; 234 int spec_id; 235 236 spec_id = __bpf_usdt_spec_id(ctx); 237 if (spec_id < 0) 238 return 0; 239 240 spec = bpf_map_lookup_elem(&__bpf_usdt_specs, &spec_id); 241 if (!spec) 242 return 0; 243 244 return spec->usdt_cookie; 245 } 246 247 /* we rely on ___bpf_apply() and ___bpf_narg() macros already defined in bpf_tracing.h */ 248 #define ___bpf_usdt_args0() ctx 249 #define ___bpf_usdt_args1(x) ___bpf_usdt_args0(), ({ long _x; bpf_usdt_arg(ctx, 0, &_x); _x; }) 250 #define ___bpf_usdt_args2(x, args...) ___bpf_usdt_args1(args), ({ long _x; bpf_usdt_arg(ctx, 1, &_x); _x; }) 251 #define ___bpf_usdt_args3(x, args...) ___bpf_usdt_args2(args), ({ long _x; bpf_usdt_arg(ctx, 2, &_x); _x; }) 252 #define ___bpf_usdt_args4(x, args...) ___bpf_usdt_args3(args), ({ long _x; bpf_usdt_arg(ctx, 3, &_x); _x; }) 253 #define ___bpf_usdt_args5(x, args...) ___bpf_usdt_args4(args), ({ long _x; bpf_usdt_arg(ctx, 4, &_x); _x; }) 254 #define ___bpf_usdt_args6(x, args...) ___bpf_usdt_args5(args), ({ long _x; bpf_usdt_arg(ctx, 5, &_x); _x; }) 255 #define ___bpf_usdt_args7(x, args...) ___bpf_usdt_args6(args), ({ long _x; bpf_usdt_arg(ctx, 6, &_x); _x; }) 256 #define ___bpf_usdt_args8(x, args...) ___bpf_usdt_args7(args), ({ long _x; bpf_usdt_arg(ctx, 7, &_x); _x; }) 257 #define ___bpf_usdt_args9(x, args...) ___bpf_usdt_args8(args), ({ long _x; bpf_usdt_arg(ctx, 8, &_x); _x; }) 258 #define ___bpf_usdt_args10(x, args...) ___bpf_usdt_args9(args), ({ long _x; bpf_usdt_arg(ctx, 9, &_x); _x; }) 259 #define ___bpf_usdt_args11(x, args...) ___bpf_usdt_args10(args), ({ long _x; bpf_usdt_arg(ctx, 10, &_x); _x; }) 260 #define ___bpf_usdt_args12(x, args...) ___bpf_usdt_args11(args), ({ long _x; bpf_usdt_arg(ctx, 11, &_x); _x; }) 261 #define ___bpf_usdt_args(args...) ___bpf_apply(___bpf_usdt_args, ___bpf_narg(args))(args) 262 263 /* 264 * BPF_USDT serves the same purpose for USDT handlers as BPF_PROG for 265 * tp_btf/fentry/fexit BPF programs and BPF_KPROBE for kprobes. 266 * Original struct pt_regs * context is preserved as 'ctx' argument. 267 */ 268 #define BPF_USDT(name, args...) \ 269 name(struct pt_regs *ctx); \ 270 static __always_inline typeof(name(0)) \ 271 ____##name(struct pt_regs *ctx, ##args); \ 272 typeof(name(0)) name(struct pt_regs *ctx) \ 273 { \ 274 _Pragma("GCC diagnostic push") \ 275 _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ 276 return ____##name(___bpf_usdt_args(args)); \ 277 _Pragma("GCC diagnostic pop") \ 278 } \ 279 static __always_inline typeof(name(0)) \ 280 ____##name(struct pt_regs *ctx, ##args) 281 282 #endif /* __USDT_BPF_H__ */ 283