1 /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ 2 #ifndef __BPF_CORE_READ_H__ 3 #define __BPF_CORE_READ_H__ 4 5 /* 6 * enum bpf_field_info_kind is passed as a second argument into 7 * __builtin_preserve_field_info() built-in to get a specific aspect of 8 * a field, captured as a first argument. __builtin_preserve_field_info(field, 9 * info_kind) returns __u32 integer and produces BTF field relocation, which 10 * is understood and processed by libbpf during BPF object loading. See 11 * selftests/bpf for examples. 12 */ 13 enum bpf_field_info_kind { 14 BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ 15 BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ 16 }; 17 18 /* 19 * Convenience macro to check that field actually exists in target kernel's. 20 * Returns: 21 * 1, if matching field is present in target kernel; 22 * 0, if no matching field found. 23 */ 24 #define bpf_core_field_exists(field) \ 25 __builtin_preserve_field_info(field, BPF_FIELD_EXISTS) 26 27 /* 28 * bpf_core_read() abstracts away bpf_probe_read() call and captures offset 29 * relocation for source address using __builtin_preserve_access_index() 30 * built-in, provided by Clang. 31 * 32 * __builtin_preserve_access_index() takes as an argument an expression of 33 * taking an address of a field within struct/union. It makes compiler emit 34 * a relocation, which records BTF type ID describing root struct/union and an 35 * accessor string which describes exact embedded field that was used to take 36 * an address. See detailed description of this relocation format and 37 * semantics in comments to struct bpf_field_reloc in libbpf_internal.h. 38 * 39 * This relocation allows libbpf to adjust BPF instruction to use correct 40 * actual field offset, based on target kernel BTF type that matches original 41 * (local) BTF, used to record relocation. 42 */ 43 #define bpf_core_read(dst, sz, src) \ 44 bpf_probe_read(dst, sz, \ 45 (const void *)__builtin_preserve_access_index(src)) 46 47 /* 48 * bpf_core_read_str() is a thin wrapper around bpf_probe_read_str() 49 * additionally emitting BPF CO-RE field relocation for specified source 50 * argument. 51 */ 52 #define bpf_core_read_str(dst, sz, src) \ 53 bpf_probe_read_str(dst, sz, \ 54 (const void *)__builtin_preserve_access_index(src)) 55 56 #define ___concat(a, b) a ## b 57 #define ___apply(fn, n) ___concat(fn, n) 58 #define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N 59 60 /* 61 * return number of provided arguments; used for switch-based variadic macro 62 * definitions (see ___last, ___arrow, etc below) 63 */ 64 #define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) 65 /* 66 * return 0 if no arguments are passed, N - otherwise; used for 67 * recursively-defined macros to specify termination (0) case, and generic 68 * (N) case (e.g., ___read_ptrs, ___core_read) 69 */ 70 #define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0) 71 72 #define ___last1(x) x 73 #define ___last2(a, x) x 74 #define ___last3(a, b, x) x 75 #define ___last4(a, b, c, x) x 76 #define ___last5(a, b, c, d, x) x 77 #define ___last6(a, b, c, d, e, x) x 78 #define ___last7(a, b, c, d, e, f, x) x 79 #define ___last8(a, b, c, d, e, f, g, x) x 80 #define ___last9(a, b, c, d, e, f, g, h, x) x 81 #define ___last10(a, b, c, d, e, f, g, h, i, x) x 82 #define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__) 83 84 #define ___nolast2(a, _) a 85 #define ___nolast3(a, b, _) a, b 86 #define ___nolast4(a, b, c, _) a, b, c 87 #define ___nolast5(a, b, c, d, _) a, b, c, d 88 #define ___nolast6(a, b, c, d, e, _) a, b, c, d, e 89 #define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f 90 #define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g 91 #define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h 92 #define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i 93 #define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__) 94 95 #define ___arrow1(a) a 96 #define ___arrow2(a, b) a->b 97 #define ___arrow3(a, b, c) a->b->c 98 #define ___arrow4(a, b, c, d) a->b->c->d 99 #define ___arrow5(a, b, c, d, e) a->b->c->d->e 100 #define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f 101 #define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g 102 #define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h 103 #define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i 104 #define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j 105 #define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__) 106 107 #define ___type(...) typeof(___arrow(__VA_ARGS__)) 108 109 #define ___read(read_fn, dst, src_type, src, accessor) \ 110 read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) 111 112 /* "recursively" read a sequence of inner pointers using local __t var */ 113 #define ___rd_first(src, a) ___read(bpf_core_read, &__t, ___type(src), src, a); 114 #define ___rd_last(...) \ 115 ___read(bpf_core_read, &__t, \ 116 ___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__)); 117 #define ___rd_p1(...) const void *__t; ___rd_first(__VA_ARGS__) 118 #define ___rd_p2(...) ___rd_p1(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 119 #define ___rd_p3(...) ___rd_p2(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 120 #define ___rd_p4(...) ___rd_p3(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 121 #define ___rd_p5(...) ___rd_p4(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 122 #define ___rd_p6(...) ___rd_p5(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 123 #define ___rd_p7(...) ___rd_p6(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 124 #define ___rd_p8(...) ___rd_p7(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 125 #define ___rd_p9(...) ___rd_p8(___nolast(__VA_ARGS__)) ___rd_last(__VA_ARGS__) 126 #define ___read_ptrs(src, ...) \ 127 ___apply(___rd_p, ___narg(__VA_ARGS__))(src, __VA_ARGS__) 128 129 #define ___core_read0(fn, dst, src, a) \ 130 ___read(fn, dst, ___type(src), src, a); 131 #define ___core_readN(fn, dst, src, ...) \ 132 ___read_ptrs(src, ___nolast(__VA_ARGS__)) \ 133 ___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \ 134 ___last(__VA_ARGS__)); 135 #define ___core_read(fn, dst, src, a, ...) \ 136 ___apply(___core_read, ___empty(__VA_ARGS__))(fn, dst, \ 137 src, a, ##__VA_ARGS__) 138 139 /* 140 * BPF_CORE_READ_INTO() is a more performance-conscious variant of 141 * BPF_CORE_READ(), in which final field is read into user-provided storage. 142 * See BPF_CORE_READ() below for more details on general usage. 143 */ 144 #define BPF_CORE_READ_INTO(dst, src, a, ...) \ 145 ({ \ 146 ___core_read(bpf_core_read, dst, src, a, ##__VA_ARGS__) \ 147 }) 148 149 /* 150 * BPF_CORE_READ_STR_INTO() does same "pointer chasing" as 151 * BPF_CORE_READ() for intermediate pointers, but then executes (and returns 152 * corresponding error code) bpf_core_read_str() for final string read. 153 */ 154 #define BPF_CORE_READ_STR_INTO(dst, src, a, ...) \ 155 ({ \ 156 ___core_read(bpf_core_read_str, dst, src, a, ##__VA_ARGS__) \ 157 }) 158 159 /* 160 * BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially 161 * when there are few pointer chasing steps. 162 * E.g., what in non-BPF world (or in BPF w/ BCC) would be something like: 163 * int x = s->a.b.c->d.e->f->g; 164 * can be succinctly achieved using BPF_CORE_READ as: 165 * int x = BPF_CORE_READ(s, a.b.c, d.e, f, g); 166 * 167 * BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF 168 * CO-RE relocatable bpf_probe_read() wrapper) calls, logically equivalent to: 169 * 1. const void *__t = s->a.b.c; 170 * 2. __t = __t->d.e; 171 * 3. __t = __t->f; 172 * 4. return __t->g; 173 * 174 * Equivalence is logical, because there is a heavy type casting/preservation 175 * involved, as well as all the reads are happening through bpf_probe_read() 176 * calls using __builtin_preserve_access_index() to emit CO-RE relocations. 177 * 178 * N.B. Only up to 9 "field accessors" are supported, which should be more 179 * than enough for any practical purpose. 180 */ 181 #define BPF_CORE_READ(src, a, ...) \ 182 ({ \ 183 ___type(src, a, ##__VA_ARGS__) __r; \ 184 BPF_CORE_READ_INTO(&__r, src, a, ##__VA_ARGS__); \ 185 __r; \ 186 }) 187 188 #endif 189 190