1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) 2 3 #ifndef _GNU_SOURCE 4 #define _GNU_SOURCE 5 #endif 6 7 #include <errno.h> 8 #include <stdlib.h> 9 #include <linux/err.h> 10 #include <linux/kernel.h> 11 #include <bpf/bpf.h> 12 #include "bpf-utils.h" 13 #include "debug.h" 14 15 struct bpil_array_desc { 16 int array_offset; /* e.g. offset of jited_prog_insns */ 17 int count_offset; /* e.g. offset of jited_prog_len */ 18 int size_offset; /* > 0: offset of rec size, 19 * < 0: fix size of -size_offset 20 */ 21 }; 22 23 static const struct bpil_array_desc bpil_array_desc[] = { 24 [PERF_BPIL_JITED_INSNS] = { 25 offsetof(struct bpf_prog_info, jited_prog_insns), 26 offsetof(struct bpf_prog_info, jited_prog_len), 27 -1, 28 }, 29 [PERF_BPIL_XLATED_INSNS] = { 30 offsetof(struct bpf_prog_info, xlated_prog_insns), 31 offsetof(struct bpf_prog_info, xlated_prog_len), 32 -1, 33 }, 34 [PERF_BPIL_MAP_IDS] = { 35 offsetof(struct bpf_prog_info, map_ids), 36 offsetof(struct bpf_prog_info, nr_map_ids), 37 -(int)sizeof(__u32), 38 }, 39 [PERF_BPIL_JITED_KSYMS] = { 40 offsetof(struct bpf_prog_info, jited_ksyms), 41 offsetof(struct bpf_prog_info, nr_jited_ksyms), 42 -(int)sizeof(__u64), 43 }, 44 [PERF_BPIL_JITED_FUNC_LENS] = { 45 offsetof(struct bpf_prog_info, jited_func_lens), 46 offsetof(struct bpf_prog_info, nr_jited_func_lens), 47 -(int)sizeof(__u32), 48 }, 49 [PERF_BPIL_FUNC_INFO] = { 50 offsetof(struct bpf_prog_info, func_info), 51 offsetof(struct bpf_prog_info, nr_func_info), 52 offsetof(struct bpf_prog_info, func_info_rec_size), 53 }, 54 [PERF_BPIL_LINE_INFO] = { 55 offsetof(struct bpf_prog_info, line_info), 56 offsetof(struct bpf_prog_info, nr_line_info), 57 offsetof(struct bpf_prog_info, line_info_rec_size), 58 }, 59 [PERF_BPIL_JITED_LINE_INFO] = { 60 offsetof(struct bpf_prog_info, jited_line_info), 61 offsetof(struct bpf_prog_info, nr_jited_line_info), 62 offsetof(struct bpf_prog_info, jited_line_info_rec_size), 63 }, 64 [PERF_BPIL_PROG_TAGS] = { 65 offsetof(struct bpf_prog_info, prog_tags), 66 offsetof(struct bpf_prog_info, nr_prog_tags), 67 -(int)sizeof(__u8) * BPF_TAG_SIZE, 68 }, 69 70 }; 71 72 static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, 73 int offset) 74 { 75 __u32 *array = (__u32 *)info; 76 77 if (offset >= 0) 78 return array[offset / sizeof(__u32)]; 79 return -(int)offset; 80 } 81 82 static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, 83 int offset) 84 { 85 __u64 *array = (__u64 *)info; 86 87 if (offset >= 0) 88 return array[offset / sizeof(__u64)]; 89 return -(int)offset; 90 } 91 92 static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, 93 __u32 val) 94 { 95 __u32 *array = (__u32 *)info; 96 97 if (offset >= 0) 98 array[offset / sizeof(__u32)] = val; 99 } 100 101 static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, 102 __u64 val) 103 { 104 __u64 *array = (__u64 *)info; 105 106 if (offset >= 0) 107 array[offset / sizeof(__u64)] = val; 108 } 109 110 struct perf_bpil * 111 get_bpf_prog_info_linear(int fd, __u64 arrays) 112 { 113 struct bpf_prog_info info = {}; 114 struct perf_bpil *info_linear; 115 __u32 info_len = sizeof(info); 116 __u32 data_len = 0; 117 int i, err; 118 __u8 *ptr; 119 120 if (arrays >> PERF_BPIL_LAST_ARRAY) 121 return ERR_PTR(-EINVAL); 122 123 /* step 1: get array dimensions */ 124 err = bpf_obj_get_info_by_fd(fd, &info, &info_len); 125 if (err) { 126 pr_debug("can't get prog info: %s", strerror(errno)); 127 return ERR_PTR(-EFAULT); 128 } 129 if (info.type >= __MAX_BPF_PROG_TYPE) 130 pr_debug("%s:%d: unexpected program type %u\n", __func__, __LINE__, info.type); 131 132 /* step 2: calculate total size of all arrays */ 133 for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { 134 const struct bpil_array_desc *desc = &bpil_array_desc[i]; 135 bool include_array = (arrays & (1UL << i)) > 0; 136 __u32 count, size; 137 138 /* kernel is too old to support this field */ 139 if (info_len < desc->array_offset + sizeof(__u32) || 140 info_len < desc->count_offset + sizeof(__u32) || 141 (desc->size_offset > 0 && info_len < (__u32)desc->size_offset)) 142 include_array = false; 143 144 if (!include_array) { 145 arrays &= ~(1UL << i); /* clear the bit */ 146 continue; 147 } 148 149 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 150 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 151 152 data_len += roundup(count * size, sizeof(__u64)); 153 } 154 155 /* step 3: allocate continuous memory */ 156 info_linear = malloc(sizeof(struct perf_bpil) + data_len); 157 if (!info_linear) 158 return ERR_PTR(-ENOMEM); 159 160 /* step 4: fill data to info_linear->info */ 161 info_linear->arrays = arrays; 162 memset(&info_linear->info, 0, sizeof(info)); 163 ptr = info_linear->data; 164 165 for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { 166 const struct bpil_array_desc *desc = &bpil_array_desc[i]; 167 __u32 count, size; 168 169 if ((arrays & (1UL << i)) == 0) 170 continue; 171 172 count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 173 size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 174 bpf_prog_info_set_offset_u32(&info_linear->info, 175 desc->count_offset, count); 176 bpf_prog_info_set_offset_u32(&info_linear->info, 177 desc->size_offset, size); 178 assert(ptr >= info_linear->data); 179 assert(ptr < &info_linear->data[data_len]); 180 bpf_prog_info_set_offset_u64(&info_linear->info, 181 desc->array_offset, 182 ptr_to_u64(ptr)); 183 ptr += roundup(count * size, sizeof(__u64)); 184 } 185 186 /* step 5: call syscall again to get required arrays */ 187 err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); 188 if (err) { 189 pr_debug("can't get prog info: %s", strerror(errno)); 190 free(info_linear); 191 return ERR_PTR(-EFAULT); 192 } 193 if (info_linear->info.type >= __MAX_BPF_PROG_TYPE) { 194 pr_debug("%s:%d: unexpected program type %u\n", 195 __func__, __LINE__, info_linear->info.type); 196 } 197 198 /* step 6: verify the data */ 199 ptr = info_linear->data; 200 for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { 201 const struct bpil_array_desc *desc = &bpil_array_desc[i]; 202 __u32 count1, count2, size1, size2; 203 __u64 ptr2; 204 205 if ((arrays & (1UL << i)) == 0) 206 continue; 207 208 count1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); 209 count2 = bpf_prog_info_read_offset_u32(&info_linear->info, 210 desc->count_offset); 211 if (count1 != count2) { 212 pr_warning("%s: mismatch in element count %u vs %u\n", __func__, count1, count2); 213 free(info_linear); 214 return ERR_PTR(-ERANGE); 215 } 216 217 size1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); 218 size2 = bpf_prog_info_read_offset_u32(&info_linear->info, 219 desc->size_offset); 220 if (size1 != size2) { 221 pr_warning("%s: mismatch in rec size %u vs %u\n", __func__, size1, size2); 222 free(info_linear); 223 return ERR_PTR(-ERANGE); 224 } 225 ptr2 = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); 226 if (ptr_to_u64(ptr) != ptr2) { 227 pr_warning("%s: mismatch in array %p vs %llx\n", __func__, ptr, ptr2); 228 free(info_linear); 229 return ERR_PTR(-ERANGE); 230 } 231 ptr += roundup(count1 * size1, sizeof(__u64)); 232 } 233 234 /* step 7: update info_len and data_len */ 235 info_linear->info_len = sizeof(struct bpf_prog_info); 236 info_linear->data_len = data_len; 237 238 return info_linear; 239 } 240 241 void bpil_addr_to_offs(struct perf_bpil *info_linear) 242 { 243 int i; 244 245 for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { 246 const struct bpil_array_desc *desc = &bpil_array_desc[i]; 247 __u64 addr, offs; 248 249 if ((info_linear->arrays & (1UL << i)) == 0) 250 continue; 251 252 addr = bpf_prog_info_read_offset_u64(&info_linear->info, 253 desc->array_offset); 254 offs = addr - ptr_to_u64(info_linear->data); 255 bpf_prog_info_set_offset_u64(&info_linear->info, 256 desc->array_offset, offs); 257 } 258 } 259 260 void bpil_offs_to_addr(struct perf_bpil *info_linear) 261 { 262 int i; 263 264 for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { 265 const struct bpil_array_desc *desc = &bpil_array_desc[i]; 266 __u64 addr, offs; 267 268 if ((info_linear->arrays & (1UL << i)) == 0) 269 continue; 270 271 offs = bpf_prog_info_read_offset_u64(&info_linear->info, 272 desc->array_offset); 273 addr = offs + ptr_to_u64(info_linear->data); 274 bpf_prog_info_set_offset_u64(&info_linear->info, 275 desc->array_offset, addr); 276 } 277 } 278