1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 Facebook */ 3 #include <linux/bpf.h> 4 #include <linux/fs.h> 5 #include <linux/filter.h> 6 #include <linux/kernel.h> 7 #include <linux/btf_ids.h> 8 9 struct bpf_iter_seq_map_info { 10 u32 map_id; 11 }; 12 13 static void *bpf_map_seq_start(struct seq_file *seq, loff_t *pos) 14 { 15 struct bpf_iter_seq_map_info *info = seq->private; 16 struct bpf_map *map; 17 18 map = bpf_map_get_curr_or_next(&info->map_id); 19 if (!map) 20 return NULL; 21 22 if (*pos == 0) 23 ++*pos; 24 return map; 25 } 26 27 static void *bpf_map_seq_next(struct seq_file *seq, void *v, loff_t *pos) 28 { 29 struct bpf_iter_seq_map_info *info = seq->private; 30 31 ++*pos; 32 ++info->map_id; 33 bpf_map_put((struct bpf_map *)v); 34 return bpf_map_get_curr_or_next(&info->map_id); 35 } 36 37 struct bpf_iter__bpf_map { 38 __bpf_md_ptr(struct bpf_iter_meta *, meta); 39 __bpf_md_ptr(struct bpf_map *, map); 40 }; 41 42 DEFINE_BPF_ITER_FUNC(bpf_map, struct bpf_iter_meta *meta, struct bpf_map *map) 43 44 static int __bpf_map_seq_show(struct seq_file *seq, void *v, bool in_stop) 45 { 46 struct bpf_iter__bpf_map ctx; 47 struct bpf_iter_meta meta; 48 struct bpf_prog *prog; 49 int ret = 0; 50 51 ctx.meta = &meta; 52 ctx.map = v; 53 meta.seq = seq; 54 prog = bpf_iter_get_info(&meta, in_stop); 55 if (prog) 56 ret = bpf_iter_run_prog(prog, &ctx); 57 58 return ret; 59 } 60 61 static int bpf_map_seq_show(struct seq_file *seq, void *v) 62 { 63 return __bpf_map_seq_show(seq, v, false); 64 } 65 66 static void bpf_map_seq_stop(struct seq_file *seq, void *v) 67 { 68 if (!v) 69 (void)__bpf_map_seq_show(seq, v, true); 70 else 71 bpf_map_put((struct bpf_map *)v); 72 } 73 74 static const struct seq_operations bpf_map_seq_ops = { 75 .start = bpf_map_seq_start, 76 .next = bpf_map_seq_next, 77 .stop = bpf_map_seq_stop, 78 .show = bpf_map_seq_show, 79 }; 80 81 BTF_ID_LIST_GLOBAL_SINGLE(btf_bpf_map_id, struct, bpf_map) 82 83 static const struct bpf_iter_seq_info bpf_map_seq_info = { 84 .seq_ops = &bpf_map_seq_ops, 85 .init_seq_private = NULL, 86 .fini_seq_private = NULL, 87 .seq_priv_size = sizeof(struct bpf_iter_seq_map_info), 88 }; 89 90 static struct bpf_iter_reg bpf_map_reg_info = { 91 .target = "bpf_map", 92 .ctx_arg_info_size = 1, 93 .ctx_arg_info = { 94 { offsetof(struct bpf_iter__bpf_map, map), 95 PTR_TO_BTF_ID_OR_NULL | PTR_TRUSTED }, 96 }, 97 .seq_info = &bpf_map_seq_info, 98 }; 99 100 static int bpf_iter_attach_map(struct bpf_prog *prog, 101 union bpf_iter_link_info *linfo, 102 struct bpf_iter_aux_info *aux) 103 { 104 u32 key_acc_size, value_acc_size, key_size, value_size; 105 struct bpf_map *map; 106 bool is_percpu = false; 107 int err = -EINVAL; 108 109 if (!linfo->map.map_fd) 110 return -EBADF; 111 112 map = bpf_map_get_with_uref(linfo->map.map_fd); 113 if (IS_ERR(map)) 114 return PTR_ERR(map); 115 if (map->excl_prog_sha) { 116 err = -EPERM; 117 goto put_map; 118 } 119 120 if (map->map_type == BPF_MAP_TYPE_PERCPU_HASH || 121 map->map_type == BPF_MAP_TYPE_LRU_PERCPU_HASH || 122 map->map_type == BPF_MAP_TYPE_PERCPU_ARRAY) 123 is_percpu = true; 124 else if (map->map_type != BPF_MAP_TYPE_HASH && 125 map->map_type != BPF_MAP_TYPE_LRU_HASH && 126 map->map_type != BPF_MAP_TYPE_ARRAY && 127 map->map_type != BPF_MAP_TYPE_RHASH) 128 goto put_map; 129 130 key_acc_size = prog->aux->max_rdonly_access; 131 value_acc_size = prog->aux->max_rdwr_access; 132 key_size = map->key_size; 133 if (!is_percpu) 134 value_size = map->value_size; 135 else 136 value_size = round_up(map->value_size, 8) * num_possible_cpus(); 137 138 if (key_acc_size > key_size || value_acc_size > value_size) { 139 err = -EACCES; 140 goto put_map; 141 } 142 143 aux->map = map; 144 return 0; 145 146 put_map: 147 bpf_map_put_with_uref(map); 148 return err; 149 } 150 151 static void bpf_iter_detach_map(struct bpf_iter_aux_info *aux) 152 { 153 bpf_map_put_with_uref(aux->map); 154 } 155 156 void bpf_iter_map_show_fdinfo(const struct bpf_iter_aux_info *aux, 157 struct seq_file *seq) 158 { 159 seq_printf(seq, "map_id:\t%u\n", aux->map->id); 160 } 161 162 int bpf_iter_map_fill_link_info(const struct bpf_iter_aux_info *aux, 163 struct bpf_link_info *info) 164 { 165 info->iter.map.map_id = aux->map->id; 166 return 0; 167 } 168 169 DEFINE_BPF_ITER_FUNC(bpf_map_elem, struct bpf_iter_meta *meta, 170 struct bpf_map *map, void *key, void *value) 171 172 static const struct bpf_iter_reg bpf_map_elem_reg_info = { 173 .target = "bpf_map_elem", 174 .attach_target = bpf_iter_attach_map, 175 .detach_target = bpf_iter_detach_map, 176 .show_fdinfo = bpf_iter_map_show_fdinfo, 177 .fill_link_info = bpf_iter_map_fill_link_info, 178 .ctx_arg_info_size = 2, 179 .ctx_arg_info = { 180 { offsetof(struct bpf_iter__bpf_map_elem, key), 181 PTR_TO_BUF | PTR_MAYBE_NULL | MEM_RDONLY }, 182 { offsetof(struct bpf_iter__bpf_map_elem, value), 183 PTR_TO_BUF | PTR_MAYBE_NULL }, 184 }, 185 }; 186 187 static int __init bpf_map_iter_init(void) 188 { 189 int ret; 190 191 bpf_map_reg_info.ctx_arg_info[0].btf_id = *btf_bpf_map_id; 192 ret = bpf_iter_reg_target(&bpf_map_reg_info); 193 if (ret) 194 return ret; 195 196 return bpf_iter_reg_target(&bpf_map_elem_reg_info); 197 } 198 199 late_initcall(bpf_map_iter_init); 200 201 __bpf_kfunc_start_defs(); 202 203 __bpf_kfunc s64 bpf_map_sum_elem_count(const struct bpf_map *map) 204 { 205 s64 *pcount; 206 s64 ret = 0; 207 int cpu; 208 209 if (!map || !map->elem_count) 210 return 0; 211 212 for_each_possible_cpu(cpu) { 213 pcount = per_cpu_ptr(map->elem_count, cpu); 214 ret += READ_ONCE(*pcount); 215 } 216 return ret; 217 } 218 219 __bpf_kfunc_end_defs(); 220 221 BTF_KFUNCS_START(bpf_map_iter_kfunc_ids) 222 BTF_ID_FLAGS(func, bpf_map_sum_elem_count) 223 BTF_KFUNCS_END(bpf_map_iter_kfunc_ids) 224 225 static const struct btf_kfunc_id_set bpf_map_iter_kfunc_set = { 226 .owner = THIS_MODULE, 227 .set = &bpf_map_iter_kfunc_ids, 228 }; 229 230 static int init_subsystem(void) 231 { 232 return register_btf_kfunc_id_set(BPF_PROG_TYPE_UNSPEC, &bpf_map_iter_kfunc_set); 233 } 234 late_initcall(init_subsystem); 235