1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2022 Meta Platforms, Inc. and affiliates. 4 */ 5 6 #include <linux/types.h> 7 #include <linux/bpf.h> 8 #include <linux/bpf_local_storage.h> 9 #include <uapi/linux/btf.h> 10 #include <linux/btf_ids.h> 11 12 DEFINE_BPF_STORAGE_CACHE(cgroup_cache); 13 14 static DEFINE_PER_CPU(int, bpf_cgrp_storage_busy); 15 16 static void bpf_cgrp_storage_lock(void) 17 { 18 cant_migrate(); 19 this_cpu_inc(bpf_cgrp_storage_busy); 20 } 21 22 static void bpf_cgrp_storage_unlock(void) 23 { 24 this_cpu_dec(bpf_cgrp_storage_busy); 25 } 26 27 static bool bpf_cgrp_storage_trylock(void) 28 { 29 cant_migrate(); 30 if (unlikely(this_cpu_inc_return(bpf_cgrp_storage_busy) != 1)) { 31 this_cpu_dec(bpf_cgrp_storage_busy); 32 return false; 33 } 34 return true; 35 } 36 37 static struct bpf_local_storage __rcu **cgroup_storage_ptr(void *owner) 38 { 39 struct cgroup *cg = owner; 40 41 return &cg->bpf_cgrp_storage; 42 } 43 44 void bpf_cgrp_storage_free(struct cgroup *cgroup) 45 { 46 struct bpf_local_storage *local_storage; 47 48 rcu_read_lock_dont_migrate(); 49 local_storage = rcu_dereference(cgroup->bpf_cgrp_storage); 50 if (!local_storage) 51 goto out; 52 53 bpf_cgrp_storage_lock(); 54 bpf_local_storage_destroy(local_storage); 55 bpf_cgrp_storage_unlock(); 56 out: 57 rcu_read_unlock_migrate(); 58 } 59 60 static struct bpf_local_storage_data * 61 cgroup_storage_lookup(struct cgroup *cgroup, struct bpf_map *map, bool cacheit_lockit) 62 { 63 struct bpf_local_storage *cgroup_storage; 64 struct bpf_local_storage_map *smap; 65 66 cgroup_storage = rcu_dereference_check(cgroup->bpf_cgrp_storage, 67 bpf_rcu_lock_held()); 68 if (!cgroup_storage) 69 return NULL; 70 71 smap = (struct bpf_local_storage_map *)map; 72 return bpf_local_storage_lookup(cgroup_storage, smap, cacheit_lockit); 73 } 74 75 static void *bpf_cgrp_storage_lookup_elem(struct bpf_map *map, void *key) 76 { 77 struct bpf_local_storage_data *sdata; 78 struct cgroup *cgroup; 79 int fd; 80 81 fd = *(int *)key; 82 cgroup = cgroup_v1v2_get_from_fd(fd); 83 if (IS_ERR(cgroup)) 84 return ERR_CAST(cgroup); 85 86 bpf_cgrp_storage_lock(); 87 sdata = cgroup_storage_lookup(cgroup, map, true); 88 bpf_cgrp_storage_unlock(); 89 cgroup_put(cgroup); 90 return sdata ? sdata->data : NULL; 91 } 92 93 static long bpf_cgrp_storage_update_elem(struct bpf_map *map, void *key, 94 void *value, u64 map_flags) 95 { 96 struct bpf_local_storage_data *sdata; 97 struct cgroup *cgroup; 98 int fd; 99 100 fd = *(int *)key; 101 cgroup = cgroup_v1v2_get_from_fd(fd); 102 if (IS_ERR(cgroup)) 103 return PTR_ERR(cgroup); 104 105 bpf_cgrp_storage_lock(); 106 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map, 107 value, map_flags, false, GFP_ATOMIC); 108 bpf_cgrp_storage_unlock(); 109 cgroup_put(cgroup); 110 return PTR_ERR_OR_ZERO(sdata); 111 } 112 113 static int cgroup_storage_delete(struct cgroup *cgroup, struct bpf_map *map) 114 { 115 struct bpf_local_storage_data *sdata; 116 117 sdata = cgroup_storage_lookup(cgroup, map, false); 118 if (!sdata) 119 return -ENOENT; 120 121 bpf_selem_unlink(SELEM(sdata), false); 122 return 0; 123 } 124 125 static long bpf_cgrp_storage_delete_elem(struct bpf_map *map, void *key) 126 { 127 struct cgroup *cgroup; 128 int err, fd; 129 130 fd = *(int *)key; 131 cgroup = cgroup_v1v2_get_from_fd(fd); 132 if (IS_ERR(cgroup)) 133 return PTR_ERR(cgroup); 134 135 bpf_cgrp_storage_lock(); 136 err = cgroup_storage_delete(cgroup, map); 137 bpf_cgrp_storage_unlock(); 138 cgroup_put(cgroup); 139 return err; 140 } 141 142 static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key) 143 { 144 return -ENOTSUPP; 145 } 146 147 static struct bpf_map *cgroup_storage_map_alloc(union bpf_attr *attr) 148 { 149 return bpf_local_storage_map_alloc(attr, &cgroup_cache, true); 150 } 151 152 static void cgroup_storage_map_free(struct bpf_map *map) 153 { 154 bpf_local_storage_map_free(map, &cgroup_cache, &bpf_cgrp_storage_busy); 155 } 156 157 /* *gfp_flags* is a hidden argument provided by the verifier */ 158 BPF_CALL_5(bpf_cgrp_storage_get, struct bpf_map *, map, struct cgroup *, cgroup, 159 void *, value, u64, flags, gfp_t, gfp_flags) 160 { 161 struct bpf_local_storage_data *sdata; 162 bool nobusy; 163 164 WARN_ON_ONCE(!bpf_rcu_lock_held()); 165 if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE)) 166 return (unsigned long)NULL; 167 168 if (!cgroup) 169 return (unsigned long)NULL; 170 171 nobusy = bpf_cgrp_storage_trylock(); 172 173 sdata = cgroup_storage_lookup(cgroup, map, nobusy); 174 if (sdata) 175 goto unlock; 176 177 /* only allocate new storage, when the cgroup is refcounted */ 178 if (!percpu_ref_is_dying(&cgroup->self.refcnt) && 179 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) && nobusy) 180 sdata = bpf_local_storage_update(cgroup, (struct bpf_local_storage_map *)map, 181 value, BPF_NOEXIST, false, gfp_flags); 182 183 unlock: 184 if (nobusy) 185 bpf_cgrp_storage_unlock(); 186 return IS_ERR_OR_NULL(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data; 187 } 188 189 BPF_CALL_2(bpf_cgrp_storage_delete, struct bpf_map *, map, struct cgroup *, cgroup) 190 { 191 int ret; 192 193 WARN_ON_ONCE(!bpf_rcu_lock_held()); 194 if (!cgroup) 195 return -EINVAL; 196 197 if (!bpf_cgrp_storage_trylock()) 198 return -EBUSY; 199 200 ret = cgroup_storage_delete(cgroup, map); 201 bpf_cgrp_storage_unlock(); 202 return ret; 203 } 204 205 const struct bpf_map_ops cgrp_storage_map_ops = { 206 .map_meta_equal = bpf_map_meta_equal, 207 .map_alloc_check = bpf_local_storage_map_alloc_check, 208 .map_alloc = cgroup_storage_map_alloc, 209 .map_free = cgroup_storage_map_free, 210 .map_get_next_key = notsupp_get_next_key, 211 .map_lookup_elem = bpf_cgrp_storage_lookup_elem, 212 .map_update_elem = bpf_cgrp_storage_update_elem, 213 .map_delete_elem = bpf_cgrp_storage_delete_elem, 214 .map_check_btf = bpf_local_storage_map_check_btf, 215 .map_mem_usage = bpf_local_storage_map_mem_usage, 216 .map_btf_id = &bpf_local_storage_map_btf_id[0], 217 .map_owner_storage_ptr = cgroup_storage_ptr, 218 }; 219 220 const struct bpf_func_proto bpf_cgrp_storage_get_proto = { 221 .func = bpf_cgrp_storage_get, 222 .gpl_only = false, 223 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, 224 .arg1_type = ARG_CONST_MAP_PTR, 225 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 226 .arg2_btf_id = &bpf_cgroup_btf_id[0], 227 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, 228 .arg4_type = ARG_ANYTHING, 229 }; 230 231 const struct bpf_func_proto bpf_cgrp_storage_delete_proto = { 232 .func = bpf_cgrp_storage_delete, 233 .gpl_only = false, 234 .ret_type = RET_INTEGER, 235 .arg1_type = ARG_CONST_MAP_PTR, 236 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL, 237 .arg2_btf_id = &bpf_cgroup_btf_id[0], 238 }; 239