1 // SPDX-License-Identifier: GPL-2.0 2 /* XSKMAP used for AF_XDP sockets 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6 #include <linux/bpf.h> 7 #include <linux/capability.h> 8 #include <net/xdp_sock.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 12 #include "xsk.h" 13 14 static struct xsk_map_node *xsk_map_node_alloc(struct xsk_map *map, 15 struct xdp_sock **map_entry) 16 { 17 struct xsk_map_node *node; 18 19 node = bpf_map_kzalloc(&map->map, sizeof(*node), 20 GFP_ATOMIC | __GFP_NOWARN); 21 if (!node) 22 return ERR_PTR(-ENOMEM); 23 24 bpf_map_inc(&map->map); 25 26 node->map = map; 27 node->map_entry = map_entry; 28 return node; 29 } 30 31 static void xsk_map_node_free(struct xsk_map_node *node) 32 { 33 bpf_map_put(&node->map->map); 34 kfree(node); 35 } 36 37 static void xsk_map_sock_add(struct xdp_sock *xs, struct xsk_map_node *node) 38 { 39 spin_lock_bh(&xs->map_list_lock); 40 list_add_tail(&node->node, &xs->map_list); 41 spin_unlock_bh(&xs->map_list_lock); 42 } 43 44 static void xsk_map_sock_delete(struct xdp_sock *xs, 45 struct xdp_sock **map_entry) 46 { 47 struct xsk_map_node *n, *tmp; 48 49 spin_lock_bh(&xs->map_list_lock); 50 list_for_each_entry_safe(n, tmp, &xs->map_list, node) { 51 if (map_entry == n->map_entry) { 52 list_del(&n->node); 53 xsk_map_node_free(n); 54 } 55 } 56 spin_unlock_bh(&xs->map_list_lock); 57 } 58 59 static struct bpf_map *xsk_map_alloc(union bpf_attr *attr) 60 { 61 struct xsk_map *m; 62 int numa_node; 63 u64 size; 64 65 if (!capable(CAP_NET_ADMIN)) 66 return ERR_PTR(-EPERM); 67 68 if (attr->max_entries == 0 || attr->key_size != 4 || 69 attr->value_size != 4 || 70 attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)) 71 return ERR_PTR(-EINVAL); 72 73 numa_node = bpf_map_attr_numa_node(attr); 74 size = struct_size(m, xsk_map, attr->max_entries); 75 76 m = bpf_map_area_alloc(size, numa_node); 77 if (!m) 78 return ERR_PTR(-ENOMEM); 79 80 bpf_map_init_from_attr(&m->map, attr); 81 spin_lock_init(&m->lock); 82 83 return &m->map; 84 } 85 86 static void xsk_map_free(struct bpf_map *map) 87 { 88 struct xsk_map *m = container_of(map, struct xsk_map, map); 89 90 synchronize_net(); 91 bpf_map_area_free(m); 92 } 93 94 static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key) 95 { 96 struct xsk_map *m = container_of(map, struct xsk_map, map); 97 u32 index = key ? *(u32 *)key : U32_MAX; 98 u32 *next = next_key; 99 100 if (index >= m->map.max_entries) { 101 *next = 0; 102 return 0; 103 } 104 105 if (index == m->map.max_entries - 1) 106 return -ENOENT; 107 *next = index + 1; 108 return 0; 109 } 110 111 static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) 112 { 113 const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2; 114 struct bpf_insn *insn = insn_buf; 115 116 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); 117 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); 118 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *))); 119 *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map)); 120 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp); 121 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(struct xsk_sock *), ret, ret, 0); 122 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 123 *insn++ = BPF_MOV64_IMM(ret, 0); 124 return insn - insn_buf; 125 } 126 127 static void *__xsk_map_lookup_elem(struct bpf_map *map, u32 key) 128 { 129 struct xsk_map *m = container_of(map, struct xsk_map, map); 130 131 if (key >= map->max_entries) 132 return NULL; 133 134 return READ_ONCE(m->xsk_map[key]); 135 } 136 137 static void *xsk_map_lookup_elem(struct bpf_map *map, void *key) 138 { 139 WARN_ON_ONCE(!rcu_read_lock_held()); 140 return __xsk_map_lookup_elem(map, *(u32 *)key); 141 } 142 143 static void *xsk_map_lookup_elem_sys_only(struct bpf_map *map, void *key) 144 { 145 return ERR_PTR(-EOPNOTSUPP); 146 } 147 148 static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value, 149 u64 map_flags) 150 { 151 struct xsk_map *m = container_of(map, struct xsk_map, map); 152 struct xdp_sock *xs, *old_xs, **map_entry; 153 u32 i = *(u32 *)key, fd = *(u32 *)value; 154 struct xsk_map_node *node; 155 struct socket *sock; 156 int err; 157 158 if (unlikely(map_flags > BPF_EXIST)) 159 return -EINVAL; 160 if (unlikely(i >= m->map.max_entries)) 161 return -E2BIG; 162 163 sock = sockfd_lookup(fd, &err); 164 if (!sock) 165 return err; 166 167 if (sock->sk->sk_family != PF_XDP) { 168 sockfd_put(sock); 169 return -EOPNOTSUPP; 170 } 171 172 xs = (struct xdp_sock *)sock->sk; 173 174 map_entry = &m->xsk_map[i]; 175 node = xsk_map_node_alloc(m, map_entry); 176 if (IS_ERR(node)) { 177 sockfd_put(sock); 178 return PTR_ERR(node); 179 } 180 181 spin_lock_bh(&m->lock); 182 old_xs = READ_ONCE(*map_entry); 183 if (old_xs == xs) { 184 err = 0; 185 goto out; 186 } else if (old_xs && map_flags == BPF_NOEXIST) { 187 err = -EEXIST; 188 goto out; 189 } else if (!old_xs && map_flags == BPF_EXIST) { 190 err = -ENOENT; 191 goto out; 192 } 193 xsk_map_sock_add(xs, node); 194 WRITE_ONCE(*map_entry, xs); 195 if (old_xs) 196 xsk_map_sock_delete(old_xs, map_entry); 197 spin_unlock_bh(&m->lock); 198 sockfd_put(sock); 199 return 0; 200 201 out: 202 spin_unlock_bh(&m->lock); 203 sockfd_put(sock); 204 xsk_map_node_free(node); 205 return err; 206 } 207 208 static int xsk_map_delete_elem(struct bpf_map *map, void *key) 209 { 210 struct xsk_map *m = container_of(map, struct xsk_map, map); 211 struct xdp_sock *old_xs, **map_entry; 212 int k = *(u32 *)key; 213 214 if (k >= map->max_entries) 215 return -EINVAL; 216 217 spin_lock_bh(&m->lock); 218 map_entry = &m->xsk_map[k]; 219 old_xs = xchg(map_entry, NULL); 220 if (old_xs) 221 xsk_map_sock_delete(old_xs, map_entry); 222 spin_unlock_bh(&m->lock); 223 224 return 0; 225 } 226 227 static int xsk_map_redirect(struct bpf_map *map, u32 ifindex, u64 flags) 228 { 229 return __bpf_xdp_redirect_map(map, ifindex, flags, __xsk_map_lookup_elem); 230 } 231 232 void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs, 233 struct xdp_sock **map_entry) 234 { 235 spin_lock_bh(&map->lock); 236 if (READ_ONCE(*map_entry) == xs) { 237 WRITE_ONCE(*map_entry, NULL); 238 xsk_map_sock_delete(xs, map_entry); 239 } 240 spin_unlock_bh(&map->lock); 241 } 242 243 static bool xsk_map_meta_equal(const struct bpf_map *meta0, 244 const struct bpf_map *meta1) 245 { 246 return meta0->max_entries == meta1->max_entries && 247 bpf_map_meta_equal(meta0, meta1); 248 } 249 250 static int xsk_map_btf_id; 251 const struct bpf_map_ops xsk_map_ops = { 252 .map_meta_equal = xsk_map_meta_equal, 253 .map_alloc = xsk_map_alloc, 254 .map_free = xsk_map_free, 255 .map_get_next_key = xsk_map_get_next_key, 256 .map_lookup_elem = xsk_map_lookup_elem, 257 .map_gen_lookup = xsk_map_gen_lookup, 258 .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only, 259 .map_update_elem = xsk_map_update_elem, 260 .map_delete_elem = xsk_map_delete_elem, 261 .map_check_btf = map_check_no_btf, 262 .map_btf_name = "xsk_map", 263 .map_btf_id = &xsk_map_btf_id, 264 .map_redirect = xsk_map_redirect, 265 }; 266