1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2023 Isovalent */ 3 4 #include <linux/bpf.h> 5 #include <linux/bpf_mprog.h> 6 7 static int bpf_mprog_link(struct bpf_tuple *tuple, 8 u32 id_or_fd, u32 flags, 9 enum bpf_prog_type type) 10 { 11 struct bpf_link *link = ERR_PTR(-EINVAL); 12 bool id = flags & BPF_F_ID; 13 14 if (id) 15 link = bpf_link_by_id(id_or_fd); 16 else if (id_or_fd) 17 link = bpf_link_get_from_fd(id_or_fd); 18 if (IS_ERR(link)) 19 return PTR_ERR(link); 20 if (type && link->prog->type != type) { 21 bpf_link_put(link); 22 return -EINVAL; 23 } 24 25 tuple->link = link; 26 tuple->prog = link->prog; 27 return 0; 28 } 29 30 static int bpf_mprog_prog(struct bpf_tuple *tuple, 31 u32 id_or_fd, u32 flags, 32 enum bpf_prog_type type) 33 { 34 struct bpf_prog *prog = ERR_PTR(-EINVAL); 35 bool id = flags & BPF_F_ID; 36 37 if (id) 38 prog = bpf_prog_by_id(id_or_fd); 39 else if (id_or_fd) 40 prog = bpf_prog_get(id_or_fd); 41 if (IS_ERR(prog)) 42 return PTR_ERR(prog); 43 if (type && prog->type != type) { 44 bpf_prog_put(prog); 45 return -EINVAL; 46 } 47 48 tuple->link = NULL; 49 tuple->prog = prog; 50 return 0; 51 } 52 53 static int bpf_mprog_tuple_relative(struct bpf_tuple *tuple, 54 u32 id_or_fd, u32 flags, 55 enum bpf_prog_type type) 56 { 57 bool link = flags & BPF_F_LINK; 58 bool id = flags & BPF_F_ID; 59 60 memset(tuple, 0, sizeof(*tuple)); 61 if (link) 62 return bpf_mprog_link(tuple, id_or_fd, flags, type); 63 /* If no relevant flag is set and no id_or_fd was passed, then 64 * tuple link/prog is just NULLed. This is the case when before/ 65 * after selects first/last position without passing fd. 66 */ 67 if (!id && !id_or_fd) 68 return 0; 69 return bpf_mprog_prog(tuple, id_or_fd, flags, type); 70 } 71 72 static void bpf_mprog_tuple_put(struct bpf_tuple *tuple) 73 { 74 if (tuple->link) 75 bpf_link_put(tuple->link); 76 else if (tuple->prog) 77 bpf_prog_put(tuple->prog); 78 } 79 80 /* The bpf_mprog_{replace,delete}() operate on exact idx position with the 81 * one exception that for deletion we support delete from front/back. In 82 * case of front idx is -1, in case of back idx is bpf_mprog_total(entry). 83 * Adjustment to first and last entry is trivial. The bpf_mprog_insert() 84 * we have to deal with the following cases: 85 * 86 * idx + before: 87 * 88 * Insert P4 before P3: idx for old array is 1, idx for new array is 2, 89 * hence we adjust target idx for the new array, so that memmove copies 90 * P1 and P2 to the new entry, and we insert P4 into idx 2. Inserting 91 * before P1 would have old idx -1 and new idx 0. 92 * 93 * +--+--+--+ +--+--+--+--+ +--+--+--+--+ 94 * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| 95 * +--+--+--+ +--+--+--+--+ +--+--+--+--+ 96 * 97 * idx + after: 98 * 99 * Insert P4 after P2: idx for old array is 2, idx for new array is 2. 100 * Again, memmove copies P1 and P2 to the new entry, and we insert P4 101 * into idx 2. Inserting after P3 would have both old/new idx at 4 aka 102 * bpf_mprog_total(entry). 103 * 104 * +--+--+--+ +--+--+--+--+ +--+--+--+--+ 105 * |P1|P2|P3| ==> |P1|P2| |P3| ==> |P1|P2|P4|P3| 106 * +--+--+--+ +--+--+--+--+ +--+--+--+--+ 107 */ 108 static int bpf_mprog_replace(struct bpf_mprog_entry *entry, 109 struct bpf_mprog_entry **entry_new, 110 struct bpf_tuple *ntuple, int idx) 111 { 112 struct bpf_mprog_fp *fp; 113 struct bpf_mprog_cp *cp; 114 struct bpf_prog *oprog; 115 116 bpf_mprog_read(entry, idx, &fp, &cp); 117 oprog = READ_ONCE(fp->prog); 118 bpf_mprog_write(fp, cp, ntuple); 119 if (!ntuple->link) { 120 WARN_ON_ONCE(cp->link); 121 bpf_prog_put(oprog); 122 } 123 *entry_new = entry; 124 return 0; 125 } 126 127 static int bpf_mprog_insert(struct bpf_mprog_entry *entry, 128 struct bpf_mprog_entry **entry_new, 129 struct bpf_tuple *ntuple, int idx, u32 flags) 130 { 131 int total = bpf_mprog_total(entry); 132 struct bpf_mprog_entry *peer; 133 struct bpf_mprog_fp *fp; 134 struct bpf_mprog_cp *cp; 135 136 peer = bpf_mprog_peer(entry); 137 bpf_mprog_entry_copy(peer, entry); 138 if (idx == total) 139 goto insert; 140 else if (flags & BPF_F_BEFORE) 141 idx += 1; 142 bpf_mprog_entry_grow(peer, idx); 143 insert: 144 bpf_mprog_read(peer, idx, &fp, &cp); 145 bpf_mprog_write(fp, cp, ntuple); 146 bpf_mprog_inc(peer); 147 *entry_new = peer; 148 return 0; 149 } 150 151 static int bpf_mprog_delete(struct bpf_mprog_entry *entry, 152 struct bpf_mprog_entry **entry_new, 153 struct bpf_tuple *dtuple, int idx) 154 { 155 int total = bpf_mprog_total(entry); 156 struct bpf_mprog_entry *peer; 157 158 peer = bpf_mprog_peer(entry); 159 bpf_mprog_entry_copy(peer, entry); 160 if (idx == -1) 161 idx = 0; 162 else if (idx == total) 163 idx = total - 1; 164 bpf_mprog_entry_shrink(peer, idx); 165 bpf_mprog_dec(peer); 166 bpf_mprog_mark_for_release(peer, dtuple); 167 *entry_new = peer; 168 return 0; 169 } 170 171 /* In bpf_mprog_pos_*() we evaluate the target position for the BPF 172 * program/link that needs to be replaced, inserted or deleted for 173 * each "rule" independently. If all rules agree on that position 174 * or existing element, then enact replacement, addition or deletion. 175 * If this is not the case, then the request cannot be satisfied and 176 * we bail out with an error. 177 */ 178 static int bpf_mprog_pos_exact(struct bpf_mprog_entry *entry, 179 struct bpf_tuple *tuple) 180 { 181 struct bpf_mprog_fp *fp; 182 struct bpf_mprog_cp *cp; 183 int i; 184 185 for (i = 0; i < bpf_mprog_total(entry); i++) { 186 bpf_mprog_read(entry, i, &fp, &cp); 187 if (tuple->prog == READ_ONCE(fp->prog)) 188 return tuple->link == cp->link ? i : -EBUSY; 189 } 190 return -ENOENT; 191 } 192 193 static int bpf_mprog_pos_before(struct bpf_mprog_entry *entry, 194 struct bpf_tuple *tuple) 195 { 196 struct bpf_mprog_fp *fp; 197 struct bpf_mprog_cp *cp; 198 int i; 199 200 for (i = 0; i < bpf_mprog_total(entry); i++) { 201 bpf_mprog_read(entry, i, &fp, &cp); 202 if (tuple->prog == READ_ONCE(fp->prog) && 203 (!tuple->link || tuple->link == cp->link)) 204 return i - 1; 205 } 206 return tuple->prog ? -ENOENT : -1; 207 } 208 209 static int bpf_mprog_pos_after(struct bpf_mprog_entry *entry, 210 struct bpf_tuple *tuple) 211 { 212 struct bpf_mprog_fp *fp; 213 struct bpf_mprog_cp *cp; 214 int i; 215 216 for (i = 0; i < bpf_mprog_total(entry); i++) { 217 bpf_mprog_read(entry, i, &fp, &cp); 218 if (tuple->prog == READ_ONCE(fp->prog) && 219 (!tuple->link || tuple->link == cp->link)) 220 return i + 1; 221 } 222 return tuple->prog ? -ENOENT : bpf_mprog_total(entry); 223 } 224 225 int bpf_mprog_attach(struct bpf_mprog_entry *entry, 226 struct bpf_mprog_entry **entry_new, 227 struct bpf_prog *prog_new, struct bpf_link *link, 228 struct bpf_prog *prog_old, 229 u32 flags, u32 id_or_fd, u64 revision) 230 { 231 struct bpf_tuple rtuple, ntuple = { 232 .prog = prog_new, 233 .link = link, 234 }, otuple = { 235 .prog = prog_old, 236 .link = link, 237 }; 238 int ret, idx = -ERANGE, tidx; 239 240 if (revision && revision != bpf_mprog_revision(entry)) 241 return -ESTALE; 242 if (bpf_mprog_exists(entry, prog_new)) 243 return -EEXIST; 244 ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, 245 flags & ~BPF_F_REPLACE, 246 prog_new->type); 247 if (ret) 248 return ret; 249 if (flags & BPF_F_REPLACE) { 250 tidx = bpf_mprog_pos_exact(entry, &otuple); 251 if (tidx < 0) { 252 ret = tidx; 253 goto out; 254 } 255 idx = tidx; 256 } else if (bpf_mprog_total(entry) == bpf_mprog_max()) { 257 ret = -ERANGE; 258 goto out; 259 } 260 if (flags & BPF_F_BEFORE) { 261 tidx = bpf_mprog_pos_before(entry, &rtuple); 262 if (tidx < -1 || (idx >= -1 && tidx != idx)) { 263 ret = tidx < -1 ? tidx : -ERANGE; 264 goto out; 265 } 266 idx = tidx; 267 } 268 if (flags & BPF_F_AFTER) { 269 tidx = bpf_mprog_pos_after(entry, &rtuple); 270 if (tidx < -1 || (idx >= -1 && tidx != idx)) { 271 ret = tidx < 0 ? tidx : -ERANGE; 272 goto out; 273 } 274 idx = tidx; 275 } 276 if (idx < -1) { 277 if (rtuple.prog || flags) { 278 ret = -EINVAL; 279 goto out; 280 } 281 idx = bpf_mprog_total(entry); 282 flags = BPF_F_AFTER; 283 } 284 if (idx >= bpf_mprog_max()) { 285 ret = -ERANGE; 286 goto out; 287 } 288 if (flags & BPF_F_REPLACE) 289 ret = bpf_mprog_replace(entry, entry_new, &ntuple, idx); 290 else 291 ret = bpf_mprog_insert(entry, entry_new, &ntuple, idx, flags); 292 out: 293 bpf_mprog_tuple_put(&rtuple); 294 return ret; 295 } 296 297 static int bpf_mprog_fetch(struct bpf_mprog_entry *entry, 298 struct bpf_tuple *tuple, int idx) 299 { 300 int total = bpf_mprog_total(entry); 301 struct bpf_mprog_cp *cp; 302 struct bpf_mprog_fp *fp; 303 struct bpf_prog *prog; 304 struct bpf_link *link; 305 306 if (idx == -1) 307 idx = 0; 308 else if (idx == total) 309 idx = total - 1; 310 bpf_mprog_read(entry, idx, &fp, &cp); 311 prog = READ_ONCE(fp->prog); 312 link = cp->link; 313 /* The deletion request can either be without filled tuple in which 314 * case it gets populated here based on idx, or with filled tuple 315 * where the only thing we end up doing is the WARN_ON_ONCE() assert. 316 * If we hit a BPF link at the given index, it must not be removed 317 * from opts path. 318 */ 319 if (link && !tuple->link) 320 return -EBUSY; 321 WARN_ON_ONCE(tuple->prog && tuple->prog != prog); 322 WARN_ON_ONCE(tuple->link && tuple->link != link); 323 tuple->prog = prog; 324 tuple->link = link; 325 return 0; 326 } 327 328 int bpf_mprog_detach(struct bpf_mprog_entry *entry, 329 struct bpf_mprog_entry **entry_new, 330 struct bpf_prog *prog, struct bpf_link *link, 331 u32 flags, u32 id_or_fd, u64 revision) 332 { 333 struct bpf_tuple rtuple, dtuple = { 334 .prog = prog, 335 .link = link, 336 }; 337 int ret, idx = -ERANGE, tidx; 338 339 if (flags & BPF_F_REPLACE) 340 return -EINVAL; 341 if (revision && revision != bpf_mprog_revision(entry)) 342 return -ESTALE; 343 if (!bpf_mprog_total(entry)) 344 return -ENOENT; 345 ret = bpf_mprog_tuple_relative(&rtuple, id_or_fd, flags, 346 prog ? prog->type : 347 BPF_PROG_TYPE_UNSPEC); 348 if (ret) 349 return ret; 350 if (dtuple.prog) { 351 tidx = bpf_mprog_pos_exact(entry, &dtuple); 352 if (tidx < 0) { 353 ret = tidx; 354 goto out; 355 } 356 idx = tidx; 357 } 358 if (flags & BPF_F_BEFORE) { 359 tidx = bpf_mprog_pos_before(entry, &rtuple); 360 if (tidx < -1 || (idx >= -1 && tidx != idx)) { 361 ret = tidx < -1 ? tidx : -ERANGE; 362 goto out; 363 } 364 idx = tidx; 365 } 366 if (flags & BPF_F_AFTER) { 367 tidx = bpf_mprog_pos_after(entry, &rtuple); 368 if (tidx < -1 || (idx >= -1 && tidx != idx)) { 369 ret = tidx < 0 ? tidx : -ERANGE; 370 goto out; 371 } 372 idx = tidx; 373 } 374 if (idx < -1) { 375 if (rtuple.prog || flags) { 376 ret = -EINVAL; 377 goto out; 378 } 379 idx = bpf_mprog_total(entry); 380 flags = BPF_F_AFTER; 381 } 382 if (idx >= bpf_mprog_max()) { 383 ret = -ERANGE; 384 goto out; 385 } 386 ret = bpf_mprog_fetch(entry, &dtuple, idx); 387 if (ret) 388 goto out; 389 ret = bpf_mprog_delete(entry, entry_new, &dtuple, idx); 390 out: 391 bpf_mprog_tuple_put(&rtuple); 392 return ret; 393 } 394 395 int bpf_mprog_query(const union bpf_attr *attr, union bpf_attr __user *uattr, 396 struct bpf_mprog_entry *entry) 397 { 398 u32 __user *uprog_flags, *ulink_flags; 399 u32 __user *uprog_id, *ulink_id; 400 struct bpf_mprog_fp *fp; 401 struct bpf_mprog_cp *cp; 402 struct bpf_prog *prog; 403 const u32 flags = 0; 404 u32 id, count = 0; 405 u64 revision = 1; 406 int i, ret = 0; 407 408 if (attr->query.query_flags || attr->query.attach_flags) 409 return -EINVAL; 410 if (entry) { 411 revision = bpf_mprog_revision(entry); 412 count = bpf_mprog_total(entry); 413 } 414 if (copy_to_user(&uattr->query.attach_flags, &flags, sizeof(flags))) 415 return -EFAULT; 416 if (copy_to_user(&uattr->query.revision, &revision, sizeof(revision))) 417 return -EFAULT; 418 if (copy_to_user(&uattr->query.count, &count, sizeof(count))) 419 return -EFAULT; 420 uprog_id = u64_to_user_ptr(attr->query.prog_ids); 421 uprog_flags = u64_to_user_ptr(attr->query.prog_attach_flags); 422 ulink_id = u64_to_user_ptr(attr->query.link_ids); 423 ulink_flags = u64_to_user_ptr(attr->query.link_attach_flags); 424 if (attr->query.count == 0 || !uprog_id || !count) 425 return 0; 426 if (attr->query.count < count) { 427 count = attr->query.count; 428 ret = -ENOSPC; 429 } 430 for (i = 0; i < bpf_mprog_max(); i++) { 431 bpf_mprog_read(entry, i, &fp, &cp); 432 prog = READ_ONCE(fp->prog); 433 if (!prog) 434 break; 435 id = prog->aux->id; 436 if (copy_to_user(uprog_id + i, &id, sizeof(id))) 437 return -EFAULT; 438 if (uprog_flags && 439 copy_to_user(uprog_flags + i, &flags, sizeof(flags))) 440 return -EFAULT; 441 id = cp->link ? cp->link->id : 0; 442 if (ulink_id && 443 copy_to_user(ulink_id + i, &id, sizeof(id))) 444 return -EFAULT; 445 if (ulink_flags && 446 copy_to_user(ulink_flags + i, &flags, sizeof(flags))) 447 return -EFAULT; 448 if (i + 1 == count) 449 break; 450 } 451 return ret; 452 } 453