1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2016-2018 Netronome Systems, Inc. */ 3 4 /* 5 * nfp_net_offload.c 6 * Netronome network device driver: TC offload functions for PF and VF 7 */ 8 9 #define pr_fmt(fmt) "NFP net bpf: " fmt 10 11 #include <linux/bpf.h> 12 #include <linux/kernel.h> 13 #include <linux/netdevice.h> 14 #include <linux/pci.h> 15 #include <linux/jiffies.h> 16 #include <linux/timer.h> 17 #include <linux/list.h> 18 #include <linux/mm.h> 19 20 #include <net/pkt_cls.h> 21 #include <net/tc_act/tc_gact.h> 22 #include <net/tc_act/tc_mirred.h> 23 24 #include "main.h" 25 #include "../ccm.h" 26 #include "../nfp_app.h" 27 #include "../nfp_net_ctrl.h" 28 #include "../nfp_net.h" 29 30 static int 31 nfp_map_ptr_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 32 struct bpf_map *map) 33 { 34 struct nfp_bpf_neutral_map *record; 35 int err; 36 37 /* Reuse path - other offloaded program is already tracking this map. */ 38 record = rhashtable_lookup_fast(&bpf->maps_neutral, &map->id, 39 nfp_bpf_maps_neutral_params); 40 if (record) { 41 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 42 record->count++; 43 return 0; 44 } 45 46 /* Grab a single ref to the map for our record. The prog destroy ndo 47 * happens after free_used_maps(). 48 */ 49 bpf_map_inc(map); 50 51 record = kmalloc_obj(*record); 52 if (!record) { 53 err = -ENOMEM; 54 goto err_map_put; 55 } 56 57 record->ptr = map; 58 record->map_id = map->id; 59 record->count = 1; 60 61 err = rhashtable_insert_fast(&bpf->maps_neutral, &record->l, 62 nfp_bpf_maps_neutral_params); 63 if (err) 64 goto err_free_rec; 65 66 nfp_prog->map_records[nfp_prog->map_records_cnt++] = record; 67 68 return 0; 69 70 err_free_rec: 71 kfree(record); 72 err_map_put: 73 bpf_map_put(map); 74 return err; 75 } 76 77 static void 78 nfp_map_ptrs_forget(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog) 79 { 80 bool freed = false; 81 int i; 82 83 for (i = 0; i < nfp_prog->map_records_cnt; i++) { 84 if (--nfp_prog->map_records[i]->count) { 85 nfp_prog->map_records[i] = NULL; 86 continue; 87 } 88 89 WARN_ON(rhashtable_remove_fast(&bpf->maps_neutral, 90 &nfp_prog->map_records[i]->l, 91 nfp_bpf_maps_neutral_params)); 92 freed = true; 93 } 94 95 if (freed) { 96 synchronize_rcu(); 97 98 for (i = 0; i < nfp_prog->map_records_cnt; i++) 99 if (nfp_prog->map_records[i]) { 100 bpf_map_put(nfp_prog->map_records[i]->ptr); 101 kfree(nfp_prog->map_records[i]); 102 } 103 } 104 105 kfree(nfp_prog->map_records); 106 nfp_prog->map_records = NULL; 107 nfp_prog->map_records_cnt = 0; 108 } 109 110 static int 111 nfp_map_ptrs_record(struct nfp_app_bpf *bpf, struct nfp_prog *nfp_prog, 112 struct bpf_prog *prog) 113 { 114 int i, cnt, err = 0; 115 116 mutex_lock(&prog->aux->used_maps_mutex); 117 118 /* Quickly count the maps we will have to remember */ 119 cnt = 0; 120 for (i = 0; i < prog->aux->used_map_cnt; i++) 121 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) 122 cnt++; 123 if (!cnt) 124 goto out; 125 126 nfp_prog->map_records = kmalloc_objs(nfp_prog->map_records[0], cnt); 127 if (!nfp_prog->map_records) { 128 err = -ENOMEM; 129 goto out; 130 } 131 132 for (i = 0; i < prog->aux->used_map_cnt; i++) 133 if (bpf_map_offload_neutral(prog->aux->used_maps[i])) { 134 err = nfp_map_ptr_record(bpf, nfp_prog, 135 prog->aux->used_maps[i]); 136 if (err) { 137 nfp_map_ptrs_forget(bpf, nfp_prog); 138 goto out; 139 } 140 } 141 WARN_ON(cnt != nfp_prog->map_records_cnt); 142 143 out: 144 mutex_unlock(&prog->aux->used_maps_mutex); 145 return err; 146 } 147 148 static int 149 nfp_prog_prepare(struct nfp_prog *nfp_prog, const struct bpf_insn *prog, 150 unsigned int cnt) 151 { 152 struct nfp_insn_meta *meta; 153 unsigned int i; 154 155 for (i = 0; i < cnt; i++) { 156 meta = kzalloc_obj(*meta); 157 if (!meta) 158 return -ENOMEM; 159 160 meta->insn = prog[i]; 161 meta->n = i; 162 if (is_mbpf_alu(meta)) { 163 meta->umin_src = U64_MAX; 164 meta->umin_dst = U64_MAX; 165 } 166 167 list_add_tail(&meta->l, &nfp_prog->insns); 168 } 169 nfp_prog->n_insns = cnt; 170 171 nfp_bpf_jit_prepare(nfp_prog); 172 173 return 0; 174 } 175 176 static void nfp_prog_free(struct nfp_prog *nfp_prog) 177 { 178 struct nfp_insn_meta *meta, *tmp; 179 180 kfree(nfp_prog->subprog); 181 182 list_for_each_entry_safe(meta, tmp, &nfp_prog->insns, l) { 183 list_del(&meta->l); 184 kfree(meta); 185 } 186 kfree(nfp_prog); 187 } 188 189 static int nfp_bpf_verifier_prep(struct bpf_prog *prog) 190 { 191 struct nfp_prog *nfp_prog; 192 int ret; 193 194 nfp_prog = kzalloc_obj(*nfp_prog); 195 if (!nfp_prog) 196 return -ENOMEM; 197 prog->aux->offload->dev_priv = nfp_prog; 198 199 INIT_LIST_HEAD(&nfp_prog->insns); 200 nfp_prog->type = prog->type; 201 nfp_prog->bpf = bpf_offload_dev_priv(prog->aux->offload->offdev); 202 203 ret = nfp_prog_prepare(nfp_prog, prog->insnsi, prog->len); 204 if (ret) 205 goto err_free; 206 207 nfp_prog->verifier_meta = nfp_prog_first_meta(nfp_prog); 208 209 return 0; 210 211 err_free: 212 nfp_prog_free(nfp_prog); 213 214 return ret; 215 } 216 217 static int nfp_bpf_translate(struct bpf_prog *prog) 218 { 219 struct nfp_net *nn = netdev_priv(prog->aux->offload->netdev); 220 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 221 unsigned int max_instr; 222 int err; 223 224 /* We depend on dead code elimination succeeding */ 225 if (prog->aux->offload->opt_failed) 226 return -EINVAL; 227 228 max_instr = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 229 nfp_prog->__prog_alloc_len = max_instr * sizeof(u64); 230 231 nfp_prog->prog = kvmalloc(nfp_prog->__prog_alloc_len, GFP_KERNEL); 232 if (!nfp_prog->prog) 233 return -ENOMEM; 234 235 err = nfp_bpf_jit(nfp_prog); 236 if (err) 237 return err; 238 239 prog->aux->offload->jited_len = nfp_prog->prog_len * sizeof(u64); 240 prog->aux->offload->jited_image = nfp_prog->prog; 241 242 return nfp_map_ptrs_record(nfp_prog->bpf, nfp_prog, prog); 243 } 244 245 static void nfp_bpf_destroy(struct bpf_prog *prog) 246 { 247 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 248 249 kvfree(nfp_prog->prog); 250 nfp_map_ptrs_forget(nfp_prog->bpf, nfp_prog); 251 nfp_prog_free(nfp_prog); 252 } 253 254 /* Atomic engine requires values to be in big endian, we need to byte swap 255 * the value words used with xadd. 256 */ 257 static void nfp_map_bpf_byte_swap(struct nfp_bpf_map *nfp_map, void *value) 258 { 259 u32 *word = value; 260 unsigned int i; 261 262 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 263 if (nfp_map->use_map[i].type == NFP_MAP_USE_ATOMIC_CNT) 264 word[i] = (__force u32)cpu_to_be32(word[i]); 265 } 266 267 /* Mark value as unsafely initialized in case it becomes atomic later 268 * and we didn't byte swap something non-byte swap neutral. 269 */ 270 static void 271 nfp_map_bpf_byte_swap_record(struct nfp_bpf_map *nfp_map, void *value) 272 { 273 u32 *word = value; 274 unsigned int i; 275 276 for (i = 0; i < DIV_ROUND_UP(nfp_map->offmap->map.value_size, 4); i++) 277 if (nfp_map->use_map[i].type == NFP_MAP_UNUSED && 278 word[i] != (__force u32)cpu_to_be32(word[i])) 279 nfp_map->use_map[i].non_zero_update = 1; 280 } 281 282 static int 283 nfp_bpf_map_lookup_entry(struct bpf_offloaded_map *offmap, 284 void *key, void *value) 285 { 286 int err; 287 288 err = nfp_bpf_ctrl_lookup_entry(offmap, key, value); 289 if (err) 290 return err; 291 292 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 293 return 0; 294 } 295 296 static int 297 nfp_bpf_map_update_entry(struct bpf_offloaded_map *offmap, 298 void *key, void *value, u64 flags) 299 { 300 nfp_map_bpf_byte_swap(offmap->dev_priv, value); 301 nfp_map_bpf_byte_swap_record(offmap->dev_priv, value); 302 return nfp_bpf_ctrl_update_entry(offmap, key, value, flags); 303 } 304 305 static int 306 nfp_bpf_map_get_next_key(struct bpf_offloaded_map *offmap, 307 void *key, void *next_key) 308 { 309 if (!key) 310 return nfp_bpf_ctrl_getfirst_entry(offmap, next_key); 311 return nfp_bpf_ctrl_getnext_entry(offmap, key, next_key); 312 } 313 314 static int 315 nfp_bpf_map_delete_elem(struct bpf_offloaded_map *offmap, void *key) 316 { 317 if (offmap->map.map_type == BPF_MAP_TYPE_ARRAY) 318 return -EINVAL; 319 return nfp_bpf_ctrl_del_entry(offmap, key); 320 } 321 322 static const struct bpf_map_dev_ops nfp_bpf_map_ops = { 323 .map_get_next_key = nfp_bpf_map_get_next_key, 324 .map_lookup_elem = nfp_bpf_map_lookup_entry, 325 .map_update_elem = nfp_bpf_map_update_entry, 326 .map_delete_elem = nfp_bpf_map_delete_elem, 327 }; 328 329 static int 330 nfp_bpf_map_alloc(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 331 { 332 struct nfp_bpf_map *nfp_map; 333 unsigned int use_map_size; 334 long long int res; 335 336 if (!bpf->maps.types) 337 return -EOPNOTSUPP; 338 339 if (offmap->map.map_flags || 340 offmap->map.numa_node != NUMA_NO_NODE) { 341 pr_info("map flags are not supported\n"); 342 return -EINVAL; 343 } 344 345 if (!(bpf->maps.types & 1 << offmap->map.map_type)) { 346 pr_info("map type not supported\n"); 347 return -EOPNOTSUPP; 348 } 349 if (bpf->maps.max_maps == bpf->maps_in_use) { 350 pr_info("too many maps for a device\n"); 351 return -ENOMEM; 352 } 353 if (bpf->maps.max_elems - bpf->map_elems_in_use < 354 offmap->map.max_entries) { 355 pr_info("map with too many elements: %u, left: %u\n", 356 offmap->map.max_entries, 357 bpf->maps.max_elems - bpf->map_elems_in_use); 358 return -ENOMEM; 359 } 360 361 if (round_up(offmap->map.key_size, 8) + 362 round_up(offmap->map.value_size, 8) > bpf->maps.max_elem_sz) { 363 pr_info("map elements too large: %u, FW max element size (key+value): %u\n", 364 round_up(offmap->map.key_size, 8) + 365 round_up(offmap->map.value_size, 8), 366 bpf->maps.max_elem_sz); 367 return -ENOMEM; 368 } 369 if (offmap->map.key_size > bpf->maps.max_key_sz) { 370 pr_info("map key size %u, FW max is %u\n", 371 offmap->map.key_size, bpf->maps.max_key_sz); 372 return -ENOMEM; 373 } 374 if (offmap->map.value_size > bpf->maps.max_val_sz) { 375 pr_info("map value size %u, FW max is %u\n", 376 offmap->map.value_size, bpf->maps.max_val_sz); 377 return -ENOMEM; 378 } 379 380 use_map_size = DIV_ROUND_UP(offmap->map.value_size, 4) * 381 sizeof_field(struct nfp_bpf_map, use_map[0]); 382 383 nfp_map = kzalloc(sizeof(*nfp_map) + use_map_size, GFP_USER); 384 if (!nfp_map) 385 return -ENOMEM; 386 387 offmap->dev_priv = nfp_map; 388 nfp_map->offmap = offmap; 389 nfp_map->bpf = bpf; 390 spin_lock_init(&nfp_map->cache_lock); 391 392 res = nfp_bpf_ctrl_alloc_map(bpf, &offmap->map); 393 if (res < 0) { 394 kfree(nfp_map); 395 return res; 396 } 397 398 nfp_map->tid = res; 399 offmap->dev_ops = &nfp_bpf_map_ops; 400 bpf->maps_in_use++; 401 bpf->map_elems_in_use += offmap->map.max_entries; 402 list_add_tail(&nfp_map->l, &bpf->map_list); 403 404 return 0; 405 } 406 407 static int 408 nfp_bpf_map_free(struct nfp_app_bpf *bpf, struct bpf_offloaded_map *offmap) 409 { 410 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 411 412 nfp_bpf_ctrl_free_map(bpf, nfp_map); 413 dev_consume_skb_any(nfp_map->cache); 414 WARN_ON_ONCE(nfp_map->cache_blockers); 415 list_del_init(&nfp_map->l); 416 bpf->map_elems_in_use -= offmap->map.max_entries; 417 bpf->maps_in_use--; 418 kfree(nfp_map); 419 420 return 0; 421 } 422 423 int nfp_ndo_bpf(struct nfp_app *app, struct nfp_net *nn, struct netdev_bpf *bpf) 424 { 425 switch (bpf->command) { 426 case BPF_OFFLOAD_MAP_ALLOC: 427 return nfp_bpf_map_alloc(app->priv, bpf->offmap); 428 case BPF_OFFLOAD_MAP_FREE: 429 return nfp_bpf_map_free(app->priv, bpf->offmap); 430 default: 431 return -EINVAL; 432 } 433 } 434 435 static unsigned long 436 nfp_bpf_perf_event_copy(void *dst, const void *src, 437 unsigned long off, unsigned long len) 438 { 439 memcpy(dst, src + off, len); 440 return 0; 441 } 442 443 int nfp_bpf_event_output(struct nfp_app_bpf *bpf, const void *data, 444 unsigned int len) 445 { 446 struct cmsg_bpf_event *cbe = (void *)data; 447 struct nfp_bpf_neutral_map *record; 448 u32 pkt_size, data_size, map_id; 449 u64 map_id_full; 450 451 if (len < sizeof(struct cmsg_bpf_event)) 452 return -EINVAL; 453 454 pkt_size = be32_to_cpu(cbe->pkt_size); 455 data_size = be32_to_cpu(cbe->data_size); 456 map_id_full = be64_to_cpu(cbe->map_ptr); 457 map_id = map_id_full; 458 459 if (size_add(pkt_size, data_size) > INT_MAX || 460 len < sizeof(struct cmsg_bpf_event) + pkt_size + data_size) 461 return -EINVAL; 462 if (cbe->hdr.ver != NFP_CCM_ABI_VERSION) 463 return -EINVAL; 464 465 rcu_read_lock(); 466 record = rhashtable_lookup(&bpf->maps_neutral, &map_id, 467 nfp_bpf_maps_neutral_params); 468 if (!record || map_id_full > U32_MAX) { 469 rcu_read_unlock(); 470 cmsg_warn(bpf, "perf event: map id %lld (0x%llx) not recognized, dropping event\n", 471 map_id_full, map_id_full); 472 return -EINVAL; 473 } 474 475 bpf_event_output(record->ptr, be32_to_cpu(cbe->cpu_id), 476 &cbe->data[round_up(pkt_size, 4)], data_size, 477 cbe->data, pkt_size, nfp_bpf_perf_event_copy); 478 rcu_read_unlock(); 479 480 return 0; 481 } 482 483 bool nfp_bpf_offload_check_mtu(struct nfp_net *nn, struct bpf_prog *prog, 484 unsigned int mtu) 485 { 486 unsigned int fw_mtu, pkt_off; 487 488 fw_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 489 pkt_off = min(prog->aux->max_pkt_offset, mtu); 490 491 return fw_mtu < pkt_off; 492 } 493 494 static int 495 nfp_net_bpf_load(struct nfp_net *nn, struct bpf_prog *prog, 496 struct netlink_ext_ack *extack) 497 { 498 struct nfp_prog *nfp_prog = prog->aux->offload->dev_priv; 499 unsigned int max_stack, max_prog_len; 500 dma_addr_t dma_addr; 501 void *img; 502 int err; 503 504 if (nfp_bpf_offload_check_mtu(nn, prog, nn->dp.netdev->mtu)) { 505 NL_SET_ERR_MSG_MOD(extack, "BPF offload not supported with potential packet access beyond HW packet split boundary"); 506 return -EOPNOTSUPP; 507 } 508 509 max_stack = nn_readb(nn, NFP_NET_CFG_BPF_STACK_SZ) * 64; 510 if (nfp_prog->stack_size > max_stack) { 511 NL_SET_ERR_MSG_MOD(extack, "stack too large"); 512 return -EOPNOTSUPP; 513 } 514 515 max_prog_len = nn_readw(nn, NFP_NET_CFG_BPF_MAX_LEN); 516 if (nfp_prog->prog_len > max_prog_len) { 517 NL_SET_ERR_MSG_MOD(extack, "program too long"); 518 return -EOPNOTSUPP; 519 } 520 521 img = nfp_bpf_relo_for_vnic(nfp_prog, nn->app_priv); 522 if (IS_ERR(img)) 523 return PTR_ERR(img); 524 525 dma_addr = dma_map_single(nn->dp.dev, img, 526 nfp_prog->prog_len * sizeof(u64), 527 DMA_TO_DEVICE); 528 if (dma_mapping_error(nn->dp.dev, dma_addr)) { 529 kfree(img); 530 return -ENOMEM; 531 } 532 533 nn_writew(nn, NFP_NET_CFG_BPF_SIZE, nfp_prog->prog_len); 534 nn_writeq(nn, NFP_NET_CFG_BPF_ADDR, dma_addr); 535 536 /* Load up the JITed code */ 537 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_BPF); 538 if (err) 539 NL_SET_ERR_MSG_MOD(extack, 540 "FW command error while loading BPF"); 541 542 dma_unmap_single(nn->dp.dev, dma_addr, nfp_prog->prog_len * sizeof(u64), 543 DMA_TO_DEVICE); 544 kfree(img); 545 546 return err; 547 } 548 549 static void 550 nfp_net_bpf_start(struct nfp_net *nn, struct netlink_ext_ack *extack) 551 { 552 int err; 553 554 /* Enable passing packets through BPF function */ 555 nn->dp.ctrl |= NFP_NET_CFG_CTRL_BPF; 556 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 557 err = nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 558 if (err) 559 NL_SET_ERR_MSG_MOD(extack, 560 "FW command error while enabling BPF"); 561 } 562 563 static int nfp_net_bpf_stop(struct nfp_net *nn) 564 { 565 if (!(nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF)) 566 return 0; 567 568 nn->dp.ctrl &= ~NFP_NET_CFG_CTRL_BPF; 569 nn_writel(nn, NFP_NET_CFG_CTRL, nn->dp.ctrl); 570 571 return nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_GEN); 572 } 573 574 int nfp_net_bpf_offload(struct nfp_net *nn, struct bpf_prog *prog, 575 bool old_prog, struct netlink_ext_ack *extack) 576 { 577 int err; 578 579 if (prog && !bpf_offload_dev_match(prog, nn->dp.netdev)) 580 return -EINVAL; 581 582 if (prog && old_prog) { 583 u8 cap; 584 585 cap = nn_readb(nn, NFP_NET_CFG_BPF_CAP); 586 if (!(cap & NFP_NET_BPF_CAP_RELO)) { 587 NL_SET_ERR_MSG_MOD(extack, 588 "FW does not support live reload"); 589 return -EBUSY; 590 } 591 } 592 593 /* Something else is loaded, different program type? */ 594 if (!old_prog && nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 595 return -EBUSY; 596 597 if (old_prog && !prog) 598 return nfp_net_bpf_stop(nn); 599 600 err = nfp_net_bpf_load(nn, prog, extack); 601 if (err) 602 return err; 603 604 if (!old_prog) 605 nfp_net_bpf_start(nn, extack); 606 607 return 0; 608 } 609 610 const struct bpf_prog_offload_ops nfp_bpf_dev_ops = { 611 .insn_hook = nfp_verify_insn, 612 .finalize = nfp_bpf_finalize, 613 .replace_insn = nfp_bpf_opt_replace_insn, 614 .remove_insns = nfp_bpf_opt_remove_insns, 615 .prepare = nfp_bpf_verifier_prep, 616 .translate = nfp_bpf_translate, 617 .destroy = nfp_bpf_destroy, 618 }; 619