1 /* 2 * Copyright (C) 2017-2018 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <net/pkt_cls.h> 35 36 #include "../nfpcore/nfp_cpp.h" 37 #include "../nfpcore/nfp_nffw.h" 38 #include "../nfpcore/nfp_nsp.h" 39 #include "../nfp_app.h" 40 #include "../nfp_main.h" 41 #include "../nfp_net.h" 42 #include "../nfp_port.h" 43 #include "fw.h" 44 #include "main.h" 45 46 const struct rhashtable_params nfp_bpf_maps_neutral_params = { 47 .nelem_hint = 4, 48 .key_len = FIELD_SIZEOF(struct bpf_map, id), 49 .key_offset = offsetof(struct nfp_bpf_neutral_map, map_id), 50 .head_offset = offsetof(struct nfp_bpf_neutral_map, l), 51 .automatic_shrinking = true, 52 }; 53 54 static bool nfp_net_ebpf_capable(struct nfp_net *nn) 55 { 56 #ifdef __LITTLE_ENDIAN 57 struct nfp_app_bpf *bpf = nn->app->priv; 58 59 return nn->cap & NFP_NET_CFG_CTRL_BPF && 60 bpf->abi_version && 61 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == bpf->abi_version; 62 #else 63 return false; 64 #endif 65 } 66 67 static int 68 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 69 struct bpf_prog *prog, struct netlink_ext_ack *extack) 70 { 71 bool running, xdp_running; 72 73 if (!nfp_net_ebpf_capable(nn)) 74 return -EINVAL; 75 76 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 77 xdp_running = running && nn->xdp_hw.prog; 78 79 if (!prog && !xdp_running) 80 return 0; 81 if (prog && running && !xdp_running) 82 return -EBUSY; 83 84 return nfp_net_bpf_offload(nn, prog, running, extack); 85 } 86 87 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 88 { 89 return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 90 } 91 92 static int 93 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 94 { 95 struct nfp_pf *pf = app->pf; 96 struct nfp_bpf_vnic *bv; 97 int err; 98 99 if (!pf->eth_tbl) { 100 nfp_err(pf->cpp, "No ETH table\n"); 101 return -EINVAL; 102 } 103 if (pf->max_data_vnics != pf->eth_tbl->count) { 104 nfp_err(pf->cpp, "ETH entries don't match vNICs (%d vs %d)\n", 105 pf->max_data_vnics, pf->eth_tbl->count); 106 return -EINVAL; 107 } 108 109 bv = kzalloc(sizeof(*bv), GFP_KERNEL); 110 if (!bv) 111 return -ENOMEM; 112 nn->app_priv = bv; 113 114 err = nfp_app_nic_vnic_alloc(app, nn, id); 115 if (err) 116 goto err_free_priv; 117 118 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); 119 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); 120 121 return 0; 122 err_free_priv: 123 kfree(nn->app_priv); 124 return err; 125 } 126 127 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) 128 { 129 struct nfp_bpf_vnic *bv = nn->app_priv; 130 131 WARN_ON(bv->tc_prog); 132 kfree(bv); 133 } 134 135 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 136 void *type_data, void *cb_priv) 137 { 138 struct tc_cls_bpf_offload *cls_bpf = type_data; 139 struct nfp_net *nn = cb_priv; 140 struct bpf_prog *oldprog; 141 struct nfp_bpf_vnic *bv; 142 int err; 143 144 if (type != TC_SETUP_CLSBPF) { 145 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 146 "only offload of BPF classifiers supported"); 147 return -EOPNOTSUPP; 148 } 149 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common)) 150 return -EOPNOTSUPP; 151 if (!nfp_net_ebpf_capable(nn)) { 152 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 153 "NFP firmware does not support eBPF offload"); 154 return -EOPNOTSUPP; 155 } 156 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { 157 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 158 "only ETH_P_ALL supported as filter protocol"); 159 return -EOPNOTSUPP; 160 } 161 162 /* Only support TC direct action */ 163 if (!cls_bpf->exts_integrated || 164 tcf_exts_has_actions(cls_bpf->exts)) { 165 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 166 "only direct action with no legacy actions supported"); 167 return -EOPNOTSUPP; 168 } 169 170 if (cls_bpf->command != TC_CLSBPF_OFFLOAD) 171 return -EOPNOTSUPP; 172 173 bv = nn->app_priv; 174 oldprog = cls_bpf->oldprog; 175 176 /* Don't remove if oldprog doesn't match driver's state */ 177 if (bv->tc_prog != oldprog) { 178 oldprog = NULL; 179 if (!cls_bpf->prog) 180 return 0; 181 } 182 183 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog, 184 cls_bpf->common.extack); 185 if (err) 186 return err; 187 188 bv->tc_prog = cls_bpf->prog; 189 nn->port->tc_offload_cnt = !!bv->tc_prog; 190 return 0; 191 } 192 193 static int nfp_bpf_setup_tc_block(struct net_device *netdev, 194 struct tc_block_offload *f) 195 { 196 struct nfp_net *nn = netdev_priv(netdev); 197 198 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 199 return -EOPNOTSUPP; 200 201 switch (f->command) { 202 case TC_BLOCK_BIND: 203 return tcf_block_cb_register(f->block, 204 nfp_bpf_setup_tc_block_cb, 205 nn, nn, f->extack); 206 case TC_BLOCK_UNBIND: 207 tcf_block_cb_unregister(f->block, 208 nfp_bpf_setup_tc_block_cb, 209 nn); 210 return 0; 211 default: 212 return -EOPNOTSUPP; 213 } 214 } 215 216 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 217 enum tc_setup_type type, void *type_data) 218 { 219 switch (type) { 220 case TC_SETUP_BLOCK: 221 return nfp_bpf_setup_tc_block(netdev, type_data); 222 default: 223 return -EOPNOTSUPP; 224 } 225 } 226 227 static int 228 nfp_bpf_check_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) 229 { 230 struct nfp_net *nn = netdev_priv(netdev); 231 unsigned int max_mtu; 232 233 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 234 return 0; 235 236 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 237 if (new_mtu > max_mtu) { 238 nn_info(nn, "BPF offload active, MTU over %u not supported\n", 239 max_mtu); 240 return -EBUSY; 241 } 242 return 0; 243 } 244 245 static int 246 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 247 u32 length) 248 { 249 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 250 struct nfp_cpp *cpp = bpf->app->pf->cpp; 251 252 if (length < sizeof(*cap)) { 253 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 254 return -EINVAL; 255 } 256 257 bpf->adjust_head.flags = readl(&cap->flags); 258 bpf->adjust_head.off_min = readl(&cap->off_min); 259 bpf->adjust_head.off_max = readl(&cap->off_max); 260 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 261 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 262 263 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 264 nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 265 return -EINVAL; 266 } 267 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 268 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 269 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 270 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 271 return 0; 272 } 273 274 return 0; 275 } 276 277 static int 278 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 279 { 280 struct nfp_bpf_cap_tlv_func __iomem *cap = value; 281 282 if (length < sizeof(*cap)) { 283 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length); 284 return -EINVAL; 285 } 286 287 switch (readl(&cap->func_id)) { 288 case BPF_FUNC_map_lookup_elem: 289 bpf->helpers.map_lookup = readl(&cap->func_addr); 290 break; 291 case BPF_FUNC_map_update_elem: 292 bpf->helpers.map_update = readl(&cap->func_addr); 293 break; 294 case BPF_FUNC_map_delete_elem: 295 bpf->helpers.map_delete = readl(&cap->func_addr); 296 break; 297 case BPF_FUNC_perf_event_output: 298 bpf->helpers.perf_event_output = readl(&cap->func_addr); 299 break; 300 } 301 302 return 0; 303 } 304 305 static int 306 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 307 { 308 struct nfp_bpf_cap_tlv_maps __iomem *cap = value; 309 310 if (length < sizeof(*cap)) { 311 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length); 312 return -EINVAL; 313 } 314 315 bpf->maps.types = readl(&cap->types); 316 bpf->maps.max_maps = readl(&cap->max_maps); 317 bpf->maps.max_elems = readl(&cap->max_elems); 318 bpf->maps.max_key_sz = readl(&cap->max_key_sz); 319 bpf->maps.max_val_sz = readl(&cap->max_val_sz); 320 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz); 321 322 return 0; 323 } 324 325 static int 326 nfp_bpf_parse_cap_random(struct nfp_app_bpf *bpf, void __iomem *value, 327 u32 length) 328 { 329 bpf->pseudo_random = true; 330 return 0; 331 } 332 333 static int 334 nfp_bpf_parse_cap_qsel(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 335 { 336 bpf->queue_select = true; 337 return 0; 338 } 339 340 static int 341 nfp_bpf_parse_cap_adjust_tail(struct nfp_app_bpf *bpf, void __iomem *value, 342 u32 length) 343 { 344 bpf->adjust_tail = true; 345 return 0; 346 } 347 348 static int 349 nfp_bpf_parse_cap_abi_version(struct nfp_app_bpf *bpf, void __iomem *value, 350 u32 length) 351 { 352 if (length < 4) { 353 nfp_err(bpf->app->cpp, "truncated ABI version TLV: %d\n", 354 length); 355 return -EINVAL; 356 } 357 358 bpf->abi_version = readl(value); 359 if (bpf->abi_version < 2 || bpf->abi_version > 3) { 360 nfp_warn(bpf->app->cpp, "unsupported BPF ABI version: %d\n", 361 bpf->abi_version); 362 bpf->abi_version = 0; 363 } 364 365 return 0; 366 } 367 368 static int nfp_bpf_parse_capabilities(struct nfp_app *app) 369 { 370 struct nfp_cpp *cpp = app->pf->cpp; 371 struct nfp_cpp_area *area; 372 u8 __iomem *mem, *start; 373 374 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 375 8, &area); 376 if (IS_ERR(mem)) 377 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 378 379 start = mem; 380 while (mem - start + 8 <= nfp_cpp_area_size(area)) { 381 u8 __iomem *value; 382 u32 type, length; 383 384 type = readl(mem); 385 length = readl(mem + 4); 386 value = mem + 8; 387 388 mem += 8 + length; 389 if (mem - start > nfp_cpp_area_size(area)) 390 goto err_release_free; 391 392 switch (type) { 393 case NFP_BPF_CAP_TYPE_FUNC: 394 if (nfp_bpf_parse_cap_func(app->priv, value, length)) 395 goto err_release_free; 396 break; 397 case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 398 if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 399 length)) 400 goto err_release_free; 401 break; 402 case NFP_BPF_CAP_TYPE_MAPS: 403 if (nfp_bpf_parse_cap_maps(app->priv, value, length)) 404 goto err_release_free; 405 break; 406 case NFP_BPF_CAP_TYPE_RANDOM: 407 if (nfp_bpf_parse_cap_random(app->priv, value, length)) 408 goto err_release_free; 409 break; 410 case NFP_BPF_CAP_TYPE_QUEUE_SELECT: 411 if (nfp_bpf_parse_cap_qsel(app->priv, value, length)) 412 goto err_release_free; 413 break; 414 case NFP_BPF_CAP_TYPE_ADJUST_TAIL: 415 if (nfp_bpf_parse_cap_adjust_tail(app->priv, value, 416 length)) 417 goto err_release_free; 418 break; 419 case NFP_BPF_CAP_TYPE_ABI_VERSION: 420 if (nfp_bpf_parse_cap_abi_version(app->priv, value, 421 length)) 422 goto err_release_free; 423 break; 424 default: 425 nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 426 break; 427 } 428 } 429 if (mem - start != nfp_cpp_area_size(area)) { 430 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 431 mem - start, nfp_cpp_area_size(area)); 432 goto err_release_free; 433 } 434 435 nfp_cpp_area_release_free(area); 436 437 return 0; 438 439 err_release_free: 440 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 441 nfp_cpp_area_release_free(area); 442 return -EINVAL; 443 } 444 445 static void nfp_bpf_init_capabilities(struct nfp_app_bpf *bpf) 446 { 447 bpf->abi_version = 2; /* Original BPF ABI version */ 448 } 449 450 static int nfp_bpf_ndo_init(struct nfp_app *app, struct net_device *netdev) 451 { 452 struct nfp_app_bpf *bpf = app->priv; 453 454 return bpf_offload_dev_netdev_register(bpf->bpf_dev, netdev); 455 } 456 457 static void nfp_bpf_ndo_uninit(struct nfp_app *app, struct net_device *netdev) 458 { 459 struct nfp_app_bpf *bpf = app->priv; 460 461 bpf_offload_dev_netdev_unregister(bpf->bpf_dev, netdev); 462 } 463 464 static int nfp_bpf_init(struct nfp_app *app) 465 { 466 struct nfp_app_bpf *bpf; 467 int err; 468 469 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 470 if (!bpf) 471 return -ENOMEM; 472 bpf->app = app; 473 app->priv = bpf; 474 475 skb_queue_head_init(&bpf->cmsg_replies); 476 init_waitqueue_head(&bpf->cmsg_wq); 477 INIT_LIST_HEAD(&bpf->map_list); 478 479 err = rhashtable_init(&bpf->maps_neutral, &nfp_bpf_maps_neutral_params); 480 if (err) 481 goto err_free_bpf; 482 483 nfp_bpf_init_capabilities(bpf); 484 485 err = nfp_bpf_parse_capabilities(app); 486 if (err) 487 goto err_free_neutral_maps; 488 489 if (bpf->abi_version < 3) { 490 bpf->cmsg_key_sz = CMSG_MAP_KEY_LW * 4; 491 bpf->cmsg_val_sz = CMSG_MAP_VALUE_LW * 4; 492 } else { 493 bpf->cmsg_key_sz = bpf->maps.max_key_sz; 494 bpf->cmsg_val_sz = bpf->maps.max_val_sz; 495 app->ctrl_mtu = nfp_bpf_ctrl_cmsg_mtu(bpf); 496 } 497 498 bpf->bpf_dev = bpf_offload_dev_create(); 499 err = PTR_ERR_OR_ZERO(bpf->bpf_dev); 500 if (err) 501 goto err_free_neutral_maps; 502 503 return 0; 504 505 err_free_neutral_maps: 506 rhashtable_destroy(&bpf->maps_neutral); 507 err_free_bpf: 508 kfree(bpf); 509 return err; 510 } 511 512 static void nfp_check_rhashtable_empty(void *ptr, void *arg) 513 { 514 WARN_ON_ONCE(1); 515 } 516 517 static void nfp_bpf_clean(struct nfp_app *app) 518 { 519 struct nfp_app_bpf *bpf = app->priv; 520 521 bpf_offload_dev_destroy(bpf->bpf_dev); 522 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies)); 523 WARN_ON(!list_empty(&bpf->map_list)); 524 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); 525 rhashtable_free_and_destroy(&bpf->maps_neutral, 526 nfp_check_rhashtable_empty, NULL); 527 kfree(bpf); 528 } 529 530 const struct nfp_app_type app_bpf = { 531 .id = NFP_APP_BPF_NIC, 532 .name = "ebpf", 533 534 .ctrl_cap_mask = 0, 535 536 .init = nfp_bpf_init, 537 .clean = nfp_bpf_clean, 538 539 .check_mtu = nfp_bpf_check_mtu, 540 541 .extra_cap = nfp_bpf_extra_cap, 542 543 .ndo_init = nfp_bpf_ndo_init, 544 .ndo_uninit = nfp_bpf_ndo_uninit, 545 546 .vnic_alloc = nfp_bpf_vnic_alloc, 547 .vnic_free = nfp_bpf_vnic_free, 548 549 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, 550 .ctrl_msg_rx_raw = nfp_bpf_ctrl_msg_rx_raw, 551 552 .setup_tc = nfp_bpf_setup_tc, 553 .bpf = nfp_ndo_bpf, 554 .xdp_offload = nfp_bpf_xdp_offload, 555 }; 556