1 /* 2 * Copyright (C) 2017 Netronome Systems, Inc. 3 * 4 * This software is dual licensed under the GNU General License Version 2, 5 * June 1991 as shown in the file COPYING in the top-level directory of this 6 * source tree or the BSD 2-Clause License provided below. You have the 7 * option to license this software under the complete terms of either license. 8 * 9 * The BSD 2-Clause License: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * 1. Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * 2. Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <net/pkt_cls.h> 35 36 #include "../nfpcore/nfp_cpp.h" 37 #include "../nfpcore/nfp_nffw.h" 38 #include "../nfp_app.h" 39 #include "../nfp_main.h" 40 #include "../nfp_net.h" 41 #include "../nfp_port.h" 42 #include "fw.h" 43 #include "main.h" 44 45 static bool nfp_net_ebpf_capable(struct nfp_net *nn) 46 { 47 #ifdef __LITTLE_ENDIAN 48 if (nn->cap & NFP_NET_CFG_CTRL_BPF && 49 nn_readb(nn, NFP_NET_CFG_BPF_ABI) == NFP_NET_BPF_ABI) 50 return true; 51 #endif 52 return false; 53 } 54 55 static int 56 nfp_bpf_xdp_offload(struct nfp_app *app, struct nfp_net *nn, 57 struct bpf_prog *prog, struct netlink_ext_ack *extack) 58 { 59 bool running, xdp_running; 60 int ret; 61 62 if (!nfp_net_ebpf_capable(nn)) 63 return -EINVAL; 64 65 running = nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF; 66 xdp_running = running && nn->dp.bpf_offload_xdp; 67 68 if (!prog && !xdp_running) 69 return 0; 70 if (prog && running && !xdp_running) 71 return -EBUSY; 72 73 ret = nfp_net_bpf_offload(nn, prog, running, extack); 74 /* Stop offload if replace not possible */ 75 if (ret && prog) 76 nfp_bpf_xdp_offload(app, nn, NULL, extack); 77 78 nn->dp.bpf_offload_xdp = prog && !ret; 79 return ret; 80 } 81 82 static const char *nfp_bpf_extra_cap(struct nfp_app *app, struct nfp_net *nn) 83 { 84 return nfp_net_ebpf_capable(nn) ? "BPF" : ""; 85 } 86 87 static int 88 nfp_bpf_vnic_alloc(struct nfp_app *app, struct nfp_net *nn, unsigned int id) 89 { 90 struct nfp_bpf_vnic *bv; 91 int err; 92 93 bv = kzalloc(sizeof(*bv), GFP_KERNEL); 94 if (!bv) 95 return -ENOMEM; 96 nn->app_priv = bv; 97 98 err = nfp_app_nic_vnic_alloc(app, nn, id); 99 if (err) 100 goto err_free_priv; 101 102 bv->start_off = nn_readw(nn, NFP_NET_CFG_BPF_START); 103 bv->tgt_done = nn_readw(nn, NFP_NET_CFG_BPF_DONE); 104 105 return 0; 106 err_free_priv: 107 kfree(nn->app_priv); 108 return err; 109 } 110 111 static void nfp_bpf_vnic_free(struct nfp_app *app, struct nfp_net *nn) 112 { 113 struct nfp_bpf_vnic *bv = nn->app_priv; 114 115 WARN_ON(bv->tc_prog); 116 kfree(bv); 117 } 118 119 static int nfp_bpf_setup_tc_block_cb(enum tc_setup_type type, 120 void *type_data, void *cb_priv) 121 { 122 struct tc_cls_bpf_offload *cls_bpf = type_data; 123 struct nfp_net *nn = cb_priv; 124 struct bpf_prog *oldprog; 125 struct nfp_bpf_vnic *bv; 126 int err; 127 128 if (type != TC_SETUP_CLSBPF) { 129 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 130 "only offload of BPF classifiers supported"); 131 return -EOPNOTSUPP; 132 } 133 if (!tc_cls_can_offload_and_chain0(nn->dp.netdev, &cls_bpf->common)) 134 return -EOPNOTSUPP; 135 if (!nfp_net_ebpf_capable(nn)) { 136 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 137 "NFP firmware does not support eBPF offload"); 138 return -EOPNOTSUPP; 139 } 140 if (cls_bpf->common.protocol != htons(ETH_P_ALL)) { 141 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 142 "only ETH_P_ALL supported as filter protocol"); 143 return -EOPNOTSUPP; 144 } 145 146 /* Only support TC direct action */ 147 if (!cls_bpf->exts_integrated || 148 tcf_exts_has_actions(cls_bpf->exts)) { 149 NL_SET_ERR_MSG_MOD(cls_bpf->common.extack, 150 "only direct action with no legacy actions supported"); 151 return -EOPNOTSUPP; 152 } 153 154 if (cls_bpf->command != TC_CLSBPF_OFFLOAD) 155 return -EOPNOTSUPP; 156 157 bv = nn->app_priv; 158 oldprog = cls_bpf->oldprog; 159 160 /* Don't remove if oldprog doesn't match driver's state */ 161 if (bv->tc_prog != oldprog) { 162 oldprog = NULL; 163 if (!cls_bpf->prog) 164 return 0; 165 } 166 167 err = nfp_net_bpf_offload(nn, cls_bpf->prog, oldprog, 168 cls_bpf->common.extack); 169 if (err) 170 return err; 171 172 bv->tc_prog = cls_bpf->prog; 173 return 0; 174 } 175 176 static int nfp_bpf_setup_tc_block(struct net_device *netdev, 177 struct tc_block_offload *f) 178 { 179 struct nfp_net *nn = netdev_priv(netdev); 180 181 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 182 return -EOPNOTSUPP; 183 184 switch (f->command) { 185 case TC_BLOCK_BIND: 186 return tcf_block_cb_register(f->block, 187 nfp_bpf_setup_tc_block_cb, 188 nn, nn); 189 case TC_BLOCK_UNBIND: 190 tcf_block_cb_unregister(f->block, 191 nfp_bpf_setup_tc_block_cb, 192 nn); 193 return 0; 194 default: 195 return -EOPNOTSUPP; 196 } 197 } 198 199 static int nfp_bpf_setup_tc(struct nfp_app *app, struct net_device *netdev, 200 enum tc_setup_type type, void *type_data) 201 { 202 switch (type) { 203 case TC_SETUP_BLOCK: 204 return nfp_bpf_setup_tc_block(netdev, type_data); 205 default: 206 return -EOPNOTSUPP; 207 } 208 } 209 210 static bool nfp_bpf_tc_busy(struct nfp_app *app, struct nfp_net *nn) 211 { 212 struct nfp_bpf_vnic *bv = nn->app_priv; 213 214 return !!bv->tc_prog; 215 } 216 217 static int 218 nfp_bpf_change_mtu(struct nfp_app *app, struct net_device *netdev, int new_mtu) 219 { 220 struct nfp_net *nn = netdev_priv(netdev); 221 unsigned int max_mtu; 222 223 if (~nn->dp.ctrl & NFP_NET_CFG_CTRL_BPF) 224 return 0; 225 226 max_mtu = nn_readb(nn, NFP_NET_CFG_BPF_INL_MTU) * 64 - 32; 227 if (new_mtu > max_mtu) { 228 nn_info(nn, "BPF offload active, MTU over %u not supported\n", 229 max_mtu); 230 return -EBUSY; 231 } 232 return 0; 233 } 234 235 static int 236 nfp_bpf_parse_cap_adjust_head(struct nfp_app_bpf *bpf, void __iomem *value, 237 u32 length) 238 { 239 struct nfp_bpf_cap_tlv_adjust_head __iomem *cap = value; 240 struct nfp_cpp *cpp = bpf->app->pf->cpp; 241 242 if (length < sizeof(*cap)) { 243 nfp_err(cpp, "truncated adjust_head TLV: %d\n", length); 244 return -EINVAL; 245 } 246 247 bpf->adjust_head.flags = readl(&cap->flags); 248 bpf->adjust_head.off_min = readl(&cap->off_min); 249 bpf->adjust_head.off_max = readl(&cap->off_max); 250 bpf->adjust_head.guaranteed_sub = readl(&cap->guaranteed_sub); 251 bpf->adjust_head.guaranteed_add = readl(&cap->guaranteed_add); 252 253 if (bpf->adjust_head.off_min > bpf->adjust_head.off_max) { 254 nfp_err(cpp, "invalid adjust_head TLV: min > max\n"); 255 return -EINVAL; 256 } 257 if (!FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_min) || 258 !FIELD_FIT(UR_REG_IMM_MAX, bpf->adjust_head.off_max)) { 259 nfp_warn(cpp, "disabling adjust_head - driver expects min/max to fit in as immediates\n"); 260 memset(&bpf->adjust_head, 0, sizeof(bpf->adjust_head)); 261 return 0; 262 } 263 264 return 0; 265 } 266 267 static int 268 nfp_bpf_parse_cap_func(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 269 { 270 struct nfp_bpf_cap_tlv_func __iomem *cap = value; 271 272 if (length < sizeof(*cap)) { 273 nfp_err(bpf->app->cpp, "truncated function TLV: %d\n", length); 274 return -EINVAL; 275 } 276 277 switch (readl(&cap->func_id)) { 278 case BPF_FUNC_map_lookup_elem: 279 bpf->helpers.map_lookup = readl(&cap->func_addr); 280 break; 281 } 282 283 return 0; 284 } 285 286 static int 287 nfp_bpf_parse_cap_maps(struct nfp_app_bpf *bpf, void __iomem *value, u32 length) 288 { 289 struct nfp_bpf_cap_tlv_maps __iomem *cap = value; 290 291 if (length < sizeof(*cap)) { 292 nfp_err(bpf->app->cpp, "truncated maps TLV: %d\n", length); 293 return -EINVAL; 294 } 295 296 bpf->maps.types = readl(&cap->types); 297 bpf->maps.max_maps = readl(&cap->max_maps); 298 bpf->maps.max_elems = readl(&cap->max_elems); 299 bpf->maps.max_key_sz = readl(&cap->max_key_sz); 300 bpf->maps.max_val_sz = readl(&cap->max_val_sz); 301 bpf->maps.max_elem_sz = readl(&cap->max_elem_sz); 302 303 return 0; 304 } 305 306 static int nfp_bpf_parse_capabilities(struct nfp_app *app) 307 { 308 struct nfp_cpp *cpp = app->pf->cpp; 309 struct nfp_cpp_area *area; 310 u8 __iomem *mem, *start; 311 312 mem = nfp_rtsym_map(app->pf->rtbl, "_abi_bpf_capabilities", "bpf.cap", 313 8, &area); 314 if (IS_ERR(mem)) 315 return PTR_ERR(mem) == -ENOENT ? 0 : PTR_ERR(mem); 316 317 start = mem; 318 while (mem - start + 8 < nfp_cpp_area_size(area)) { 319 u8 __iomem *value; 320 u32 type, length; 321 322 type = readl(mem); 323 length = readl(mem + 4); 324 value = mem + 8; 325 326 mem += 8 + length; 327 if (mem - start > nfp_cpp_area_size(area)) 328 goto err_release_free; 329 330 switch (type) { 331 case NFP_BPF_CAP_TYPE_FUNC: 332 if (nfp_bpf_parse_cap_func(app->priv, value, length)) 333 goto err_release_free; 334 break; 335 case NFP_BPF_CAP_TYPE_ADJUST_HEAD: 336 if (nfp_bpf_parse_cap_adjust_head(app->priv, value, 337 length)) 338 goto err_release_free; 339 break; 340 case NFP_BPF_CAP_TYPE_MAPS: 341 if (nfp_bpf_parse_cap_maps(app->priv, value, length)) 342 goto err_release_free; 343 break; 344 default: 345 nfp_dbg(cpp, "unknown BPF capability: %d\n", type); 346 break; 347 } 348 } 349 if (mem - start != nfp_cpp_area_size(area)) { 350 nfp_err(cpp, "BPF capabilities left after parsing, parsed:%zd total length:%zu\n", 351 mem - start, nfp_cpp_area_size(area)); 352 goto err_release_free; 353 } 354 355 nfp_cpp_area_release_free(area); 356 357 return 0; 358 359 err_release_free: 360 nfp_err(cpp, "invalid BPF capabilities at offset:%zd\n", mem - start); 361 nfp_cpp_area_release_free(area); 362 return -EINVAL; 363 } 364 365 static int nfp_bpf_init(struct nfp_app *app) 366 { 367 struct nfp_app_bpf *bpf; 368 int err; 369 370 bpf = kzalloc(sizeof(*bpf), GFP_KERNEL); 371 if (!bpf) 372 return -ENOMEM; 373 bpf->app = app; 374 app->priv = bpf; 375 376 skb_queue_head_init(&bpf->cmsg_replies); 377 init_waitqueue_head(&bpf->cmsg_wq); 378 INIT_LIST_HEAD(&bpf->map_list); 379 380 err = nfp_bpf_parse_capabilities(app); 381 if (err) 382 goto err_free_bpf; 383 384 return 0; 385 386 err_free_bpf: 387 kfree(bpf); 388 return err; 389 } 390 391 static void nfp_bpf_clean(struct nfp_app *app) 392 { 393 struct nfp_app_bpf *bpf = app->priv; 394 395 WARN_ON(!skb_queue_empty(&bpf->cmsg_replies)); 396 WARN_ON(!list_empty(&bpf->map_list)); 397 WARN_ON(bpf->maps_in_use || bpf->map_elems_in_use); 398 kfree(bpf); 399 } 400 401 const struct nfp_app_type app_bpf = { 402 .id = NFP_APP_BPF_NIC, 403 .name = "ebpf", 404 405 .ctrl_cap_mask = 0, 406 407 .init = nfp_bpf_init, 408 .clean = nfp_bpf_clean, 409 410 .change_mtu = nfp_bpf_change_mtu, 411 412 .extra_cap = nfp_bpf_extra_cap, 413 414 .vnic_alloc = nfp_bpf_vnic_alloc, 415 .vnic_free = nfp_bpf_vnic_free, 416 417 .ctrl_msg_rx = nfp_bpf_ctrl_msg_rx, 418 419 .setup_tc = nfp_bpf_setup_tc, 420 .tc_busy = nfp_bpf_tc_busy, 421 .bpf = nfp_ndo_bpf, 422 .xdp_offload = nfp_bpf_xdp_offload, 423 }; 424