1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2017-2018 Netronome Systems, Inc. */ 3 4 #include <linux/bpf.h> 5 #include <linux/bitops.h> 6 #include <linux/bug.h> 7 #include <linux/jiffies.h> 8 #include <linux/skbuff.h> 9 #include <linux/timekeeping.h> 10 11 #include "../ccm.h" 12 #include "../nfp_app.h" 13 #include "../nfp_net.h" 14 #include "fw.h" 15 #include "main.h" 16 17 static struct sk_buff * 18 nfp_bpf_cmsg_alloc(struct nfp_app_bpf *bpf, unsigned int size) 19 { 20 struct sk_buff *skb; 21 22 skb = nfp_app_ctrl_msg_alloc(bpf->app, size, GFP_KERNEL); 23 skb_put(skb, size); 24 25 return skb; 26 } 27 28 static unsigned int 29 nfp_bpf_cmsg_map_req_size(struct nfp_app_bpf *bpf, unsigned int n) 30 { 31 unsigned int size; 32 33 size = sizeof(struct cmsg_req_map_op); 34 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n; 35 36 return size; 37 } 38 39 static struct sk_buff * 40 nfp_bpf_cmsg_map_req_alloc(struct nfp_app_bpf *bpf, unsigned int n) 41 { 42 return nfp_bpf_cmsg_alloc(bpf, nfp_bpf_cmsg_map_req_size(bpf, n)); 43 } 44 45 static unsigned int 46 nfp_bpf_cmsg_map_reply_size(struct nfp_app_bpf *bpf, unsigned int n) 47 { 48 unsigned int size; 49 50 size = sizeof(struct cmsg_reply_map_op); 51 size += (bpf->cmsg_key_sz + bpf->cmsg_val_sz) * n; 52 53 return size; 54 } 55 56 static int 57 nfp_bpf_ctrl_rc_to_errno(struct nfp_app_bpf *bpf, 58 struct cmsg_reply_map_simple *reply) 59 { 60 static const int res_table[] = { 61 [CMSG_RC_SUCCESS] = 0, 62 [CMSG_RC_ERR_MAP_FD] = -EBADFD, 63 [CMSG_RC_ERR_MAP_NOENT] = -ENOENT, 64 [CMSG_RC_ERR_MAP_ERR] = -EINVAL, 65 [CMSG_RC_ERR_MAP_PARSE] = -EIO, 66 [CMSG_RC_ERR_MAP_EXIST] = -EEXIST, 67 [CMSG_RC_ERR_MAP_NOMEM] = -ENOMEM, 68 [CMSG_RC_ERR_MAP_E2BIG] = -E2BIG, 69 }; 70 u32 rc; 71 72 rc = be32_to_cpu(reply->rc); 73 if (rc >= ARRAY_SIZE(res_table)) { 74 cmsg_warn(bpf, "FW responded with invalid status: %u\n", rc); 75 return -EIO; 76 } 77 78 return res_table[rc]; 79 } 80 81 long long int 82 nfp_bpf_ctrl_alloc_map(struct nfp_app_bpf *bpf, struct bpf_map *map) 83 { 84 struct cmsg_reply_map_alloc_tbl *reply; 85 struct cmsg_req_map_alloc_tbl *req; 86 struct sk_buff *skb; 87 u32 tid; 88 int err; 89 90 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req)); 91 if (!skb) 92 return -ENOMEM; 93 94 req = (void *)skb->data; 95 req->key_size = cpu_to_be32(map->key_size); 96 req->value_size = cpu_to_be32(map->value_size); 97 req->max_entries = cpu_to_be32(map->max_entries); 98 req->map_type = cpu_to_be32(map->map_type); 99 req->map_flags = 0; 100 101 skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_ALLOC, 102 sizeof(*reply)); 103 if (IS_ERR(skb)) 104 return PTR_ERR(skb); 105 106 reply = (void *)skb->data; 107 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 108 if (err) 109 goto err_free; 110 111 tid = be32_to_cpu(reply->tid); 112 dev_consume_skb_any(skb); 113 114 return tid; 115 err_free: 116 dev_kfree_skb_any(skb); 117 return err; 118 } 119 120 void nfp_bpf_ctrl_free_map(struct nfp_app_bpf *bpf, struct nfp_bpf_map *nfp_map) 121 { 122 struct cmsg_reply_map_free_tbl *reply; 123 struct cmsg_req_map_free_tbl *req; 124 struct sk_buff *skb; 125 int err; 126 127 skb = nfp_bpf_cmsg_alloc(bpf, sizeof(*req)); 128 if (!skb) { 129 cmsg_warn(bpf, "leaking map - failed to allocate msg\n"); 130 return; 131 } 132 133 req = (void *)skb->data; 134 req->tid = cpu_to_be32(nfp_map->tid); 135 136 skb = nfp_ccm_communicate(&bpf->ccm, skb, NFP_CCM_TYPE_BPF_MAP_FREE, 137 sizeof(*reply)); 138 if (IS_ERR(skb)) { 139 cmsg_warn(bpf, "leaking map - I/O error\n"); 140 return; 141 } 142 143 reply = (void *)skb->data; 144 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 145 if (err) 146 cmsg_warn(bpf, "leaking map - FW responded with: %d\n", err); 147 148 dev_consume_skb_any(skb); 149 } 150 151 static void * 152 nfp_bpf_ctrl_req_key(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req, 153 unsigned int n) 154 { 155 return &req->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n]; 156 } 157 158 static void * 159 nfp_bpf_ctrl_req_val(struct nfp_app_bpf *bpf, struct cmsg_req_map_op *req, 160 unsigned int n) 161 { 162 return &req->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n]; 163 } 164 165 static void * 166 nfp_bpf_ctrl_reply_key(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply, 167 unsigned int n) 168 { 169 return &reply->data[bpf->cmsg_key_sz * n + bpf->cmsg_val_sz * n]; 170 } 171 172 static void * 173 nfp_bpf_ctrl_reply_val(struct nfp_app_bpf *bpf, struct cmsg_reply_map_op *reply, 174 unsigned int n) 175 { 176 return &reply->data[bpf->cmsg_key_sz * (n + 1) + bpf->cmsg_val_sz * n]; 177 } 178 179 static bool nfp_bpf_ctrl_op_cache_invalidate(enum nfp_ccm_type op) 180 { 181 return op == NFP_CCM_TYPE_BPF_MAP_UPDATE || 182 op == NFP_CCM_TYPE_BPF_MAP_DELETE; 183 } 184 185 static bool nfp_bpf_ctrl_op_cache_capable(enum nfp_ccm_type op) 186 { 187 return op == NFP_CCM_TYPE_BPF_MAP_LOOKUP || 188 op == NFP_CCM_TYPE_BPF_MAP_GETNEXT; 189 } 190 191 static bool nfp_bpf_ctrl_op_cache_fill(enum nfp_ccm_type op) 192 { 193 return op == NFP_CCM_TYPE_BPF_MAP_GETFIRST || 194 op == NFP_CCM_TYPE_BPF_MAP_GETNEXT; 195 } 196 197 static unsigned int 198 nfp_bpf_ctrl_op_cache_get(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op, 199 const u8 *key, u8 *out_key, u8 *out_value, 200 u32 *cache_gen) 201 { 202 struct bpf_map *map = &nfp_map->offmap->map; 203 struct nfp_app_bpf *bpf = nfp_map->bpf; 204 unsigned int i, count, n_entries; 205 struct cmsg_reply_map_op *reply; 206 207 n_entries = nfp_bpf_ctrl_op_cache_fill(op) ? bpf->cmsg_cache_cnt : 1; 208 209 spin_lock(&nfp_map->cache_lock); 210 *cache_gen = nfp_map->cache_gen; 211 if (nfp_map->cache_blockers) 212 n_entries = 1; 213 214 if (nfp_bpf_ctrl_op_cache_invalidate(op)) 215 goto exit_block; 216 if (!nfp_bpf_ctrl_op_cache_capable(op)) 217 goto exit_unlock; 218 219 if (!nfp_map->cache) 220 goto exit_unlock; 221 if (nfp_map->cache_to < ktime_get_ns()) 222 goto exit_invalidate; 223 224 reply = (void *)nfp_map->cache->data; 225 count = be32_to_cpu(reply->count); 226 227 for (i = 0; i < count; i++) { 228 void *cached_key; 229 230 cached_key = nfp_bpf_ctrl_reply_key(bpf, reply, i); 231 if (memcmp(cached_key, key, map->key_size)) 232 continue; 233 234 if (op == NFP_CCM_TYPE_BPF_MAP_LOOKUP) 235 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, i), 236 map->value_size); 237 if (op == NFP_CCM_TYPE_BPF_MAP_GETNEXT) { 238 if (i + 1 == count) 239 break; 240 241 memcpy(out_key, 242 nfp_bpf_ctrl_reply_key(bpf, reply, i + 1), 243 map->key_size); 244 } 245 246 n_entries = 0; 247 goto exit_unlock; 248 } 249 goto exit_unlock; 250 251 exit_block: 252 nfp_map->cache_blockers++; 253 exit_invalidate: 254 dev_consume_skb_any(nfp_map->cache); 255 nfp_map->cache = NULL; 256 exit_unlock: 257 spin_unlock(&nfp_map->cache_lock); 258 return n_entries; 259 } 260 261 static void 262 nfp_bpf_ctrl_op_cache_put(struct nfp_bpf_map *nfp_map, enum nfp_ccm_type op, 263 struct sk_buff *skb, u32 cache_gen) 264 { 265 bool blocker, filler; 266 267 blocker = nfp_bpf_ctrl_op_cache_invalidate(op); 268 filler = nfp_bpf_ctrl_op_cache_fill(op); 269 if (blocker || filler) { 270 u64 to = 0; 271 272 if (filler) 273 to = ktime_get_ns() + NFP_BPF_MAP_CACHE_TIME_NS; 274 275 spin_lock(&nfp_map->cache_lock); 276 if (blocker) { 277 nfp_map->cache_blockers--; 278 nfp_map->cache_gen++; 279 } 280 if (filler && !nfp_map->cache_blockers && 281 nfp_map->cache_gen == cache_gen) { 282 nfp_map->cache_to = to; 283 swap(nfp_map->cache, skb); 284 } 285 spin_unlock(&nfp_map->cache_lock); 286 } 287 288 dev_consume_skb_any(skb); 289 } 290 291 static int 292 nfp_bpf_ctrl_entry_op(struct bpf_offloaded_map *offmap, enum nfp_ccm_type op, 293 u8 *key, u8 *value, u64 flags, u8 *out_key, u8 *out_value) 294 { 295 struct nfp_bpf_map *nfp_map = offmap->dev_priv; 296 unsigned int n_entries, reply_entries, count; 297 struct nfp_app_bpf *bpf = nfp_map->bpf; 298 struct bpf_map *map = &offmap->map; 299 struct cmsg_reply_map_op *reply; 300 struct cmsg_req_map_op *req; 301 struct sk_buff *skb; 302 u32 cache_gen; 303 int err; 304 305 /* FW messages have no space for more than 32 bits of flags */ 306 if (flags >> 32) 307 return -EOPNOTSUPP; 308 309 /* Handle op cache */ 310 n_entries = nfp_bpf_ctrl_op_cache_get(nfp_map, op, key, out_key, 311 out_value, &cache_gen); 312 if (!n_entries) 313 return 0; 314 315 skb = nfp_bpf_cmsg_map_req_alloc(bpf, 1); 316 if (!skb) { 317 err = -ENOMEM; 318 goto err_cache_put; 319 } 320 321 req = (void *)skb->data; 322 req->tid = cpu_to_be32(nfp_map->tid); 323 req->count = cpu_to_be32(n_entries); 324 req->flags = cpu_to_be32(flags); 325 326 /* Copy inputs */ 327 if (key) 328 memcpy(nfp_bpf_ctrl_req_key(bpf, req, 0), key, map->key_size); 329 if (value) 330 memcpy(nfp_bpf_ctrl_req_val(bpf, req, 0), value, 331 map->value_size); 332 333 skb = nfp_ccm_communicate(&bpf->ccm, skb, op, 0); 334 if (IS_ERR(skb)) { 335 err = PTR_ERR(skb); 336 goto err_cache_put; 337 } 338 339 if (skb->len < sizeof(*reply)) { 340 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d!\n", 341 op, skb->len); 342 err = -EIO; 343 goto err_free; 344 } 345 346 reply = (void *)skb->data; 347 count = be32_to_cpu(reply->count); 348 err = nfp_bpf_ctrl_rc_to_errno(bpf, &reply->reply_hdr); 349 /* FW responds with message sized to hold the good entries, 350 * plus one extra entry if there was an error. 351 */ 352 reply_entries = count + !!err; 353 if (n_entries > 1 && count) 354 err = 0; 355 if (err) 356 goto err_free; 357 358 if (skb->len != nfp_bpf_cmsg_map_reply_size(bpf, reply_entries)) { 359 cmsg_warn(bpf, "cmsg drop - type 0x%02x too short %d for %d entries!\n", 360 op, skb->len, reply_entries); 361 err = -EIO; 362 goto err_free; 363 } 364 365 /* Copy outputs */ 366 if (out_key) 367 memcpy(out_key, nfp_bpf_ctrl_reply_key(bpf, reply, 0), 368 map->key_size); 369 if (out_value) 370 memcpy(out_value, nfp_bpf_ctrl_reply_val(bpf, reply, 0), 371 map->value_size); 372 373 nfp_bpf_ctrl_op_cache_put(nfp_map, op, skb, cache_gen); 374 375 return 0; 376 err_free: 377 dev_kfree_skb_any(skb); 378 err_cache_put: 379 nfp_bpf_ctrl_op_cache_put(nfp_map, op, NULL, cache_gen); 380 return err; 381 } 382 383 int nfp_bpf_ctrl_update_entry(struct bpf_offloaded_map *offmap, 384 void *key, void *value, u64 flags) 385 { 386 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_UPDATE, 387 key, value, flags, NULL, NULL); 388 } 389 390 int nfp_bpf_ctrl_del_entry(struct bpf_offloaded_map *offmap, void *key) 391 { 392 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_DELETE, 393 key, NULL, 0, NULL, NULL); 394 } 395 396 int nfp_bpf_ctrl_lookup_entry(struct bpf_offloaded_map *offmap, 397 void *key, void *value) 398 { 399 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_LOOKUP, 400 key, NULL, 0, NULL, value); 401 } 402 403 int nfp_bpf_ctrl_getfirst_entry(struct bpf_offloaded_map *offmap, 404 void *next_key) 405 { 406 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETFIRST, 407 NULL, NULL, 0, next_key, NULL); 408 } 409 410 int nfp_bpf_ctrl_getnext_entry(struct bpf_offloaded_map *offmap, 411 void *key, void *next_key) 412 { 413 return nfp_bpf_ctrl_entry_op(offmap, NFP_CCM_TYPE_BPF_MAP_GETNEXT, 414 key, NULL, 0, next_key, NULL); 415 } 416 417 unsigned int nfp_bpf_ctrl_cmsg_min_mtu(struct nfp_app_bpf *bpf) 418 { 419 return max(nfp_bpf_cmsg_map_req_size(bpf, 1), 420 nfp_bpf_cmsg_map_reply_size(bpf, 1)); 421 } 422 423 unsigned int nfp_bpf_ctrl_cmsg_mtu(struct nfp_app_bpf *bpf) 424 { 425 return max3(NFP_NET_DEFAULT_MTU, 426 nfp_bpf_cmsg_map_req_size(bpf, NFP_BPF_MAP_CACHE_CNT), 427 nfp_bpf_cmsg_map_reply_size(bpf, NFP_BPF_MAP_CACHE_CNT)); 428 } 429 430 unsigned int nfp_bpf_ctrl_cmsg_cache_cnt(struct nfp_app_bpf *bpf) 431 { 432 unsigned int mtu, req_max, reply_max, entry_sz; 433 434 mtu = bpf->app->ctrl->dp.mtu; 435 entry_sz = bpf->cmsg_key_sz + bpf->cmsg_val_sz; 436 req_max = (mtu - sizeof(struct cmsg_req_map_op)) / entry_sz; 437 reply_max = (mtu - sizeof(struct cmsg_reply_map_op)) / entry_sz; 438 439 return min3(req_max, reply_max, NFP_BPF_MAP_CACHE_CNT); 440 } 441 442 void nfp_bpf_ctrl_msg_rx(struct nfp_app *app, struct sk_buff *skb) 443 { 444 struct nfp_app_bpf *bpf = app->priv; 445 446 if (unlikely(skb->len < sizeof(struct cmsg_reply_map_simple))) { 447 cmsg_warn(bpf, "cmsg drop - too short %d!\n", skb->len); 448 dev_kfree_skb_any(skb); 449 return; 450 } 451 452 if (nfp_ccm_get_type(skb) == NFP_CCM_TYPE_BPF_BPF_EVENT) { 453 if (!nfp_bpf_event_output(bpf, skb->data, skb->len)) 454 dev_consume_skb_any(skb); 455 else 456 dev_kfree_skb_any(skb); 457 } 458 459 nfp_ccm_rx(&bpf->ccm, skb); 460 } 461 462 void 463 nfp_bpf_ctrl_msg_rx_raw(struct nfp_app *app, const void *data, unsigned int len) 464 { 465 const struct nfp_ccm_hdr *hdr = data; 466 struct nfp_app_bpf *bpf = app->priv; 467 468 if (unlikely(len < sizeof(struct cmsg_reply_map_simple))) { 469 cmsg_warn(bpf, "cmsg drop - too short %d!\n", len); 470 return; 471 } 472 473 if (hdr->type == NFP_CCM_TYPE_BPF_BPF_EVENT) 474 nfp_bpf_event_output(bpf, data, len); 475 else 476 cmsg_warn(bpf, "cmsg drop - msg type %d with raw buffer!\n", 477 hdr->type); 478 } 479