1 // SPDX-License-Identifier: GPL-2.0-only 2 /**************************************************************************** 3 * Driver for Solarflare network controllers and boards 4 * Copyright 2023, Advanced Micro Devices, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms of the GNU General Public License version 2 as published 8 * by the Free Software Foundation, incorporated herein by reference. 9 */ 10 11 #include "tc_encap_actions.h" 12 #include "tc.h" 13 #include "mae.h" 14 #include <net/vxlan.h> 15 #include <net/geneve.h> 16 #include <net/netevent.h> 17 #include <net/arp.h> 18 19 static const struct rhashtable_params efx_neigh_ht_params = { 20 .key_len = offsetof(struct efx_neigh_binder, ha), 21 .key_offset = 0, 22 .head_offset = offsetof(struct efx_neigh_binder, linkage), 23 }; 24 25 static const struct rhashtable_params efx_tc_encap_ht_params = { 26 .key_len = offsetofend(struct efx_tc_encap_action, key), 27 .key_offset = 0, 28 .head_offset = offsetof(struct efx_tc_encap_action, linkage), 29 }; 30 31 static void efx_tc_encap_free(void *ptr, void *__unused) 32 { 33 struct efx_tc_encap_action *enc = ptr; 34 35 WARN_ON(refcount_read(&enc->ref)); 36 kfree(enc); 37 } 38 39 static void efx_neigh_free(void *ptr, void *__unused) 40 { 41 struct efx_neigh_binder *neigh = ptr; 42 43 WARN_ON(refcount_read(&neigh->ref)); 44 WARN_ON(!list_empty(&neigh->users)); 45 put_net_track(neigh->net, &neigh->ns_tracker); 46 netdev_put(neigh->egdev, &neigh->dev_tracker); 47 kfree(neigh); 48 } 49 50 int efx_tc_init_encap_actions(struct efx_nic *efx) 51 { 52 int rc; 53 54 rc = rhashtable_init(&efx->tc->neigh_ht, &efx_neigh_ht_params); 55 if (rc < 0) 56 goto fail_neigh_ht; 57 rc = rhashtable_init(&efx->tc->encap_ht, &efx_tc_encap_ht_params); 58 if (rc < 0) 59 goto fail_encap_ht; 60 return 0; 61 fail_encap_ht: 62 rhashtable_destroy(&efx->tc->neigh_ht); 63 fail_neigh_ht: 64 return rc; 65 } 66 67 /* Only call this in init failure teardown. 68 * Normal exit should fini instead as there may be entries in the table. 69 */ 70 void efx_tc_destroy_encap_actions(struct efx_nic *efx) 71 { 72 rhashtable_destroy(&efx->tc->encap_ht); 73 rhashtable_destroy(&efx->tc->neigh_ht); 74 } 75 76 void efx_tc_fini_encap_actions(struct efx_nic *efx) 77 { 78 rhashtable_free_and_destroy(&efx->tc->encap_ht, efx_tc_encap_free, NULL); 79 rhashtable_free_and_destroy(&efx->tc->neigh_ht, efx_neigh_free, NULL); 80 } 81 82 static void efx_neigh_update(struct work_struct *work); 83 84 static int efx_bind_neigh(struct efx_nic *efx, 85 struct efx_tc_encap_action *encap, struct net *net, 86 struct netlink_ext_ack *extack) 87 { 88 struct efx_neigh_binder *neigh, *old; 89 struct flowi6 flow6 = {}; 90 struct flowi4 flow4 = {}; 91 int rc; 92 93 /* GCC stupidly thinks that only values explicitly listed in the enum 94 * definition can _possibly_ be sensible case values, so without this 95 * cast it complains about the IPv6 versions. 96 */ 97 switch ((int)encap->type) { 98 case EFX_ENCAP_TYPE_VXLAN: 99 case EFX_ENCAP_TYPE_GENEVE: 100 flow4.flowi4_proto = IPPROTO_UDP; 101 flow4.fl4_dport = encap->key.tp_dst; 102 flow4.flowi4_tos = encap->key.tos; 103 flow4.daddr = encap->key.u.ipv4.dst; 104 flow4.saddr = encap->key.u.ipv4.src; 105 break; 106 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6: 107 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6: 108 flow6.flowi6_proto = IPPROTO_UDP; 109 flow6.fl6_dport = encap->key.tp_dst; 110 flow6.flowlabel = ip6_make_flowinfo(encap->key.tos, 111 encap->key.label); 112 flow6.daddr = encap->key.u.ipv6.dst; 113 flow6.saddr = encap->key.u.ipv6.src; 114 break; 115 default: 116 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported encap type %d", 117 (int)encap->type); 118 return -EOPNOTSUPP; 119 } 120 121 neigh = kzalloc(sizeof(*neigh), GFP_KERNEL_ACCOUNT); 122 if (!neigh) 123 return -ENOMEM; 124 neigh->net = get_net_track(net, &neigh->ns_tracker, GFP_KERNEL_ACCOUNT); 125 neigh->dst_ip = flow4.daddr; 126 neigh->dst_ip6 = flow6.daddr; 127 128 old = rhashtable_lookup_get_insert_fast(&efx->tc->neigh_ht, 129 &neigh->linkage, 130 efx_neigh_ht_params); 131 if (old) { 132 /* don't need our new entry */ 133 put_net_track(neigh->net, &neigh->ns_tracker); 134 kfree(neigh); 135 if (IS_ERR(old)) /* oh dear, it's actually an error */ 136 return PTR_ERR(old); 137 if (!refcount_inc_not_zero(&old->ref)) 138 return -EAGAIN; 139 /* existing entry found, ref taken */ 140 neigh = old; 141 } else { 142 /* New entry. We need to initiate a lookup */ 143 struct neighbour *n; 144 struct rtable *rt; 145 146 if (encap->type & EFX_ENCAP_FLAG_IPV6) { 147 #if IS_ENABLED(CONFIG_IPV6) 148 struct dst_entry *dst; 149 150 dst = ipv6_stub->ipv6_dst_lookup_flow(net, NULL, &flow6, 151 NULL); 152 rc = PTR_ERR_OR_ZERO(dst); 153 if (rc) { 154 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for IPv6 encap"); 155 goto out_free; 156 } 157 neigh->egdev = dst->dev; 158 netdev_hold(neigh->egdev, &neigh->dev_tracker, 159 GFP_KERNEL_ACCOUNT); 160 neigh->ttl = ip6_dst_hoplimit(dst); 161 n = dst_neigh_lookup(dst, &flow6.daddr); 162 dst_release(dst); 163 #else 164 /* We shouldn't ever get here, because if IPv6 isn't 165 * enabled how did someone create an IPv6 tunnel_key? 166 */ 167 rc = -EOPNOTSUPP; 168 NL_SET_ERR_MSG_MOD(extack, "No IPv6 support (neigh bind)"); 169 goto out_free; 170 #endif 171 } else { 172 rt = ip_route_output_key(net, &flow4); 173 if (IS_ERR_OR_NULL(rt)) { 174 rc = PTR_ERR_OR_ZERO(rt); 175 if (!rc) 176 rc = -EIO; 177 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for encap"); 178 goto out_free; 179 } 180 neigh->egdev = rt->dst.dev; 181 netdev_hold(neigh->egdev, &neigh->dev_tracker, 182 GFP_KERNEL_ACCOUNT); 183 neigh->ttl = ip4_dst_hoplimit(&rt->dst); 184 n = dst_neigh_lookup(&rt->dst, &flow4.daddr); 185 ip_rt_put(rt); 186 } 187 if (!n) { 188 rc = -ENETUNREACH; 189 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup neighbour for encap"); 190 netdev_put(neigh->egdev, &neigh->dev_tracker); 191 goto out_free; 192 } 193 refcount_set(&neigh->ref, 1); 194 INIT_LIST_HEAD(&neigh->users); 195 read_lock_bh(&n->lock); 196 ether_addr_copy(neigh->ha, n->ha); 197 neigh->n_valid = n->nud_state & NUD_VALID; 198 read_unlock_bh(&n->lock); 199 rwlock_init(&neigh->lock); 200 INIT_WORK(&neigh->work, efx_neigh_update); 201 neigh->efx = efx; 202 neigh->used = jiffies; 203 if (!neigh->n_valid) 204 /* Prod ARP to find us a neighbour */ 205 neigh_event_send(n, NULL); 206 neigh_release(n); 207 } 208 /* Add us to this neigh */ 209 encap->neigh = neigh; 210 list_add_tail(&encap->list, &neigh->users); 211 return 0; 212 213 out_free: 214 /* cleanup common to several error paths */ 215 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage, 216 efx_neigh_ht_params); 217 synchronize_rcu(); 218 put_net_track(net, &neigh->ns_tracker); 219 kfree(neigh); 220 return rc; 221 } 222 223 static void efx_free_neigh(struct efx_neigh_binder *neigh) 224 { 225 struct efx_nic *efx = neigh->efx; 226 227 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage, 228 efx_neigh_ht_params); 229 synchronize_rcu(); 230 netdev_put(neigh->egdev, &neigh->dev_tracker); 231 put_net_track(neigh->net, &neigh->ns_tracker); 232 kfree(neigh); 233 } 234 235 static void efx_release_neigh(struct efx_nic *efx, 236 struct efx_tc_encap_action *encap) 237 { 238 struct efx_neigh_binder *neigh = encap->neigh; 239 240 if (!neigh) 241 return; 242 list_del(&encap->list); 243 encap->neigh = NULL; 244 if (!refcount_dec_and_test(&neigh->ref)) 245 return; /* still in use */ 246 efx_free_neigh(neigh); 247 } 248 249 static void efx_gen_tun_header_eth(struct efx_tc_encap_action *encap, u16 proto) 250 { 251 struct efx_neigh_binder *neigh = encap->neigh; 252 struct ethhdr *eth; 253 254 encap->encap_hdr_len = sizeof(*eth); 255 eth = (struct ethhdr *)encap->encap_hdr; 256 257 if (encap->neigh->n_valid) 258 ether_addr_copy(eth->h_dest, neigh->ha); 259 else 260 eth_zero_addr(eth->h_dest); 261 ether_addr_copy(eth->h_source, neigh->egdev->dev_addr); 262 eth->h_proto = htons(proto); 263 } 264 265 static void efx_gen_tun_header_ipv4(struct efx_tc_encap_action *encap, u8 ipproto, u8 len) 266 { 267 struct efx_neigh_binder *neigh = encap->neigh; 268 struct ip_tunnel_key *key = &encap->key; 269 struct iphdr *ip; 270 271 ip = (struct iphdr *)(encap->encap_hdr + encap->encap_hdr_len); 272 encap->encap_hdr_len += sizeof(*ip); 273 274 ip->daddr = key->u.ipv4.dst; 275 ip->saddr = key->u.ipv4.src; 276 ip->ttl = neigh->ttl; 277 ip->protocol = ipproto; 278 ip->version = 0x4; 279 ip->ihl = 0x5; 280 ip->tot_len = cpu_to_be16(ip->ihl * 4 + len); 281 ip_send_check(ip); 282 } 283 284 #ifdef CONFIG_IPV6 285 static void efx_gen_tun_header_ipv6(struct efx_tc_encap_action *encap, u8 ipproto, u8 len) 286 { 287 struct efx_neigh_binder *neigh = encap->neigh; 288 struct ip_tunnel_key *key = &encap->key; 289 struct ipv6hdr *ip; 290 291 ip = (struct ipv6hdr *)(encap->encap_hdr + encap->encap_hdr_len); 292 encap->encap_hdr_len += sizeof(*ip); 293 294 ip6_flow_hdr(ip, key->tos, key->label); 295 ip->daddr = key->u.ipv6.dst; 296 ip->saddr = key->u.ipv6.src; 297 ip->hop_limit = neigh->ttl; 298 ip->nexthdr = ipproto; 299 ip->version = 0x6; 300 ip->payload_len = cpu_to_be16(len); 301 } 302 #endif 303 304 static void efx_gen_tun_header_udp(struct efx_tc_encap_action *encap, u8 len) 305 { 306 struct ip_tunnel_key *key = &encap->key; 307 struct udphdr *udp; 308 309 udp = (struct udphdr *)(encap->encap_hdr + encap->encap_hdr_len); 310 encap->encap_hdr_len += sizeof(*udp); 311 312 udp->dest = key->tp_dst; 313 udp->len = cpu_to_be16(sizeof(*udp) + len); 314 } 315 316 static void efx_gen_tun_header_vxlan(struct efx_tc_encap_action *encap) 317 { 318 struct ip_tunnel_key *key = &encap->key; 319 struct vxlanhdr *vxlan; 320 321 vxlan = (struct vxlanhdr *)(encap->encap_hdr + encap->encap_hdr_len); 322 encap->encap_hdr_len += sizeof(*vxlan); 323 324 vxlan->vx_flags = VXLAN_HF_VNI; 325 vxlan->vx_vni = vxlan_vni_field(tunnel_id_to_key32(key->tun_id)); 326 } 327 328 static void efx_gen_tun_header_geneve(struct efx_tc_encap_action *encap) 329 { 330 struct ip_tunnel_key *key = &encap->key; 331 struct genevehdr *geneve; 332 u32 vni; 333 334 geneve = (struct genevehdr *)(encap->encap_hdr + encap->encap_hdr_len); 335 encap->encap_hdr_len += sizeof(*geneve); 336 337 geneve->proto_type = htons(ETH_P_TEB); 338 /* convert tun_id to host-endian so we can use host arithmetic to 339 * extract individual bytes. 340 */ 341 vni = ntohl(tunnel_id_to_key32(key->tun_id)); 342 geneve->vni[0] = vni >> 16; 343 geneve->vni[1] = vni >> 8; 344 geneve->vni[2] = vni; 345 } 346 347 #define vxlan_header_l4_len (sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 348 #define vxlan4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + vxlan_header_l4_len) 349 static void efx_gen_vxlan_header_ipv4(struct efx_tc_encap_action *encap) 350 { 351 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan4_header_len); 352 efx_gen_tun_header_eth(encap, ETH_P_IP); 353 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, vxlan_header_l4_len); 354 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr)); 355 efx_gen_tun_header_vxlan(encap); 356 } 357 358 #define geneve_header_l4_len (sizeof(struct udphdr) + sizeof(struct genevehdr)) 359 #define geneve4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + geneve_header_l4_len) 360 static void efx_gen_geneve_header_ipv4(struct efx_tc_encap_action *encap) 361 { 362 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve4_header_len); 363 efx_gen_tun_header_eth(encap, ETH_P_IP); 364 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, geneve_header_l4_len); 365 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr)); 366 efx_gen_tun_header_geneve(encap); 367 } 368 369 #ifdef CONFIG_IPV6 370 #define vxlan6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + vxlan_header_l4_len) 371 static void efx_gen_vxlan_header_ipv6(struct efx_tc_encap_action *encap) 372 { 373 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan6_header_len); 374 efx_gen_tun_header_eth(encap, ETH_P_IPV6); 375 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, vxlan_header_l4_len); 376 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr)); 377 efx_gen_tun_header_vxlan(encap); 378 } 379 380 #define geneve6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + geneve_header_l4_len) 381 static void efx_gen_geneve_header_ipv6(struct efx_tc_encap_action *encap) 382 { 383 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve6_header_len); 384 efx_gen_tun_header_eth(encap, ETH_P_IPV6); 385 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, geneve_header_l4_len); 386 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr)); 387 efx_gen_tun_header_geneve(encap); 388 } 389 #endif 390 391 static void efx_gen_encap_header(struct efx_nic *efx, 392 struct efx_tc_encap_action *encap) 393 { 394 encap->n_valid = encap->neigh->n_valid; 395 396 /* GCC stupidly thinks that only values explicitly listed in the enum 397 * definition can _possibly_ be sensible case values, so without this 398 * cast it complains about the IPv6 versions. 399 */ 400 switch ((int)encap->type) { 401 case EFX_ENCAP_TYPE_VXLAN: 402 efx_gen_vxlan_header_ipv4(encap); 403 break; 404 case EFX_ENCAP_TYPE_GENEVE: 405 efx_gen_geneve_header_ipv4(encap); 406 break; 407 #ifdef CONFIG_IPV6 408 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6: 409 efx_gen_vxlan_header_ipv6(encap); 410 break; 411 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6: 412 efx_gen_geneve_header_ipv6(encap); 413 break; 414 #endif 415 default: 416 /* unhandled encap type, can't happen */ 417 if (net_ratelimit()) 418 netif_err(efx, drv, efx->net_dev, 419 "Bogus encap type %d, can't generate\n", 420 encap->type); 421 422 /* Use fallback action. */ 423 encap->n_valid = false; 424 break; 425 } 426 } 427 428 static void efx_tc_update_encap(struct efx_nic *efx, 429 struct efx_tc_encap_action *encap) 430 { 431 struct efx_tc_action_set_list *acts, *fallback; 432 struct efx_tc_flow_rule *rule; 433 struct efx_tc_action_set *act; 434 int rc; 435 436 if (encap->n_valid) { 437 /* Make sure no rules are using this encap while we change it */ 438 list_for_each_entry(act, &encap->users, encap_user) { 439 acts = act->user; 440 if (WARN_ON(!acts)) /* can't happen */ 441 continue; 442 rule = container_of(acts, struct efx_tc_flow_rule, acts); 443 if (rule->fallback) 444 fallback = rule->fallback; 445 else /* fallback fallback: deliver to PF */ 446 fallback = &efx->tc->facts.pf; 447 rc = efx_mae_update_rule(efx, fallback->fw_id, 448 rule->fw_id); 449 if (rc) 450 netif_err(efx, drv, efx->net_dev, 451 "Failed to update (f) rule %08x rc %d\n", 452 rule->fw_id, rc); 453 else 454 netif_dbg(efx, drv, efx->net_dev, "Updated (f) rule %08x\n", 455 rule->fw_id); 456 } 457 } 458 459 /* Make sure we don't leak arbitrary bytes on the wire; 460 * set an all-0s ethernet header. A successful call to 461 * efx_gen_encap_header() will overwrite this. 462 */ 463 memset(encap->encap_hdr, 0, sizeof(encap->encap_hdr)); 464 encap->encap_hdr_len = ETH_HLEN; 465 466 if (encap->neigh) { 467 read_lock_bh(&encap->neigh->lock); 468 efx_gen_encap_header(efx, encap); 469 read_unlock_bh(&encap->neigh->lock); 470 } else { 471 encap->n_valid = false; 472 } 473 474 rc = efx_mae_update_encap_md(efx, encap); 475 if (rc) { 476 netif_err(efx, drv, efx->net_dev, 477 "Failed to update encap hdr %08x rc %d\n", 478 encap->fw_id, rc); 479 return; 480 } 481 netif_dbg(efx, drv, efx->net_dev, "Updated encap hdr %08x\n", 482 encap->fw_id); 483 if (!encap->n_valid) 484 return; 485 /* Update rule users: use the action if they are now ready */ 486 list_for_each_entry(act, &encap->users, encap_user) { 487 acts = act->user; 488 if (WARN_ON(!acts)) /* can't happen */ 489 continue; 490 rule = container_of(acts, struct efx_tc_flow_rule, acts); 491 if (!efx_tc_check_ready(efx, rule)) 492 continue; 493 rc = efx_mae_update_rule(efx, acts->fw_id, rule->fw_id); 494 if (rc) 495 netif_err(efx, drv, efx->net_dev, 496 "Failed to update rule %08x rc %d\n", 497 rule->fw_id, rc); 498 else 499 netif_dbg(efx, drv, efx->net_dev, "Updated rule %08x\n", 500 rule->fw_id); 501 } 502 } 503 504 static void efx_neigh_update(struct work_struct *work) 505 { 506 struct efx_neigh_binder *neigh = container_of(work, struct efx_neigh_binder, work); 507 struct efx_tc_encap_action *encap; 508 struct efx_nic *efx = neigh->efx; 509 510 mutex_lock(&efx->tc->mutex); 511 list_for_each_entry(encap, &neigh->users, list) 512 efx_tc_update_encap(neigh->efx, encap); 513 /* release ref taken in efx_neigh_event() */ 514 if (refcount_dec_and_test(&neigh->ref)) 515 efx_free_neigh(neigh); 516 mutex_unlock(&efx->tc->mutex); 517 } 518 519 static int efx_neigh_event(struct efx_nic *efx, struct neighbour *n) 520 { 521 struct efx_neigh_binder keys = {NULL}, *neigh; 522 bool n_valid, ipv6 = false; 523 char ha[ETH_ALEN]; 524 size_t keysize; 525 526 if (WARN_ON(!efx->tc)) 527 return NOTIFY_DONE; 528 529 if (n->tbl == &arp_tbl) { 530 keysize = sizeof(keys.dst_ip); 531 #if IS_ENABLED(CONFIG_IPV6) 532 } else if (n->tbl == ipv6_stub->nd_tbl) { 533 ipv6 = true; 534 keysize = sizeof(keys.dst_ip6); 535 #endif 536 } else { 537 return NOTIFY_DONE; 538 } 539 if (!n->parms) { 540 netif_warn(efx, drv, efx->net_dev, "neigh_event with no parms!\n"); 541 return NOTIFY_DONE; 542 } 543 keys.net = read_pnet(&n->parms->net); 544 if (n->tbl->key_len != keysize) { 545 netif_warn(efx, drv, efx->net_dev, "neigh_event with bad key_len %u\n", 546 n->tbl->key_len); 547 return NOTIFY_DONE; 548 } 549 read_lock_bh(&n->lock); /* Get a consistent view */ 550 memcpy(ha, n->ha, ETH_ALEN); 551 n_valid = (n->nud_state & NUD_VALID) && !n->dead; 552 read_unlock_bh(&n->lock); 553 if (ipv6) 554 memcpy(&keys.dst_ip6, n->primary_key, n->tbl->key_len); 555 else 556 memcpy(&keys.dst_ip, n->primary_key, n->tbl->key_len); 557 rcu_read_lock(); 558 neigh = rhashtable_lookup_fast(&efx->tc->neigh_ht, &keys, 559 efx_neigh_ht_params); 560 if (!neigh || neigh->dying) 561 /* We're not interested in this neighbour */ 562 goto done; 563 write_lock_bh(&neigh->lock); 564 if (n_valid == neigh->n_valid && !memcmp(ha, neigh->ha, ETH_ALEN)) { 565 write_unlock_bh(&neigh->lock); 566 /* Nothing has changed; no work to do */ 567 goto done; 568 } 569 neigh->n_valid = n_valid; 570 memcpy(neigh->ha, ha, ETH_ALEN); 571 write_unlock_bh(&neigh->lock); 572 if (refcount_inc_not_zero(&neigh->ref)) { 573 rcu_read_unlock(); 574 if (!schedule_work(&neigh->work)) 575 /* failed to schedule, release the ref we just took */ 576 if (refcount_dec_and_test(&neigh->ref)) 577 efx_free_neigh(neigh); 578 } else { 579 done: 580 rcu_read_unlock(); 581 } 582 return NOTIFY_DONE; 583 } 584 585 bool efx_tc_check_ready(struct efx_nic *efx, struct efx_tc_flow_rule *rule) 586 { 587 struct efx_tc_action_set *act; 588 589 /* Encap actions can only be offloaded if they have valid 590 * neighbour info for the outer Ethernet header. 591 */ 592 list_for_each_entry(act, &rule->acts.list, list) 593 if (act->encap_md && !act->encap_md->n_valid) 594 return false; 595 return true; 596 } 597 598 struct efx_tc_encap_action *efx_tc_flower_create_encap_md( 599 struct efx_nic *efx, const struct ip_tunnel_info *info, 600 struct net_device *egdev, struct netlink_ext_ack *extack) 601 { 602 enum efx_encap_type type = efx_tc_indr_netdev_type(egdev); 603 struct efx_tc_encap_action *encap, *old; 604 struct efx_rep *to_efv; 605 s64 rc; 606 607 if (type == EFX_ENCAP_TYPE_NONE) { 608 /* dest is not an encap device */ 609 NL_SET_ERR_MSG_MOD(extack, "Not a (supported) tunnel device but tunnel_key is set"); 610 return ERR_PTR(-EOPNOTSUPP); 611 } 612 rc = efx_mae_check_encap_type_supported(efx, type); 613 if (rc < 0) { 614 NL_SET_ERR_MSG_MOD(extack, "Firmware reports no support for this tunnel type"); 615 return ERR_PTR(rc); 616 } 617 /* No support yet for Geneve options */ 618 if (info->options_len) { 619 NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel options"); 620 return ERR_PTR(-EOPNOTSUPP); 621 } 622 switch (info->mode) { 623 case IP_TUNNEL_INFO_TX: 624 break; 625 case IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_IPV6: 626 type |= EFX_ENCAP_FLAG_IPV6; 627 break; 628 default: 629 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported tunnel mode %u", 630 info->mode); 631 return ERR_PTR(-EOPNOTSUPP); 632 } 633 encap = kzalloc(sizeof(*encap), GFP_KERNEL_ACCOUNT); 634 if (!encap) 635 return ERR_PTR(-ENOMEM); 636 encap->type = type; 637 encap->key = info->key; 638 INIT_LIST_HEAD(&encap->users); 639 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_ht, 640 &encap->linkage, 641 efx_tc_encap_ht_params); 642 if (old) { 643 /* don't need our new entry */ 644 kfree(encap); 645 if (IS_ERR(old)) /* oh dear, it's actually an error */ 646 return ERR_CAST(old); 647 if (!refcount_inc_not_zero(&old->ref)) 648 return ERR_PTR(-EAGAIN); 649 /* existing entry found, ref taken */ 650 return old; 651 } 652 653 rc = efx_bind_neigh(efx, encap, dev_net(egdev), extack); 654 if (rc < 0) 655 goto out_remove; 656 to_efv = efx_tc_flower_lookup_efv(efx, encap->neigh->egdev); 657 if (IS_ERR(to_efv)) { 658 /* neigh->egdev isn't ours */ 659 NL_SET_ERR_MSG_MOD(extack, "Tunnel egress device not on switch"); 660 rc = PTR_ERR(to_efv); 661 goto out_release; 662 } 663 rc = efx_tc_flower_external_mport(efx, to_efv); 664 if (rc < 0) { 665 NL_SET_ERR_MSG_MOD(extack, "Failed to identify tunnel egress m-port"); 666 goto out_release; 667 } 668 encap->dest_mport = rc; 669 read_lock_bh(&encap->neigh->lock); 670 efx_gen_encap_header(efx, encap); 671 read_unlock_bh(&encap->neigh->lock); 672 673 rc = efx_mae_allocate_encap_md(efx, encap); 674 if (rc < 0) { 675 NL_SET_ERR_MSG_MOD(extack, "Failed to write tunnel header to hw"); 676 goto out_release; 677 } 678 679 /* ref and return */ 680 refcount_set(&encap->ref, 1); 681 return encap; 682 out_release: 683 efx_release_neigh(efx, encap); 684 out_remove: 685 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage, 686 efx_tc_encap_ht_params); 687 kfree(encap); 688 return ERR_PTR(rc); 689 } 690 691 void efx_tc_flower_release_encap_md(struct efx_nic *efx, 692 struct efx_tc_encap_action *encap) 693 { 694 if (!refcount_dec_and_test(&encap->ref)) 695 return; /* still in use */ 696 efx_release_neigh(efx, encap); 697 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage, 698 efx_tc_encap_ht_params); 699 efx_mae_free_encap_md(efx, encap); 700 kfree(encap); 701 } 702 703 static void efx_tc_remove_neigh_users(struct efx_nic *efx, struct efx_neigh_binder *neigh) 704 { 705 struct efx_tc_encap_action *encap, *next; 706 707 list_for_each_entry_safe(encap, next, &neigh->users, list) { 708 /* Should cause neigh usage count to fall to zero, freeing it */ 709 efx_release_neigh(efx, encap); 710 /* The encap has lost its neigh, so it's now unready */ 711 efx_tc_update_encap(efx, encap); 712 } 713 } 714 715 void efx_tc_unregister_egdev(struct efx_nic *efx, struct net_device *net_dev) 716 { 717 struct efx_neigh_binder *neigh; 718 struct rhashtable_iter walk; 719 720 mutex_lock(&efx->tc->mutex); 721 rhashtable_walk_enter(&efx->tc->neigh_ht, &walk); 722 rhashtable_walk_start(&walk); 723 while ((neigh = rhashtable_walk_next(&walk)) != NULL) { 724 if (IS_ERR(neigh)) 725 continue; 726 if (neigh->egdev != net_dev) 727 continue; 728 neigh->dying = true; 729 rhashtable_walk_stop(&walk); 730 synchronize_rcu(); /* Make sure any updates see dying flag */ 731 efx_tc_remove_neigh_users(efx, neigh); /* might sleep */ 732 rhashtable_walk_start(&walk); 733 } 734 rhashtable_walk_stop(&walk); 735 rhashtable_walk_exit(&walk); 736 mutex_unlock(&efx->tc->mutex); 737 } 738 739 int efx_tc_netevent_event(struct efx_nic *efx, unsigned long event, 740 void *ptr) 741 { 742 if (efx->type->is_vf) 743 return NOTIFY_DONE; 744 745 switch (event) { 746 case NETEVENT_NEIGH_UPDATE: 747 return efx_neigh_event(efx, ptr); 748 default: 749 return NOTIFY_DONE; 750 } 751 } 752