1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2023, Advanced Micro Devices, Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11 #include "tc_encap_actions.h"
12 #include "tc.h"
13 #include "mae.h"
14 #include <net/flow.h>
15 #include <net/inet_dscp.h>
16 #include <net/vxlan.h>
17 #include <net/geneve.h>
18 #include <net/netevent.h>
19 #include <net/arp.h>
20
21 static const struct rhashtable_params efx_neigh_ht_params = {
22 .key_len = offsetof(struct efx_neigh_binder, ha),
23 .key_offset = 0,
24 .head_offset = offsetof(struct efx_neigh_binder, linkage),
25 };
26
27 static const struct rhashtable_params efx_tc_encap_ht_params = {
28 .key_len = offsetofend(struct efx_tc_encap_action, key),
29 .key_offset = 0,
30 .head_offset = offsetof(struct efx_tc_encap_action, linkage),
31 };
32
efx_tc_encap_free(void * ptr,void * __unused)33 static void efx_tc_encap_free(void *ptr, void *__unused)
34 {
35 struct efx_tc_encap_action *enc = ptr;
36
37 WARN_ON(refcount_read(&enc->ref));
38 kfree(enc);
39 }
40
efx_neigh_free(void * ptr,void * __unused)41 static void efx_neigh_free(void *ptr, void *__unused)
42 {
43 struct efx_neigh_binder *neigh = ptr;
44
45 WARN_ON(refcount_read(&neigh->ref));
46 WARN_ON(!list_empty(&neigh->users));
47 put_net_track(neigh->net, &neigh->ns_tracker);
48 netdev_put(neigh->egdev, &neigh->dev_tracker);
49 kfree(neigh);
50 }
51
efx_tc_init_encap_actions(struct efx_nic * efx)52 int efx_tc_init_encap_actions(struct efx_nic *efx)
53 {
54 int rc;
55
56 rc = rhashtable_init(&efx->tc->neigh_ht, &efx_neigh_ht_params);
57 if (rc < 0)
58 goto fail_neigh_ht;
59 rc = rhashtable_init(&efx->tc->encap_ht, &efx_tc_encap_ht_params);
60 if (rc < 0)
61 goto fail_encap_ht;
62 return 0;
63 fail_encap_ht:
64 rhashtable_destroy(&efx->tc->neigh_ht);
65 fail_neigh_ht:
66 return rc;
67 }
68
69 /* Only call this in init failure teardown.
70 * Normal exit should fini instead as there may be entries in the table.
71 */
efx_tc_destroy_encap_actions(struct efx_nic * efx)72 void efx_tc_destroy_encap_actions(struct efx_nic *efx)
73 {
74 rhashtable_destroy(&efx->tc->encap_ht);
75 rhashtable_destroy(&efx->tc->neigh_ht);
76 }
77
efx_tc_fini_encap_actions(struct efx_nic * efx)78 void efx_tc_fini_encap_actions(struct efx_nic *efx)
79 {
80 rhashtable_free_and_destroy(&efx->tc->encap_ht, efx_tc_encap_free, NULL);
81 rhashtable_free_and_destroy(&efx->tc->neigh_ht, efx_neigh_free, NULL);
82 }
83
84 static void efx_neigh_update(struct work_struct *work);
85
efx_bind_neigh(struct efx_nic * efx,struct efx_tc_encap_action * encap,struct net * net,struct netlink_ext_ack * extack)86 static int efx_bind_neigh(struct efx_nic *efx,
87 struct efx_tc_encap_action *encap, struct net *net,
88 struct netlink_ext_ack *extack)
89 {
90 struct efx_neigh_binder *neigh, *old;
91 struct flowi6 flow6 = {};
92 struct flowi4 flow4 = {};
93 int rc;
94
95 /* GCC stupidly thinks that only values explicitly listed in the enum
96 * definition can _possibly_ be sensible case values, so without this
97 * cast it complains about the IPv6 versions.
98 */
99 switch ((int)encap->type) {
100 case EFX_ENCAP_TYPE_VXLAN:
101 case EFX_ENCAP_TYPE_GENEVE:
102 flow4.flowi4_proto = IPPROTO_UDP;
103 flow4.fl4_dport = encap->key.tp_dst;
104 flow4.flowi4_dscp = inet_dsfield_to_dscp(encap->key.tos);
105 flow4.daddr = encap->key.u.ipv4.dst;
106 flow4.saddr = encap->key.u.ipv4.src;
107 break;
108 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6:
109 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6:
110 flow6.flowi6_proto = IPPROTO_UDP;
111 flow6.fl6_dport = encap->key.tp_dst;
112 flow6.flowlabel = ip6_make_flowinfo(encap->key.tos,
113 encap->key.label);
114 flow6.daddr = encap->key.u.ipv6.dst;
115 flow6.saddr = encap->key.u.ipv6.src;
116 break;
117 default:
118 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported encap type %d",
119 (int)encap->type);
120 return -EOPNOTSUPP;
121 }
122
123 neigh = kzalloc_obj(*neigh, GFP_KERNEL_ACCOUNT);
124 if (!neigh)
125 return -ENOMEM;
126 neigh->net = get_net_track(net, &neigh->ns_tracker, GFP_KERNEL_ACCOUNT);
127 neigh->dst_ip = flow4.daddr;
128 neigh->dst_ip6 = flow6.daddr;
129
130 old = rhashtable_lookup_get_insert_fast(&efx->tc->neigh_ht,
131 &neigh->linkage,
132 efx_neigh_ht_params);
133 if (old) {
134 /* don't need our new entry */
135 put_net_track(neigh->net, &neigh->ns_tracker);
136 kfree(neigh);
137 if (IS_ERR(old)) /* oh dear, it's actually an error */
138 return PTR_ERR(old);
139 if (!refcount_inc_not_zero(&old->ref))
140 return -EAGAIN;
141 /* existing entry found, ref taken */
142 neigh = old;
143 } else {
144 /* New entry. We need to initiate a lookup */
145 struct neighbour *n;
146 struct rtable *rt;
147
148 if (encap->type & EFX_ENCAP_FLAG_IPV6) {
149 #if IS_ENABLED(CONFIG_IPV6)
150 struct dst_entry *dst;
151
152 dst = ip6_dst_lookup_flow(net, NULL, &flow6, NULL);
153 rc = PTR_ERR_OR_ZERO(dst);
154 if (rc) {
155 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for IPv6 encap");
156 goto out_free;
157 }
158 neigh->egdev = dst->dev;
159 netdev_hold(neigh->egdev, &neigh->dev_tracker,
160 GFP_KERNEL_ACCOUNT);
161 neigh->ttl = ip6_dst_hoplimit(dst);
162 n = dst_neigh_lookup(dst, &flow6.daddr);
163 dst_release(dst);
164 #else
165 /* We shouldn't ever get here, because if IPv6 isn't
166 * enabled how did someone create an IPv6 tunnel_key?
167 */
168 rc = -EOPNOTSUPP;
169 NL_SET_ERR_MSG_MOD(extack, "No IPv6 support (neigh bind)");
170 goto out_free;
171 #endif
172 } else {
173 rt = ip_route_output_key(net, &flow4);
174 if (IS_ERR_OR_NULL(rt)) {
175 rc = PTR_ERR_OR_ZERO(rt);
176 if (!rc)
177 rc = -EIO;
178 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup route for encap");
179 goto out_free;
180 }
181 neigh->egdev = rt->dst.dev;
182 netdev_hold(neigh->egdev, &neigh->dev_tracker,
183 GFP_KERNEL_ACCOUNT);
184 neigh->ttl = ip4_dst_hoplimit(&rt->dst);
185 n = dst_neigh_lookup(&rt->dst, &flow4.daddr);
186 ip_rt_put(rt);
187 }
188 if (!n) {
189 rc = -ENETUNREACH;
190 NL_SET_ERR_MSG_MOD(extack, "Failed to lookup neighbour for encap");
191 netdev_put(neigh->egdev, &neigh->dev_tracker);
192 goto out_free;
193 }
194 refcount_set(&neigh->ref, 1);
195 INIT_LIST_HEAD(&neigh->users);
196 read_lock_bh(&n->lock);
197 ether_addr_copy(neigh->ha, n->ha);
198 neigh->n_valid = n->nud_state & NUD_VALID;
199 read_unlock_bh(&n->lock);
200 rwlock_init(&neigh->lock);
201 INIT_WORK(&neigh->work, efx_neigh_update);
202 neigh->efx = efx;
203 neigh->used = jiffies;
204 if (!neigh->n_valid)
205 /* Prod ARP to find us a neighbour */
206 neigh_event_send(n, NULL);
207 neigh_release(n);
208 }
209 /* Add us to this neigh */
210 encap->neigh = neigh;
211 list_add_tail(&encap->list, &neigh->users);
212 return 0;
213
214 out_free:
215 /* cleanup common to several error paths */
216 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage,
217 efx_neigh_ht_params);
218 synchronize_rcu();
219 put_net_track(net, &neigh->ns_tracker);
220 kfree(neigh);
221 return rc;
222 }
223
efx_free_neigh(struct efx_neigh_binder * neigh)224 static void efx_free_neigh(struct efx_neigh_binder *neigh)
225 {
226 struct efx_nic *efx = neigh->efx;
227
228 rhashtable_remove_fast(&efx->tc->neigh_ht, &neigh->linkage,
229 efx_neigh_ht_params);
230 synchronize_rcu();
231 netdev_put(neigh->egdev, &neigh->dev_tracker);
232 put_net_track(neigh->net, &neigh->ns_tracker);
233 kfree(neigh);
234 }
235
efx_release_neigh(struct efx_nic * efx,struct efx_tc_encap_action * encap)236 static void efx_release_neigh(struct efx_nic *efx,
237 struct efx_tc_encap_action *encap)
238 {
239 struct efx_neigh_binder *neigh = encap->neigh;
240
241 if (!neigh)
242 return;
243 list_del(&encap->list);
244 encap->neigh = NULL;
245 if (!refcount_dec_and_test(&neigh->ref))
246 return; /* still in use */
247 efx_free_neigh(neigh);
248 }
249
efx_gen_tun_header_eth(struct efx_tc_encap_action * encap,u16 proto)250 static void efx_gen_tun_header_eth(struct efx_tc_encap_action *encap, u16 proto)
251 {
252 struct efx_neigh_binder *neigh = encap->neigh;
253 struct ethhdr *eth;
254
255 encap->encap_hdr_len = sizeof(*eth);
256 eth = (struct ethhdr *)encap->encap_hdr;
257
258 if (encap->neigh->n_valid)
259 ether_addr_copy(eth->h_dest, neigh->ha);
260 else
261 eth_zero_addr(eth->h_dest);
262 ether_addr_copy(eth->h_source, neigh->egdev->dev_addr);
263 eth->h_proto = htons(proto);
264 }
265
efx_gen_tun_header_ipv4(struct efx_tc_encap_action * encap,u8 ipproto,u8 len)266 static void efx_gen_tun_header_ipv4(struct efx_tc_encap_action *encap, u8 ipproto, u8 len)
267 {
268 struct efx_neigh_binder *neigh = encap->neigh;
269 struct ip_tunnel_key *key = &encap->key;
270 struct iphdr *ip;
271
272 ip = (struct iphdr *)(encap->encap_hdr + encap->encap_hdr_len);
273 encap->encap_hdr_len += sizeof(*ip);
274
275 ip->daddr = key->u.ipv4.dst;
276 ip->saddr = key->u.ipv4.src;
277 ip->ttl = neigh->ttl;
278 ip->protocol = ipproto;
279 ip->version = 0x4;
280 ip->ihl = 0x5;
281 ip->tot_len = cpu_to_be16(ip->ihl * 4 + len);
282 ip_send_check(ip);
283 }
284
285 #ifdef CONFIG_IPV6
efx_gen_tun_header_ipv6(struct efx_tc_encap_action * encap,u8 ipproto,u8 len)286 static void efx_gen_tun_header_ipv6(struct efx_tc_encap_action *encap, u8 ipproto, u8 len)
287 {
288 struct efx_neigh_binder *neigh = encap->neigh;
289 struct ip_tunnel_key *key = &encap->key;
290 struct ipv6hdr *ip;
291
292 ip = (struct ipv6hdr *)(encap->encap_hdr + encap->encap_hdr_len);
293 encap->encap_hdr_len += sizeof(*ip);
294
295 ip6_flow_hdr(ip, key->tos, key->label);
296 ip->daddr = key->u.ipv6.dst;
297 ip->saddr = key->u.ipv6.src;
298 ip->hop_limit = neigh->ttl;
299 ip->nexthdr = ipproto;
300 ip->version = 0x6;
301 ip->payload_len = cpu_to_be16(len);
302 }
303 #endif
304
efx_gen_tun_header_udp(struct efx_tc_encap_action * encap,u8 len)305 static void efx_gen_tun_header_udp(struct efx_tc_encap_action *encap, u8 len)
306 {
307 struct ip_tunnel_key *key = &encap->key;
308 struct udphdr *udp;
309
310 udp = (struct udphdr *)(encap->encap_hdr + encap->encap_hdr_len);
311 encap->encap_hdr_len += sizeof(*udp);
312
313 udp->dest = key->tp_dst;
314 udp_set_len_short(udp, sizeof(*udp) + len);
315 }
316
efx_gen_tun_header_vxlan(struct efx_tc_encap_action * encap)317 static void efx_gen_tun_header_vxlan(struct efx_tc_encap_action *encap)
318 {
319 struct ip_tunnel_key *key = &encap->key;
320 struct vxlanhdr *vxlan;
321
322 vxlan = (struct vxlanhdr *)(encap->encap_hdr + encap->encap_hdr_len);
323 encap->encap_hdr_len += sizeof(*vxlan);
324
325 vxlan->vx_flags = VXLAN_HF_VNI;
326 vxlan->vx_vni = vxlan_vni_field(tunnel_id_to_key32(key->tun_id));
327 }
328
efx_gen_tun_header_geneve(struct efx_tc_encap_action * encap)329 static void efx_gen_tun_header_geneve(struct efx_tc_encap_action *encap)
330 {
331 struct ip_tunnel_key *key = &encap->key;
332 struct genevehdr *geneve;
333 u32 vni;
334
335 geneve = (struct genevehdr *)(encap->encap_hdr + encap->encap_hdr_len);
336 encap->encap_hdr_len += sizeof(*geneve);
337
338 geneve->proto_type = htons(ETH_P_TEB);
339 /* convert tun_id to host-endian so we can use host arithmetic to
340 * extract individual bytes.
341 */
342 vni = ntohl(tunnel_id_to_key32(key->tun_id));
343 geneve->vni[0] = vni >> 16;
344 geneve->vni[1] = vni >> 8;
345 geneve->vni[2] = vni;
346 }
347
348 #define vxlan_header_l4_len (sizeof(struct udphdr) + sizeof(struct vxlanhdr))
349 #define vxlan4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + vxlan_header_l4_len)
efx_gen_vxlan_header_ipv4(struct efx_tc_encap_action * encap)350 static void efx_gen_vxlan_header_ipv4(struct efx_tc_encap_action *encap)
351 {
352 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan4_header_len);
353 efx_gen_tun_header_eth(encap, ETH_P_IP);
354 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, vxlan_header_l4_len);
355 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr));
356 efx_gen_tun_header_vxlan(encap);
357 }
358
359 #define geneve_header_l4_len (sizeof(struct udphdr) + sizeof(struct genevehdr))
360 #define geneve4_header_len (sizeof(struct ethhdr) + sizeof(struct iphdr) + geneve_header_l4_len)
efx_gen_geneve_header_ipv4(struct efx_tc_encap_action * encap)361 static void efx_gen_geneve_header_ipv4(struct efx_tc_encap_action *encap)
362 {
363 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve4_header_len);
364 efx_gen_tun_header_eth(encap, ETH_P_IP);
365 efx_gen_tun_header_ipv4(encap, IPPROTO_UDP, geneve_header_l4_len);
366 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr));
367 efx_gen_tun_header_geneve(encap);
368 }
369
370 #ifdef CONFIG_IPV6
371 #define vxlan6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + vxlan_header_l4_len)
efx_gen_vxlan_header_ipv6(struct efx_tc_encap_action * encap)372 static void efx_gen_vxlan_header_ipv6(struct efx_tc_encap_action *encap)
373 {
374 BUILD_BUG_ON(sizeof(encap->encap_hdr) < vxlan6_header_len);
375 efx_gen_tun_header_eth(encap, ETH_P_IPV6);
376 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, vxlan_header_l4_len);
377 efx_gen_tun_header_udp(encap, sizeof(struct vxlanhdr));
378 efx_gen_tun_header_vxlan(encap);
379 }
380
381 #define geneve6_header_len (sizeof(struct ethhdr) + sizeof(struct ipv6hdr) + geneve_header_l4_len)
efx_gen_geneve_header_ipv6(struct efx_tc_encap_action * encap)382 static void efx_gen_geneve_header_ipv6(struct efx_tc_encap_action *encap)
383 {
384 BUILD_BUG_ON(sizeof(encap->encap_hdr) < geneve6_header_len);
385 efx_gen_tun_header_eth(encap, ETH_P_IPV6);
386 efx_gen_tun_header_ipv6(encap, IPPROTO_UDP, geneve_header_l4_len);
387 efx_gen_tun_header_udp(encap, sizeof(struct genevehdr));
388 efx_gen_tun_header_geneve(encap);
389 }
390 #endif
391
efx_gen_encap_header(struct efx_nic * efx,struct efx_tc_encap_action * encap)392 static void efx_gen_encap_header(struct efx_nic *efx,
393 struct efx_tc_encap_action *encap)
394 {
395 encap->n_valid = encap->neigh->n_valid;
396
397 /* GCC stupidly thinks that only values explicitly listed in the enum
398 * definition can _possibly_ be sensible case values, so without this
399 * cast it complains about the IPv6 versions.
400 */
401 switch ((int)encap->type) {
402 case EFX_ENCAP_TYPE_VXLAN:
403 efx_gen_vxlan_header_ipv4(encap);
404 break;
405 case EFX_ENCAP_TYPE_GENEVE:
406 efx_gen_geneve_header_ipv4(encap);
407 break;
408 #ifdef CONFIG_IPV6
409 case EFX_ENCAP_TYPE_VXLAN | EFX_ENCAP_FLAG_IPV6:
410 efx_gen_vxlan_header_ipv6(encap);
411 break;
412 case EFX_ENCAP_TYPE_GENEVE | EFX_ENCAP_FLAG_IPV6:
413 efx_gen_geneve_header_ipv6(encap);
414 break;
415 #endif
416 default:
417 /* unhandled encap type, can't happen */
418 if (net_ratelimit())
419 netif_err(efx, drv, efx->net_dev,
420 "Bogus encap type %d, can't generate\n",
421 encap->type);
422
423 /* Use fallback action. */
424 encap->n_valid = false;
425 break;
426 }
427 }
428
efx_tc_update_encap(struct efx_nic * efx,struct efx_tc_encap_action * encap)429 static void efx_tc_update_encap(struct efx_nic *efx,
430 struct efx_tc_encap_action *encap)
431 {
432 struct efx_tc_action_set_list *acts, *fallback;
433 struct efx_tc_flow_rule *rule;
434 struct efx_tc_action_set *act;
435 int rc;
436
437 if (encap->n_valid) {
438 /* Make sure no rules are using this encap while we change it */
439 list_for_each_entry(act, &encap->users, encap_user) {
440 acts = act->user;
441 if (WARN_ON(!acts)) /* can't happen */
442 continue;
443 rule = container_of(acts, struct efx_tc_flow_rule, acts);
444 if (rule->fallback)
445 fallback = rule->fallback;
446 else /* fallback of the fallback: deliver to PF */
447 fallback = &efx->tc->facts.pf;
448 rc = efx_mae_update_rule(efx, fallback->fw_id,
449 rule->fw_id);
450 if (rc)
451 netif_err(efx, drv, efx->net_dev,
452 "Failed to update (f) rule %08x rc %d\n",
453 rule->fw_id, rc);
454 else
455 netif_dbg(efx, drv, efx->net_dev, "Updated (f) rule %08x\n",
456 rule->fw_id);
457 }
458 }
459
460 /* Make sure we don't leak arbitrary bytes on the wire;
461 * set an all-0s ethernet header. A successful call to
462 * efx_gen_encap_header() will overwrite this.
463 */
464 memset(encap->encap_hdr, 0, sizeof(encap->encap_hdr));
465 encap->encap_hdr_len = ETH_HLEN;
466
467 if (encap->neigh) {
468 read_lock_bh(&encap->neigh->lock);
469 efx_gen_encap_header(efx, encap);
470 read_unlock_bh(&encap->neigh->lock);
471 } else {
472 encap->n_valid = false;
473 }
474
475 rc = efx_mae_update_encap_md(efx, encap);
476 if (rc) {
477 netif_err(efx, drv, efx->net_dev,
478 "Failed to update encap hdr %08x rc %d\n",
479 encap->fw_id, rc);
480 return;
481 }
482 netif_dbg(efx, drv, efx->net_dev, "Updated encap hdr %08x\n",
483 encap->fw_id);
484 if (!encap->n_valid)
485 return;
486 /* Update rule users: use the action if they are now ready */
487 list_for_each_entry(act, &encap->users, encap_user) {
488 acts = act->user;
489 if (WARN_ON(!acts)) /* can't happen */
490 continue;
491 rule = container_of(acts, struct efx_tc_flow_rule, acts);
492 if (!efx_tc_check_ready(efx, rule))
493 continue;
494 rc = efx_mae_update_rule(efx, acts->fw_id, rule->fw_id);
495 if (rc)
496 netif_err(efx, drv, efx->net_dev,
497 "Failed to update rule %08x rc %d\n",
498 rule->fw_id, rc);
499 else
500 netif_dbg(efx, drv, efx->net_dev, "Updated rule %08x\n",
501 rule->fw_id);
502 }
503 }
504
efx_neigh_update(struct work_struct * work)505 static void efx_neigh_update(struct work_struct *work)
506 {
507 struct efx_neigh_binder *neigh = container_of(work, struct efx_neigh_binder, work);
508 struct efx_tc_encap_action *encap;
509 struct efx_nic *efx = neigh->efx;
510
511 mutex_lock(&efx->tc->mutex);
512 list_for_each_entry(encap, &neigh->users, list)
513 efx_tc_update_encap(neigh->efx, encap);
514 /* release ref taken in efx_neigh_event() */
515 if (refcount_dec_and_test(&neigh->ref))
516 efx_free_neigh(neigh);
517 mutex_unlock(&efx->tc->mutex);
518 }
519
efx_neigh_event(struct efx_nic * efx,struct neighbour * n)520 static int efx_neigh_event(struct efx_nic *efx, struct neighbour *n)
521 {
522 struct efx_neigh_binder keys = {NULL}, *neigh;
523 bool n_valid, ipv6 = false;
524 char ha[ETH_ALEN];
525 size_t keysize;
526
527 if (WARN_ON(!efx->tc))
528 return NOTIFY_DONE;
529
530 if (n->tbl == &arp_tbl) {
531 keysize = sizeof(keys.dst_ip);
532 #if IS_ENABLED(CONFIG_IPV6)
533 } else if (n->tbl == &nd_tbl) {
534 ipv6 = true;
535 keysize = sizeof(keys.dst_ip6);
536 #endif
537 } else {
538 return NOTIFY_DONE;
539 }
540 if (!n->parms) {
541 netif_warn(efx, drv, efx->net_dev, "neigh_event with no parms!\n");
542 return NOTIFY_DONE;
543 }
544 keys.net = read_pnet(&n->parms->net);
545 if (n->tbl->key_len != keysize) {
546 netif_warn(efx, drv, efx->net_dev, "neigh_event with bad key_len %u\n",
547 n->tbl->key_len);
548 return NOTIFY_DONE;
549 }
550 read_lock_bh(&n->lock); /* Get a consistent view */
551 memcpy(ha, n->ha, ETH_ALEN);
552 n_valid = (n->nud_state & NUD_VALID) && !n->dead;
553 read_unlock_bh(&n->lock);
554 if (ipv6)
555 memcpy(&keys.dst_ip6, n->primary_key, n->tbl->key_len);
556 else
557 memcpy(&keys.dst_ip, n->primary_key, n->tbl->key_len);
558 rcu_read_lock();
559 neigh = rhashtable_lookup_fast(&efx->tc->neigh_ht, &keys,
560 efx_neigh_ht_params);
561 if (!neigh || neigh->dying)
562 /* We're not interested in this neighbour */
563 goto done;
564 write_lock_bh(&neigh->lock);
565 if (n_valid == neigh->n_valid && !memcmp(ha, neigh->ha, ETH_ALEN)) {
566 write_unlock_bh(&neigh->lock);
567 /* Nothing has changed; no work to do */
568 goto done;
569 }
570 neigh->n_valid = n_valid;
571 memcpy(neigh->ha, ha, ETH_ALEN);
572 write_unlock_bh(&neigh->lock);
573 if (refcount_inc_not_zero(&neigh->ref)) {
574 rcu_read_unlock();
575 if (!schedule_work(&neigh->work))
576 /* failed to schedule, release the ref we just took */
577 if (refcount_dec_and_test(&neigh->ref))
578 efx_free_neigh(neigh);
579 } else {
580 done:
581 rcu_read_unlock();
582 }
583 return NOTIFY_DONE;
584 }
585
efx_tc_check_ready(struct efx_nic * efx,struct efx_tc_flow_rule * rule)586 bool efx_tc_check_ready(struct efx_nic *efx, struct efx_tc_flow_rule *rule)
587 {
588 struct efx_tc_action_set *act;
589
590 /* Encap actions can only be offloaded if they have valid
591 * neighbour info for the outer Ethernet header.
592 */
593 list_for_each_entry(act, &rule->acts.list, list)
594 if (act->encap_md && !act->encap_md->n_valid)
595 return false;
596 return true;
597 }
598
efx_tc_flower_create_encap_md(struct efx_nic * efx,const struct ip_tunnel_info * info,struct net_device * egdev,struct netlink_ext_ack * extack)599 struct efx_tc_encap_action *efx_tc_flower_create_encap_md(
600 struct efx_nic *efx, const struct ip_tunnel_info *info,
601 struct net_device *egdev, struct netlink_ext_ack *extack)
602 {
603 enum efx_encap_type type = efx_tc_indr_netdev_type(egdev);
604 struct efx_tc_encap_action *encap, *old;
605 struct efx_rep *to_efv;
606 s64 rc;
607
608 if (type == EFX_ENCAP_TYPE_NONE) {
609 /* dest is not an encap device */
610 NL_SET_ERR_MSG_MOD(extack, "Not a (supported) tunnel device but tunnel_key is set");
611 return ERR_PTR(-EOPNOTSUPP);
612 }
613 rc = efx_mae_check_encap_type_supported(efx, type);
614 if (rc < 0) {
615 NL_SET_ERR_MSG_MOD(extack, "Firmware reports no support for this tunnel type");
616 return ERR_PTR(rc);
617 }
618 /* No support yet for Geneve options */
619 if (info->options_len) {
620 NL_SET_ERR_MSG_MOD(extack, "Unsupported tunnel options");
621 return ERR_PTR(-EOPNOTSUPP);
622 }
623 switch (info->mode) {
624 case IP_TUNNEL_INFO_TX:
625 break;
626 case IP_TUNNEL_INFO_TX | IP_TUNNEL_INFO_IPV6:
627 type |= EFX_ENCAP_FLAG_IPV6;
628 break;
629 default:
630 NL_SET_ERR_MSG_FMT_MOD(extack, "Unsupported tunnel mode %u",
631 info->mode);
632 return ERR_PTR(-EOPNOTSUPP);
633 }
634 encap = kzalloc_obj(*encap, GFP_KERNEL_ACCOUNT);
635 if (!encap)
636 return ERR_PTR(-ENOMEM);
637 encap->type = type;
638 encap->key = info->key;
639 INIT_LIST_HEAD(&encap->users);
640 old = rhashtable_lookup_get_insert_fast(&efx->tc->encap_ht,
641 &encap->linkage,
642 efx_tc_encap_ht_params);
643 if (old) {
644 /* don't need our new entry */
645 kfree(encap);
646 if (IS_ERR(old)) /* oh dear, it's actually an error */
647 return ERR_CAST(old);
648 if (!refcount_inc_not_zero(&old->ref))
649 return ERR_PTR(-EAGAIN);
650 /* existing entry found, ref taken */
651 return old;
652 }
653
654 rc = efx_bind_neigh(efx, encap, dev_net(egdev), extack);
655 if (rc < 0)
656 goto out_remove;
657 to_efv = efx_tc_flower_lookup_efv(efx, encap->neigh->egdev);
658 if (IS_ERR(to_efv)) {
659 /* neigh->egdev isn't ours */
660 NL_SET_ERR_MSG_MOD(extack, "Tunnel egress device not on switch");
661 rc = PTR_ERR(to_efv);
662 goto out_release;
663 }
664 rc = efx_tc_flower_external_mport(efx, to_efv);
665 if (rc < 0) {
666 NL_SET_ERR_MSG_MOD(extack, "Failed to identify tunnel egress m-port");
667 goto out_release;
668 }
669 encap->dest_mport = rc;
670 read_lock_bh(&encap->neigh->lock);
671 efx_gen_encap_header(efx, encap);
672 read_unlock_bh(&encap->neigh->lock);
673
674 rc = efx_mae_allocate_encap_md(efx, encap);
675 if (rc < 0) {
676 NL_SET_ERR_MSG_MOD(extack, "Failed to write tunnel header to hw");
677 goto out_release;
678 }
679
680 /* ref and return */
681 refcount_set(&encap->ref, 1);
682 return encap;
683 out_release:
684 efx_release_neigh(efx, encap);
685 out_remove:
686 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage,
687 efx_tc_encap_ht_params);
688 kfree(encap);
689 return ERR_PTR(rc);
690 }
691
efx_tc_flower_release_encap_md(struct efx_nic * efx,struct efx_tc_encap_action * encap)692 void efx_tc_flower_release_encap_md(struct efx_nic *efx,
693 struct efx_tc_encap_action *encap)
694 {
695 if (!refcount_dec_and_test(&encap->ref))
696 return; /* still in use */
697 efx_release_neigh(efx, encap);
698 rhashtable_remove_fast(&efx->tc->encap_ht, &encap->linkage,
699 efx_tc_encap_ht_params);
700 efx_mae_free_encap_md(efx, encap);
701 kfree(encap);
702 }
703
efx_tc_remove_neigh_users(struct efx_nic * efx,struct efx_neigh_binder * neigh)704 static void efx_tc_remove_neigh_users(struct efx_nic *efx, struct efx_neigh_binder *neigh)
705 {
706 struct efx_tc_encap_action *encap, *next;
707
708 list_for_each_entry_safe(encap, next, &neigh->users, list) {
709 /* Should cause neigh usage count to fall to zero, freeing it */
710 efx_release_neigh(efx, encap);
711 /* The encap has lost its neigh, so it's now unready */
712 efx_tc_update_encap(efx, encap);
713 }
714 }
715
efx_tc_unregister_egdev(struct efx_nic * efx,struct net_device * net_dev)716 void efx_tc_unregister_egdev(struct efx_nic *efx, struct net_device *net_dev)
717 {
718 struct efx_neigh_binder *neigh;
719 struct rhashtable_iter walk;
720
721 mutex_lock(&efx->tc->mutex);
722 rhashtable_walk_enter(&efx->tc->neigh_ht, &walk);
723 rhashtable_walk_start(&walk);
724 while ((neigh = rhashtable_walk_next(&walk)) != NULL) {
725 if (IS_ERR(neigh))
726 continue;
727 if (neigh->egdev != net_dev)
728 continue;
729 neigh->dying = true;
730 rhashtable_walk_stop(&walk);
731 synchronize_rcu(); /* Make sure any updates see dying flag */
732 efx_tc_remove_neigh_users(efx, neigh); /* might sleep */
733 rhashtable_walk_start(&walk);
734 }
735 rhashtable_walk_stop(&walk);
736 rhashtable_walk_exit(&walk);
737 mutex_unlock(&efx->tc->mutex);
738 }
739
efx_tc_netevent_event(struct efx_nic * efx,unsigned long event,void * ptr)740 int efx_tc_netevent_event(struct efx_nic *efx, unsigned long event,
741 void *ptr)
742 {
743 if (efx->type->is_vf)
744 return NOTIFY_DONE;
745
746 switch (event) {
747 case NETEVENT_NEIGH_UPDATE:
748 return efx_neigh_event(efx, ptr);
749 default:
750 return NOTIFY_DONE;
751 }
752 }
753