1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2019 Mellanox Technologies. */
3
4 #include <net/netfilter/nf_conntrack.h>
5 #include <net/netfilter/nf_conntrack_core.h>
6 #include <net/netfilter/nf_conntrack_zones.h>
7 #include <net/netfilter/nf_conntrack_labels.h>
8 #include <net/netfilter/nf_conntrack_helper.h>
9 #include <net/netfilter/nf_conntrack_acct.h>
10 #include <uapi/linux/tc_act/tc_pedit.h>
11 #include <net/tc_act/tc_ct.h>
12 #include <net/flow_offload.h>
13 #include <net/netfilter/nf_flow_table.h>
14 #include <linux/workqueue.h>
15 #include <linux/refcount.h>
16 #include <linux/xarray.h>
17 #include <linux/if_macvlan.h>
18 #include <linux/debugfs.h>
19
20 #include "lib/fs_chains.h"
21 #include "en/tc_ct.h"
22 #include "en/tc/ct_fs.h"
23 #include "en/tc_priv.h"
24 #include "en/mod_hdr.h"
25 #include "en/mapping.h"
26 #include "en/tc/post_act.h"
27 #include "en.h"
28 #include "en_tc.h"
29 #include "en_rep.h"
30 #include "fs_core.h"
31
32 #define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
33 #define MLX5_CT_STATE_TRK_BIT BIT(2)
34 #define MLX5_CT_STATE_NAT_BIT BIT(3)
35 #define MLX5_CT_STATE_REPLY_BIT BIT(4)
36 #define MLX5_CT_STATE_RELATED_BIT BIT(5)
37 #define MLX5_CT_STATE_INVALID_BIT BIT(6)
38 #define MLX5_CT_STATE_NEW_BIT BIT(7)
39
40 #define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG)
41 #define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG)
42
43 /* Statically allocate modify actions for
44 * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10.
45 * This will be increased dynamically if needed (for the ipv6 snat + dnat).
46 */
47 #define MLX5_CT_MIN_MOD_ACTS 10
48
49 #define ct_dbg(fmt, args...)\
50 netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args)
51
52 struct mlx5_tc_ct_debugfs {
53 struct {
54 atomic_t offloaded;
55 atomic_t rx_dropped;
56 } stats;
57
58 struct dentry *root;
59 };
60
61 struct mlx5_tc_ct_priv {
62 struct mlx5_core_dev *dev;
63 struct mlx5e_priv *priv;
64 const struct net_device *netdev;
65 struct mod_hdr_tbl *mod_hdr_tbl;
66 struct xarray tuple_ids;
67 struct rhashtable zone_ht;
68 struct rhashtable ct_tuples_ht;
69 struct rhashtable ct_tuples_nat_ht;
70 struct mlx5_flow_table *ct;
71 struct mlx5_flow_table *ct_nat;
72 struct mlx5_flow_group *ct_nat_miss_group;
73 struct mlx5_flow_handle *ct_nat_miss_rule;
74 struct mlx5e_post_act *post_act;
75 struct mutex control_lock; /* guards parallel adds/dels */
76 struct mapping_ctx *zone_mapping;
77 struct mapping_ctx *labels_mapping;
78 enum mlx5_flow_namespace_type ns_type;
79 struct mlx5_fs_chains *chains;
80 struct mlx5_ct_fs *fs;
81 struct mlx5_ct_fs_ops *fs_ops;
82 spinlock_t ht_lock; /* protects ft entries */
83 struct workqueue_struct *wq;
84
85 struct mlx5_tc_ct_debugfs debugfs;
86 };
87
88 struct mlx5_ct_zone_rule {
89 struct mlx5_ct_fs_rule *rule;
90 struct mlx5e_mod_hdr_handle *mh;
91 struct mlx5_flow_attr *attr;
92 bool nat;
93 };
94
95 struct mlx5_tc_ct_pre {
96 struct mlx5_flow_table *ft;
97 struct mlx5_flow_group *flow_grp;
98 struct mlx5_flow_group *miss_grp;
99 struct mlx5_flow_handle *flow_rule;
100 struct mlx5_flow_handle *miss_rule;
101 struct mlx5_modify_hdr *modify_hdr;
102 };
103
104 struct mlx5_ct_ft {
105 struct rhash_head node;
106 u16 zone;
107 u32 zone_restore_id;
108 refcount_t refcount;
109 struct nf_flowtable *nf_ft;
110 struct mlx5_tc_ct_priv *ct_priv;
111 struct rhashtable ct_entries_ht;
112 struct mlx5_tc_ct_pre pre_ct;
113 struct mlx5_tc_ct_pre pre_ct_nat;
114 };
115
116 struct mlx5_ct_tuple {
117 u16 addr_type;
118 __be16 n_proto;
119 u8 ip_proto;
120 struct {
121 union {
122 __be32 src_v4;
123 struct in6_addr src_v6;
124 };
125 union {
126 __be32 dst_v4;
127 struct in6_addr dst_v6;
128 };
129 } ip;
130 struct {
131 __be16 src;
132 __be16 dst;
133 } port;
134
135 u16 zone;
136 };
137
138 struct mlx5_ct_counter {
139 struct mlx5_fc *counter;
140 refcount_t refcount;
141 bool is_shared;
142 };
143
144 enum {
145 MLX5_CT_ENTRY_FLAG_VALID,
146 MLX5_CT_ENTRY_IN_CT_TABLE,
147 MLX5_CT_ENTRY_IN_CT_NAT_TABLE,
148 };
149
150 struct mlx5_ct_entry {
151 struct rhash_head node;
152 struct rhash_head tuple_node;
153 struct rhash_head tuple_nat_node;
154 struct mlx5_ct_counter *counter;
155 unsigned long cookie;
156 unsigned long restore_cookie;
157 struct mlx5_ct_tuple tuple;
158 struct mlx5_ct_tuple tuple_nat;
159 struct mlx5_ct_zone_rule zone_rules[2];
160
161 struct mlx5_tc_ct_priv *ct_priv;
162 struct work_struct work;
163
164 refcount_t refcnt;
165 unsigned long flags;
166 };
167
168 static void
169 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
170 struct mlx5_flow_attr *attr,
171 struct mlx5e_mod_hdr_handle *mh);
172
173 static const struct rhashtable_params cts_ht_params = {
174 .head_offset = offsetof(struct mlx5_ct_entry, node),
175 .key_offset = offsetof(struct mlx5_ct_entry, cookie),
176 .key_len = sizeof(((struct mlx5_ct_entry *)0)->cookie),
177 .automatic_shrinking = true,
178 .min_size = 16 * 1024,
179 };
180
181 static const struct rhashtable_params zone_params = {
182 .head_offset = offsetof(struct mlx5_ct_ft, node),
183 .key_offset = offsetof(struct mlx5_ct_ft, zone),
184 .key_len = sizeof(((struct mlx5_ct_ft *)0)->zone),
185 .automatic_shrinking = true,
186 };
187
188 static const struct rhashtable_params tuples_ht_params = {
189 .head_offset = offsetof(struct mlx5_ct_entry, tuple_node),
190 .key_offset = offsetof(struct mlx5_ct_entry, tuple),
191 .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple),
192 .automatic_shrinking = true,
193 .min_size = 16 * 1024,
194 };
195
196 static const struct rhashtable_params tuples_nat_ht_params = {
197 .head_offset = offsetof(struct mlx5_ct_entry, tuple_nat_node),
198 .key_offset = offsetof(struct mlx5_ct_entry, tuple_nat),
199 .key_len = sizeof(((struct mlx5_ct_entry *)0)->tuple_nat),
200 .automatic_shrinking = true,
201 .min_size = 16 * 1024,
202 };
203
204 static bool
mlx5_tc_ct_entry_in_ct_table(struct mlx5_ct_entry * entry)205 mlx5_tc_ct_entry_in_ct_table(struct mlx5_ct_entry *entry)
206 {
207 return test_bit(MLX5_CT_ENTRY_IN_CT_TABLE, &entry->flags);
208 }
209
210 static bool
mlx5_tc_ct_entry_in_ct_nat_table(struct mlx5_ct_entry * entry)211 mlx5_tc_ct_entry_in_ct_nat_table(struct mlx5_ct_entry *entry)
212 {
213 return test_bit(MLX5_CT_ENTRY_IN_CT_NAT_TABLE, &entry->flags);
214 }
215
216 static int
mlx5_get_label_mapping(struct mlx5_tc_ct_priv * ct_priv,u32 * labels,u32 * id)217 mlx5_get_label_mapping(struct mlx5_tc_ct_priv *ct_priv,
218 u32 *labels, u32 *id)
219 {
220 if (!memchr_inv(labels, 0, sizeof(u32) * 4)) {
221 *id = 0;
222 return 0;
223 }
224
225 if (mapping_add(ct_priv->labels_mapping, labels, id))
226 return -EOPNOTSUPP;
227
228 return 0;
229 }
230
231 static void
mlx5_put_label_mapping(struct mlx5_tc_ct_priv * ct_priv,u32 id)232 mlx5_put_label_mapping(struct mlx5_tc_ct_priv *ct_priv, u32 id)
233 {
234 if (id)
235 mapping_remove(ct_priv->labels_mapping, id);
236 }
237
238 static int
mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple * tuple,struct flow_rule * rule)239 mlx5_tc_ct_rule_to_tuple(struct mlx5_ct_tuple *tuple, struct flow_rule *rule)
240 {
241 struct flow_match_control control;
242 struct flow_match_basic basic;
243
244 flow_rule_match_basic(rule, &basic);
245 flow_rule_match_control(rule, &control);
246
247 tuple->n_proto = basic.key->n_proto;
248 tuple->ip_proto = basic.key->ip_proto;
249 tuple->addr_type = control.key->addr_type;
250
251 if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
252 struct flow_match_ipv4_addrs match;
253
254 flow_rule_match_ipv4_addrs(rule, &match);
255 tuple->ip.src_v4 = match.key->src;
256 tuple->ip.dst_v4 = match.key->dst;
257 } else if (tuple->addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
258 struct flow_match_ipv6_addrs match;
259
260 flow_rule_match_ipv6_addrs(rule, &match);
261 tuple->ip.src_v6 = match.key->src;
262 tuple->ip.dst_v6 = match.key->dst;
263 } else {
264 return -EOPNOTSUPP;
265 }
266
267 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
268 struct flow_match_ports match;
269
270 flow_rule_match_ports(rule, &match);
271 switch (tuple->ip_proto) {
272 case IPPROTO_TCP:
273 case IPPROTO_UDP:
274 tuple->port.src = match.key->src;
275 tuple->port.dst = match.key->dst;
276 break;
277 default:
278 return -EOPNOTSUPP;
279 }
280 } else {
281 if (tuple->ip_proto != IPPROTO_GRE)
282 return -EOPNOTSUPP;
283 }
284
285 return 0;
286 }
287
288 static int
mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple * tuple,struct flow_rule * rule)289 mlx5_tc_ct_rule_to_tuple_nat(struct mlx5_ct_tuple *tuple,
290 struct flow_rule *rule)
291 {
292 struct flow_action *flow_action = &rule->action;
293 struct flow_action_entry *act;
294 u32 offset, val, ip6_offset;
295 int i;
296
297 flow_action_for_each(i, act, flow_action) {
298 if (act->id != FLOW_ACTION_MANGLE)
299 continue;
300
301 offset = act->mangle.offset;
302 val = act->mangle.val;
303 switch (act->mangle.htype) {
304 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
305 if (offset == offsetof(struct iphdr, saddr))
306 tuple->ip.src_v4 = cpu_to_be32(val);
307 else if (offset == offsetof(struct iphdr, daddr))
308 tuple->ip.dst_v4 = cpu_to_be32(val);
309 else
310 return -EOPNOTSUPP;
311 break;
312
313 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
314 ip6_offset = (offset - offsetof(struct ipv6hdr, saddr));
315 ip6_offset /= 4;
316 if (ip6_offset < 4)
317 tuple->ip.src_v6.s6_addr32[ip6_offset] = cpu_to_be32(val);
318 else if (ip6_offset < 8)
319 tuple->ip.dst_v6.s6_addr32[ip6_offset - 4] = cpu_to_be32(val);
320 else
321 return -EOPNOTSUPP;
322 break;
323
324 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
325 if (offset == offsetof(struct tcphdr, source))
326 tuple->port.src = cpu_to_be16(val);
327 else if (offset == offsetof(struct tcphdr, dest))
328 tuple->port.dst = cpu_to_be16(val);
329 else
330 return -EOPNOTSUPP;
331 break;
332
333 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
334 if (offset == offsetof(struct udphdr, source))
335 tuple->port.src = cpu_to_be16(val);
336 else if (offset == offsetof(struct udphdr, dest))
337 tuple->port.dst = cpu_to_be16(val);
338 else
339 return -EOPNOTSUPP;
340 break;
341
342 default:
343 return -EOPNOTSUPP;
344 }
345 }
346
347 return 0;
348 }
349
350 static int
mlx5_tc_ct_get_flow_source_match(struct mlx5_tc_ct_priv * ct_priv,struct net_device * ndev)351 mlx5_tc_ct_get_flow_source_match(struct mlx5_tc_ct_priv *ct_priv,
352 struct net_device *ndev)
353 {
354 struct mlx5e_priv *other_priv = netdev_priv(ndev);
355 struct mlx5_core_dev *mdev = ct_priv->dev;
356 bool vf_rep, uplink_rep;
357
358 vf_rep = mlx5e_eswitch_vf_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev);
359 uplink_rep = mlx5e_eswitch_uplink_rep(ndev) && mlx5_same_hw_devs(mdev, other_priv->mdev);
360
361 if (vf_rep)
362 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
363 if (uplink_rep)
364 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
365 if (is_vlan_dev(ndev))
366 return mlx5_tc_ct_get_flow_source_match(ct_priv, vlan_dev_real_dev(ndev));
367 if (netif_is_macvlan(ndev))
368 return mlx5_tc_ct_get_flow_source_match(ct_priv, macvlan_dev_real_dev(ndev));
369 if (mlx5e_get_tc_tun(ndev) || netif_is_lag_master(ndev))
370 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK;
371
372 return MLX5_FLOW_CONTEXT_FLOW_SOURCE_ANY_VPORT;
373 }
374
375 static int
mlx5_tc_ct_set_tuple_match(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_flow_spec * spec,struct flow_rule * rule)376 mlx5_tc_ct_set_tuple_match(struct mlx5_tc_ct_priv *ct_priv,
377 struct mlx5_flow_spec *spec,
378 struct flow_rule *rule)
379 {
380 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
381 outer_headers);
382 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
383 outer_headers);
384 u16 addr_type = 0;
385 u8 ip_proto = 0;
386
387 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
388 struct flow_match_basic match;
389
390 flow_rule_match_basic(rule, &match);
391
392 mlx5e_tc_set_ethertype(ct_priv->dev, &match, true, headers_c, headers_v);
393 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_protocol,
394 match.mask->ip_proto);
395 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol,
396 match.key->ip_proto);
397
398 ip_proto = match.key->ip_proto;
399 }
400
401 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
402 struct flow_match_control match;
403
404 flow_rule_match_control(rule, &match);
405 addr_type = match.key->addr_type;
406 }
407
408 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
409 struct flow_match_ipv4_addrs match;
410
411 flow_rule_match_ipv4_addrs(rule, &match);
412 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
413 src_ipv4_src_ipv6.ipv4_layout.ipv4),
414 &match.mask->src, sizeof(match.mask->src));
415 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
416 src_ipv4_src_ipv6.ipv4_layout.ipv4),
417 &match.key->src, sizeof(match.key->src));
418 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
419 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
420 &match.mask->dst, sizeof(match.mask->dst));
421 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
422 dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
423 &match.key->dst, sizeof(match.key->dst));
424 }
425
426 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
427 struct flow_match_ipv6_addrs match;
428
429 flow_rule_match_ipv6_addrs(rule, &match);
430 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
431 src_ipv4_src_ipv6.ipv6_layout.ipv6),
432 &match.mask->src, sizeof(match.mask->src));
433 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
434 src_ipv4_src_ipv6.ipv6_layout.ipv6),
435 &match.key->src, sizeof(match.key->src));
436
437 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
438 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
439 &match.mask->dst, sizeof(match.mask->dst));
440 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
441 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
442 &match.key->dst, sizeof(match.key->dst));
443 }
444
445 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
446 struct flow_match_ports match;
447
448 flow_rule_match_ports(rule, &match);
449 switch (ip_proto) {
450 case IPPROTO_TCP:
451 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
452 tcp_sport, ntohs(match.mask->src));
453 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
454 tcp_sport, ntohs(match.key->src));
455
456 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
457 tcp_dport, ntohs(match.mask->dst));
458 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
459 tcp_dport, ntohs(match.key->dst));
460 break;
461
462 case IPPROTO_UDP:
463 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
464 udp_sport, ntohs(match.mask->src));
465 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
466 udp_sport, ntohs(match.key->src));
467
468 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
469 udp_dport, ntohs(match.mask->dst));
470 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
471 udp_dport, ntohs(match.key->dst));
472 break;
473 default:
474 break;
475 }
476 }
477
478 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_TCP)) {
479 struct flow_match_tcp match;
480
481 flow_rule_match_tcp(rule, &match);
482 MLX5_SET(fte_match_set_lyr_2_4, headers_c, tcp_flags,
483 ntohs(match.mask->flags));
484 MLX5_SET(fte_match_set_lyr_2_4, headers_v, tcp_flags,
485 ntohs(match.key->flags));
486 }
487
488 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META)) {
489 struct flow_match_meta match;
490
491 flow_rule_match_meta(rule, &match);
492
493 if (match.key->ingress_ifindex & match.mask->ingress_ifindex) {
494 struct net_device *dev;
495
496 dev = dev_get_by_index(&init_net, match.key->ingress_ifindex);
497 if (dev && MLX5_CAP_ESW_FLOWTABLE(ct_priv->dev, flow_source))
498 spec->flow_context.flow_source =
499 mlx5_tc_ct_get_flow_source_match(ct_priv, dev);
500
501 dev_put(dev);
502 }
503 }
504
505 return 0;
506 }
507
508 static void
mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_entry * entry)509 mlx5_tc_ct_counter_put(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_entry *entry)
510 {
511 if (entry->counter->is_shared &&
512 !refcount_dec_and_test(&entry->counter->refcount))
513 return;
514
515 mlx5_fc_destroy(ct_priv->dev, entry->counter->counter);
516 kfree(entry->counter);
517 }
518
519 static void
mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_entry * entry,bool nat)520 mlx5_tc_ct_entry_del_rule(struct mlx5_tc_ct_priv *ct_priv,
521 struct mlx5_ct_entry *entry,
522 bool nat)
523 {
524 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
525 struct mlx5_flow_attr *attr = zone_rule->attr;
526
527 ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone);
528
529 ct_priv->fs_ops->ct_rule_del(ct_priv->fs, zone_rule->rule);
530 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
531 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
532 kfree(attr);
533 }
534
535 static void
mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_entry * entry)536 mlx5_tc_ct_entry_del_rules(struct mlx5_tc_ct_priv *ct_priv,
537 struct mlx5_ct_entry *entry)
538 {
539 if (mlx5_tc_ct_entry_in_ct_nat_table(entry))
540 mlx5_tc_ct_entry_del_rule(ct_priv, entry, true);
541 if (mlx5_tc_ct_entry_in_ct_table(entry))
542 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
543
544 atomic_dec(&ct_priv->debugfs.stats.offloaded);
545 }
546
547 static struct flow_action_entry *
mlx5_tc_ct_get_ct_metadata_action(struct flow_rule * flow_rule)548 mlx5_tc_ct_get_ct_metadata_action(struct flow_rule *flow_rule)
549 {
550 struct flow_action *flow_action = &flow_rule->action;
551 struct flow_action_entry *act;
552 int i;
553
554 flow_action_for_each(i, act, flow_action) {
555 if (act->id == FLOW_ACTION_CT_METADATA)
556 return act;
557 }
558
559 return NULL;
560 }
561
562 static int
mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv * ct_priv,struct mlx5e_tc_mod_hdr_acts * mod_acts,u8 ct_state,u32 mark,u32 labels_id,u8 zone_restore_id)563 mlx5_tc_ct_entry_set_registers(struct mlx5_tc_ct_priv *ct_priv,
564 struct mlx5e_tc_mod_hdr_acts *mod_acts,
565 u8 ct_state,
566 u32 mark,
567 u32 labels_id,
568 u8 zone_restore_id)
569 {
570 enum mlx5_flow_namespace_type ns = ct_priv->ns_type;
571 struct mlx5_core_dev *dev = ct_priv->dev;
572 int err;
573
574 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
575 CTSTATE_TO_REG, ct_state);
576 if (err)
577 return err;
578
579 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
580 MARK_TO_REG, mark);
581 if (err)
582 return err;
583
584 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
585 LABELS_TO_REG, labels_id);
586 if (err)
587 return err;
588
589 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
590 ZONE_RESTORE_TO_REG, zone_restore_id);
591 if (err)
592 return err;
593
594 /* Make another copy of zone id in reg_b for
595 * NIC rx flows since we don't copy reg_c1 to
596 * reg_b upon miss.
597 */
598 if (ns != MLX5_FLOW_NAMESPACE_FDB) {
599 err = mlx5e_tc_match_to_reg_set(dev, mod_acts, ns,
600 NIC_ZONE_RESTORE_TO_REG, zone_restore_id);
601 if (err)
602 return err;
603 }
604 return 0;
605 }
606
607 static int
mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry * act,char * modact)608 mlx5_tc_ct_parse_mangle_to_mod_act(struct flow_action_entry *act,
609 char *modact)
610 {
611 u32 offset = act->mangle.offset, field;
612
613 switch (act->mangle.htype) {
614 case FLOW_ACT_MANGLE_HDR_TYPE_IP4:
615 MLX5_SET(set_action_in, modact, length, 0);
616 if (offset == offsetof(struct iphdr, saddr))
617 field = MLX5_ACTION_IN_FIELD_OUT_SIPV4;
618 else if (offset == offsetof(struct iphdr, daddr))
619 field = MLX5_ACTION_IN_FIELD_OUT_DIPV4;
620 else
621 return -EOPNOTSUPP;
622 break;
623
624 case FLOW_ACT_MANGLE_HDR_TYPE_IP6:
625 MLX5_SET(set_action_in, modact, length, 0);
626 if (offset == offsetof(struct ipv6hdr, saddr) + 12)
627 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_31_0;
628 else if (offset == offsetof(struct ipv6hdr, saddr) + 8)
629 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_63_32;
630 else if (offset == offsetof(struct ipv6hdr, saddr) + 4)
631 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_95_64;
632 else if (offset == offsetof(struct ipv6hdr, saddr))
633 field = MLX5_ACTION_IN_FIELD_OUT_SIPV6_127_96;
634 else if (offset == offsetof(struct ipv6hdr, daddr) + 12)
635 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_31_0;
636 else if (offset == offsetof(struct ipv6hdr, daddr) + 8)
637 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_63_32;
638 else if (offset == offsetof(struct ipv6hdr, daddr) + 4)
639 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_95_64;
640 else if (offset == offsetof(struct ipv6hdr, daddr))
641 field = MLX5_ACTION_IN_FIELD_OUT_DIPV6_127_96;
642 else
643 return -EOPNOTSUPP;
644 break;
645
646 case FLOW_ACT_MANGLE_HDR_TYPE_TCP:
647 MLX5_SET(set_action_in, modact, length, 16);
648 if (offset == offsetof(struct tcphdr, source))
649 field = MLX5_ACTION_IN_FIELD_OUT_TCP_SPORT;
650 else if (offset == offsetof(struct tcphdr, dest))
651 field = MLX5_ACTION_IN_FIELD_OUT_TCP_DPORT;
652 else
653 return -EOPNOTSUPP;
654 break;
655
656 case FLOW_ACT_MANGLE_HDR_TYPE_UDP:
657 MLX5_SET(set_action_in, modact, length, 16);
658 if (offset == offsetof(struct udphdr, source))
659 field = MLX5_ACTION_IN_FIELD_OUT_UDP_SPORT;
660 else if (offset == offsetof(struct udphdr, dest))
661 field = MLX5_ACTION_IN_FIELD_OUT_UDP_DPORT;
662 else
663 return -EOPNOTSUPP;
664 break;
665
666 default:
667 return -EOPNOTSUPP;
668 }
669
670 MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
671 MLX5_SET(set_action_in, modact, offset, 0);
672 MLX5_SET(set_action_in, modact, field, field);
673 MLX5_SET(set_action_in, modact, data, act->mangle.val);
674
675 return 0;
676 }
677
678 static int
mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv * ct_priv,struct flow_rule * flow_rule,struct mlx5e_tc_mod_hdr_acts * mod_acts)679 mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv,
680 struct flow_rule *flow_rule,
681 struct mlx5e_tc_mod_hdr_acts *mod_acts)
682 {
683 struct flow_action *flow_action = &flow_rule->action;
684 struct mlx5_core_dev *mdev = ct_priv->dev;
685 struct flow_action_entry *act;
686 char *modact;
687 int err, i;
688
689 flow_action_for_each(i, act, flow_action) {
690 switch (act->id) {
691 case FLOW_ACTION_MANGLE: {
692 modact = mlx5e_mod_hdr_alloc(mdev, ct_priv->ns_type, mod_acts);
693 if (IS_ERR(modact))
694 return PTR_ERR(modact);
695
696 err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact);
697 if (err)
698 return err;
699
700 mod_acts->num_actions++;
701 }
702 break;
703
704 case FLOW_ACTION_CT_METADATA:
705 /* Handled earlier */
706 continue;
707 default:
708 return -EOPNOTSUPP;
709 }
710 }
711
712 return 0;
713 }
714
715 static int
mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_flow_attr * attr,struct flow_rule * flow_rule,struct mlx5e_mod_hdr_handle ** mh,u8 zone_restore_id,bool nat_table,bool has_nat)716 mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
717 struct mlx5_flow_attr *attr,
718 struct flow_rule *flow_rule,
719 struct mlx5e_mod_hdr_handle **mh,
720 u8 zone_restore_id, bool nat_table, bool has_nat)
721 {
722 DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS);
723 DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr);
724 struct flow_action_entry *meta;
725 enum ip_conntrack_info ctinfo;
726 u16 ct_state = 0;
727 int err;
728
729 meta = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
730 if (!meta)
731 return -EOPNOTSUPP;
732 ctinfo = meta->ct_metadata.cookie & NFCT_INFOMASK;
733
734 err = mlx5_get_label_mapping(ct_priv, meta->ct_metadata.labels,
735 &attr->ct_attr.ct_labels_id);
736 if (err)
737 return -EOPNOTSUPP;
738 if (nat_table) {
739 if (has_nat) {
740 err = mlx5_tc_ct_entry_create_nat(ct_priv, flow_rule, &mod_acts);
741 if (err)
742 goto err_mapping;
743 }
744
745 ct_state |= MLX5_CT_STATE_NAT_BIT;
746 }
747
748 ct_state |= MLX5_CT_STATE_TRK_BIT;
749 ct_state |= ctinfo == IP_CT_NEW ? MLX5_CT_STATE_NEW_BIT : MLX5_CT_STATE_ESTABLISHED_BIT;
750 ct_state |= meta->ct_metadata.orig_dir ? 0 : MLX5_CT_STATE_REPLY_BIT;
751 err = mlx5_tc_ct_entry_set_registers(ct_priv, &mod_acts,
752 ct_state,
753 meta->ct_metadata.mark,
754 attr->ct_attr.ct_labels_id,
755 zone_restore_id);
756 if (err)
757 goto err_mapping;
758
759 if (nat_table && has_nat) {
760 attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type,
761 mod_acts.num_actions,
762 mod_acts.actions);
763 if (IS_ERR(attr->modify_hdr)) {
764 err = PTR_ERR(attr->modify_hdr);
765 goto err_mapping;
766 }
767
768 *mh = NULL;
769 } else {
770 *mh = mlx5e_mod_hdr_attach(ct_priv->dev,
771 ct_priv->mod_hdr_tbl,
772 ct_priv->ns_type,
773 &mod_acts);
774 if (IS_ERR(*mh)) {
775 err = PTR_ERR(*mh);
776 goto err_mapping;
777 }
778 attr->modify_hdr = mlx5e_mod_hdr_get(*mh);
779 }
780
781 mlx5e_mod_hdr_dealloc(&mod_acts);
782 return 0;
783
784 err_mapping:
785 mlx5e_mod_hdr_dealloc(&mod_acts);
786 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
787 return err;
788 }
789
790 static void
mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_flow_attr * attr,struct mlx5e_mod_hdr_handle * mh)791 mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
792 struct mlx5_flow_attr *attr,
793 struct mlx5e_mod_hdr_handle *mh)
794 {
795 if (mh)
796 mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh);
797 else
798 mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr);
799 }
800
801 static int
mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv * ct_priv,struct flow_rule * flow_rule,struct mlx5_ct_entry * entry,bool nat,u8 zone_restore_id)802 mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
803 struct flow_rule *flow_rule,
804 struct mlx5_ct_entry *entry,
805 bool nat, u8 zone_restore_id)
806 {
807 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
808 struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
809 struct mlx5_flow_spec *spec = NULL;
810 struct mlx5_flow_attr *attr;
811 int err;
812
813 zone_rule->nat = nat;
814
815 spec = kvzalloc_obj(*spec);
816 if (!spec)
817 return -ENOMEM;
818
819 attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
820 if (!attr) {
821 err = -ENOMEM;
822 goto err_attr;
823 }
824
825 err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule,
826 &zone_rule->mh,
827 zone_restore_id,
828 nat,
829 mlx5_tc_ct_entry_in_ct_nat_table(entry));
830 if (err) {
831 ct_dbg("Failed to create ct entry mod hdr");
832 goto err_mod_hdr;
833 }
834
835 attr->action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR |
836 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
837 MLX5_FLOW_CONTEXT_ACTION_COUNT;
838 attr->dest_chain = 0;
839 attr->dest_ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
840 attr->ft = nat ? ct_priv->ct_nat : ct_priv->ct;
841 if (entry->tuple.ip_proto == IPPROTO_TCP ||
842 entry->tuple.ip_proto == IPPROTO_UDP)
843 attr->outer_match_level = MLX5_MATCH_L4;
844 else
845 attr->outer_match_level = MLX5_MATCH_L3;
846 attr->counter = entry->counter->counter;
847 attr->flags |= MLX5_ATTR_FLAG_NO_IN_PORT;
848 if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB)
849 attr->esw_attr->in_mdev = priv->mdev;
850
851 mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule);
852 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);
853
854 zone_rule->rule = ct_priv->fs_ops->ct_rule_add(ct_priv->fs, spec, attr, flow_rule);
855 if (IS_ERR(zone_rule->rule)) {
856 err = PTR_ERR(zone_rule->rule);
857 ct_dbg("Failed to add ct entry rule, nat: %d", nat);
858 goto err_rule;
859 }
860
861 zone_rule->attr = attr;
862
863 kvfree(spec);
864 ct_dbg("Offloaded ct entry rule in zone %d", entry->tuple.zone);
865
866 return 0;
867
868 err_rule:
869 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, attr, zone_rule->mh);
870 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
871 err_mod_hdr:
872 kfree(attr);
873 err_attr:
874 kvfree(spec);
875 return err;
876 }
877
878 static int
mlx5_tc_ct_entry_update_rule(struct mlx5_tc_ct_priv * ct_priv,struct flow_rule * flow_rule,struct mlx5_ct_entry * entry,bool nat,u8 zone_restore_id)879 mlx5_tc_ct_entry_update_rule(struct mlx5_tc_ct_priv *ct_priv,
880 struct flow_rule *flow_rule,
881 struct mlx5_ct_entry *entry,
882 bool nat, u8 zone_restore_id)
883 {
884 struct mlx5_ct_zone_rule *zone_rule = &entry->zone_rules[nat];
885 struct mlx5_flow_attr *attr = zone_rule->attr, *old_attr;
886 struct mlx5e_mod_hdr_handle *mh;
887 struct mlx5_flow_spec *spec;
888 int err;
889
890 spec = kvzalloc_obj(*spec);
891 if (!spec)
892 return -ENOMEM;
893
894 old_attr = mlx5_alloc_flow_attr(ct_priv->ns_type);
895 if (!old_attr) {
896 err = -ENOMEM;
897 goto err_attr;
898 }
899 *old_attr = *attr;
900
901 err = mlx5_tc_ct_entry_create_mod_hdr(ct_priv, attr, flow_rule, &mh, zone_restore_id,
902 nat, mlx5_tc_ct_entry_in_ct_nat_table(entry));
903 if (err) {
904 ct_dbg("Failed to create ct entry mod hdr, err: %d", err);
905 goto err_mod_hdr;
906 }
907
908 mlx5_tc_ct_set_tuple_match(ct_priv, spec, flow_rule);
909 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG, entry->tuple.zone, MLX5_CT_ZONE_MASK);
910
911 err = ct_priv->fs_ops->ct_rule_update(ct_priv->fs, zone_rule->rule, spec, attr);
912 if (err) {
913 ct_dbg("Failed to update ct entry rule, nat: %d, err: %d", nat, err);
914 goto err_rule;
915 }
916
917 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, old_attr, zone_rule->mh);
918 zone_rule->mh = mh;
919 mlx5_put_label_mapping(ct_priv, old_attr->ct_attr.ct_labels_id);
920
921 kfree(old_attr);
922 kvfree(spec);
923 ct_dbg("Updated ct entry rule in zone %d", entry->tuple.zone);
924
925 return 0;
926
927 err_rule:
928 mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, mh);
929 mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
930 err_mod_hdr:
931 *attr = *old_attr;
932 kfree(old_attr);
933 err_attr:
934 kvfree(spec);
935 return err;
936 }
937
938 static bool
mlx5_tc_ct_entry_valid(struct mlx5_ct_entry * entry)939 mlx5_tc_ct_entry_valid(struct mlx5_ct_entry *entry)
940 {
941 return test_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
942 }
943
944 static struct mlx5_ct_entry *
mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_tuple * tuple)945 mlx5_tc_ct_entry_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple *tuple)
946 {
947 struct mlx5_ct_entry *entry;
948
949 entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_ht, tuple,
950 tuples_ht_params);
951 if (entry && mlx5_tc_ct_entry_valid(entry) &&
952 refcount_inc_not_zero(&entry->refcnt)) {
953 return entry;
954 } else if (!entry) {
955 entry = rhashtable_lookup_fast(&ct_priv->ct_tuples_nat_ht,
956 tuple, tuples_nat_ht_params);
957 if (entry && mlx5_tc_ct_entry_valid(entry) &&
958 refcount_inc_not_zero(&entry->refcnt))
959 return entry;
960 }
961
962 return entry ? ERR_PTR(-EINVAL) : NULL;
963 }
964
mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry * entry)965 static void mlx5_tc_ct_entry_remove_from_tuples(struct mlx5_ct_entry *entry)
966 {
967 struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
968
969 if (mlx5_tc_ct_entry_in_ct_nat_table(entry))
970 rhashtable_remove_fast(&ct_priv->ct_tuples_nat_ht,
971 &entry->tuple_nat_node,
972 tuples_nat_ht_params);
973 if (mlx5_tc_ct_entry_in_ct_table(entry))
974 rhashtable_remove_fast(&ct_priv->ct_tuples_ht, &entry->tuple_node,
975 tuples_ht_params);
976 }
977
mlx5_tc_ct_entry_del(struct mlx5_ct_entry * entry)978 static void mlx5_tc_ct_entry_del(struct mlx5_ct_entry *entry)
979 {
980 struct mlx5_tc_ct_priv *ct_priv = entry->ct_priv;
981
982 mlx5_tc_ct_entry_del_rules(ct_priv, entry);
983
984 spin_lock_bh(&ct_priv->ht_lock);
985 mlx5_tc_ct_entry_remove_from_tuples(entry);
986 spin_unlock_bh(&ct_priv->ht_lock);
987
988 mlx5_tc_ct_counter_put(ct_priv, entry);
989 kfree(entry);
990 }
991
992 static void
mlx5_tc_ct_entry_put(struct mlx5_ct_entry * entry)993 mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
994 {
995 if (!refcount_dec_and_test(&entry->refcnt))
996 return;
997
998 mlx5_tc_ct_entry_del(entry);
999 }
1000
mlx5_tc_ct_entry_del_work(struct work_struct * work)1001 static void mlx5_tc_ct_entry_del_work(struct work_struct *work)
1002 {
1003 struct mlx5_ct_entry *entry = container_of(work, struct mlx5_ct_entry, work);
1004
1005 mlx5_tc_ct_entry_del(entry);
1006 }
1007
1008 static void
__mlx5_tc_ct_entry_put(struct mlx5_ct_entry * entry)1009 __mlx5_tc_ct_entry_put(struct mlx5_ct_entry *entry)
1010 {
1011 if (!refcount_dec_and_test(&entry->refcnt))
1012 return;
1013
1014 INIT_WORK(&entry->work, mlx5_tc_ct_entry_del_work);
1015 queue_work(entry->ct_priv->wq, &entry->work);
1016 }
1017
1018 static struct mlx5_ct_counter *
mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv * ct_priv)1019 mlx5_tc_ct_counter_create(struct mlx5_tc_ct_priv *ct_priv)
1020 {
1021 struct mlx5_ct_counter *counter;
1022 int ret;
1023
1024 counter = kzalloc_obj(*counter);
1025 if (!counter)
1026 return ERR_PTR(-ENOMEM);
1027
1028 counter->is_shared = false;
1029 counter->counter = mlx5_fc_create(ct_priv->dev, true);
1030 if (IS_ERR(counter->counter)) {
1031 ct_dbg("Failed to create counter for ct entry");
1032 ret = PTR_ERR(counter->counter);
1033 kfree(counter);
1034 return ERR_PTR(ret);
1035 }
1036
1037 return counter;
1038 }
1039
1040 static struct mlx5_ct_counter *
mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_entry * entry)1041 mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv,
1042 struct mlx5_ct_entry *entry)
1043 {
1044 struct mlx5_ct_tuple rev_tuple = entry->tuple;
1045 struct mlx5_ct_counter *shared_counter;
1046 struct mlx5_ct_entry *rev_entry;
1047
1048 /* get the reversed tuple */
1049 swap(rev_tuple.port.src, rev_tuple.port.dst);
1050
1051 if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
1052 __be32 tmp_addr = rev_tuple.ip.src_v4;
1053
1054 rev_tuple.ip.src_v4 = rev_tuple.ip.dst_v4;
1055 rev_tuple.ip.dst_v4 = tmp_addr;
1056 } else if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
1057 struct in6_addr tmp_addr = rev_tuple.ip.src_v6;
1058
1059 rev_tuple.ip.src_v6 = rev_tuple.ip.dst_v6;
1060 rev_tuple.ip.dst_v6 = tmp_addr;
1061 } else {
1062 return ERR_PTR(-EOPNOTSUPP);
1063 }
1064
1065 /* Use the same counter as the reverse direction */
1066 spin_lock_bh(&ct_priv->ht_lock);
1067 rev_entry = mlx5_tc_ct_entry_get(ct_priv, &rev_tuple);
1068
1069 if (IS_ERR(rev_entry)) {
1070 spin_unlock_bh(&ct_priv->ht_lock);
1071 goto create_counter;
1072 }
1073
1074 if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) {
1075 ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry);
1076 shared_counter = rev_entry->counter;
1077 spin_unlock_bh(&ct_priv->ht_lock);
1078
1079 mlx5_tc_ct_entry_put(rev_entry);
1080 return shared_counter;
1081 }
1082
1083 spin_unlock_bh(&ct_priv->ht_lock);
1084
1085 if (rev_entry)
1086 mlx5_tc_ct_entry_put(rev_entry);
1087
1088 create_counter:
1089
1090 shared_counter = mlx5_tc_ct_counter_create(ct_priv);
1091 if (IS_ERR(shared_counter))
1092 return shared_counter;
1093
1094 shared_counter->is_shared = true;
1095 refcount_set(&shared_counter->refcount, 1);
1096 return shared_counter;
1097 }
1098
1099 static int
mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv * ct_priv,struct flow_rule * flow_rule,struct mlx5_ct_entry * entry,u8 zone_restore_id)1100 mlx5_tc_ct_entry_add_rules(struct mlx5_tc_ct_priv *ct_priv,
1101 struct flow_rule *flow_rule,
1102 struct mlx5_ct_entry *entry,
1103 u8 zone_restore_id)
1104 {
1105 int err;
1106
1107 if (nf_ct_acct_enabled(dev_net(ct_priv->netdev)))
1108 entry->counter = mlx5_tc_ct_counter_create(ct_priv);
1109 else
1110 entry->counter = mlx5_tc_ct_shared_counter_get(ct_priv, entry);
1111
1112 if (IS_ERR(entry->counter)) {
1113 err = PTR_ERR(entry->counter);
1114 return err;
1115 }
1116
1117 if (mlx5_tc_ct_entry_in_ct_table(entry)) {
1118 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, false,
1119 zone_restore_id);
1120 if (err)
1121 goto err_orig;
1122 }
1123
1124 if (mlx5_tc_ct_entry_in_ct_nat_table(entry)) {
1125 err = mlx5_tc_ct_entry_add_rule(ct_priv, flow_rule, entry, true,
1126 zone_restore_id);
1127 if (err)
1128 goto err_nat;
1129 }
1130
1131 atomic_inc(&ct_priv->debugfs.stats.offloaded);
1132 return 0;
1133
1134 err_nat:
1135 if (mlx5_tc_ct_entry_in_ct_table(entry))
1136 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
1137 err_orig:
1138 mlx5_tc_ct_counter_put(ct_priv, entry);
1139 return err;
1140 }
1141
1142 static int
mlx5_tc_ct_entry_update_rules(struct mlx5_tc_ct_priv * ct_priv,struct flow_rule * flow_rule,struct mlx5_ct_entry * entry,u8 zone_restore_id)1143 mlx5_tc_ct_entry_update_rules(struct mlx5_tc_ct_priv *ct_priv,
1144 struct flow_rule *flow_rule,
1145 struct mlx5_ct_entry *entry,
1146 u8 zone_restore_id)
1147 {
1148 int err = 0;
1149
1150 if (mlx5_tc_ct_entry_in_ct_table(entry)) {
1151 err = mlx5_tc_ct_entry_update_rule(ct_priv, flow_rule, entry, false,
1152 zone_restore_id);
1153 if (err)
1154 return err;
1155 }
1156
1157 if (mlx5_tc_ct_entry_in_ct_nat_table(entry)) {
1158 err = mlx5_tc_ct_entry_update_rule(ct_priv, flow_rule, entry, true,
1159 zone_restore_id);
1160 if (err && mlx5_tc_ct_entry_in_ct_table(entry))
1161 mlx5_tc_ct_entry_del_rule(ct_priv, entry, false);
1162 }
1163 return err;
1164 }
1165
1166 static int
mlx5_tc_ct_block_flow_offload_update(struct mlx5_ct_ft * ft,struct flow_rule * flow_rule,struct mlx5_ct_entry * entry,unsigned long cookie)1167 mlx5_tc_ct_block_flow_offload_update(struct mlx5_ct_ft *ft, struct flow_rule *flow_rule,
1168 struct mlx5_ct_entry *entry, unsigned long cookie)
1169 {
1170 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1171 int err;
1172
1173 err = mlx5_tc_ct_entry_update_rules(ct_priv, flow_rule, entry, ft->zone_restore_id);
1174 if (!err)
1175 return 0;
1176
1177 /* If failed to update the entry, then look it up again under ht_lock
1178 * protection and properly delete it.
1179 */
1180 spin_lock_bh(&ct_priv->ht_lock);
1181 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1182 if (entry) {
1183 rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
1184 spin_unlock_bh(&ct_priv->ht_lock);
1185 mlx5_tc_ct_entry_put(entry);
1186 } else {
1187 spin_unlock_bh(&ct_priv->ht_lock);
1188 }
1189 return err;
1190 }
1191
1192 static int
mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft * ft,struct flow_cls_offload * flow)1193 mlx5_tc_ct_block_flow_offload_add(struct mlx5_ct_ft *ft,
1194 struct flow_cls_offload *flow)
1195 {
1196 struct flow_rule *flow_rule = flow_cls_offload_flow_rule(flow);
1197 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1198 struct flow_action_entry *meta_action;
1199 unsigned long cookie = flow->cookie;
1200 struct mlx5_ct_entry *entry;
1201 bool has_nat;
1202 int err;
1203
1204 meta_action = mlx5_tc_ct_get_ct_metadata_action(flow_rule);
1205 if (!meta_action)
1206 return -EOPNOTSUPP;
1207
1208 spin_lock_bh(&ct_priv->ht_lock);
1209 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1210 if (entry && refcount_inc_not_zero(&entry->refcnt)) {
1211 if (entry->restore_cookie == meta_action->ct_metadata.cookie) {
1212 spin_unlock_bh(&ct_priv->ht_lock);
1213 mlx5_tc_ct_entry_put(entry);
1214 return -EEXIST;
1215 }
1216 entry->restore_cookie = meta_action->ct_metadata.cookie;
1217 spin_unlock_bh(&ct_priv->ht_lock);
1218
1219 err = mlx5_tc_ct_block_flow_offload_update(ft, flow_rule, entry, cookie);
1220 mlx5_tc_ct_entry_put(entry);
1221 return err;
1222 }
1223 spin_unlock_bh(&ct_priv->ht_lock);
1224
1225 entry = kzalloc_obj(*entry);
1226 if (!entry)
1227 return -ENOMEM;
1228
1229 entry->tuple.zone = ft->zone;
1230 entry->cookie = flow->cookie;
1231 entry->restore_cookie = meta_action->ct_metadata.cookie;
1232 refcount_set(&entry->refcnt, 2);
1233 entry->ct_priv = ct_priv;
1234
1235 err = mlx5_tc_ct_rule_to_tuple(&entry->tuple, flow_rule);
1236 if (err)
1237 goto err_set;
1238
1239 memcpy(&entry->tuple_nat, &entry->tuple, sizeof(entry->tuple));
1240 err = mlx5_tc_ct_rule_to_tuple_nat(&entry->tuple_nat, flow_rule);
1241 if (err)
1242 goto err_set;
1243 has_nat = memcmp(&entry->tuple, &entry->tuple_nat,
1244 sizeof(entry->tuple));
1245
1246 spin_lock_bh(&ct_priv->ht_lock);
1247
1248 err = rhashtable_lookup_insert_fast(&ft->ct_entries_ht, &entry->node,
1249 cts_ht_params);
1250 if (err)
1251 goto err_entries;
1252
1253 if (has_nat) {
1254 err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_nat_ht,
1255 &entry->tuple_nat_node,
1256 tuples_nat_ht_params);
1257 if (err)
1258 goto err_tuple_nat;
1259
1260 set_bit(MLX5_CT_ENTRY_IN_CT_NAT_TABLE, &entry->flags);
1261 }
1262
1263 if (!mlx5_tc_ct_entry_in_ct_nat_table(entry)) {
1264 err = rhashtable_lookup_insert_fast(&ct_priv->ct_tuples_ht,
1265 &entry->tuple_node,
1266 tuples_ht_params);
1267 if (err)
1268 goto err_tuple;
1269
1270 set_bit(MLX5_CT_ENTRY_IN_CT_TABLE, &entry->flags);
1271 }
1272 spin_unlock_bh(&ct_priv->ht_lock);
1273
1274 err = mlx5_tc_ct_entry_add_rules(ct_priv, flow_rule, entry,
1275 ft->zone_restore_id);
1276 if (err)
1277 goto err_rules;
1278
1279 set_bit(MLX5_CT_ENTRY_FLAG_VALID, &entry->flags);
1280 mlx5_tc_ct_entry_put(entry); /* this function reference */
1281
1282 return 0;
1283
1284 err_rules:
1285 spin_lock_bh(&ct_priv->ht_lock);
1286 err_tuple:
1287 mlx5_tc_ct_entry_remove_from_tuples(entry);
1288 err_tuple_nat:
1289 rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
1290 err_entries:
1291 spin_unlock_bh(&ct_priv->ht_lock);
1292 err_set:
1293 kfree(entry);
1294 if (err != -EEXIST)
1295 netdev_warn(ct_priv->netdev, "Failed to offload ct entry, err: %d\n", err);
1296 return err;
1297 }
1298
1299 static int
mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft * ft,struct flow_cls_offload * flow)1300 mlx5_tc_ct_block_flow_offload_del(struct mlx5_ct_ft *ft,
1301 struct flow_cls_offload *flow)
1302 {
1303 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1304 unsigned long cookie = flow->cookie;
1305 struct mlx5_ct_entry *entry;
1306
1307 spin_lock_bh(&ct_priv->ht_lock);
1308 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1309 if (!entry) {
1310 spin_unlock_bh(&ct_priv->ht_lock);
1311 return -ENOENT;
1312 }
1313
1314 if (!mlx5_tc_ct_entry_valid(entry)) {
1315 spin_unlock_bh(&ct_priv->ht_lock);
1316 return -EINVAL;
1317 }
1318
1319 rhashtable_remove_fast(&ft->ct_entries_ht, &entry->node, cts_ht_params);
1320 spin_unlock_bh(&ct_priv->ht_lock);
1321
1322 mlx5_tc_ct_entry_put(entry);
1323
1324 return 0;
1325 }
1326
1327 static int
mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft * ft,struct flow_cls_offload * f)1328 mlx5_tc_ct_block_flow_offload_stats(struct mlx5_ct_ft *ft,
1329 struct flow_cls_offload *f)
1330 {
1331 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1332 unsigned long cookie = f->cookie;
1333 struct mlx5_ct_entry *entry;
1334 u64 lastuse, packets, bytes;
1335
1336 spin_lock_bh(&ct_priv->ht_lock);
1337 entry = rhashtable_lookup_fast(&ft->ct_entries_ht, &cookie, cts_ht_params);
1338 if (!entry) {
1339 spin_unlock_bh(&ct_priv->ht_lock);
1340 return -ENOENT;
1341 }
1342
1343 if (!mlx5_tc_ct_entry_valid(entry) || !refcount_inc_not_zero(&entry->refcnt)) {
1344 spin_unlock_bh(&ct_priv->ht_lock);
1345 return -EINVAL;
1346 }
1347
1348 spin_unlock_bh(&ct_priv->ht_lock);
1349
1350 mlx5_fc_query_cached(entry->counter->counter, &bytes, &packets, &lastuse);
1351 flow_stats_update(&f->stats, bytes, packets, 0, lastuse,
1352 FLOW_ACTION_HW_STATS_DELAYED);
1353
1354 mlx5_tc_ct_entry_put(entry);
1355 return 0;
1356 }
1357
1358 static bool
mlx5_tc_ct_filter_legacy_non_nic_flows(struct mlx5_ct_ft * ft,struct flow_cls_offload * flow)1359 mlx5_tc_ct_filter_legacy_non_nic_flows(struct mlx5_ct_ft *ft,
1360 struct flow_cls_offload *flow)
1361 {
1362 struct flow_rule *rule = flow_cls_offload_flow_rule(flow);
1363 struct mlx5_tc_ct_priv *ct_priv = ft->ct_priv;
1364 struct flow_match_meta match;
1365 struct net_device *netdev;
1366 bool same_dev = false;
1367
1368 if (!is_mdev_legacy_mode(ct_priv->dev) ||
1369 !flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_META))
1370 return true;
1371
1372 flow_rule_match_meta(rule, &match);
1373
1374 if (!(match.key->ingress_ifindex & match.mask->ingress_ifindex))
1375 return true;
1376
1377 netdev = dev_get_by_index(&init_net, match.key->ingress_ifindex);
1378 same_dev = ct_priv->netdev == netdev;
1379 dev_put(netdev);
1380
1381 return same_dev;
1382 }
1383
1384 static int
mlx5_tc_ct_block_flow_offload(enum tc_setup_type type,void * type_data,void * cb_priv)1385 mlx5_tc_ct_block_flow_offload(enum tc_setup_type type, void *type_data,
1386 void *cb_priv)
1387 {
1388 struct flow_cls_offload *f = type_data;
1389 struct mlx5_ct_ft *ft = cb_priv;
1390
1391 if (type != TC_SETUP_CLSFLOWER)
1392 return -EOPNOTSUPP;
1393
1394 switch (f->command) {
1395 case FLOW_CLS_REPLACE:
1396 if (!mlx5_tc_ct_filter_legacy_non_nic_flows(ft, f))
1397 return -EOPNOTSUPP;
1398
1399 return mlx5_tc_ct_block_flow_offload_add(ft, f);
1400 case FLOW_CLS_DESTROY:
1401 return mlx5_tc_ct_block_flow_offload_del(ft, f);
1402 case FLOW_CLS_STATS:
1403 return mlx5_tc_ct_block_flow_offload_stats(ft, f);
1404 default:
1405 break;
1406 }
1407
1408 return -EOPNOTSUPP;
1409 }
1410
1411 static bool
mlx5_tc_ct_skb_to_tuple(struct sk_buff * skb,struct mlx5_ct_tuple * tuple,u16 zone)1412 mlx5_tc_ct_skb_to_tuple(struct sk_buff *skb, struct mlx5_ct_tuple *tuple,
1413 u16 zone)
1414 {
1415 struct flow_keys flow_keys;
1416
1417 skb_reset_network_header(skb);
1418 skb_flow_dissect_flow_keys(skb, &flow_keys, FLOW_DISSECTOR_F_STOP_BEFORE_ENCAP);
1419
1420 tuple->zone = zone;
1421
1422 if (flow_keys.basic.ip_proto != IPPROTO_TCP &&
1423 flow_keys.basic.ip_proto != IPPROTO_UDP &&
1424 flow_keys.basic.ip_proto != IPPROTO_GRE)
1425 return false;
1426
1427 if (flow_keys.basic.ip_proto == IPPROTO_TCP ||
1428 flow_keys.basic.ip_proto == IPPROTO_UDP) {
1429 tuple->port.src = flow_keys.ports.src;
1430 tuple->port.dst = flow_keys.ports.dst;
1431 }
1432 tuple->n_proto = flow_keys.basic.n_proto;
1433 tuple->ip_proto = flow_keys.basic.ip_proto;
1434
1435 switch (flow_keys.basic.n_proto) {
1436 case htons(ETH_P_IP):
1437 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
1438 tuple->ip.src_v4 = flow_keys.addrs.v4addrs.src;
1439 tuple->ip.dst_v4 = flow_keys.addrs.v4addrs.dst;
1440 break;
1441
1442 case htons(ETH_P_IPV6):
1443 tuple->addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
1444 tuple->ip.src_v6 = flow_keys.addrs.v6addrs.src;
1445 tuple->ip.dst_v6 = flow_keys.addrs.v6addrs.dst;
1446 break;
1447 default:
1448 goto out;
1449 }
1450
1451 return true;
1452
1453 out:
1454 return false;
1455 }
1456
mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec * spec)1457 int mlx5_tc_ct_add_no_trk_match(struct mlx5_flow_spec *spec)
1458 {
1459 u32 ctstate = 0, ctstate_mask = 0;
1460
1461 mlx5e_tc_match_to_reg_get_match(spec, CTSTATE_TO_REG,
1462 &ctstate, &ctstate_mask);
1463
1464 if ((ctstate & ctstate_mask) == MLX5_CT_STATE_TRK_BIT)
1465 return -EOPNOTSUPP;
1466
1467 ctstate_mask |= MLX5_CT_STATE_TRK_BIT;
1468 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1469 ctstate, ctstate_mask);
1470
1471 return 0;
1472 }
1473
mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv * priv,struct mlx5_ct_attr * ct_attr)1474 void mlx5_tc_ct_match_del(struct mlx5_tc_ct_priv *priv, struct mlx5_ct_attr *ct_attr)
1475 {
1476 if (!priv || !ct_attr->ct_labels_id)
1477 return;
1478
1479 mlx5_put_label_mapping(priv, ct_attr->ct_labels_id);
1480 }
1481
1482 int
mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,struct mlx5_ct_attr * ct_attr,struct netlink_ext_ack * extack)1483 mlx5_tc_ct_match_add(struct mlx5_tc_ct_priv *priv,
1484 struct mlx5_flow_spec *spec,
1485 struct flow_cls_offload *f,
1486 struct mlx5_ct_attr *ct_attr,
1487 struct netlink_ext_ack *extack)
1488 {
1489 bool trk, est, untrk, unnew, unest, new, rpl, unrpl, rel, unrel, inv, uninv;
1490 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
1491 struct flow_dissector_key_ct *mask, *key;
1492 u32 ctstate = 0, ctstate_mask = 0;
1493 u16 ct_state_on, ct_state_off;
1494 u16 ct_state, ct_state_mask;
1495 struct flow_match_ct match;
1496 u32 ct_labels[4];
1497
1498 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CT))
1499 return 0;
1500
1501 if (!priv) {
1502 NL_SET_ERR_MSG_MOD(extack,
1503 "offload of ct matching isn't available");
1504 return -EOPNOTSUPP;
1505 }
1506
1507 flow_rule_match_ct(rule, &match);
1508
1509 key = match.key;
1510 mask = match.mask;
1511
1512 ct_state = key->ct_state;
1513 ct_state_mask = mask->ct_state;
1514
1515 if (ct_state_mask & ~(TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
1516 TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED |
1517 TCA_FLOWER_KEY_CT_FLAGS_NEW |
1518 TCA_FLOWER_KEY_CT_FLAGS_REPLY |
1519 TCA_FLOWER_KEY_CT_FLAGS_RELATED |
1520 TCA_FLOWER_KEY_CT_FLAGS_INVALID)) {
1521 NL_SET_ERR_MSG_MOD(extack,
1522 "only ct_state trk, est, new and rpl are supported for offload");
1523 return -EOPNOTSUPP;
1524 }
1525
1526 ct_state_on = ct_state & ct_state_mask;
1527 ct_state_off = (ct_state & ct_state_mask) ^ ct_state_mask;
1528 trk = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1529 new = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_NEW;
1530 est = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1531 rpl = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1532 rel = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1533 inv = ct_state_on & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1534 untrk = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_TRACKED;
1535 unnew = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_NEW;
1536 unest = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED;
1537 unrpl = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_REPLY;
1538 unrel = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_RELATED;
1539 uninv = ct_state_off & TCA_FLOWER_KEY_CT_FLAGS_INVALID;
1540
1541 ctstate |= trk ? MLX5_CT_STATE_TRK_BIT : 0;
1542 ctstate |= new ? MLX5_CT_STATE_NEW_BIT : 0;
1543 ctstate |= est ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1544 ctstate |= rpl ? MLX5_CT_STATE_REPLY_BIT : 0;
1545 ctstate_mask |= (untrk || trk) ? MLX5_CT_STATE_TRK_BIT : 0;
1546 ctstate_mask |= (unnew || new) ? MLX5_CT_STATE_NEW_BIT : 0;
1547 ctstate_mask |= (unest || est) ? MLX5_CT_STATE_ESTABLISHED_BIT : 0;
1548 ctstate_mask |= (unrpl || rpl) ? MLX5_CT_STATE_REPLY_BIT : 0;
1549 ctstate_mask |= unrel ? MLX5_CT_STATE_RELATED_BIT : 0;
1550 ctstate_mask |= uninv ? MLX5_CT_STATE_INVALID_BIT : 0;
1551
1552 if (rel) {
1553 NL_SET_ERR_MSG_MOD(extack,
1554 "matching on ct_state +rel isn't supported");
1555 return -EOPNOTSUPP;
1556 }
1557
1558 if (inv) {
1559 NL_SET_ERR_MSG_MOD(extack,
1560 "matching on ct_state +inv isn't supported");
1561 return -EOPNOTSUPP;
1562 }
1563
1564 if (mask->ct_zone)
1565 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1566 key->ct_zone, MLX5_CT_ZONE_MASK);
1567 if (ctstate_mask)
1568 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG,
1569 ctstate, ctstate_mask);
1570 if (mask->ct_mark)
1571 mlx5e_tc_match_to_reg_match(spec, MARK_TO_REG,
1572 key->ct_mark, mask->ct_mark);
1573 if (mask->ct_labels[0] || mask->ct_labels[1] || mask->ct_labels[2] ||
1574 mask->ct_labels[3]) {
1575 ct_labels[0] = key->ct_labels[0] & mask->ct_labels[0];
1576 ct_labels[1] = key->ct_labels[1] & mask->ct_labels[1];
1577 ct_labels[2] = key->ct_labels[2] & mask->ct_labels[2];
1578 ct_labels[3] = key->ct_labels[3] & mask->ct_labels[3];
1579 if (mlx5_get_label_mapping(priv, ct_labels, &ct_attr->ct_labels_id))
1580 return -EOPNOTSUPP;
1581 mlx5e_tc_match_to_reg_match(spec, LABELS_TO_REG, ct_attr->ct_labels_id,
1582 MLX5_CT_LABELS_MASK);
1583 }
1584
1585 return 0;
1586 }
1587
1588 int
mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv * priv,struct mlx5_flow_attr * attr,const struct flow_action_entry * act,struct netlink_ext_ack * extack)1589 mlx5_tc_ct_parse_action(struct mlx5_tc_ct_priv *priv,
1590 struct mlx5_flow_attr *attr,
1591 const struct flow_action_entry *act,
1592 struct netlink_ext_ack *extack)
1593 {
1594 if (!priv) {
1595 NL_SET_ERR_MSG_MOD(extack,
1596 "offload of ct action isn't available");
1597 return -EOPNOTSUPP;
1598 }
1599
1600 attr->ct_attr.ct_action |= act->ct.action; /* So we can have clear + ct */
1601 attr->ct_attr.zone = act->ct.zone;
1602 if (!(act->ct.action & TCA_CT_ACT_CLEAR))
1603 attr->ct_attr.nf_ft = act->ct.flow_table;
1604 attr->ct_attr.act_miss_cookie = act->miss_cookie;
1605
1606 return 0;
1607 }
1608
tc_ct_pre_ct_add_rules(struct mlx5_ct_ft * ct_ft,struct mlx5_tc_ct_pre * pre_ct,bool nat)1609 static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft,
1610 struct mlx5_tc_ct_pre *pre_ct,
1611 bool nat)
1612 {
1613 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1614 struct mlx5e_tc_mod_hdr_acts pre_mod_acts = {};
1615 struct mlx5_core_dev *dev = ct_priv->dev;
1616 struct mlx5_flow_table *ft = pre_ct->ft;
1617 struct mlx5_flow_destination dest = {};
1618 struct mlx5_flow_act flow_act = {};
1619 struct mlx5_modify_hdr *mod_hdr;
1620 struct mlx5_flow_handle *rule;
1621 struct mlx5_flow_spec *spec;
1622 u32 ctstate;
1623 u16 zone;
1624 int err;
1625
1626 spec = kvzalloc_obj(*spec);
1627 if (!spec)
1628 return -ENOMEM;
1629
1630 zone = ct_ft->zone & MLX5_CT_ZONE_MASK;
1631 err = mlx5e_tc_match_to_reg_set(dev, &pre_mod_acts, ct_priv->ns_type,
1632 ZONE_TO_REG, zone);
1633 if (err) {
1634 ct_dbg("Failed to set zone register mapping");
1635 goto err_mapping;
1636 }
1637
1638 mod_hdr = mlx5_modify_header_alloc(dev, ct_priv->ns_type,
1639 pre_mod_acts.num_actions,
1640 pre_mod_acts.actions);
1641
1642 if (IS_ERR(mod_hdr)) {
1643 err = PTR_ERR(mod_hdr);
1644 ct_dbg("Failed to create pre ct mod hdr");
1645 goto err_mapping;
1646 }
1647 pre_ct->modify_hdr = mod_hdr;
1648
1649 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1650 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1651 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
1652 flow_act.modify_hdr = mod_hdr;
1653 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1654
1655 /* add flow rule */
1656 mlx5e_tc_match_to_reg_match(spec, ZONE_TO_REG,
1657 zone, MLX5_CT_ZONE_MASK);
1658 ctstate = MLX5_CT_STATE_TRK_BIT;
1659 if (nat)
1660 ctstate |= MLX5_CT_STATE_NAT_BIT;
1661 mlx5e_tc_match_to_reg_match(spec, CTSTATE_TO_REG, ctstate, ctstate);
1662
1663 dest.ft = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
1664 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
1665 if (IS_ERR(rule)) {
1666 err = PTR_ERR(rule);
1667 ct_dbg("Failed to add pre ct flow rule zone %d", zone);
1668 goto err_flow_rule;
1669 }
1670 pre_ct->flow_rule = rule;
1671
1672 /* add miss rule */
1673 dest.ft = nat ? ct_priv->ct_nat : ct_priv->ct;
1674 rule = mlx5_add_flow_rules(ft, NULL, &flow_act, &dest, 1);
1675 if (IS_ERR(rule)) {
1676 err = PTR_ERR(rule);
1677 ct_dbg("Failed to add pre ct miss rule zone %d", zone);
1678 goto err_miss_rule;
1679 }
1680 pre_ct->miss_rule = rule;
1681
1682 mlx5e_mod_hdr_dealloc(&pre_mod_acts);
1683 kvfree(spec);
1684 return 0;
1685
1686 err_miss_rule:
1687 mlx5_del_flow_rules(pre_ct->flow_rule);
1688 err_flow_rule:
1689 mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1690 err_mapping:
1691 mlx5e_mod_hdr_dealloc(&pre_mod_acts);
1692 kvfree(spec);
1693 return err;
1694 }
1695
1696 static void
tc_ct_pre_ct_del_rules(struct mlx5_ct_ft * ct_ft,struct mlx5_tc_ct_pre * pre_ct)1697 tc_ct_pre_ct_del_rules(struct mlx5_ct_ft *ct_ft,
1698 struct mlx5_tc_ct_pre *pre_ct)
1699 {
1700 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1701 struct mlx5_core_dev *dev = ct_priv->dev;
1702
1703 mlx5_del_flow_rules(pre_ct->flow_rule);
1704 mlx5_del_flow_rules(pre_ct->miss_rule);
1705 mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr);
1706 }
1707
1708 static int
mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft * ct_ft,struct mlx5_tc_ct_pre * pre_ct,bool nat)1709 mlx5_tc_ct_alloc_pre_ct(struct mlx5_ct_ft *ct_ft,
1710 struct mlx5_tc_ct_pre *pre_ct,
1711 bool nat)
1712 {
1713 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1714 struct mlx5_tc_ct_priv *ct_priv = ct_ft->ct_priv;
1715 struct mlx5_core_dev *dev = ct_priv->dev;
1716 struct mlx5_flow_table_attr ft_attr = {};
1717 struct mlx5_flow_namespace *ns;
1718 struct mlx5_flow_table *ft;
1719 struct mlx5_flow_group *g;
1720 u32 metadata_reg_c_2_mask;
1721 u32 *flow_group_in;
1722 void *misc;
1723 int err;
1724
1725 ns = mlx5_get_flow_namespace(dev, ct_priv->ns_type);
1726 if (!ns) {
1727 err = -EOPNOTSUPP;
1728 ct_dbg("Failed to get flow namespace");
1729 return err;
1730 }
1731
1732 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1733 if (!flow_group_in)
1734 return -ENOMEM;
1735
1736 ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED;
1737 ft_attr.prio = ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB ?
1738 FDB_TC_OFFLOAD : MLX5E_TC_PRIO;
1739 ft_attr.max_fte = 2;
1740 ft_attr.level = 1;
1741 ft = mlx5_create_flow_table(ns, &ft_attr);
1742 if (IS_ERR(ft)) {
1743 err = PTR_ERR(ft);
1744 ct_dbg("Failed to create pre ct table");
1745 goto out_free;
1746 }
1747 pre_ct->ft = ft;
1748
1749 /* create flow group */
1750 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1751 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
1752 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1753 MLX5_MATCH_MISC_PARAMETERS_2);
1754
1755 misc = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1756 match_criteria.misc_parameters_2);
1757
1758 metadata_reg_c_2_mask = MLX5_CT_ZONE_MASK;
1759 metadata_reg_c_2_mask |= (MLX5_CT_STATE_TRK_BIT << 16);
1760 if (nat)
1761 metadata_reg_c_2_mask |= (MLX5_CT_STATE_NAT_BIT << 16);
1762
1763 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_2,
1764 metadata_reg_c_2_mask);
1765
1766 g = mlx5_create_flow_group(ft, flow_group_in);
1767 if (IS_ERR(g)) {
1768 err = PTR_ERR(g);
1769 ct_dbg("Failed to create pre ct group");
1770 goto err_flow_grp;
1771 }
1772 pre_ct->flow_grp = g;
1773
1774 /* create miss group */
1775 memset(flow_group_in, 0, inlen);
1776 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 1);
1777 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1);
1778 g = mlx5_create_flow_group(ft, flow_group_in);
1779 if (IS_ERR(g)) {
1780 err = PTR_ERR(g);
1781 ct_dbg("Failed to create pre ct miss group");
1782 goto err_miss_grp;
1783 }
1784 pre_ct->miss_grp = g;
1785
1786 err = tc_ct_pre_ct_add_rules(ct_ft, pre_ct, nat);
1787 if (err)
1788 goto err_add_rules;
1789
1790 kvfree(flow_group_in);
1791 return 0;
1792
1793 err_add_rules:
1794 mlx5_destroy_flow_group(pre_ct->miss_grp);
1795 err_miss_grp:
1796 mlx5_destroy_flow_group(pre_ct->flow_grp);
1797 err_flow_grp:
1798 mlx5_destroy_flow_table(ft);
1799 out_free:
1800 kvfree(flow_group_in);
1801 return err;
1802 }
1803
1804 static void
mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft * ct_ft,struct mlx5_tc_ct_pre * pre_ct)1805 mlx5_tc_ct_free_pre_ct(struct mlx5_ct_ft *ct_ft,
1806 struct mlx5_tc_ct_pre *pre_ct)
1807 {
1808 tc_ct_pre_ct_del_rules(ct_ft, pre_ct);
1809 mlx5_destroy_flow_group(pre_ct->miss_grp);
1810 mlx5_destroy_flow_group(pre_ct->flow_grp);
1811 mlx5_destroy_flow_table(pre_ct->ft);
1812 }
1813
1814 static int
mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft * ft)1815 mlx5_tc_ct_alloc_pre_ct_tables(struct mlx5_ct_ft *ft)
1816 {
1817 int err;
1818
1819 err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct, false);
1820 if (err)
1821 return err;
1822
1823 err = mlx5_tc_ct_alloc_pre_ct(ft, &ft->pre_ct_nat, true);
1824 if (err)
1825 goto err_pre_ct_nat;
1826
1827 return 0;
1828
1829 err_pre_ct_nat:
1830 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1831 return err;
1832 }
1833
1834 static void
mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft * ft)1835 mlx5_tc_ct_free_pre_ct_tables(struct mlx5_ct_ft *ft)
1836 {
1837 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct_nat);
1838 mlx5_tc_ct_free_pre_ct(ft, &ft->pre_ct);
1839 }
1840
1841 /* To avoid false lock dependency warning set the ct_entries_ht lock
1842 * class different than the lock class of the ht being used when deleting
1843 * last flow from a group and then deleting a group, we get into del_sw_flow_group()
1844 * which call rhashtable_destroy on fg->ftes_hash which will take ht->mutex but
1845 * it's different than the ht->mutex here.
1846 */
1847 static struct lock_class_key ct_entries_ht_lock_key;
1848
1849 static struct mlx5_ct_ft *
mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv * ct_priv,u16 zone,struct nf_flowtable * nf_ft)1850 mlx5_tc_ct_add_ft_cb(struct mlx5_tc_ct_priv *ct_priv, u16 zone,
1851 struct nf_flowtable *nf_ft)
1852 {
1853 struct mlx5_ct_ft *ft;
1854 int err;
1855
1856 ft = rhashtable_lookup_fast(&ct_priv->zone_ht, &zone, zone_params);
1857 if (ft) {
1858 refcount_inc(&ft->refcount);
1859 return ft;
1860 }
1861
1862 ft = kzalloc_obj(*ft);
1863 if (!ft)
1864 return ERR_PTR(-ENOMEM);
1865
1866 err = mapping_add(ct_priv->zone_mapping, &zone, &ft->zone_restore_id);
1867 if (err)
1868 goto err_mapping;
1869
1870 ft->zone = zone;
1871 ft->nf_ft = nf_ft;
1872 ft->ct_priv = ct_priv;
1873 refcount_set(&ft->refcount, 1);
1874
1875 err = mlx5_tc_ct_alloc_pre_ct_tables(ft);
1876 if (err)
1877 goto err_alloc_pre_ct;
1878
1879 err = rhashtable_init(&ft->ct_entries_ht, &cts_ht_params);
1880 if (err)
1881 goto err_init;
1882
1883 lockdep_set_class(&ft->ct_entries_ht.mutex, &ct_entries_ht_lock_key);
1884
1885 err = rhashtable_insert_fast(&ct_priv->zone_ht, &ft->node,
1886 zone_params);
1887 if (err)
1888 goto err_insert;
1889
1890 err = nf_flow_table_offload_add_cb(ft->nf_ft,
1891 mlx5_tc_ct_block_flow_offload, ft);
1892 if (err)
1893 goto err_add_cb;
1894
1895 return ft;
1896
1897 err_add_cb:
1898 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1899 err_insert:
1900 rhashtable_destroy(&ft->ct_entries_ht);
1901 err_init:
1902 mlx5_tc_ct_free_pre_ct_tables(ft);
1903 err_alloc_pre_ct:
1904 mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1905 err_mapping:
1906 kfree(ft);
1907 return ERR_PTR(err);
1908 }
1909
1910 static void
mlx5_tc_ct_flush_ft_entry(void * ptr,void * arg)1911 mlx5_tc_ct_flush_ft_entry(void *ptr, void *arg)
1912 {
1913 struct mlx5_ct_entry *entry = ptr;
1914
1915 mlx5_tc_ct_entry_put(entry);
1916 }
1917
1918 static void
mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_ct_ft * ft)1919 mlx5_tc_ct_del_ft_cb(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_ft *ft)
1920 {
1921 if (!refcount_dec_and_test(&ft->refcount))
1922 return;
1923
1924 flush_workqueue(ct_priv->wq);
1925 nf_flow_table_offload_del_cb(ft->nf_ft,
1926 mlx5_tc_ct_block_flow_offload, ft);
1927 rhashtable_remove_fast(&ct_priv->zone_ht, &ft->node, zone_params);
1928 rhashtable_free_and_destroy(&ft->ct_entries_ht,
1929 mlx5_tc_ct_flush_ft_entry,
1930 ct_priv);
1931 mlx5_tc_ct_free_pre_ct_tables(ft);
1932 mapping_remove(ct_priv->zone_mapping, ft->zone_restore_id);
1933 kfree(ft);
1934 }
1935
1936 /* We translate the tc filter with CT action to the following HW model:
1937 *
1938 * +-----------------------+
1939 * + rule (either original +
1940 * + or post_act rule) +
1941 * +-----------------------+
1942 * | set act_miss_cookie mapping
1943 * | set fte_id
1944 * | set tunnel_id
1945 * | rest of actions before the CT action (for this orig/post_act rule)
1946 * |
1947 * +-------------+
1948 * | Chain 0 |
1949 * | optimization|
1950 * | v
1951 * | +---------------------+
1952 * | + pre_ct/pre_ct_nat + if matches +----------------------+
1953 * | + zone+nat match +---------------->+ post_act (see below) +
1954 * | +---------------------+ set zone +----------------------+
1955 * | |
1956 * +-------------+ set zone
1957 * |
1958 * v
1959 * +--------------------+
1960 * + CT (nat or no nat) +
1961 * + tuple + zone match +
1962 * +--------------------+
1963 * | set mark
1964 * | set labels_id
1965 * | set established
1966 * | set zone_restore
1967 * | do nat (if needed)
1968 * v
1969 * +--------------+
1970 * + post_act + rest of parsed filter's actions
1971 * + fte_id match +------------------------>
1972 * +--------------+
1973 *
1974 */
1975 static int
__mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_flow_attr * attr)1976 __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv,
1977 struct mlx5_flow_attr *attr)
1978 {
1979 bool nat = attr->ct_attr.ct_action & TCA_CT_ACT_NAT;
1980 struct mlx5e_priv *priv = netdev_priv(ct_priv->netdev);
1981 int act_miss_mapping = 0, err;
1982 struct mlx5_ct_ft *ft;
1983 u16 zone;
1984
1985 /* Register for CT established events */
1986 ft = mlx5_tc_ct_add_ft_cb(ct_priv, attr->ct_attr.zone,
1987 attr->ct_attr.nf_ft);
1988 if (IS_ERR(ft)) {
1989 err = PTR_ERR(ft);
1990 ct_dbg("Failed to register to ft callback");
1991 goto err_ft;
1992 }
1993 attr->ct_attr.ft = ft;
1994
1995 err = mlx5e_tc_action_miss_mapping_get(ct_priv->priv, attr, attr->ct_attr.act_miss_cookie,
1996 &act_miss_mapping);
1997 if (err) {
1998 ct_dbg("Failed to get register mapping for act miss");
1999 goto err_get_act_miss;
2000 }
2001
2002 err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts,
2003 ct_priv->ns_type, MAPPED_OBJ_TO_REG, act_miss_mapping);
2004 if (err) {
2005 ct_dbg("Failed to set act miss register mapping");
2006 goto err_mapping;
2007 }
2008
2009 /* Chain 0 sets the zone and jumps to ct table
2010 * Other chains jump to pre_ct table to align with act_ct cached logic
2011 */
2012 if (!attr->chain) {
2013 zone = ft->zone & MLX5_CT_ZONE_MASK;
2014 err = mlx5e_tc_match_to_reg_set(priv->mdev, &attr->parse_attr->mod_hdr_acts,
2015 ct_priv->ns_type, ZONE_TO_REG, zone);
2016 if (err) {
2017 ct_dbg("Failed to set zone register mapping");
2018 goto err_mapping;
2019 }
2020
2021 attr->dest_ft = nat ? ct_priv->ct_nat : ct_priv->ct;
2022 } else {
2023 attr->dest_ft = nat ? ft->pre_ct_nat.ft : ft->pre_ct.ft;
2024 }
2025
2026 attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2027 attr->ct_attr.act_miss_mapping = act_miss_mapping;
2028
2029 return 0;
2030
2031 err_mapping:
2032 mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, act_miss_mapping);
2033 err_get_act_miss:
2034 mlx5_tc_ct_del_ft_cb(ct_priv, ft);
2035 err_ft:
2036 netdev_warn(priv->netdev, "Failed to offload ct flow, err %d\n", err);
2037 return err;
2038 }
2039
2040 int
mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv * priv,struct mlx5_flow_attr * attr)2041 mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *priv, struct mlx5_flow_attr *attr)
2042 {
2043 int err;
2044
2045 if (!priv)
2046 return -EOPNOTSUPP;
2047
2048 if (attr->ct_attr.offloaded)
2049 return 0;
2050
2051 if (attr->ct_attr.ct_action & TCA_CT_ACT_CLEAR) {
2052 err = mlx5_tc_ct_entry_set_registers(priv, &attr->parse_attr->mod_hdr_acts,
2053 0, 0, 0, 0);
2054 if (err)
2055 return err;
2056
2057 attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
2058 }
2059
2060 if (!attr->ct_attr.nf_ft) { /* means only ct clear action, and not ct_clear,ct() */
2061 attr->ct_attr.offloaded = true;
2062 return 0;
2063 }
2064
2065 mutex_lock(&priv->control_lock);
2066 err = __mlx5_tc_ct_flow_offload(priv, attr);
2067 if (!err)
2068 attr->ct_attr.offloaded = true;
2069 mutex_unlock(&priv->control_lock);
2070
2071 return err;
2072 }
2073
2074 static void
__mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv * ct_priv,struct mlx5_flow_attr * attr)2075 __mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *ct_priv,
2076 struct mlx5_flow_attr *attr)
2077 {
2078 mlx5e_tc_action_miss_mapping_put(ct_priv->priv, attr, attr->ct_attr.act_miss_mapping);
2079 mlx5_tc_ct_del_ft_cb(ct_priv, attr->ct_attr.ft);
2080 }
2081
2082 void
mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv * priv,struct mlx5_flow_attr * attr)2083 mlx5_tc_ct_delete_flow(struct mlx5_tc_ct_priv *priv,
2084 struct mlx5_flow_attr *attr)
2085 {
2086 if (!attr->ct_attr.offloaded) /* no ct action, return */
2087 return;
2088 if (!attr->ct_attr.nf_ft) /* means only ct clear action, and not ct_clear,ct() */
2089 return;
2090
2091 mutex_lock(&priv->control_lock);
2092 __mlx5_tc_ct_delete_flow(priv, attr);
2093 mutex_unlock(&priv->control_lock);
2094 }
2095
2096 static int
mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv * ct_priv)2097 mlx5_tc_ct_fs_init(struct mlx5_tc_ct_priv *ct_priv)
2098 {
2099 struct mlx5_flow_table *post_ct = mlx5e_tc_post_act_get_ft(ct_priv->post_act);
2100 struct mlx5_ct_fs_ops *fs_ops = mlx5_ct_fs_dmfs_ops_get();
2101 int err;
2102
2103 if (ct_priv->ns_type == MLX5_FLOW_NAMESPACE_FDB) {
2104 if (ct_priv->dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_HMFS) {
2105 ct_dbg("Using HMFS ct flow steering provider");
2106 fs_ops = mlx5_ct_fs_hmfs_ops_get();
2107 } else if (ct_priv->dev->priv.steering->mode == MLX5_FLOW_STEERING_MODE_SMFS) {
2108 ct_dbg("Using SMFS ct flow steering provider");
2109 fs_ops = mlx5_ct_fs_smfs_ops_get();
2110 }
2111
2112 if (!fs_ops) {
2113 ct_dbg("Requested flow steering mode is not enabled.");
2114 return -EOPNOTSUPP;
2115 }
2116 }
2117
2118 ct_priv->fs = kzalloc(sizeof(*ct_priv->fs) + fs_ops->priv_size, GFP_KERNEL);
2119 if (!ct_priv->fs)
2120 return -ENOMEM;
2121
2122 ct_priv->fs->netdev = ct_priv->netdev;
2123 ct_priv->fs->dev = ct_priv->dev;
2124 ct_priv->fs_ops = fs_ops;
2125
2126 err = ct_priv->fs_ops->init(ct_priv->fs, ct_priv->ct, ct_priv->ct_nat, post_ct);
2127 if (err)
2128 goto err_init;
2129
2130 return 0;
2131
2132 err_init:
2133 kfree(ct_priv->fs);
2134 return err;
2135 }
2136
2137 static int
mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch * esw,const char ** err_msg)2138 mlx5_tc_ct_init_check_esw_support(struct mlx5_eswitch *esw,
2139 const char **err_msg)
2140 {
2141 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) {
2142 /* vlan workaround should be avoided for multi chain rules.
2143 * This is just a sanity check as pop vlan action should
2144 * be supported by any FW that supports ignore_flow_level
2145 */
2146
2147 *err_msg = "firmware vlan actions support is missing";
2148 return -EOPNOTSUPP;
2149 }
2150
2151 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev,
2152 fdb_modify_header_fwd_to_table)) {
2153 /* CT always writes to registers which are mod header actions.
2154 * Therefore, mod header and goto is required
2155 */
2156
2157 *err_msg = "firmware fwd and modify support is missing";
2158 return -EOPNOTSUPP;
2159 }
2160
2161 if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
2162 *err_msg = "register loopback isn't supported";
2163 return -EOPNOTSUPP;
2164 }
2165
2166 return 0;
2167 }
2168
2169 static int
mlx5_tc_ct_init_check_support(struct mlx5e_priv * priv,enum mlx5_flow_namespace_type ns_type,struct mlx5e_post_act * post_act)2170 mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv,
2171 enum mlx5_flow_namespace_type ns_type,
2172 struct mlx5e_post_act *post_act)
2173 {
2174 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
2175 const char *err_msg = NULL;
2176 int err = 0;
2177
2178 if (IS_ERR_OR_NULL(post_act)) {
2179 /* Ignore_flow_level support isn't supported by default for VFs and so post_act
2180 * won't be supported. Skip showing error msg.
2181 */
2182 if (priv->mdev->coredev_type == MLX5_COREDEV_PF)
2183 err_msg = "post action is missing";
2184 err = -EOPNOTSUPP;
2185 goto out_err;
2186 }
2187
2188 if (ns_type == MLX5_FLOW_NAMESPACE_FDB)
2189 err = mlx5_tc_ct_init_check_esw_support(esw, &err_msg);
2190
2191 out_err:
2192 if (err && err_msg)
2193 netdev_dbg(priv->netdev, "tc ct offload not supported, %s\n", err_msg);
2194 return err;
2195 }
2196
2197 static void
mlx5_ct_tc_create_dbgfs(struct mlx5_tc_ct_priv * ct_priv)2198 mlx5_ct_tc_create_dbgfs(struct mlx5_tc_ct_priv *ct_priv)
2199 {
2200 struct mlx5_tc_ct_debugfs *ct_dbgfs = &ct_priv->debugfs;
2201
2202 ct_dbgfs->root = debugfs_create_dir("ct", mlx5_debugfs_get_dev_root(ct_priv->dev));
2203 debugfs_create_atomic_t("offloaded", 0400, ct_dbgfs->root,
2204 &ct_dbgfs->stats.offloaded);
2205 debugfs_create_atomic_t("rx_dropped", 0400, ct_dbgfs->root,
2206 &ct_dbgfs->stats.rx_dropped);
2207 }
2208
2209 static void
mlx5_ct_tc_remove_dbgfs(struct mlx5_tc_ct_priv * ct_priv)2210 mlx5_ct_tc_remove_dbgfs(struct mlx5_tc_ct_priv *ct_priv)
2211 {
2212 debugfs_remove_recursive(ct_priv->debugfs.root);
2213 }
2214
2215 static struct mlx5_flow_handle *
tc_ct_add_miss_rule(struct mlx5_flow_table * ft,struct mlx5_flow_table * next_ft)2216 tc_ct_add_miss_rule(struct mlx5_flow_table *ft,
2217 struct mlx5_flow_table *next_ft)
2218 {
2219 struct mlx5_flow_destination dest = {};
2220 struct mlx5_flow_act act = {};
2221
2222 act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL | FLOW_ACT_NO_APPEND;
2223 act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2224 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
2225 dest.ft = next_ft;
2226
2227 return mlx5_add_flow_rules(ft, NULL, &act, &dest, 1);
2228 }
2229
2230 static int
tc_ct_add_ct_table_miss_rule(struct mlx5_flow_table * from,struct mlx5_flow_table * to,struct mlx5_flow_group ** miss_group,struct mlx5_flow_handle ** miss_rule)2231 tc_ct_add_ct_table_miss_rule(struct mlx5_flow_table *from,
2232 struct mlx5_flow_table *to,
2233 struct mlx5_flow_group **miss_group,
2234 struct mlx5_flow_handle **miss_rule)
2235 {
2236 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2237 struct mlx5_flow_group *group;
2238 struct mlx5_flow_handle *rule;
2239 unsigned int max_fte = from->max_fte;
2240 u32 *flow_group_in;
2241 int err = 0;
2242
2243 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2244 if (!flow_group_in)
2245 return -ENOMEM;
2246
2247 /* create miss group */
2248 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index,
2249 max_fte - 2);
2250 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2251 max_fte - 1);
2252 group = mlx5_create_flow_group(from, flow_group_in);
2253 if (IS_ERR(group)) {
2254 err = PTR_ERR(group);
2255 goto err_miss_grp;
2256 }
2257
2258 /* add miss rule to next fdb */
2259 rule = tc_ct_add_miss_rule(from, to);
2260 if (IS_ERR(rule)) {
2261 err = PTR_ERR(rule);
2262 goto err_miss_rule;
2263 }
2264
2265 *miss_group = group;
2266 *miss_rule = rule;
2267 kvfree(flow_group_in);
2268 return 0;
2269
2270 err_miss_rule:
2271 mlx5_destroy_flow_group(group);
2272 err_miss_grp:
2273 kvfree(flow_group_in);
2274 return err;
2275 }
2276
2277 static void
tc_ct_del_ct_table_miss_rule(struct mlx5_flow_group * miss_group,struct mlx5_flow_handle * miss_rule)2278 tc_ct_del_ct_table_miss_rule(struct mlx5_flow_group *miss_group,
2279 struct mlx5_flow_handle *miss_rule)
2280 {
2281 mlx5_del_flow_rules(miss_rule);
2282 mlx5_destroy_flow_group(miss_group);
2283 }
2284
2285 #define INIT_ERR_PREFIX "tc ct offload init failed"
2286
2287 struct mlx5_tc_ct_priv *
mlx5_tc_ct_init(struct mlx5e_priv * priv,struct mlx5_fs_chains * chains,struct mod_hdr_tbl * mod_hdr,enum mlx5_flow_namespace_type ns_type,struct mlx5e_post_act * post_act)2288 mlx5_tc_ct_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains,
2289 struct mod_hdr_tbl *mod_hdr,
2290 enum mlx5_flow_namespace_type ns_type,
2291 struct mlx5e_post_act *post_act)
2292 {
2293 u8 mapping_id[MLX5_SW_IMAGE_GUID_MAX_BYTES];
2294 struct mlx5_tc_ct_priv *ct_priv;
2295 struct mlx5_core_dev *dev;
2296 u8 id_len;
2297 int err;
2298
2299 dev = priv->mdev;
2300 err = mlx5_tc_ct_init_check_support(priv, ns_type, post_act);
2301 if (err)
2302 goto err_support;
2303
2304 ct_priv = kzalloc_obj(*ct_priv);
2305 if (!ct_priv)
2306 goto err_alloc;
2307
2308 mlx5_query_nic_sw_system_image_guid(dev, mapping_id, &id_len);
2309
2310 ct_priv->zone_mapping = mapping_create_for_id(mapping_id, id_len,
2311 MAPPING_TYPE_ZONE,
2312 sizeof(u16), 0, true);
2313 if (IS_ERR(ct_priv->zone_mapping)) {
2314 err = PTR_ERR(ct_priv->zone_mapping);
2315 goto err_mapping_zone;
2316 }
2317
2318 ct_priv->labels_mapping = mapping_create_for_id(mapping_id, id_len,
2319 MAPPING_TYPE_LABELS,
2320 sizeof(u32) * 4, 0, true);
2321 if (IS_ERR(ct_priv->labels_mapping)) {
2322 err = PTR_ERR(ct_priv->labels_mapping);
2323 goto err_mapping_labels;
2324 }
2325
2326 spin_lock_init(&ct_priv->ht_lock);
2327 ct_priv->priv = priv;
2328 ct_priv->ns_type = ns_type;
2329 ct_priv->chains = chains;
2330 ct_priv->netdev = priv->netdev;
2331 ct_priv->dev = priv->mdev;
2332 ct_priv->mod_hdr_tbl = mod_hdr;
2333 ct_priv->ct = mlx5_chains_create_global_table(chains);
2334 if (IS_ERR(ct_priv->ct)) {
2335 err = PTR_ERR(ct_priv->ct);
2336 mlx5_core_warn(dev,
2337 "%s, failed to create ct table err: %d\n",
2338 INIT_ERR_PREFIX, err);
2339 goto err_ct_tbl;
2340 }
2341
2342 ct_priv->ct_nat = mlx5_chains_create_global_table(chains);
2343 if (IS_ERR(ct_priv->ct_nat)) {
2344 err = PTR_ERR(ct_priv->ct_nat);
2345 mlx5_core_warn(dev,
2346 "%s, failed to create ct nat table err: %d\n",
2347 INIT_ERR_PREFIX, err);
2348 goto err_ct_nat_tbl;
2349 }
2350
2351 err = tc_ct_add_ct_table_miss_rule(ct_priv->ct_nat, ct_priv->ct,
2352 &ct_priv->ct_nat_miss_group,
2353 &ct_priv->ct_nat_miss_rule);
2354 if (err)
2355 goto err_ct_zone_ht;
2356
2357 ct_priv->post_act = post_act;
2358 mutex_init(&ct_priv->control_lock);
2359 if (rhashtable_init(&ct_priv->zone_ht, &zone_params))
2360 goto err_ct_zone_ht;
2361 if (rhashtable_init(&ct_priv->ct_tuples_ht, &tuples_ht_params))
2362 goto err_ct_tuples_ht;
2363 if (rhashtable_init(&ct_priv->ct_tuples_nat_ht, &tuples_nat_ht_params))
2364 goto err_ct_tuples_nat_ht;
2365
2366 ct_priv->wq = alloc_ordered_workqueue("mlx5e_ct_priv_wq", 0);
2367 if (!ct_priv->wq) {
2368 err = -ENOMEM;
2369 goto err_wq;
2370 }
2371
2372 err = mlx5_tc_ct_fs_init(ct_priv);
2373 if (err)
2374 goto err_init_fs;
2375
2376 mlx5_ct_tc_create_dbgfs(ct_priv);
2377 return ct_priv;
2378
2379 err_init_fs:
2380 destroy_workqueue(ct_priv->wq);
2381 err_wq:
2382 rhashtable_destroy(&ct_priv->ct_tuples_nat_ht);
2383 err_ct_tuples_nat_ht:
2384 rhashtable_destroy(&ct_priv->ct_tuples_ht);
2385 err_ct_tuples_ht:
2386 rhashtable_destroy(&ct_priv->zone_ht);
2387 err_ct_zone_ht:
2388 mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2389 err_ct_nat_tbl:
2390 mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2391 err_ct_tbl:
2392 mapping_destroy(ct_priv->labels_mapping);
2393 err_mapping_labels:
2394 mapping_destroy(ct_priv->zone_mapping);
2395 err_mapping_zone:
2396 kfree(ct_priv);
2397 err_alloc:
2398 err_support:
2399
2400 return NULL;
2401 }
2402
2403 void
mlx5_tc_ct_clean(struct mlx5_tc_ct_priv * ct_priv)2404 mlx5_tc_ct_clean(struct mlx5_tc_ct_priv *ct_priv)
2405 {
2406 struct mlx5_fs_chains *chains;
2407
2408 if (!ct_priv)
2409 return;
2410
2411 destroy_workqueue(ct_priv->wq);
2412 mlx5_ct_tc_remove_dbgfs(ct_priv);
2413 chains = ct_priv->chains;
2414
2415 ct_priv->fs_ops->destroy(ct_priv->fs);
2416 kfree(ct_priv->fs);
2417
2418 tc_ct_del_ct_table_miss_rule(ct_priv->ct_nat_miss_group, ct_priv->ct_nat_miss_rule);
2419 mlx5_chains_destroy_global_table(chains, ct_priv->ct_nat);
2420 mlx5_chains_destroy_global_table(chains, ct_priv->ct);
2421 mapping_destroy(ct_priv->zone_mapping);
2422 mapping_destroy(ct_priv->labels_mapping);
2423
2424 rhashtable_destroy(&ct_priv->ct_tuples_ht);
2425 rhashtable_destroy(&ct_priv->ct_tuples_nat_ht);
2426 rhashtable_destroy(&ct_priv->zone_ht);
2427 mutex_destroy(&ct_priv->control_lock);
2428 kfree(ct_priv);
2429 }
2430
2431 bool
mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv * ct_priv,struct sk_buff * skb,u8 zone_restore_id)2432 mlx5e_tc_ct_restore_flow(struct mlx5_tc_ct_priv *ct_priv,
2433 struct sk_buff *skb, u8 zone_restore_id)
2434 {
2435 struct mlx5_ct_tuple tuple = {};
2436 struct mlx5_ct_entry *entry;
2437 u16 zone;
2438
2439 if (!ct_priv || !zone_restore_id)
2440 return true;
2441
2442 if (mapping_find(ct_priv->zone_mapping, zone_restore_id, &zone))
2443 goto out_inc_drop;
2444
2445 if (!mlx5_tc_ct_skb_to_tuple(skb, &tuple, zone))
2446 goto out_inc_drop;
2447
2448 spin_lock(&ct_priv->ht_lock);
2449
2450 entry = mlx5_tc_ct_entry_get(ct_priv, &tuple);
2451 if (!entry) {
2452 spin_unlock(&ct_priv->ht_lock);
2453 goto out_inc_drop;
2454 }
2455
2456 if (IS_ERR(entry)) {
2457 spin_unlock(&ct_priv->ht_lock);
2458 goto out_inc_drop;
2459 }
2460 spin_unlock(&ct_priv->ht_lock);
2461
2462 tcf_ct_flow_table_restore_skb(skb, entry->restore_cookie);
2463 __mlx5_tc_ct_entry_put(entry);
2464
2465 return true;
2466
2467 out_inc_drop:
2468 atomic_inc(&ct_priv->debugfs.stats.rx_dropped);
2469 return false;
2470 }
2471
mlx5e_tc_ct_valid_used_dissector_keys(const u64 used_keys)2472 static bool mlx5e_tc_ct_valid_used_dissector_keys(const u64 used_keys)
2473 {
2474 #define DISS_BIT(name) BIT_ULL(FLOW_DISSECTOR_KEY_ ## name)
2475 const u64 basic_keys = DISS_BIT(BASIC) | DISS_BIT(CONTROL) |
2476 DISS_BIT(META);
2477 const u64 ipv4_tcp = basic_keys | DISS_BIT(IPV4_ADDRS) |
2478 DISS_BIT(PORTS) | DISS_BIT(TCP);
2479 const u64 ipv6_tcp = basic_keys | DISS_BIT(IPV6_ADDRS) |
2480 DISS_BIT(PORTS) | DISS_BIT(TCP);
2481 const u64 ipv4_udp = basic_keys | DISS_BIT(IPV4_ADDRS) |
2482 DISS_BIT(PORTS);
2483 const u64 ipv6_udp = basic_keys | DISS_BIT(IPV6_ADDRS) |
2484 DISS_BIT(PORTS);
2485 const u64 ipv4_gre = basic_keys | DISS_BIT(IPV4_ADDRS);
2486 const u64 ipv6_gre = basic_keys | DISS_BIT(IPV6_ADDRS);
2487
2488 return (used_keys == ipv4_tcp || used_keys == ipv4_udp || used_keys == ipv6_tcp ||
2489 used_keys == ipv6_udp || used_keys == ipv4_gre || used_keys == ipv6_gre);
2490 }
2491
mlx5e_tc_ct_is_valid_flow_rule(const struct net_device * dev,struct flow_rule * flow_rule)2492 bool mlx5e_tc_ct_is_valid_flow_rule(const struct net_device *dev, struct flow_rule *flow_rule)
2493 {
2494 struct flow_match_ipv4_addrs ipv4_addrs;
2495 struct flow_match_ipv6_addrs ipv6_addrs;
2496 struct flow_match_control control;
2497 struct flow_match_basic basic;
2498 struct flow_match_ports ports;
2499 struct flow_match_tcp tcp;
2500
2501 if (!mlx5e_tc_ct_valid_used_dissector_keys(flow_rule->match.dissector->used_keys)) {
2502 netdev_dbg(dev, "ct_debug: rule uses unexpected dissectors (0x%016llx)",
2503 flow_rule->match.dissector->used_keys);
2504 return false;
2505 }
2506
2507 flow_rule_match_basic(flow_rule, &basic);
2508 flow_rule_match_control(flow_rule, &control);
2509 flow_rule_match_ipv4_addrs(flow_rule, &ipv4_addrs);
2510 flow_rule_match_ipv6_addrs(flow_rule, &ipv6_addrs);
2511 if (basic.key->ip_proto != IPPROTO_GRE)
2512 flow_rule_match_ports(flow_rule, &ports);
2513 if (basic.key->ip_proto == IPPROTO_TCP)
2514 flow_rule_match_tcp(flow_rule, &tcp);
2515
2516 if (basic.mask->n_proto != htons(0xFFFF) ||
2517 (basic.key->n_proto != htons(ETH_P_IP) && basic.key->n_proto != htons(ETH_P_IPV6)) ||
2518 basic.mask->ip_proto != 0xFF ||
2519 (basic.key->ip_proto != IPPROTO_UDP && basic.key->ip_proto != IPPROTO_TCP &&
2520 basic.key->ip_proto != IPPROTO_GRE)) {
2521 netdev_dbg(dev, "ct_debug: rule uses unexpected basic match (n_proto 0x%04x/0x%04x, ip_proto 0x%02x/0x%02x)",
2522 ntohs(basic.key->n_proto), ntohs(basic.mask->n_proto),
2523 basic.key->ip_proto, basic.mask->ip_proto);
2524 return false;
2525 }
2526
2527 if (basic.key->ip_proto != IPPROTO_GRE &&
2528 (ports.mask->src != htons(0xFFFF) || ports.mask->dst != htons(0xFFFF))) {
2529 netdev_dbg(dev, "ct_debug: rule uses ports match (src 0x%04x, dst 0x%04x)",
2530 ports.mask->src, ports.mask->dst);
2531 return false;
2532 }
2533
2534 if (basic.key->ip_proto == IPPROTO_TCP && tcp.mask->flags != MLX5_CT_TCP_FLAGS_MASK) {
2535 netdev_dbg(dev, "ct_debug: rule uses unexpected tcp match (flags 0x%02x)",
2536 tcp.mask->flags);
2537 return false;
2538 }
2539
2540 return true;
2541 }
2542