1 /*-
2 * Copyright (c) 2020-2021, Mellanox Technologies, Ltd.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_inet.h"
27 #include "opt_inet6.h"
28
29 #include <dev/mlx5/mlx5_en/en.h>
30
31 #include <dev/mlx5/mlx5_core/fs_core.h>
32 #include <dev/mlx5/mlx5_core/fs_tcp.h>
33 #include <dev/mlx5/device.h>
34
35 #include <sys/domain.h>
36
37 #include <netinet/in_pcb.h>
38
39 #if defined(INET) || defined(INET6)
40 static void
accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec * spec,struct inpcb * inp)41 accel_fs_tcp_set_ipv4_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
42 {
43 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
44 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
45 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
46 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4);
47 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
48 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4),
49 &inp->inp_faddr, 4);
50 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
51 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
52 &inp->inp_laddr, 4);
53 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
54 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4);
55 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
56 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
57 }
58 #endif
59
60 #ifdef INET6
61 static void
accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec * spec,struct inpcb * inp)62 accel_fs_tcp_set_ipv6_flow(struct mlx5_flow_spec *spec, struct inpcb *inp)
63 {
64 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
65 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_TCP);
66 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
67 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6);
68 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
69 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
70 &inp->in6p_faddr, 16);
71 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value,
72 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
73 &inp->in6p_laddr, 16);
74 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
75 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6),
76 0xff, 16);
77 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
78 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
79 0xff, 16);
80 }
81 #endif
82
83 void
mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle * rule)84 mlx5e_accel_fs_del_inpcb(struct mlx5_flow_handle *rule)
85 {
86 mlx5_del_flow_rules(&rule);
87 }
88
89 struct mlx5_flow_handle *
mlx5e_accel_fs_add_inpcb(struct mlx5e_priv * priv,struct inpcb * inp,uint32_t tirn,uint32_t flow_tag,uint16_t vlan_id)90 mlx5e_accel_fs_add_inpcb(struct mlx5e_priv *priv,
91 struct inpcb *inp, uint32_t tirn, uint32_t flow_tag,
92 uint16_t vlan_id)
93 {
94 struct mlx5_flow_destination dest = {};
95 struct mlx5e_flow_table *ft = NULL;
96 #if defined(INET) || defined(INET6)
97 struct mlx5e_accel_fs_tcp *fs_tcp = &priv->fts.accel_tcp;
98 #endif
99 struct mlx5_flow_handle *flow;
100 struct mlx5_flow_spec *spec;
101 struct mlx5_flow_act flow_act = {};
102
103 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
104 if (!spec)
105 return (ERR_PTR(-ENOMEM));
106
107 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
108 spec->flow_context.flags = FLOW_CONTEXT_HAS_TAG;
109 spec->flow_context.flow_tag = flow_tag;
110
111 INP_RLOCK(inp);
112 /* Set VLAN ID to match, if any. */
113 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag);
114 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.first_vid);
115 if (vlan_id != MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN) {
116 MLX5_SET_TO_ONES(fte_match_param, spec->match_value, outer_headers.cvlan_tag);
117 MLX5_SET(fte_match_param, spec->match_value, outer_headers.first_vid, vlan_id);
118 }
119
120 /* Set TCP port numbers. */
121 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
122 outer_headers.tcp_dport);
123 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
124 outer_headers.tcp_sport);
125 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_dport,
126 ntohs(inp->inp_lport));
127 MLX5_SET(fte_match_param, spec->match_value, outer_headers.tcp_sport,
128 ntohs(inp->inp_fport));
129
130 /* Set IP addresses. */
131 switch (INP_SOCKAF(inp->inp_socket)) {
132 #ifdef INET
133 case AF_INET:
134 accel_fs_tcp_set_ipv4_flow(spec, inp);
135 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
136 break;
137 #endif
138 #ifdef INET6
139 case AF_INET6:
140 if ((inp->inp_flags & IN6P_IPV6_V6ONLY) == 0 &&
141 IN6_IS_ADDR_V4MAPPED(&inp->in6p_faddr)) {
142 accel_fs_tcp_set_ipv4_flow(spec, inp);
143 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV4_TCP];
144 } else {
145 accel_fs_tcp_set_ipv6_flow(spec, inp);
146 ft = &fs_tcp->tables[MLX5E_ACCEL_FS_IPV6_TCP];
147 }
148 break;
149 #endif
150 default:
151 break;
152 }
153 INP_RUNLOCK(inp);
154
155 if (!ft) {
156 flow = ERR_PTR(-EINVAL);
157 goto out;
158 }
159
160 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
161 dest.tir_num = tirn;
162 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
163
164 flow = mlx5_add_flow_rules(ft->t, spec, &flow_act, &dest, 1);
165 out:
166 kvfree(spec);
167 return (flow);
168 }
169
170 static int
accel_fs_tcp_add_default_rule(struct mlx5e_priv * priv,int type)171 accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, int type)
172 {
173 static struct mlx5_flow_spec spec = {};
174 struct mlx5_flow_destination dest = {};
175 struct mlx5e_accel_fs_tcp *fs_tcp;
176 struct mlx5_flow_handle *rule;
177 struct mlx5_flow_act flow_act = {
178 .action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
179 };
180
181 fs_tcp = &priv->fts.accel_tcp;
182
183 spec.flow_context.flags = FLOW_CONTEXT_HAS_TAG;
184 spec.flow_context.flow_tag = MLX5_FS_DEFAULT_FLOW_TAG;
185 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
186
187 /*
188 * Traffic not matched by flow table rules should be forwarded
189 * to the next flow table in order to not be dropped by the
190 * default action. Refer to the diagram in
191 * mlx5_en_flow_table.c for more information about the order
192 * of flow tables.
193 */
194 dest.ft = (type == MLX5E_ACCEL_FS_TCP_NUM_TYPES - 1) ?
195 ((priv->fts.ipsec_ft) ? priv->fts.ipsec_ft : priv->fts.vlan.t) :
196 fs_tcp->tables[type + 1].t;
197
198 rule = mlx5_add_flow_rules(fs_tcp->tables[type].t, &spec, &flow_act,
199 &dest, 1);
200 if (IS_ERR(rule))
201 return (PTR_ERR(rule));
202
203 fs_tcp->default_rules[type] = rule;
204 return (0);
205 }
206
207 #define MLX5E_ACCEL_FS_TCP_NUM_GROUPS (2)
208 #define MLX5E_ACCEL_FS_TCP_GROUP1_SIZE (BIT(16) - 1)
209 #define MLX5E_ACCEL_FS_TCP_GROUP2_SIZE (BIT(0))
210 #define MLX5E_ACCEL_FS_TCP_TABLE_SIZE (MLX5E_ACCEL_FS_TCP_GROUP1_SIZE +\
211 MLX5E_ACCEL_FS_TCP_GROUP2_SIZE)
212 static int
accel_fs_tcp_create_groups(struct mlx5e_flow_table * ft,int type)213 accel_fs_tcp_create_groups(struct mlx5e_flow_table *ft, int type)
214 {
215 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
216 void *outer_headers_c;
217 int ix = 0;
218 u32 *in;
219 int err;
220 u8 *mc;
221
222 ft->g = kcalloc(MLX5E_ACCEL_FS_TCP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
223 in = kvzalloc(inlen, GFP_KERNEL);
224 if (!in || !ft->g) {
225 kfree(ft->g);
226 kvfree(in);
227 return (-ENOMEM);
228 }
229
230 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
231 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
232 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
233 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
234 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, cvlan_tag);
235 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, first_vid);
236
237 switch (type) {
238 case MLX5E_ACCEL_FS_IPV4_TCP:
239 case MLX5E_ACCEL_FS_IPV6_TCP:
240 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport);
241 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport);
242 break;
243 default:
244 err = -EINVAL;
245 goto out;
246 }
247
248 switch (type) {
249 case MLX5E_ACCEL_FS_IPV4_TCP:
250 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
251 src_ipv4_src_ipv6.ipv4_layout.ipv4);
252 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c,
253 dst_ipv4_dst_ipv6.ipv4_layout.ipv4);
254 break;
255 case MLX5E_ACCEL_FS_IPV6_TCP:
256 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
257 src_ipv4_src_ipv6.ipv6_layout.ipv6),
258 0xff, 16);
259 memset(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
260 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
261 0xff, 16);
262 break;
263 default:
264 err = -EINVAL;
265 goto out;
266 }
267
268 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
269 MLX5_SET_CFG(in, start_flow_index, ix);
270 ix += MLX5E_ACCEL_FS_TCP_GROUP1_SIZE;
271 MLX5_SET_CFG(in, end_flow_index, ix - 1);
272 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
273 if (IS_ERR(ft->g[ft->num_groups]))
274 goto err;
275 ft->num_groups++;
276
277 /* Default Flow Group */
278 memset(in, 0, inlen);
279 MLX5_SET_CFG(in, start_flow_index, ix);
280 ix += MLX5E_ACCEL_FS_TCP_GROUP2_SIZE;
281 MLX5_SET_CFG(in, end_flow_index, ix - 1);
282 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
283 if (IS_ERR(ft->g[ft->num_groups]))
284 goto err;
285 ft->num_groups++;
286
287 kvfree(in);
288 return (0);
289
290 err:
291 err = PTR_ERR(ft->g[ft->num_groups]);
292 ft->g[ft->num_groups] = NULL;
293 out:
294 kvfree(in);
295
296 return (err);
297 }
298
299 static void
accel_fs_tcp_destroy_groups(struct mlx5e_flow_table * ft)300 accel_fs_tcp_destroy_groups(struct mlx5e_flow_table *ft)
301 {
302 int i;
303
304 for (i = ft->num_groups - 1; i >= 0; i--) {
305 if (!IS_ERR_OR_NULL(ft->g[i]))
306 mlx5_destroy_flow_group(ft->g[i]);
307 ft->g[i] = NULL;
308 }
309 ft->num_groups = 0;
310 }
311
312 static int
accel_fs_tcp_create_table(struct mlx5e_priv * priv,int type)313 accel_fs_tcp_create_table(struct mlx5e_priv *priv, int type)
314 {
315 struct mlx5e_flow_table *ft = &priv->fts.accel_tcp.tables[type];
316 struct mlx5_flow_table_attr ft_attr = {};
317 int err;
318
319 ft->num_groups = 0;
320 ft_attr.max_fte = MLX5E_ACCEL_FS_TCP_TABLE_SIZE;
321 ft_attr.level = type;
322 ft->t = mlx5_create_flow_table(priv->fts.accel_tcp.ns, &ft_attr);
323 if (IS_ERR(ft->t)) {
324 err = PTR_ERR(ft->t);
325 ft->t = NULL;
326 return (err);
327 }
328
329 err = accel_fs_tcp_create_groups(ft, type);
330 if (err)
331 goto err_destroy_flow_table;
332
333 return (0);
334
335 err_destroy_flow_table:
336 mlx5_destroy_flow_table(ft->t);
337 ft->t = NULL;
338 return (err);
339 }
340
341 static void
accel_fs_tcp_destroy_table(struct mlx5e_priv * priv,int i)342 accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i)
343 {
344 struct mlx5e_accel_fs_tcp *fs_tcp;
345 struct mlx5e_flow_table *ft;
346
347 fs_tcp = &priv->fts.accel_tcp;
348 ft = fs_tcp->tables + i;
349
350 accel_fs_tcp_destroy_groups(ft);
351 kfree(ft->g);
352 ft->g = NULL;
353 mlx5_destroy_flow_table(ft->t);
354 ft->t = NULL;
355 }
356
357 void
mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv * priv)358 mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv)
359 {
360 int i;
361
362 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
363 return;
364
365 for (i = 0; i < MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
366 mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
367 accel_fs_tcp_destroy_table(priv, i);
368 }
369 }
370
371 int
mlx5e_accel_fs_tcp_create(struct mlx5e_priv * priv)372 mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv)
373 {
374 int i, err;
375
376 if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version))
377 return (0);
378
379 /* Setup namespace pointer. */
380 priv->fts.accel_tcp.ns = mlx5_get_flow_namespace(
381 priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS);
382
383 /*
384 * Create flow tables first, because the priority level is
385 * assigned at allocation time.
386 */
387 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
388 err = accel_fs_tcp_create_table(priv, i);
389 if (err)
390 goto err_destroy_tables;
391 }
392
393 /* Create default rules last. */
394 for (i = 0; i != MLX5E_ACCEL_FS_TCP_NUM_TYPES; i++) {
395 err = accel_fs_tcp_add_default_rule(priv, i);
396 if (err)
397 goto err_destroy_rules;
398 }
399 return (0);
400
401 err_destroy_rules:
402 while (i--)
403 mlx5_del_flow_rules(&priv->fts.accel_tcp.default_rules[i]);
404 i = MLX5E_ACCEL_FS_TCP_NUM_TYPES;
405
406 err_destroy_tables:
407 while (i--)
408 accel_fs_tcp_destroy_table(priv, i);
409 return (err);
410 }
411