1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */
3 #include <linux/mlx5/device.h>
4 #include <net/psp.h>
5 #include <linux/psp.h>
6 #include "mlx5_core.h"
7 #include "psp.h"
8 #include "lib/crypto.h"
9 #include "en_accel/psp.h"
10 #include "fs_core.h"
11
12 enum accel_fs_psp_type {
13 ACCEL_FS_PSP4,
14 ACCEL_FS_PSP6,
15 ACCEL_FS_PSP_NUM_TYPES,
16 };
17
18 enum accel_psp_syndrome {
19 PSP_OK = 0,
20 PSP_ICV_FAIL,
21 PSP_BAD_TRAILER,
22 };
23
24 struct mlx5e_psp_tx_table {
25 struct mlx5_flow_namespace *ns;
26 struct mlx5_flow_table *ft;
27 struct mlx5_flow_group *fg;
28 struct mlx5_flow_handle *rule;
29 };
30
31 struct mlx5e_psp_rx_check_table {
32 struct mlx5_flow_table *ft;
33 struct mlx5_flow_group *drop_group;
34 struct mlx5_flow_handle *rule;
35 struct mlx5_flow_handle *auth_fail_rule;
36 struct mlx5_flow_handle *err_rule;
37 struct mlx5_flow_handle *bad_rule;
38 };
39
40 struct mlx5e_psp_rx_decrypt_table {
41 struct mlx5_flow_table *ft;
42 struct mlx5_flow_group *miss_group;
43 struct mlx5_flow_handle *miss_rule;
44 struct mlx5_modify_hdr *rx_modify_hdr;
45 struct mlx5_flow_handle *rule;
46 };
47
48 struct mlx5e_psp_rx_table {
49 struct mlx5_flow_table *ft;
50 struct mlx5_flow_group *miss_group;
51 struct mlx5_flow_handle *miss_rule;
52 struct mlx5_flow_handle *udp_rules[ACCEL_FS_PSP_NUM_TYPES];
53 };
54
55 struct mlx5e_psp_fs {
56 struct mlx5_core_dev *mdev;
57 struct mlx5_fc *tx_counter;
58 struct mlx5e_psp_tx_table tx;
59
60 /* Rx */
61 struct mlx5e_flow_steering *fs;
62 struct mlx5_fc *rx_counter;
63 struct mlx5_fc *rx_auth_fail_counter;
64 struct mlx5_fc *rx_err_counter;
65 struct mlx5_fc *rx_bad_counter;
66
67 struct mlx5e_psp_rx_decrypt_table decrypt[ACCEL_FS_PSP_NUM_TYPES];
68 struct mlx5e_psp_rx_check_table check;
69 struct mlx5e_psp_rx_table rx;
70 };
71
72 /* PSP RX flow steering */
fs_psp2tt(enum accel_fs_psp_type i)73 static enum mlx5_traffic_types fs_psp2tt(enum accel_fs_psp_type i)
74 {
75 if (i == ACCEL_FS_PSP4)
76 return MLX5_TT_IPV4_UDP;
77
78 return MLX5_TT_IPV6_UDP;
79 }
80
accel_psp_fs_create_ft(struct mlx5e_psp_fs * fs,struct mlx5_flow_table_attr * ft_attr,struct mlx5_flow_table ** ft)81 static int accel_psp_fs_create_ft(struct mlx5e_psp_fs *fs,
82 struct mlx5_flow_table_attr *ft_attr,
83 struct mlx5_flow_table **ft)
84 {
85 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(fs->fs, false);
86 int err = 0;
87
88 *ft = mlx5_create_auto_grouped_flow_table(ns, ft_attr);
89 if (IS_ERR(*ft)) {
90 err = PTR_ERR(*ft);
91 *ft = NULL;
92 }
93
94 return err;
95 }
96
accel_psp_fs_destroy_ft(struct mlx5_flow_table ** table)97 static void accel_psp_fs_destroy_ft(struct mlx5_flow_table **table)
98 {
99 if (*table) {
100 mlx5_destroy_flow_table(*table);
101 *table = NULL;
102 }
103 }
104
accel_psp_fs_del_flow_rule(struct mlx5_flow_handle ** rule)105 static void accel_psp_fs_del_flow_rule(struct mlx5_flow_handle **rule)
106 {
107 if (*rule) {
108 mlx5_del_flow_rules(*rule);
109 *rule = NULL;
110 }
111 }
112
accel_psp_fs_create_miss_group(struct mlx5_flow_table * ft,struct mlx5_flow_group ** group)113 static int accel_psp_fs_create_miss_group(struct mlx5_flow_table *ft,
114 struct mlx5_flow_group **group)
115 {
116 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
117 u32 *in = kvzalloc(inlen, GFP_KERNEL);
118 int err = 0;
119
120 if (!in)
121 return -ENOMEM;
122
123 MLX5_SET(create_flow_group_in, in, start_flow_index, ft->max_fte - 1);
124 MLX5_SET(create_flow_group_in, in, end_flow_index, ft->max_fte - 1);
125 *group = mlx5_create_flow_group(ft, in);
126 if (IS_ERR(*group)) {
127 err = PTR_ERR(*group);
128 *group = NULL;
129 }
130 kvfree(in);
131
132 return err;
133 }
134
accel_psp_fs_destroy_flow_group(struct mlx5_flow_group ** group)135 static void accel_psp_fs_destroy_flow_group(struct mlx5_flow_group **group)
136 {
137 if (*group) {
138 mlx5_destroy_flow_group(*group);
139 *group = NULL;
140 }
141 }
142
accel_psp_fs_create_counter(struct mlx5_core_dev * dev,struct mlx5_fc ** counter)143 static int accel_psp_fs_create_counter(struct mlx5_core_dev *dev,
144 struct mlx5_fc **counter)
145 {
146 *counter = mlx5_fc_create(dev, false);
147 if (IS_ERR(*counter)) {
148 int err = PTR_ERR(*counter);
149
150 *counter = NULL;
151 return err;
152 }
153
154 return 0;
155 }
156
accel_psp_fs_destroy_counter(struct mlx5_core_dev * dev,struct mlx5_fc ** counter)157 static void accel_psp_fs_destroy_counter(struct mlx5_core_dev *dev,
158 struct mlx5_fc **counter)
159 {
160 if (*counter) {
161 mlx5_fc_destroy(dev, *counter);
162 *counter = NULL;
163 }
164 }
165
accel_psp_fs_rx_ft_destroy(struct mlx5e_psp_rx_table * rx)166 static void accel_psp_fs_rx_ft_destroy(struct mlx5e_psp_rx_table *rx)
167 {
168 int i;
169
170 for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++)
171 accel_psp_fs_del_flow_rule(&rx->udp_rules[i]);
172 accel_psp_fs_del_flow_rule(&rx->miss_rule);
173 accel_psp_fs_destroy_flow_group(&rx->miss_group);
174 accel_psp_fs_destroy_ft(&rx->ft);
175 }
176
accel_psp_fs_rx_ft_create(struct mlx5e_psp_fs * fs,struct mlx5e_psp_rx_table * rx)177 static int accel_psp_fs_rx_ft_create(struct mlx5e_psp_fs *fs,
178 struct mlx5e_psp_rx_table *rx)
179 {
180 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false);
181 struct mlx5_flow_destination dest[2] = {};
182 struct mlx5_flow_table_attr ft_attr = {};
183 struct mlx5_core_dev *mdev = fs->mdev;
184 MLX5_DECLARE_FLOW_ACT(flow_act);
185 struct mlx5_flow_handle *rule;
186 struct mlx5_flow_spec *spec;
187 int i, err = 0;
188
189 spec = kzalloc_obj(*spec);
190 if (!spec)
191 return -ENOMEM;
192
193 ft_attr.max_fte = 1 + ACCEL_FS_PSP_NUM_TYPES;
194 ft_attr.level = MLX5E_ACCEL_FS_PSP_RX_FT_LEVEL;
195 ft_attr.prio = MLX5E_NIC_PRIO;
196 ft_attr.autogroup.num_reserved_entries = 1;
197 err = accel_psp_fs_create_ft(fs, &ft_attr, &rx->ft);
198 if (err) {
199 mlx5_core_err(mdev, "fail to create psp rx ft err=%d\n", err);
200 goto out_err;
201 }
202
203 err = accel_psp_fs_create_miss_group(rx->ft, &rx->miss_group);
204 if (err) {
205 mlx5_core_err(mdev, "fail to create psp rx miss_group err=%d\n",
206 err);
207 goto out_err;
208 }
209
210 /* Add miss rule */
211 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
212 MLX5_FLOW_CONTEXT_ACTION_COUNT;
213 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL;
214 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
215 dest[0].ft = mlx5_get_ttc_flow_table(ttc);
216 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
217 dest[1].counter = fs->rx_counter;
218 rule = mlx5_add_flow_rules(rx->ft, NULL, &flow_act, dest, 2);
219 if (IS_ERR(rule)) {
220 err = PTR_ERR(rule);
221 mlx5_core_err(mdev, "fail to create psp rx rule, err=%d\n",
222 err);
223 goto out_err;
224 }
225 rx->miss_rule = rule;
226
227 /* Add UDP v4/v6 rules */
228 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
229 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria,
230 outer_headers.ip_version);
231 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria,
232 ip_protocol);
233 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol,
234 IPPROTO_UDP);
235 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
236 MLX5_FLOW_CONTEXT_ACTION_COUNT;
237 flow_act.flags = 0;
238 for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) {
239 int version = i == ACCEL_FS_PSP4 ? 4 : 6;
240
241 MLX5_SET(fte_match_param, spec->match_value,
242 outer_headers.ip_version, version);
243 dest[0] = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(i));
244 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
245 dest[1].counter = fs->rx_counter;
246 rule = mlx5_add_flow_rules(rx->ft, spec, &flow_act, dest,
247 2);
248 if (IS_ERR(rule)) {
249 err = PTR_ERR(rule);
250 mlx5_core_err(mdev,
251 "fail to create psp rx UDP%d rule err=%d\n",
252 version, err);
253 goto out_err;
254 }
255 rx->udp_rules[i] = rule;
256 }
257 goto out_spec;
258
259 out_err:
260 accel_psp_fs_rx_ft_destroy(rx);
261 out_spec:
262 kvfree(spec);
263 return err;
264 }
265
266 static
accel_psp_fs_rx_check_ft_destroy(struct mlx5e_psp_rx_check_table * check)267 void accel_psp_fs_rx_check_ft_destroy(struct mlx5e_psp_rx_check_table *check)
268 {
269 accel_psp_fs_del_flow_rule(&check->bad_rule);
270 accel_psp_fs_del_flow_rule(&check->err_rule);
271 accel_psp_fs_del_flow_rule(&check->auth_fail_rule);
272 accel_psp_fs_del_flow_rule(&check->rule);
273 accel_psp_fs_destroy_flow_group(&check->drop_group);
274 accel_psp_fs_destroy_ft(&check->ft);
275 }
276
accel_psp_setup_syndrome_match(struct mlx5_flow_spec * spec,enum accel_psp_syndrome syndrome)277 static void accel_psp_setup_syndrome_match(struct mlx5_flow_spec *spec,
278 enum accel_psp_syndrome syndrome)
279 {
280 void *misc_params_2;
281
282 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
283 misc_params_2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
284 MLX5_SET_TO_ONES(fte_match_set_misc2, misc_params_2, psp_syndrome);
285 misc_params_2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
286 MLX5_SET(fte_match_set_misc2, misc_params_2, psp_syndrome, syndrome);
287 }
288
accel_psp_add_drop_rule(struct mlx5_flow_table * ft,struct mlx5_flow_spec * spec,struct mlx5_fc * counter,struct mlx5_flow_handle ** rule)289 static int accel_psp_add_drop_rule(struct mlx5_flow_table *ft,
290 struct mlx5_flow_spec *spec,
291 struct mlx5_fc *counter,
292 struct mlx5_flow_handle **rule)
293 {
294 struct mlx5_flow_destination dest = {};
295 struct mlx5_flow_act flow_act = {};
296 int err = 0;
297
298 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP |
299 MLX5_FLOW_CONTEXT_ACTION_COUNT;
300 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
301 dest.counter = counter;
302 *rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
303 if (IS_ERR(*rule)) {
304 err = PTR_ERR(*rule);
305 *rule = NULL;
306 }
307 return err;
308 }
309
310 static
accel_psp_fs_rx_check_ft_create(struct mlx5e_psp_fs * fs,struct mlx5e_psp_rx_check_table * check)311 int accel_psp_fs_rx_check_ft_create(struct mlx5e_psp_fs *fs,
312 struct mlx5e_psp_rx_check_table *check)
313 {
314 struct mlx5_flow_table_attr ft_attr = {};
315 struct mlx5_flow_destination dest = {};
316 struct mlx5_core_dev *mdev = fs->mdev;
317 struct mlx5_flow_act flow_act = {};
318 struct mlx5_flow_handle *fte;
319 struct mlx5_flow_spec *spec;
320 int err = 0;
321
322 spec = kzalloc_obj(*spec);
323 if (!spec)
324 return -ENOMEM;
325
326 ft_attr.max_fte = 4;
327 ft_attr.autogroup.num_reserved_entries = 1;
328 ft_attr.autogroup.max_num_groups = 2;
329 ft_attr.level = MLX5E_ACCEL_FS_PSP_ERR_FT_LEVEL;
330 ft_attr.prio = MLX5E_NIC_PRIO;
331 err = accel_psp_fs_create_ft(fs, &ft_attr, &check->ft);
332 if (err) {
333 mlx5_core_err(fs->mdev,
334 "fail to create psp rx check ft err=%d\n", err);
335 goto out_err;
336 }
337
338 err = accel_psp_fs_create_miss_group(check->ft, &check->drop_group);
339 if (err) {
340 mlx5_core_err(fs->mdev,
341 "fail to create psp rx check drop group err=%d\n",
342 err);
343 goto out_err;
344 }
345
346 accel_psp_setup_syndrome_match(spec, PSP_OK);
347 /* create fte */
348 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
349 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
350 dest.ft = fs->rx.ft;
351 fte = mlx5_add_flow_rules(check->ft, spec, &flow_act, &dest, 1);
352 if (IS_ERR(fte)) {
353 err = PTR_ERR(fte);
354 mlx5_core_err(mdev, "fail to add psp rx check ok rule err=%d\n",
355 err);
356 goto out_err;
357 }
358 check->rule = fte;
359
360 /* add auth fail drop rule */
361 memset(spec, 0, sizeof(*spec));
362 accel_psp_setup_syndrome_match(spec, PSP_ICV_FAIL);
363 err = accel_psp_add_drop_rule(check->ft, spec,
364 fs->rx_auth_fail_counter,
365 &check->auth_fail_rule);
366 if (err) {
367 mlx5_core_err(mdev,
368 "fail to add psp rx check auth fail drop rule err=%d\n",
369 err);
370 goto out_err;
371 }
372
373 /* add framing drop rule */
374 memset(spec, 0, sizeof(*spec));
375 accel_psp_setup_syndrome_match(spec, PSP_BAD_TRAILER);
376 err = accel_psp_add_drop_rule(check->ft, spec, fs->rx_err_counter,
377 &check->err_rule);
378 if (err) {
379 mlx5_core_err(mdev,
380 "fail to add psp rx check framing drop rule err=%d\n",
381 err);
382 goto out_err;
383 }
384
385 /* add misc. errors drop rule */
386 memset(spec, 0, sizeof(*spec));
387 err = accel_psp_add_drop_rule(check->ft, spec, fs->rx_bad_counter,
388 &check->bad_rule);
389 if (err) {
390 mlx5_core_err(mdev,
391 "fail to add psp rx check misc. err drop rule err=%d\n",
392 err);
393 goto out_err;
394 }
395
396 goto out_spec;
397
398 out_err:
399 accel_psp_fs_rx_check_ft_destroy(check);
400 out_spec:
401 kfree(spec);
402 return err;
403 }
404
405 static void
accel_psp_fs_rx_decrypt_ft_destroy(struct mlx5e_psp_fs * fs,struct mlx5e_psp_rx_decrypt_table * decrypt)406 accel_psp_fs_rx_decrypt_ft_destroy(struct mlx5e_psp_fs *fs,
407 struct mlx5e_psp_rx_decrypt_table *decrypt)
408 {
409 accel_psp_fs_del_flow_rule(&decrypt->rule);
410 if (decrypt->rx_modify_hdr) {
411 mlx5_modify_header_dealloc(fs->mdev, decrypt->rx_modify_hdr);
412 decrypt->rx_modify_hdr = NULL;
413 }
414 accel_psp_fs_del_flow_rule(&decrypt->miss_rule);
415 accel_psp_fs_destroy_flow_group(&decrypt->miss_group);
416 accel_psp_fs_destroy_ft(&decrypt->ft);
417 }
418
setup_fte_udp_psp(struct mlx5_flow_spec * spec,u16 udp_port)419 static void setup_fte_udp_psp(struct mlx5_flow_spec *spec, u16 udp_port)
420 {
421 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
422 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, udp_dport, 0xffff);
423 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, udp_dport, udp_port);
424 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol);
425 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, IPPROTO_UDP);
426 }
427
428 static int
accel_psp_fs_rx_decrypt_ft_create(struct mlx5e_psp_fs * fs,struct mlx5e_psp_rx_decrypt_table * decrypt,struct mlx5_flow_destination * default_dest)429 accel_psp_fs_rx_decrypt_ft_create(struct mlx5e_psp_fs *fs,
430 struct mlx5e_psp_rx_decrypt_table *decrypt,
431 struct mlx5_flow_destination *default_dest)
432 {
433 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
434 struct mlx5_modify_hdr *modify_hdr = NULL;
435 struct mlx5_flow_table_attr ft_attr = {};
436 struct mlx5_flow_destination dest = {};
437 struct mlx5_core_dev *mdev = fs->mdev;
438 MLX5_DECLARE_FLOW_ACT(flow_act);
439 struct mlx5_flow_handle *rule;
440 struct mlx5_flow_spec *spec;
441 int err = 0;
442
443 spec = kvzalloc_obj(*spec);
444 if (!spec)
445 return -ENOMEM;
446
447 /* Create FT */
448 ft_attr.max_fte = 2;
449 ft_attr.level = MLX5E_ACCEL_FS_PSP_FT_LEVEL;
450 ft_attr.autogroup.num_reserved_entries = 1;
451 ft_attr.autogroup.max_num_groups = 1;
452 ft_attr.prio = MLX5E_NIC_PRIO;
453 err = accel_psp_fs_create_ft(fs, &ft_attr, &decrypt->ft);
454 if (err) {
455 mlx5_core_err(mdev, "fail to create psp rx decrypt ft err=%d\n",
456 err);
457 goto out_err;
458 }
459
460 /* Create miss_group */
461 err = accel_psp_fs_create_miss_group(decrypt->ft, &decrypt->miss_group);
462 if (err) {
463 mlx5_core_err(mdev,
464 "fail to create psp rx decrypt miss_group err=%d\n",
465 err);
466 goto out_err;
467 }
468
469 /* Create miss rule */
470 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
471 rule = mlx5_add_flow_rules(decrypt->ft, spec, &flow_act, default_dest,
472 1);
473 if (IS_ERR(rule)) {
474 err = PTR_ERR(rule);
475 mlx5_core_err(mdev,
476 "fail to create psp rx decrypt miss_rule err=%d\n",
477 err);
478 goto out_err;
479 }
480 decrypt->miss_rule = rule;
481
482 /* Add PSP RX decrypt rule */
483 setup_fte_udp_psp(spec, PSP_DEFAULT_UDP_PORT);
484 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_PSP;
485 /* Set bit[31, 30] PSP marker */
486 #define MLX5E_PSP_MARKER_BIT (BIT(30) | BIT(31))
487 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET);
488 MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B);
489 MLX5_SET(set_action_in, action, data, MLX5E_PSP_MARKER_BIT);
490 MLX5_SET(set_action_in, action, offset, 0);
491 MLX5_SET(set_action_in, action, length, 32);
492
493 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 1, action);
494 if (IS_ERR(modify_hdr)) {
495 err = PTR_ERR(modify_hdr);
496 mlx5_core_err(mdev, "fail to alloc psp set modify_header_id err=%d\n", err);
497 modify_hdr = NULL;
498 goto out_err;
499 }
500 decrypt->rx_modify_hdr = modify_hdr;
501
502 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
503 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT |
504 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
505 flow_act.modify_hdr = modify_hdr;
506 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
507 dest.ft = fs->check.ft;
508 rule = mlx5_add_flow_rules(decrypt->ft, spec, &flow_act, &dest, 1);
509 if (IS_ERR(rule)) {
510 err = PTR_ERR(rule);
511 mlx5_core_err(mdev, "fail to add psp rx decrypt rule, err=%d\n",
512 err);
513 goto out_err;
514 }
515
516 decrypt->rule = rule;
517 goto out_spec;
518
519 out_err:
520 accel_psp_fs_rx_decrypt_ft_destroy(fs, decrypt);
521 out_spec:
522 kvfree(spec);
523 return err;
524 }
525
accel_psp_fs_rx_destroy(struct mlx5e_psp_fs * fs)526 static void accel_psp_fs_rx_destroy(struct mlx5e_psp_fs *fs)
527 {
528 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false);
529 int i;
530
531 /* disconnect */
532 for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) {
533 mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(i));
534 accel_psp_fs_rx_decrypt_ft_destroy(fs, &fs->decrypt[i]);
535 }
536 accel_psp_fs_rx_check_ft_destroy(&fs->check);
537 accel_psp_fs_rx_ft_destroy(&fs->rx);
538 }
539
accel_psp_fs_rx_create(struct mlx5e_psp_fs * fs,struct netlink_ext_ack * extack)540 static int accel_psp_fs_rx_create(struct mlx5e_psp_fs *fs,
541 struct netlink_ext_ack *extack)
542 {
543 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(fs->fs, false);
544 int i, err;
545
546 err = accel_psp_fs_rx_ft_create(fs, &fs->rx);
547 if (err) {
548 NL_SET_ERR_MSG(extack, "Failed creating RX steering table");
549 return err;
550 }
551
552 err = accel_psp_fs_rx_check_ft_create(fs, &fs->check);
553 if (err) {
554 NL_SET_ERR_MSG(extack,
555 "Failed creating RX check steering table");
556 goto err_ft;
557 }
558
559 for (i = 0; i < ACCEL_FS_PSP_NUM_TYPES; i++) {
560 struct mlx5_flow_destination dest;
561
562 dest = mlx5_ttc_get_default_dest(ttc, fs_psp2tt(i));
563 err = accel_psp_fs_rx_decrypt_ft_create(fs, &fs->decrypt[i],
564 &dest);
565 if (err) {
566 NL_SET_ERR_MSG(extack,
567 "Failed creating RX decrypt steering table");
568 goto err_decrypt_ft;
569 }
570
571 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
572 dest.ft = fs->decrypt[i].ft;
573 mlx5_ttc_fwd_dest(ttc, fs_psp2tt(i), &dest);
574 }
575
576 return 0;
577
578 err_decrypt_ft:
579 while (--i >= 0) {
580 mlx5_ttc_fwd_default_dest(ttc, fs_psp2tt(i));
581 accel_psp_fs_rx_decrypt_ft_destroy(fs, &fs->decrypt[i]);
582 }
583 accel_psp_fs_rx_check_ft_destroy(&fs->check);
584 err_ft:
585 accel_psp_fs_rx_ft_destroy(&fs->rx);
586 return err;
587 }
588
accel_psp_fs_rx_cleanup(struct mlx5e_psp_fs * fs)589 static void accel_psp_fs_rx_cleanup(struct mlx5e_psp_fs *fs)
590 {
591 accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_bad_counter);
592 accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_err_counter);
593 accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_auth_fail_counter);
594 accel_psp_fs_destroy_counter(fs->mdev, &fs->rx_counter);
595 }
596
accel_psp_fs_rx_init(struct mlx5e_psp_fs * fs)597 static int accel_psp_fs_rx_init(struct mlx5e_psp_fs *fs)
598 {
599 struct mlx5_core_dev *mdev = fs->mdev;
600 int err;
601
602 err = accel_psp_fs_create_counter(mdev, &fs->rx_counter);
603 if (err) {
604 mlx5_core_warn(mdev,
605 "fail to create psp rx flow counter err=%d\n",
606 err);
607 goto out_err;
608 }
609
610 err = accel_psp_fs_create_counter(mdev, &fs->rx_auth_fail_counter);
611 if (err) {
612 mlx5_core_warn(mdev,
613 "fail to create psp rx auth fail flow counter err=%d\n",
614 err);
615 goto out_err;
616 }
617
618 err = accel_psp_fs_create_counter(mdev, &fs->rx_err_counter);
619 if (err) {
620 mlx5_core_warn(mdev,
621 "fail to create psp rx error flow counter err=%d\n",
622 err);
623 goto out_err;
624 }
625
626 err = accel_psp_fs_create_counter(mdev, &fs->rx_bad_counter);
627 if (err) {
628 mlx5_core_warn(mdev,
629 "fail to create psp rx bad flow counter err=%d\n",
630 err);
631 goto out_err;
632 }
633
634 return 0;
635
636 out_err:
637 accel_psp_fs_rx_cleanup(fs);
638 return err;
639 }
640
mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv * priv)641 void mlx5_accel_psp_fs_cleanup_rx_tables(struct mlx5e_priv *priv)
642 {
643 if (!priv->psp)
644 return;
645
646 netdev_lock(priv->netdev);
647 accel_psp_fs_rx_destroy(priv->psp->fs);
648 netdev_unlock(priv->netdev);
649 }
650
accel_psp_fs_tx_ft_create(struct mlx5e_psp_fs * fs,struct mlx5e_psp_tx_table * tx)651 static int accel_psp_fs_tx_ft_create(struct mlx5e_psp_fs *fs,
652 struct mlx5e_psp_tx_table *tx)
653 {
654 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
655 struct mlx5_flow_table_attr ft_attr = {};
656 struct mlx5_flow_destination dest = {};
657 struct mlx5_core_dev *mdev = fs->mdev;
658 struct mlx5_flow_act flow_act = {};
659 u32 *in, *mc, *outer_headers_c;
660 struct mlx5_flow_handle *rule;
661 struct mlx5_flow_spec *spec;
662 struct mlx5_flow_table *ft;
663 struct mlx5_flow_group *fg;
664 int err = 0;
665
666 spec = kvzalloc_obj(*spec);
667 in = kvzalloc(inlen, GFP_KERNEL);
668 if (!spec || !in) {
669 err = -ENOMEM;
670 goto out;
671 }
672
673 ft_attr.max_fte = 1;
674 #define MLX5E_PSP_PRIO 0
675 ft_attr.prio = MLX5E_PSP_PRIO;
676 #define MLX5E_PSP_LEVEL 0
677 ft_attr.level = MLX5E_PSP_LEVEL;
678 ft_attr.autogroup.max_num_groups = 1;
679
680 ft = mlx5_create_flow_table(tx->ns, &ft_attr);
681 if (IS_ERR(ft)) {
682 err = PTR_ERR(ft);
683 mlx5_core_err(mdev, "PSP: fail to add psp tx flow table, err = %d\n", err);
684 goto out;
685 }
686
687 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
688 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
689 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
690 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
691 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
692 fg = mlx5_create_flow_group(ft, in);
693 if (IS_ERR(fg)) {
694 err = PTR_ERR(fg);
695 mlx5_core_err(mdev, "PSP: fail to add psp tx flow group, err = %d\n", err);
696 goto err_create_fg;
697 }
698
699 setup_fte_udp_psp(spec, PSP_DEFAULT_UDP_PORT);
700 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_PSP;
701 flow_act.flags |= FLOW_ACT_NO_APPEND;
702 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW |
703 MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT |
704 MLX5_FLOW_CONTEXT_ACTION_COUNT;
705 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
706 dest.counter = fs->tx_counter;
707 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
708 if (IS_ERR(rule)) {
709 err = PTR_ERR(rule);
710 mlx5_core_err(mdev, "PSP: fail to add psp tx flow rule, err = %d\n", err);
711 goto err_add_flow_rule;
712 }
713
714 tx->ft = ft;
715 tx->fg = fg;
716 tx->rule = rule;
717 goto out;
718
719 err_add_flow_rule:
720 mlx5_destroy_flow_group(fg);
721 err_create_fg:
722 mlx5_destroy_flow_table(ft);
723 out:
724 kvfree(in);
725 kvfree(spec);
726 return err;
727 }
728
accel_psp_fs_tx_ft_destroy(struct mlx5e_psp_tx_table * tx)729 static void accel_psp_fs_tx_ft_destroy(struct mlx5e_psp_tx_table *tx)
730 {
731 accel_psp_fs_del_flow_rule(&tx->rule);
732 accel_psp_fs_destroy_flow_group(&tx->fg);
733 accel_psp_fs_destroy_ft(&tx->ft);
734 }
735
accel_psp_fs_tx_cleanup(struct mlx5e_psp_fs * fs)736 static void accel_psp_fs_tx_cleanup(struct mlx5e_psp_fs *fs)
737 {
738 accel_psp_fs_destroy_counter(fs->mdev, &fs->tx_counter);
739 }
740
accel_psp_fs_tx_init(struct mlx5e_psp_fs * fs)741 static int accel_psp_fs_tx_init(struct mlx5e_psp_fs *fs)
742 {
743 struct mlx5_core_dev *mdev = fs->mdev;
744 int err;
745
746 fs->tx.ns = mlx5_get_flow_namespace(mdev,
747 MLX5_FLOW_NAMESPACE_EGRESS_IPSEC);
748 if (!fs->tx.ns)
749 return -EOPNOTSUPP;
750
751 err = accel_psp_fs_create_counter(mdev, &fs->tx_counter);
752 if (err) {
753 mlx5_core_warn(mdev,
754 "fail to create psp tx flow counter err=%d\n",
755 err);
756 return err;
757 }
758 return 0;
759 }
760
761 static void
mlx5e_accel_psp_fs_get_stats_fill(struct mlx5e_priv * priv,struct mlx5e_psp_stats * stats)762 mlx5e_accel_psp_fs_get_stats_fill(struct mlx5e_priv *priv,
763 struct mlx5e_psp_stats *stats)
764 {
765 struct mlx5e_psp_fs *fs = priv->psp->fs;
766 struct mlx5_core_dev *mdev = priv->mdev;
767
768 if (fs->tx_counter)
769 mlx5_fc_query(mdev, fs->tx_counter, &stats->psp_tx_pkts,
770 &stats->psp_tx_bytes);
771
772 if (fs->rx_counter)
773 mlx5_fc_query(mdev, fs->rx_counter, &stats->psp_rx_pkts,
774 &stats->psp_rx_bytes);
775
776 if (fs->rx_auth_fail_counter)
777 mlx5_fc_query(mdev, fs->rx_auth_fail_counter,
778 &stats->psp_rx_pkts_auth_fail,
779 &stats->psp_rx_bytes_auth_fail);
780
781 if (fs->rx_err_counter)
782 mlx5_fc_query(mdev, fs->rx_err_counter,
783 &stats->psp_rx_pkts_frame_err,
784 &stats->psp_rx_bytes_frame_err);
785
786 if (fs->rx_bad_counter)
787 mlx5_fc_query(mdev, fs->rx_bad_counter,
788 &stats->psp_rx_pkts_drop,
789 &stats->psp_rx_bytes_drop);
790 }
791
mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv * priv)792 void mlx5_accel_psp_fs_cleanup_tx_tables(struct mlx5e_priv *priv)
793 {
794 if (!priv->psp)
795 return;
796
797 netdev_lock(priv->netdev);
798 accel_psp_fs_tx_ft_destroy(&priv->psp->fs->tx);
799 netdev_unlock(priv->netdev);
800 }
801
mlx5e_accel_psp_fs_cleanup(struct mlx5e_psp_fs * fs)802 static void mlx5e_accel_psp_fs_cleanup(struct mlx5e_psp_fs *fs)
803 {
804 accel_psp_fs_rx_cleanup(fs);
805 accel_psp_fs_tx_cleanup(fs);
806 kfree(fs);
807 }
808
mlx5e_accel_psp_fs_init(struct mlx5e_priv * priv)809 static struct mlx5e_psp_fs *mlx5e_accel_psp_fs_init(struct mlx5e_priv *priv)
810 {
811 struct mlx5e_psp_fs *fs;
812 int err = 0;
813
814 fs = kzalloc_obj(*fs);
815 if (!fs)
816 return ERR_PTR(-ENOMEM);
817
818 fs->mdev = priv->mdev;
819 err = accel_psp_fs_tx_init(fs);
820 if (err)
821 goto err_tx;
822
823 fs->fs = priv->fs;
824 err = accel_psp_fs_rx_init(fs);
825 if (err)
826 goto err_rx;
827
828 return fs;
829
830 err_rx:
831 accel_psp_fs_tx_cleanup(fs);
832 err_tx:
833 kfree(fs);
834 return ERR_PTR(err);
835 }
836
accel_psp_fs_create(struct mlx5e_priv * priv,struct netlink_ext_ack * extack)837 static int accel_psp_fs_create(struct mlx5e_priv *priv,
838 struct netlink_ext_ack *extack)
839 {
840 int err;
841
842 err = accel_psp_fs_rx_create(priv->psp->fs, extack);
843 if (err)
844 return err;
845
846 err = accel_psp_fs_tx_ft_create(priv->psp->fs, &priv->psp->fs->tx);
847 if (err) {
848 NL_SET_ERR_MSG(extack, "Failed creating TX steering table");
849 accel_psp_fs_rx_destroy(priv->psp->fs);
850 }
851 return err;
852 }
853
accel_psp_fs_destroy(struct mlx5e_priv * priv)854 static void accel_psp_fs_destroy(struct mlx5e_priv *priv)
855 {
856 accel_psp_fs_tx_ft_destroy(&priv->psp->fs->tx);
857 accel_psp_fs_rx_destroy(priv->psp->fs);
858 }
859
860 static int
mlx5e_psp_set_config(struct psp_dev * psd,struct psp_dev_config * conf,struct netlink_ext_ack * extack)861 mlx5e_psp_set_config(struct psp_dev *psd, struct psp_dev_config *conf,
862 struct netlink_ext_ack *extack)
863 {
864 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
865 bool psp_enabled = psd->config.versions;
866 bool enable_psp = conf->versions;
867 int err = 0;
868
869 netdev_lock(priv->netdev);
870 if (!psp_enabled && enable_psp)
871 err = accel_psp_fs_create(priv, extack);
872 else if (psp_enabled && !enable_psp)
873 accel_psp_fs_destroy(priv);
874 netdev_unlock(priv->netdev);
875 return err;
876 }
877
878 static int
mlx5e_psp_generate_key_spi(struct mlx5_core_dev * mdev,enum mlx5_psp_gen_spi_in_key_size keysz,unsigned int keysz_bytes,struct psp_key_parsed * key)879 mlx5e_psp_generate_key_spi(struct mlx5_core_dev *mdev,
880 enum mlx5_psp_gen_spi_in_key_size keysz,
881 unsigned int keysz_bytes,
882 struct psp_key_parsed *key)
883 {
884 u32 out[MLX5_ST_SZ_DW(psp_gen_spi_out) + MLX5_ST_SZ_DW(key_spi)] = {};
885 u32 in[MLX5_ST_SZ_DW(psp_gen_spi_in)] = {};
886 void *outkey;
887 int err;
888
889 WARN_ON_ONCE(keysz_bytes > PSP_MAX_KEY);
890
891 MLX5_SET(psp_gen_spi_in, in, opcode, MLX5_CMD_OP_PSP_GEN_SPI);
892 MLX5_SET(psp_gen_spi_in, in, key_size, keysz);
893 MLX5_SET(psp_gen_spi_in, in, num_of_spi, 1);
894 err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
895 if (err)
896 return err;
897
898 outkey = MLX5_ADDR_OF(psp_gen_spi_out, out, key_spi);
899 key->spi = cpu_to_be32(MLX5_GET(key_spi, outkey, spi));
900 memcpy(key->key, MLX5_ADDR_OF(key_spi, outkey, key) + 32 - keysz_bytes,
901 keysz_bytes);
902
903 return 0;
904 }
905
906 static int
mlx5e_psp_rx_spi_alloc(struct psp_dev * psd,u32 version,struct psp_key_parsed * assoc,struct netlink_ext_ack * extack)907 mlx5e_psp_rx_spi_alloc(struct psp_dev *psd, u32 version,
908 struct psp_key_parsed *assoc,
909 struct netlink_ext_ack *extack)
910 {
911 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
912 enum mlx5_psp_gen_spi_in_key_size keysz;
913 u8 keysz_bytes;
914
915 switch (version) {
916 case PSP_VERSION_HDR0_AES_GCM_128:
917 keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_128;
918 keysz_bytes = 16;
919 break;
920 case PSP_VERSION_HDR0_AES_GCM_256:
921 keysz = MLX5_PSP_GEN_SPI_IN_KEY_SIZE_256;
922 keysz_bytes = 32;
923 break;
924 default:
925 return -EINVAL;
926 }
927
928 return mlx5e_psp_generate_key_spi(priv->mdev, keysz, keysz_bytes, assoc);
929 }
930
931 struct psp_key {
932 u32 id;
933 };
934
mlx5e_psp_assoc_add(struct psp_dev * psd,struct psp_assoc * pas,struct netlink_ext_ack * extack)935 static int mlx5e_psp_assoc_add(struct psp_dev *psd, struct psp_assoc *pas,
936 struct netlink_ext_ack *extack)
937 {
938 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
939 struct mlx5_core_dev *mdev = priv->mdev;
940 struct psp_key_parsed *tx = &pas->tx;
941 struct mlx5e_psp *psp = priv->psp;
942 struct psp_key *nkey;
943 int err;
944
945 mdev = priv->mdev;
946 nkey = (struct psp_key *)pas->drv_data;
947
948 err = mlx5_create_encryption_key(mdev, tx->key,
949 psp_key_size(pas->version),
950 MLX5_ACCEL_OBJ_PSP_KEY,
951 &nkey->id);
952 if (err) {
953 mlx5_core_err(mdev, "Failed to create encryption key (err = %d)\n", err);
954 return err;
955 }
956
957 atomic_inc(&psp->tx_key_cnt);
958 return 0;
959 }
960
mlx5e_psp_assoc_del(struct psp_dev * psd,struct psp_assoc * pas)961 static void mlx5e_psp_assoc_del(struct psp_dev *psd, struct psp_assoc *pas)
962 {
963 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
964 struct mlx5e_psp *psp = priv->psp;
965 struct psp_key *nkey;
966
967 nkey = (struct psp_key *)pas->drv_data;
968 mlx5_destroy_encryption_key(priv->mdev, nkey->id);
969 atomic_dec(&psp->tx_key_cnt);
970 }
971
mlx5e_psp_rotate_key(struct mlx5_core_dev * mdev)972 static int mlx5e_psp_rotate_key(struct mlx5_core_dev *mdev)
973 {
974 u32 in[MLX5_ST_SZ_DW(psp_rotate_key_in)] = {};
975 u32 out[MLX5_ST_SZ_DW(psp_rotate_key_out)];
976
977 MLX5_SET(psp_rotate_key_in, in, opcode,
978 MLX5_CMD_OP_PSP_ROTATE_KEY);
979
980 return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
981 }
982
983 static int
mlx5e_psp_key_rotate(struct psp_dev * psd,struct netlink_ext_ack * exack)984 mlx5e_psp_key_rotate(struct psp_dev *psd, struct netlink_ext_ack *exack)
985 {
986 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
987
988 /* no support for protecting against external rotations */
989 psd->generation = 0;
990
991 return mlx5e_psp_rotate_key(priv->mdev);
992 }
993
994 static void
mlx5e_psp_get_stats(struct psp_dev * psd,struct psp_dev_stats * stats)995 mlx5e_psp_get_stats(struct psp_dev *psd, struct psp_dev_stats *stats)
996 {
997 struct mlx5e_priv *priv = netdev_priv(psd->main_netdev);
998 struct mlx5e_psp_stats nstats;
999
1000 mlx5e_accel_psp_fs_get_stats_fill(priv, &nstats);
1001 stats->rx_packets = nstats.psp_rx_pkts;
1002 stats->rx_bytes = nstats.psp_rx_bytes;
1003 stats->rx_auth_fail = nstats.psp_rx_pkts_auth_fail;
1004 stats->rx_error = nstats.psp_rx_pkts_frame_err;
1005 stats->rx_bad = nstats.psp_rx_pkts_drop;
1006 stats->tx_packets = nstats.psp_tx_pkts;
1007 stats->tx_bytes = nstats.psp_tx_bytes;
1008 stats->tx_error = atomic_read(&priv->psp->tx_drop);
1009 }
1010
1011 static struct psp_dev_ops mlx5_psp_ops = {
1012 .set_config = mlx5e_psp_set_config,
1013 .rx_spi_alloc = mlx5e_psp_rx_spi_alloc,
1014 .tx_key_add = mlx5e_psp_assoc_add,
1015 .tx_key_del = mlx5e_psp_assoc_del,
1016 .key_rotate = mlx5e_psp_key_rotate,
1017 .get_stats = mlx5e_psp_get_stats,
1018 };
1019
mlx5e_psp_unregister(struct mlx5e_priv * priv)1020 void mlx5e_psp_unregister(struct mlx5e_priv *priv)
1021 {
1022 struct mlx5e_psp *psp = priv->psp;
1023
1024 if (!psp || !psp->psd)
1025 return;
1026
1027 psp_dev_unregister(psp->psd);
1028 psp->psd = NULL;
1029 }
1030
mlx5e_psp_register(struct mlx5e_priv * priv)1031 int mlx5e_psp_register(struct mlx5e_priv *priv)
1032 {
1033 struct mlx5e_psp *psp = priv->psp;
1034 struct psp_dev *psd;
1035
1036 /* FW Caps missing */
1037 if (!priv->psp)
1038 return 0;
1039
1040 psp->caps.assoc_drv_spc = sizeof(u32);
1041 psp->caps.versions = 1 << PSP_VERSION_HDR0_AES_GCM_128;
1042 if (MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_encrypt) &&
1043 MLX5_CAP_PSP(priv->mdev, psp_crypto_esp_aes_gcm_256_decrypt))
1044 psp->caps.versions |= 1 << PSP_VERSION_HDR0_AES_GCM_256;
1045
1046 psd = psp_dev_create(priv->netdev, &mlx5_psp_ops, &psp->caps, NULL);
1047 if (IS_ERR(psd)) {
1048 mlx5_core_err(priv->mdev, "PSP failed to register due to %pe\n",
1049 psd);
1050 return PTR_ERR(psd);
1051 }
1052 psp->psd = psd;
1053
1054 return 0;
1055 }
1056
mlx5e_psp_init(struct mlx5e_priv * priv)1057 int mlx5e_psp_init(struct mlx5e_priv *priv)
1058 {
1059 struct mlx5_core_dev *mdev = priv->mdev;
1060 struct mlx5e_psp_fs *fs;
1061 struct mlx5e_psp *psp;
1062 int err;
1063
1064 if (!mlx5_is_psp_device(mdev)) {
1065 mlx5_core_dbg(mdev, "PSP offload not supported\n");
1066 return 0;
1067 }
1068
1069 if (!MLX5_CAP_ETH(mdev, swp)) {
1070 mlx5_core_dbg(mdev, "SWP not supported\n");
1071 return 0;
1072 }
1073
1074 if (!MLX5_CAP_ETH(mdev, swp_csum)) {
1075 mlx5_core_dbg(mdev, "SWP checksum not supported\n");
1076 return 0;
1077 }
1078
1079 if (!MLX5_CAP_ETH(mdev, swp_csum_l4_partial)) {
1080 mlx5_core_dbg(mdev, "SWP L4 partial checksum not supported\n");
1081 return 0;
1082 }
1083
1084 if (!MLX5_CAP_ETH(mdev, swp_lso)) {
1085 mlx5_core_dbg(mdev, "PSP LSO not supported\n");
1086 return 0;
1087 }
1088
1089 psp = kzalloc_obj(*psp);
1090 if (!psp)
1091 return -ENOMEM;
1092
1093 fs = mlx5e_accel_psp_fs_init(priv);
1094 if (IS_ERR(fs)) {
1095 err = PTR_ERR(fs);
1096 kfree(psp);
1097 return err;
1098 }
1099
1100 psp->fs = fs;
1101 priv->psp = psp;
1102
1103 mlx5_core_dbg(priv->mdev, "PSP attached to netdevice\n");
1104 return 0;
1105 }
1106
mlx5e_psp_cleanup(struct mlx5e_priv * priv)1107 void mlx5e_psp_cleanup(struct mlx5e_priv *priv)
1108 {
1109 struct mlx5e_psp *psp = priv->psp;
1110
1111 if (!psp)
1112 return;
1113
1114 WARN_ON(atomic_read(&psp->tx_key_cnt));
1115 mlx5e_accel_psp_fs_cleanup(psp->fs);
1116 priv->psp = NULL;
1117 kfree(psp);
1118 }
1119