1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES.
3
4 #include "rss.h"
5
6 #define mlx5e_rss_warn(__dev, format, ...) \
7 dev_warn((__dev)->device, "%s:%d:(pid %d): " format, \
8 __func__, __LINE__, current->pid, \
9 ##__VA_ARGS__)
10
11 static const struct mlx5e_rss_params_traffic_type rss_default_config[MLX5E_NUM_INDIR_TIRS] = {
12 [MLX5_TT_IPV4_TCP] = {
13 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
14 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
15 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
16 },
17 [MLX5_TT_IPV6_TCP] = {
18 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
19 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP,
20 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
21 },
22 [MLX5_TT_IPV4_UDP] = {
23 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
24 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
25 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
26 },
27 [MLX5_TT_IPV6_UDP] = {
28 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
29 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP,
30 .rx_hash_fields = MLX5_HASH_IP_L4PORTS,
31 },
32 [MLX5_TT_IPV4_IPSEC_AH] = {
33 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
34 .l4_prot_type = 0,
35 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
36 },
37 [MLX5_TT_IPV6_IPSEC_AH] = {
38 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
39 .l4_prot_type = 0,
40 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
41 },
42 [MLX5_TT_IPV4_IPSEC_ESP] = {
43 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
44 .l4_prot_type = 0,
45 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
46 },
47 [MLX5_TT_IPV6_IPSEC_ESP] = {
48 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
49 .l4_prot_type = 0,
50 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI,
51 },
52 [MLX5_TT_IPV4] = {
53 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4,
54 .l4_prot_type = 0,
55 .rx_hash_fields = MLX5_HASH_IP,
56 },
57 [MLX5_TT_IPV6] = {
58 .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6,
59 .l4_prot_type = 0,
60 .rx_hash_fields = MLX5_HASH_IP,
61 },
62 };
63
64 struct mlx5e_rss_params_traffic_type
mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt)65 mlx5e_rss_get_default_tt_config(enum mlx5_traffic_types tt)
66 {
67 return rss_default_config[tt];
68 }
69
70 struct mlx5e_rss {
71 struct mlx5e_rss_params_hash hash;
72 struct mlx5e_rss_params_indir indir;
73 u32 rx_hash_fields[MLX5E_NUM_INDIR_TIRS];
74 struct mlx5e_tir *tir[MLX5E_NUM_INDIR_TIRS];
75 struct mlx5e_tir *inner_tir[MLX5E_NUM_INDIR_TIRS];
76 struct mlx5e_rqt rqt;
77 struct mlx5_core_dev *mdev; /* primary */
78 struct mlx5e_rss_params params;
79 bool enabled;
80 refcount_t refcnt;
81 };
82
mlx5e_rss_get_inner_ft_support(struct mlx5e_rss * rss)83 bool mlx5e_rss_get_inner_ft_support(struct mlx5e_rss *rss)
84 {
85 return rss->params.inner_ft_support;
86 }
87
mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss * rss,u32 num_channels)88 void mlx5e_rss_params_indir_modify_actual_size(struct mlx5e_rss *rss, u32 num_channels)
89 {
90 rss->indir.actual_table_size = mlx5e_rqt_size(rss->mdev, num_channels);
91 }
92
mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir * indir,u32 actual_table_size,u32 max_table_size)93 int mlx5e_rss_params_indir_init(struct mlx5e_rss_params_indir *indir,
94 u32 actual_table_size, u32 max_table_size)
95 {
96 indir->table = kvmalloc_objs(*indir->table, max_table_size);
97 if (!indir->table)
98 return -ENOMEM;
99
100 indir->max_table_size = max_table_size;
101 indir->actual_table_size = actual_table_size;
102
103 return 0;
104 }
105
mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir * indir)106 void mlx5e_rss_params_indir_cleanup(struct mlx5e_rss_params_indir *indir)
107 {
108 kvfree(indir->table);
109 }
110
mlx5e_rss_copy(struct mlx5e_rss * to,const struct mlx5e_rss * from)111 static int mlx5e_rss_copy(struct mlx5e_rss *to, const struct mlx5e_rss *from)
112 {
113 u32 *dst_indir_table;
114
115 if (to->indir.actual_table_size != from->indir.actual_table_size ||
116 to->indir.max_table_size != from->indir.max_table_size) {
117 mlx5e_rss_warn(to->mdev,
118 "Failed to copy RSS due to size mismatch, src (actual %u, max %u) != dst (actual %u, max %u)\n",
119 from->indir.actual_table_size, from->indir.max_table_size,
120 to->indir.actual_table_size, to->indir.max_table_size);
121 return -EINVAL;
122 }
123
124 dst_indir_table = to->indir.table;
125 *to = *from;
126 to->indir.table = dst_indir_table;
127 memcpy(to->indir.table, from->indir.table,
128 from->indir.actual_table_size * sizeof(*from->indir.table));
129 return 0;
130 }
131
mlx5e_rss_init_copy(const struct mlx5e_rss * from)132 static struct mlx5e_rss *mlx5e_rss_init_copy(const struct mlx5e_rss *from)
133 {
134 struct mlx5e_rss *rss;
135 int err;
136
137 rss = kvzalloc_obj(*rss);
138 if (!rss)
139 return ERR_PTR(-ENOMEM);
140
141 err = mlx5e_rss_params_indir_init(&rss->indir,
142 from->indir.actual_table_size,
143 from->indir.max_table_size);
144 if (err)
145 goto err_free_rss;
146
147 err = mlx5e_rss_copy(rss, from);
148 if (err)
149 goto err_free_indir;
150
151 return rss;
152
153 err_free_indir:
154 mlx5e_rss_params_indir_cleanup(&rss->indir);
155 err_free_rss:
156 kvfree(rss);
157 return ERR_PTR(err);
158 }
159
mlx5e_rss_params_init(struct mlx5e_rss * rss)160 static void mlx5e_rss_params_init(struct mlx5e_rss *rss)
161 {
162 enum mlx5_traffic_types tt;
163
164 rss->hash.symmetric = true;
165 rss->hash.hfunc = ETH_RSS_HASH_TOP;
166 netdev_rss_key_fill(rss->hash.toeplitz_hash_key,
167 sizeof(rss->hash.toeplitz_hash_key));
168 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
169 rss->rx_hash_fields[tt] =
170 mlx5e_rss_get_default_tt_config(tt).rx_hash_fields;
171 }
172
rss_get_tirp(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)173 static struct mlx5e_tir **rss_get_tirp(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
174 bool inner)
175 {
176 return inner ? &rss->inner_tir[tt] : &rss->tir[tt];
177 }
178
rss_get_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)179 static struct mlx5e_tir *rss_get_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
180 bool inner)
181 {
182 return *rss_get_tirp(rss, tt, inner);
183 }
184
185 static struct mlx5e_rss_params_traffic_type
mlx5e_rss_get_tt_config(struct mlx5e_rss * rss,enum mlx5_traffic_types tt)186 mlx5e_rss_get_tt_config(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
187 {
188 struct mlx5e_rss_params_traffic_type rss_tt;
189
190 rss_tt = mlx5e_rss_get_default_tt_config(tt);
191 rss_tt.rx_hash_fields = rss->rx_hash_fields[tt];
192 return rss_tt;
193 }
194
195 static int
mlx5e_rss_create_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,const struct mlx5e_packet_merge_param * pkt_merge_param,bool inner)196 mlx5e_rss_create_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
197 const struct mlx5e_packet_merge_param *pkt_merge_param,
198 bool inner)
199 {
200 bool rss_inner = rss->params.inner_ft_support;
201 struct mlx5e_rss_params_traffic_type rss_tt;
202 struct mlx5e_tir_builder *builder;
203 struct mlx5e_tir **tir_p;
204 struct mlx5e_tir *tir;
205 u32 rqtn;
206 int err;
207
208 if (inner && !rss_inner) {
209 mlx5e_rss_warn(rss->mdev,
210 "Cannot create inner indirect TIR[%d], RSS inner FT is not supported.\n",
211 tt);
212 return -EINVAL;
213 }
214
215 tir_p = rss_get_tirp(rss, tt, inner);
216 if (*tir_p)
217 return -EINVAL;
218
219 tir = kvzalloc_obj(*tir);
220 if (!tir)
221 return -ENOMEM;
222
223 builder = mlx5e_tir_builder_alloc(false);
224 if (!builder) {
225 err = -ENOMEM;
226 goto free_tir;
227 }
228
229 rqtn = mlx5e_rqt_get_rqtn(&rss->rqt);
230 mlx5e_tir_builder_build_rqt(builder, rss->mdev->mlx5e_res.hw_objs.td.tdn,
231 rqtn, rss_inner);
232 mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
233 rss_tt = mlx5e_rss_get_tt_config(rss, tt);
234 mlx5e_tir_builder_build_self_lb_block(builder, rss->params.self_lb_blk,
235 rss->params.self_lb_blk);
236 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
237
238 err = mlx5e_tir_init(tir, builder, rss->mdev, true);
239 mlx5e_tir_builder_free(builder);
240 if (err) {
241 mlx5e_rss_warn(rss->mdev, "Failed to create %sindirect TIR: err = %d, tt = %d\n",
242 inner ? "inner " : "", err, tt);
243 goto free_tir;
244 }
245
246 *tir_p = tir;
247 return 0;
248
249 free_tir:
250 kvfree(tir);
251 return err;
252 }
253
mlx5e_rss_destroy_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)254 static void mlx5e_rss_destroy_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
255 bool inner)
256 {
257 struct mlx5e_tir **tir_p;
258 struct mlx5e_tir *tir;
259
260 tir_p = rss_get_tirp(rss, tt, inner);
261 if (!*tir_p)
262 return;
263
264 tir = *tir_p;
265 mlx5e_tir_destroy(tir);
266 kvfree(tir);
267 *tir_p = NULL;
268 }
269
270 static int
mlx5e_rss_create_tirs(struct mlx5e_rss * rss,const struct mlx5e_packet_merge_param * pkt_merge_param,bool inner)271 mlx5e_rss_create_tirs(struct mlx5e_rss *rss,
272 const struct mlx5e_packet_merge_param *pkt_merge_param,
273 bool inner)
274 {
275 enum mlx5_traffic_types tt, max_tt;
276 int err;
277
278 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
279 err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner);
280 if (err)
281 goto err_destroy_tirs;
282 }
283
284 return 0;
285
286 err_destroy_tirs:
287 max_tt = tt;
288 for (tt = 0; tt < max_tt; tt++)
289 mlx5e_rss_destroy_tir(rss, tt, inner);
290 return err;
291 }
292
mlx5e_rss_destroy_tirs(struct mlx5e_rss * rss,bool inner)293 static void mlx5e_rss_destroy_tirs(struct mlx5e_rss *rss, bool inner)
294 {
295 enum mlx5_traffic_types tt;
296
297 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++)
298 mlx5e_rss_destroy_tir(rss, tt, inner);
299 }
300
mlx5e_rss_update_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)301 static int mlx5e_rss_update_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
302 bool inner)
303 {
304 struct mlx5e_rss_params_traffic_type rss_tt;
305 struct mlx5e_tir_builder *builder;
306 struct mlx5e_tir *tir;
307 int err;
308
309 tir = rss_get_tir(rss, tt, inner);
310 if (!tir)
311 return 0;
312
313 builder = mlx5e_tir_builder_alloc(true);
314 if (!builder)
315 return -ENOMEM;
316
317 rss_tt = mlx5e_rss_get_tt_config(rss, tt);
318
319 mlx5e_tir_builder_build_rss(builder, &rss->hash, &rss_tt, inner);
320 err = mlx5e_tir_modify(tir, builder);
321
322 mlx5e_tir_builder_free(builder);
323 return err;
324 }
325
mlx5e_rss_update_tirs(struct mlx5e_rss * rss)326 static int mlx5e_rss_update_tirs(struct mlx5e_rss *rss)
327 {
328 enum mlx5_traffic_types tt;
329 int err, retval;
330
331 retval = 0;
332
333 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
334 err = mlx5e_rss_update_tir(rss, tt, false);
335 if (err) {
336 retval = retval ? : err;
337 mlx5e_rss_warn(rss->mdev,
338 "Failed to update RSS hash of indirect TIR for traffic type %d: err = %d\n",
339 tt, err);
340 }
341
342 if (!rss->params.inner_ft_support)
343 continue;
344
345 err = mlx5e_rss_update_tir(rss, tt, true);
346 if (err) {
347 retval = retval ? : err;
348 mlx5e_rss_warn(rss->mdev,
349 "Failed to update RSS hash of inner indirect TIR for traffic type %d: err = %d\n",
350 tt, err);
351 }
352 }
353 return retval;
354 }
355
mlx5e_rss_init_no_tirs(struct mlx5e_rss * rss)356 static int mlx5e_rss_init_no_tirs(struct mlx5e_rss *rss)
357 {
358 mlx5e_rss_params_init(rss);
359 refcount_set(&rss->refcnt, 1);
360
361 return mlx5e_rqt_init_direct(&rss->rqt, rss->mdev, true,
362 rss->params.drop_rqn,
363 rss->indir.max_table_size);
364 }
365
366 struct mlx5e_rss *
mlx5e_rss_init(struct mlx5_core_dev * mdev,const struct mlx5e_rss_params * params,const struct mlx5e_rss_init_params * init_params)367 mlx5e_rss_init(struct mlx5_core_dev *mdev,
368 const struct mlx5e_rss_params *params,
369 const struct mlx5e_rss_init_params *init_params)
370 {
371 u32 rqt_max_size, rqt_size;
372 struct mlx5e_rss *rss;
373 int err;
374
375 rss = kvzalloc_obj(*rss);
376 if (!rss)
377 return ERR_PTR(-ENOMEM);
378
379 rqt_size = mlx5e_rqt_size(mdev, init_params->nch);
380 rqt_max_size = mlx5e_rqt_size(mdev, init_params->max_nch);
381 err = mlx5e_rss_params_indir_init(&rss->indir, rqt_size, rqt_max_size);
382 if (err)
383 goto err_free_rss;
384
385 rss->mdev = mdev;
386 rss->params = *params;
387
388 err = mlx5e_rss_init_no_tirs(rss);
389 if (err)
390 goto err_free_indir;
391
392 if (init_params->type == MLX5E_RSS_INIT_NO_TIRS)
393 goto out;
394
395 err = mlx5e_rss_create_tirs(rss, init_params->pkt_merge_param,
396 false);
397 if (err)
398 goto err_destroy_rqt;
399
400 if (params->inner_ft_support) {
401 err = mlx5e_rss_create_tirs(rss,
402 init_params->pkt_merge_param,
403 true);
404 if (err)
405 goto err_destroy_tirs;
406 }
407
408 out:
409 return rss;
410
411 err_destroy_tirs:
412 mlx5e_rss_destroy_tirs(rss, false);
413 err_destroy_rqt:
414 mlx5e_rqt_destroy(&rss->rqt);
415 err_free_indir:
416 mlx5e_rss_params_indir_cleanup(&rss->indir);
417 err_free_rss:
418 kvfree(rss);
419 return ERR_PTR(err);
420 }
421
mlx5e_rss_cleanup(struct mlx5e_rss * rss)422 int mlx5e_rss_cleanup(struct mlx5e_rss *rss)
423 {
424 if (!refcount_dec_if_one(&rss->refcnt))
425 return -EBUSY;
426
427 mlx5e_rss_destroy_tirs(rss, false);
428
429 if (rss->params.inner_ft_support)
430 mlx5e_rss_destroy_tirs(rss, true);
431
432 mlx5e_rqt_destroy(&rss->rqt);
433 mlx5e_rss_params_indir_cleanup(&rss->indir);
434 kvfree(rss);
435
436 return 0;
437 }
438
mlx5e_rss_refcnt_inc(struct mlx5e_rss * rss)439 void mlx5e_rss_refcnt_inc(struct mlx5e_rss *rss)
440 {
441 refcount_inc(&rss->refcnt);
442 }
443
mlx5e_rss_refcnt_dec(struct mlx5e_rss * rss)444 void mlx5e_rss_refcnt_dec(struct mlx5e_rss *rss)
445 {
446 refcount_dec(&rss->refcnt);
447 }
448
mlx5e_rss_refcnt_read(struct mlx5e_rss * rss)449 unsigned int mlx5e_rss_refcnt_read(struct mlx5e_rss *rss)
450 {
451 return refcount_read(&rss->refcnt);
452 }
453
mlx5e_rss_get_tirn(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)454 u32 mlx5e_rss_get_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
455 bool inner)
456 {
457 struct mlx5e_tir *tir;
458
459 WARN_ON(inner && !rss->params.inner_ft_support);
460 tir = rss_get_tir(rss, tt, inner);
461 WARN_ON(!tir);
462
463 return mlx5e_tir_get_tirn(tir);
464 }
465
mlx5e_rss_get_rqtn(struct mlx5e_rss * rss)466 u32 mlx5e_rss_get_rqtn(struct mlx5e_rss *rss)
467 {
468 return mlx5e_rqt_get_rqtn(&rss->rqt);
469 }
470
mlx5e_rss_valid_tir(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,bool inner)471 bool mlx5e_rss_valid_tir(struct mlx5e_rss *rss, enum mlx5_traffic_types tt, bool inner)
472 {
473 return !!rss_get_tir(rss, tt, inner);
474 }
475
476 /* Fill the "tirn" output parameter.
477 * Create the requested TIR if it's its first usage.
478 */
479 int
mlx5e_rss_obtain_tirn(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,const struct mlx5e_packet_merge_param * pkt_merge_param,bool inner,u32 * tirn)480 mlx5e_rss_obtain_tirn(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
481 const struct mlx5e_packet_merge_param *pkt_merge_param,
482 bool inner, u32 *tirn)
483 {
484 struct mlx5e_tir *tir;
485
486 tir = rss_get_tir(rss, tt, inner);
487 if (!tir) { /* TIR doesn't exist, create one */
488 int err;
489
490 err = mlx5e_rss_create_tir(rss, tt, pkt_merge_param, inner);
491 if (err)
492 return err;
493 tir = rss_get_tir(rss, tt, inner);
494 }
495
496 *tirn = mlx5e_tir_get_tirn(tir);
497 return 0;
498 }
499
mlx5e_rss_apply(struct mlx5e_rss * rss,u32 * rqns,u32 * vhca_ids,unsigned int num_rqns)500 static int mlx5e_rss_apply(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns)
501 {
502 int err;
503
504 err = mlx5e_rqt_redirect_indir(&rss->rqt, rqns, vhca_ids, num_rqns, rss->hash.hfunc,
505 &rss->indir);
506 if (err)
507 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to channels: err = %d\n",
508 mlx5e_rqt_get_rqtn(&rss->rqt), err);
509 return err;
510 }
511
mlx5e_rss_enable(struct mlx5e_rss * rss,u32 * rqns,u32 * vhca_ids,unsigned int num_rqns)512 void mlx5e_rss_enable(struct mlx5e_rss *rss, u32 *rqns, u32 *vhca_ids, unsigned int num_rqns)
513 {
514 rss->enabled = true;
515 mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns);
516 }
517
mlx5e_rss_disable(struct mlx5e_rss * rss)518 void mlx5e_rss_disable(struct mlx5e_rss *rss)
519 {
520 int err;
521
522 rss->enabled = false;
523 err = mlx5e_rqt_redirect_direct(&rss->rqt, rss->params.drop_rqn, NULL);
524 if (err)
525 mlx5e_rss_warn(rss->mdev, "Failed to redirect RQT %#x to drop RQ %#x: err = %d\n",
526 mlx5e_rqt_get_rqtn(&rss->rqt),
527 rss->params.drop_rqn, err);
528 }
529
mlx5e_rss_packet_merge_set_param(struct mlx5e_rss * rss,struct mlx5e_packet_merge_param * pkt_merge_param)530 int mlx5e_rss_packet_merge_set_param(struct mlx5e_rss *rss,
531 struct mlx5e_packet_merge_param *pkt_merge_param)
532 {
533 struct mlx5e_tir_builder *builder;
534 enum mlx5_traffic_types tt;
535 int err, final_err;
536
537 builder = mlx5e_tir_builder_alloc(true);
538 if (!builder)
539 return -ENOMEM;
540
541 mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param);
542
543 final_err = 0;
544
545 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
546 struct mlx5e_tir *tir;
547
548 tir = rss_get_tir(rss, tt, false);
549 if (!tir)
550 goto inner_tir;
551 err = mlx5e_tir_modify(tir, builder);
552 if (err) {
553 mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of indirect TIR %#x for traffic type %d: err = %d\n",
554 mlx5e_tir_get_tirn(tir), tt, err);
555 if (!final_err)
556 final_err = err;
557 }
558
559 inner_tir:
560 if (!rss->params.inner_ft_support)
561 continue;
562
563 tir = rss_get_tir(rss, tt, true);
564 if (!tir)
565 continue;
566 err = mlx5e_tir_modify(tir, builder);
567 if (err) {
568 mlx5e_rss_warn(rss->mdev, "Failed to update packet merge state of inner indirect TIR %#x for traffic type %d: err = %d\n",
569 mlx5e_tir_get_tirn(tir), tt, err);
570 if (!final_err)
571 final_err = err;
572 }
573 }
574
575 mlx5e_tir_builder_free(builder);
576 return final_err;
577 }
578
mlx5e_rss_get_rxfh(struct mlx5e_rss * rss,u32 * indir,u8 * key,u8 * hfunc,bool * symmetric)579 void mlx5e_rss_get_rxfh(struct mlx5e_rss *rss, u32 *indir, u8 *key, u8 *hfunc,
580 bool *symmetric)
581 {
582 if (indir)
583 memcpy(indir, rss->indir.table,
584 rss->indir.actual_table_size * sizeof(*rss->indir.table));
585
586 if (key)
587 memcpy(key, rss->hash.toeplitz_hash_key,
588 sizeof(rss->hash.toeplitz_hash_key));
589
590 if (hfunc)
591 *hfunc = rss->hash.hfunc;
592
593 if (symmetric)
594 *symmetric = rss->hash.symmetric;
595 }
596
mlx5e_rss_set_rxfh(struct mlx5e_rss * rss,const u32 * indir,const u8 * key,const u8 * hfunc,const bool * symmetric,u32 * rqns,u32 * vhca_ids,unsigned int num_rqns)597 int mlx5e_rss_set_rxfh(struct mlx5e_rss *rss, const u32 *indir,
598 const u8 *key, const u8 *hfunc, const bool *symmetric,
599 u32 *rqns, u32 *vhca_ids, unsigned int num_rqns)
600 {
601 bool changed_indir = false;
602 bool changed_hash = false;
603 struct mlx5e_rss *old_rss;
604 int err = 0;
605
606 old_rss = mlx5e_rss_init_copy(rss);
607 if (IS_ERR(old_rss))
608 return PTR_ERR(old_rss);
609
610 if (hfunc && *hfunc != rss->hash.hfunc) {
611 switch (*hfunc) {
612 case ETH_RSS_HASH_XOR:
613 case ETH_RSS_HASH_TOP:
614 break;
615 default:
616 err = -EINVAL;
617 goto out;
618 }
619 changed_hash = true;
620 changed_indir = true;
621 rss->hash.hfunc = *hfunc;
622 }
623
624 if (key) {
625 if (rss->hash.hfunc == ETH_RSS_HASH_TOP)
626 changed_hash = true;
627 memcpy(rss->hash.toeplitz_hash_key, key,
628 sizeof(rss->hash.toeplitz_hash_key));
629 }
630
631 if (indir) {
632 changed_indir = true;
633
634 memcpy(rss->indir.table, indir,
635 rss->indir.actual_table_size * sizeof(*rss->indir.table));
636 }
637
638 if (symmetric) {
639 rss->hash.symmetric = *symmetric;
640 changed_hash = true;
641 }
642
643 if (changed_indir && rss->enabled) {
644 err = mlx5e_rss_apply(rss, rqns, vhca_ids, num_rqns);
645 if (err) {
646 mlx5e_rss_copy(rss, old_rss);
647 goto out;
648 }
649 }
650
651 if (changed_hash)
652 mlx5e_rss_update_tirs(rss);
653
654 out:
655 mlx5e_rss_params_indir_cleanup(&old_rss->indir);
656 kvfree(old_rss);
657
658 return err;
659 }
660
mlx5e_rss_get_hash(struct mlx5e_rss * rss)661 struct mlx5e_rss_params_hash mlx5e_rss_get_hash(struct mlx5e_rss *rss)
662 {
663 return rss->hash;
664 }
665
mlx5e_rss_get_hash_fields(struct mlx5e_rss * rss,enum mlx5_traffic_types tt)666 u8 mlx5e_rss_get_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt)
667 {
668 return rss->rx_hash_fields[tt];
669 }
670
mlx5e_rss_set_hash_fields(struct mlx5e_rss * rss,enum mlx5_traffic_types tt,u8 rx_hash_fields)671 int mlx5e_rss_set_hash_fields(struct mlx5e_rss *rss, enum mlx5_traffic_types tt,
672 u8 rx_hash_fields)
673 {
674 u8 old_rx_hash_fields;
675 int err;
676
677 old_rx_hash_fields = rss->rx_hash_fields[tt];
678
679 if (old_rx_hash_fields == rx_hash_fields)
680 return 0;
681
682 rss->rx_hash_fields[tt] = rx_hash_fields;
683
684 err = mlx5e_rss_update_tir(rss, tt, false);
685 if (err) {
686 rss->rx_hash_fields[tt] = old_rx_hash_fields;
687 mlx5e_rss_warn(rss->mdev,
688 "Failed to update RSS hash fields of indirect TIR for traffic type %d: err = %d\n",
689 tt, err);
690 return err;
691 }
692
693 if (!(rss->params.inner_ft_support))
694 return 0;
695
696 err = mlx5e_rss_update_tir(rss, tt, true);
697 if (err) {
698 /* Partial update happened. Try to revert - it may fail too, but
699 * there is nothing more we can do.
700 */
701 rss->rx_hash_fields[tt] = old_rx_hash_fields;
702 mlx5e_rss_warn(rss->mdev,
703 "Failed to update RSS hash fields of inner indirect TIR for traffic type %d: err = %d\n",
704 tt, err);
705 if (mlx5e_rss_update_tir(rss, tt, false))
706 mlx5e_rss_warn(rss->mdev,
707 "Partial update of RSS hash fields happened: failed to revert indirect TIR for traffic type %d to the old values\n",
708 tt);
709 }
710
711 return err;
712 }
713
mlx5e_rss_set_indir_uniform(struct mlx5e_rss * rss,unsigned int nch)714 void mlx5e_rss_set_indir_uniform(struct mlx5e_rss *rss, unsigned int nch)
715 {
716 mlx5e_rss_params_indir_init_uniform(&rss->indir, nch);
717 }
718