1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3
4 #include "rqt.h"
5 #include <linux/mlx5/transobj.h>
6
verify_num_vhca_ids(struct mlx5_core_dev * mdev,u32 * vhca_ids,unsigned int size)7 static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
8 unsigned int size)
9 {
10 unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id);
11 int i;
12
13 /* Verify that all vhca_ids are in range [0, max_num_vhca_ids - 1] */
14 for (i = 0; i < size; i++)
15 if (vhca_ids[i] >= max_num_vhca_id)
16 return false;
17 return true;
18 }
19
rqt_verify_vhca_ids(struct mlx5_core_dev * mdev,u32 * vhca_ids,unsigned int size)20 static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids,
21 unsigned int size)
22 {
23 if (!vhca_ids)
24 return true;
25
26 if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt))
27 return false;
28 if (!verify_num_vhca_ids(mdev, vhca_ids, size))
29 return false;
30
31 return true;
32 }
33
mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir * indir,unsigned int num_channels)34 void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir,
35 unsigned int num_channels)
36 {
37 unsigned int i;
38
39 for (i = 0; i < indir->actual_table_size; i++)
40 indir->table[i] = i % num_channels;
41 }
42
fill_rqn_list(void * rqtc,u32 * rqns,u32 * vhca_ids,unsigned int size)43 static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size)
44 {
45 unsigned int i;
46
47 if (vhca_ids) {
48 MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1);
49 for (i = 0; i < size; i++) {
50 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]);
51 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]);
52 }
53 } else {
54 for (i = 0; i < size; i++)
55 MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]);
56 }
57 }
mlx5e_rqt_init(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,u16 max_size,u32 * init_rqns,u32 * init_vhca_ids,u16 init_size)58 static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
59 u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size)
60 {
61 int entry_sz;
62 void *rqtc;
63 int inlen;
64 int err;
65 u32 *in;
66
67 if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size))
68 return -EOPNOTSUPP;
69
70 rqt->mdev = mdev;
71 rqt->size = max_size;
72
73 entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
74 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size;
75 in = kvzalloc(inlen, GFP_KERNEL);
76 if (!in)
77 return -ENOMEM;
78
79 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
80
81 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size);
82 MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size);
83
84 fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size);
85
86 err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn);
87
88 kvfree(in);
89 return err;
90 }
91
mlx5e_rqt_init_direct(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,bool indir_enabled,u32 init_rqn,u32 indir_table_size)92 int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
93 bool indir_enabled, u32 init_rqn, u32 indir_table_size)
94 {
95 u16 max_size = indir_enabled ? indir_table_size : 1;
96
97 return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1);
98 }
99
mlx5e_bits_invert(unsigned long a,int size)100 static int mlx5e_bits_invert(unsigned long a, int size)
101 {
102 int inv = 0;
103 int i;
104
105 for (i = 0; i < size; i++)
106 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
107
108 return inv;
109 }
110
mlx5e_calc_indir_rqns(u32 * rss_rqns,u32 * rqns,u32 * rss_vhca_ids,u32 * vhca_ids,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)111 static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids,
112 unsigned int num_rqns,
113 u8 hfunc, struct mlx5e_rss_params_indir *indir)
114 {
115 unsigned int i;
116
117 for (i = 0; i < indir->actual_table_size; i++) {
118 unsigned int ix = i;
119
120 if (hfunc == ETH_RSS_HASH_XOR)
121 ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size));
122
123 ix = indir->table[ix];
124
125 if (WARN_ON(ix >= num_rqns))
126 /* Could be a bug in the driver or in the kernel part of
127 * ethtool: indir table refers to non-existent RQs.
128 */
129 return -EINVAL;
130 rss_rqns[i] = rqns[ix];
131 if (vhca_ids)
132 rss_vhca_ids[i] = vhca_ids[ix];
133 }
134
135 return 0;
136 }
137
mlx5e_rqt_init_indir(struct mlx5e_rqt * rqt,struct mlx5_core_dev * mdev,u32 * rqns,u32 * vhca_ids,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)138 int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev,
139 u32 *rqns, u32 *vhca_ids, unsigned int num_rqns,
140 u8 hfunc, struct mlx5e_rss_params_indir *indir)
141 {
142 u32 *rss_rqns, *rss_vhca_ids = NULL;
143 int err;
144
145 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
146 if (!rss_rqns)
147 return -ENOMEM;
148
149 if (vhca_ids) {
150 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids),
151 GFP_KERNEL);
152 if (!rss_vhca_ids) {
153 kvfree(rss_rqns);
154 return -ENOMEM;
155 }
156 }
157
158 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
159 if (err)
160 goto out;
161
162 err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids,
163 indir->actual_table_size);
164
165 out:
166 kvfree(rss_vhca_ids);
167 kvfree(rss_rqns);
168 return err;
169 }
170
171 #define MLX5E_UNIFORM_SPREAD_RQT_FACTOR 2
172
mlx5e_rqt_size(struct mlx5_core_dev * mdev,unsigned int num_channels)173 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels)
174 {
175 u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE,
176 roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR));
177 u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size);
178
179 return min_t(u32, rqt_size, max_cap_rqt_size);
180 }
181
182 #define MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH 256
183
mlx5e_rqt_max_num_channels_allowed_for_xor8(void)184 unsigned int mlx5e_rqt_max_num_channels_allowed_for_xor8(void)
185 {
186 return MLX5E_MAX_RQT_SIZE_ALLOWED_WITH_XOR8_HASH / MLX5E_UNIFORM_SPREAD_RQT_FACTOR;
187 }
188
mlx5e_rqt_destroy(struct mlx5e_rqt * rqt)189 void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt)
190 {
191 mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn);
192 }
193
mlx5e_rqt_redirect(struct mlx5e_rqt * rqt,u32 * rqns,u32 * vhca_ids,unsigned int size)194 static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
195 unsigned int size)
196 {
197 int entry_sz;
198 void *rqtc;
199 int inlen;
200 u32 *in;
201 int err;
202
203 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size))
204 return -EINVAL;
205
206 entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num);
207 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size;
208 in = kvzalloc(inlen, GFP_KERNEL);
209 if (!in)
210 return -ENOMEM;
211
212 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
213
214 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
215 MLX5_SET(rqtc, rqtc, rqt_actual_size, size);
216
217 fill_rqn_list(rqtc, rqns, vhca_ids, size);
218
219 err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen);
220
221 kvfree(in);
222 return err;
223 }
224
mlx5e_rqt_redirect_direct(struct mlx5e_rqt * rqt,u32 rqn,u32 * vhca_id)225 int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id)
226 {
227 return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1);
228 }
229
mlx5e_rqt_redirect_indir(struct mlx5e_rqt * rqt,u32 * rqns,u32 * vhca_ids,unsigned int num_rqns,u8 hfunc,struct mlx5e_rss_params_indir * indir)230 int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids,
231 unsigned int num_rqns,
232 u8 hfunc, struct mlx5e_rss_params_indir *indir)
233 {
234 u32 *rss_rqns, *rss_vhca_ids = NULL;
235 int err;
236
237 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns))
238 return -EINVAL;
239
240 if (WARN_ON(rqt->size != indir->max_table_size))
241 return -EINVAL;
242
243 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL);
244 if (!rss_rqns)
245 return -ENOMEM;
246
247 if (vhca_ids) {
248 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids),
249 GFP_KERNEL);
250 if (!rss_vhca_ids) {
251 kvfree(rss_rqns);
252 return -ENOMEM;
253 }
254 }
255
256 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir);
257 if (err)
258 goto out;
259
260 err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size);
261
262 out:
263 kvfree(rss_vhca_ids);
264 kvfree(rss_rqns);
265 return err;
266 }
267