1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "rqt.h" 5 #include <linux/mlx5/transobj.h> 6 7 static bool verify_num_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, 8 unsigned int size) 9 { 10 unsigned int max_num_vhca_id = MLX5_CAP_GEN_2(mdev, max_rqt_vhca_id); 11 unsigned int unique_count = 0; 12 int i, j; 13 14 /* Count unique vhca_ids */ 15 for (i = 0; i < size; i++) { 16 bool is_unique = true; 17 18 /* Check if vhca_ids[i] was already seen */ 19 for (j = 0; j < i; j++) { 20 if (vhca_ids[j] == vhca_ids[i]) { 21 is_unique = false; 22 break; 23 } 24 } 25 if (is_unique) 26 unique_count++; 27 } 28 29 /* Verify that number of unique vhca_ids doesn't exceed 30 * max_num_vhca_id 31 */ 32 return unique_count <= max_num_vhca_id; 33 } 34 35 static bool rqt_verify_vhca_ids(struct mlx5_core_dev *mdev, u32 *vhca_ids, 36 unsigned int size) 37 { 38 if (!vhca_ids) 39 return true; 40 41 if (!MLX5_CAP_GEN(mdev, cross_vhca_rqt)) 42 return false; 43 if (!verify_num_vhca_ids(mdev, vhca_ids, size)) 44 return false; 45 46 return true; 47 } 48 49 void mlx5e_rss_params_indir_init_uniform(struct mlx5e_rss_params_indir *indir, 50 unsigned int num_channels) 51 { 52 unsigned int i; 53 54 for (i = 0; i < indir->actual_table_size; i++) 55 indir->table[i] = i % num_channels; 56 } 57 58 static void fill_rqn_list(void *rqtc, u32 *rqns, u32 *vhca_ids, unsigned int size) 59 { 60 unsigned int i; 61 62 if (vhca_ids) { 63 MLX5_SET(rqtc, rqtc, rq_vhca_id_format, 1); 64 for (i = 0; i < size; i++) { 65 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_num, rqns[i]); 66 MLX5_SET(rqtc, rqtc, rq_vhca[i].rq_vhca_id, vhca_ids[i]); 67 } 68 } else { 69 for (i = 0; i < size; i++) 70 MLX5_SET(rqtc, rqtc, rq_num[i], rqns[i]); 71 } 72 } 73 static int mlx5e_rqt_init(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 74 u16 max_size, u32 *init_rqns, u32 *init_vhca_ids, u16 init_size) 75 { 76 int entry_sz; 77 void *rqtc; 78 int inlen; 79 int err; 80 u32 *in; 81 82 if (!rqt_verify_vhca_ids(mdev, init_vhca_ids, init_size)) 83 return -EOPNOTSUPP; 84 85 rqt->mdev = mdev; 86 rqt->size = max_size; 87 88 entry_sz = init_vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); 89 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + entry_sz * init_size; 90 in = kvzalloc(inlen, GFP_KERNEL); 91 if (!in) 92 return -ENOMEM; 93 94 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 95 96 MLX5_SET(rqtc, rqtc, rqt_max_size, rqt->size); 97 MLX5_SET(rqtc, rqtc, rqt_actual_size, init_size); 98 99 fill_rqn_list(rqtc, init_rqns, init_vhca_ids, init_size); 100 101 err = mlx5_core_create_rqt(rqt->mdev, in, inlen, &rqt->rqtn); 102 103 kvfree(in); 104 return err; 105 } 106 107 int mlx5e_rqt_init_direct(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 108 bool indir_enabled, u32 init_rqn, u32 indir_table_size) 109 { 110 u16 max_size = indir_enabled ? indir_table_size : 1; 111 112 return mlx5e_rqt_init(rqt, mdev, max_size, &init_rqn, NULL, 1); 113 } 114 115 static int mlx5e_bits_invert(unsigned long a, int size) 116 { 117 int inv = 0; 118 int i; 119 120 for (i = 0; i < size; i++) 121 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; 122 123 return inv; 124 } 125 126 static int mlx5e_calc_indir_rqns(u32 *rss_rqns, u32 *rqns, u32 *rss_vhca_ids, u32 *vhca_ids, 127 unsigned int num_rqns, 128 u8 hfunc, struct mlx5e_rss_params_indir *indir) 129 { 130 unsigned int i; 131 132 for (i = 0; i < indir->actual_table_size; i++) { 133 unsigned int ix = i; 134 135 if (hfunc == ETH_RSS_HASH_XOR) 136 ix = mlx5e_bits_invert(ix, ilog2(indir->actual_table_size)); 137 138 ix = indir->table[ix]; 139 140 if (WARN_ON(ix >= num_rqns)) 141 /* Could be a bug in the driver or in the kernel part of 142 * ethtool: indir table refers to non-existent RQs. 143 */ 144 return -EINVAL; 145 rss_rqns[i] = rqns[ix]; 146 if (vhca_ids) 147 rss_vhca_ids[i] = vhca_ids[ix]; 148 } 149 150 return 0; 151 } 152 153 int mlx5e_rqt_init_indir(struct mlx5e_rqt *rqt, struct mlx5_core_dev *mdev, 154 u32 *rqns, u32 *vhca_ids, unsigned int num_rqns, 155 u8 hfunc, struct mlx5e_rss_params_indir *indir) 156 { 157 u32 *rss_rqns, *rss_vhca_ids = NULL; 158 int err; 159 160 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); 161 if (!rss_rqns) 162 return -ENOMEM; 163 164 if (vhca_ids) { 165 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), 166 GFP_KERNEL); 167 if (!rss_vhca_ids) { 168 kvfree(rss_rqns); 169 return -ENOMEM; 170 } 171 } 172 173 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); 174 if (err) 175 goto out; 176 177 err = mlx5e_rqt_init(rqt, mdev, indir->max_table_size, rss_rqns, rss_vhca_ids, 178 indir->actual_table_size); 179 180 out: 181 kvfree(rss_vhca_ids); 182 kvfree(rss_rqns); 183 return err; 184 } 185 186 u32 mlx5e_rqt_size(struct mlx5_core_dev *mdev, unsigned int num_channels) 187 { 188 u32 rqt_size = max_t(u32, MLX5E_INDIR_MIN_RQT_SIZE, 189 roundup_pow_of_two(num_channels * MLX5E_UNIFORM_SPREAD_RQT_FACTOR)); 190 u32 max_cap_rqt_size = 1 << MLX5_CAP_GEN(mdev, log_max_rqt_size); 191 192 return min_t(u32, rqt_size, max_cap_rqt_size); 193 } 194 195 void mlx5e_rqt_destroy(struct mlx5e_rqt *rqt) 196 { 197 mlx5_core_destroy_rqt(rqt->mdev, rqt->rqtn); 198 } 199 200 static int mlx5e_rqt_redirect(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, 201 unsigned int size) 202 { 203 int entry_sz; 204 void *rqtc; 205 int inlen; 206 u32 *in; 207 int err; 208 209 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, size)) 210 return -EINVAL; 211 212 entry_sz = vhca_ids ? MLX5_ST_SZ_BYTES(rq_vhca) : MLX5_ST_SZ_BYTES(rq_num); 213 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + entry_sz * size; 214 in = kvzalloc(inlen, GFP_KERNEL); 215 if (!in) 216 return -ENOMEM; 217 218 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 219 220 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); 221 MLX5_SET(rqtc, rqtc, rqt_actual_size, size); 222 223 fill_rqn_list(rqtc, rqns, vhca_ids, size); 224 225 err = mlx5_core_modify_rqt(rqt->mdev, rqt->rqtn, in, inlen); 226 227 kvfree(in); 228 return err; 229 } 230 231 int mlx5e_rqt_redirect_direct(struct mlx5e_rqt *rqt, u32 rqn, u32 *vhca_id) 232 { 233 return mlx5e_rqt_redirect(rqt, &rqn, vhca_id, 1); 234 } 235 236 int mlx5e_rqt_redirect_indir(struct mlx5e_rqt *rqt, u32 *rqns, u32 *vhca_ids, 237 unsigned int num_rqns, 238 u8 hfunc, struct mlx5e_rss_params_indir *indir) 239 { 240 u32 *rss_rqns, *rss_vhca_ids = NULL; 241 int err; 242 243 if (!rqt_verify_vhca_ids(rqt->mdev, vhca_ids, num_rqns)) 244 return -EINVAL; 245 246 if (WARN_ON(rqt->size != indir->max_table_size)) 247 return -EINVAL; 248 249 rss_rqns = kvmalloc_array(indir->actual_table_size, sizeof(*rss_rqns), GFP_KERNEL); 250 if (!rss_rqns) 251 return -ENOMEM; 252 253 if (vhca_ids) { 254 rss_vhca_ids = kvmalloc_array(indir->actual_table_size, sizeof(*rss_vhca_ids), 255 GFP_KERNEL); 256 if (!rss_vhca_ids) { 257 kvfree(rss_rqns); 258 return -ENOMEM; 259 } 260 } 261 262 err = mlx5e_calc_indir_rqns(rss_rqns, rqns, rss_vhca_ids, vhca_ids, num_rqns, hfunc, indir); 263 if (err) 264 goto out; 265 266 err = mlx5e_rqt_redirect(rqt, rss_rqns, rss_vhca_ids, indir->actual_table_size); 267 268 out: 269 kvfree(rss_vhca_ids); 270 kvfree(rss_rqns); 271 return err; 272 } 273