1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4 */
5
6 #include "mlx5_ib.h"
7 #include <linux/mlx5/eswitch.h>
8 #include <linux/mlx5/vport.h>
9 #include "counters.h"
10 #include "ib_rep.h"
11 #include "qp.h"
12
13 struct mlx5_ib_counter {
14 const char *name;
15 size_t offset;
16 u32 type;
17 };
18
19 #define INIT_Q_COUNTER(_name) \
20 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)}
21
22 #define INIT_VPORT_Q_COUNTER(_name) \
23 { .name = "vport_" #_name, .offset = \
24 MLX5_BYTE_OFF(query_q_counter_out, _name)}
25
26 static const struct mlx5_ib_counter basic_q_cnts[] = {
27 INIT_Q_COUNTER(rx_write_requests),
28 INIT_Q_COUNTER(rx_read_requests),
29 INIT_Q_COUNTER(rx_atomic_requests),
30 INIT_Q_COUNTER(rx_dct_connect),
31 INIT_Q_COUNTER(out_of_buffer),
32 };
33
34 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = {
35 INIT_Q_COUNTER(out_of_sequence),
36 };
37
38 static const struct mlx5_ib_counter retrans_q_cnts[] = {
39 INIT_Q_COUNTER(duplicate_request),
40 INIT_Q_COUNTER(rnr_nak_retry_err),
41 INIT_Q_COUNTER(packet_seq_err),
42 INIT_Q_COUNTER(implied_nak_seq_err),
43 INIT_Q_COUNTER(local_ack_timeout_err),
44 };
45
46 static const struct mlx5_ib_counter vport_basic_q_cnts[] = {
47 INIT_VPORT_Q_COUNTER(rx_write_requests),
48 INIT_VPORT_Q_COUNTER(rx_read_requests),
49 INIT_VPORT_Q_COUNTER(rx_atomic_requests),
50 INIT_VPORT_Q_COUNTER(rx_dct_connect),
51 INIT_VPORT_Q_COUNTER(out_of_buffer),
52 };
53
54 static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = {
55 INIT_VPORT_Q_COUNTER(out_of_sequence),
56 };
57
58 static const struct mlx5_ib_counter vport_retrans_q_cnts[] = {
59 INIT_VPORT_Q_COUNTER(duplicate_request),
60 INIT_VPORT_Q_COUNTER(rnr_nak_retry_err),
61 INIT_VPORT_Q_COUNTER(packet_seq_err),
62 INIT_VPORT_Q_COUNTER(implied_nak_seq_err),
63 INIT_VPORT_Q_COUNTER(local_ack_timeout_err),
64 };
65
66 #define INIT_CONG_COUNTER(_name) \
67 { .name = #_name, .offset = \
68 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)}
69
70 static const struct mlx5_ib_counter cong_cnts[] = {
71 INIT_CONG_COUNTER(rp_cnp_ignored),
72 INIT_CONG_COUNTER(rp_cnp_handled),
73 INIT_CONG_COUNTER(np_ecn_marked_roce_packets),
74 INIT_CONG_COUNTER(np_cnp_sent),
75 };
76
77 static const struct mlx5_ib_counter extended_err_cnts[] = {
78 INIT_Q_COUNTER(resp_local_length_error),
79 INIT_Q_COUNTER(resp_cqe_error),
80 INIT_Q_COUNTER(req_cqe_error),
81 INIT_Q_COUNTER(req_remote_invalid_request),
82 INIT_Q_COUNTER(req_remote_access_errors),
83 INIT_Q_COUNTER(resp_remote_access_errors),
84 INIT_Q_COUNTER(resp_cqe_flush_error),
85 INIT_Q_COUNTER(req_cqe_flush_error),
86 INIT_Q_COUNTER(req_transport_retries_exceeded),
87 INIT_Q_COUNTER(req_rnr_retries_exceeded),
88 };
89
90 static const struct mlx5_ib_counter roce_accl_cnts[] = {
91 INIT_Q_COUNTER(roce_adp_retrans),
92 INIT_Q_COUNTER(roce_adp_retrans_to),
93 INIT_Q_COUNTER(roce_slow_restart),
94 INIT_Q_COUNTER(roce_slow_restart_cnps),
95 INIT_Q_COUNTER(roce_slow_restart_trans),
96 };
97
98 static const struct mlx5_ib_counter vport_extended_err_cnts[] = {
99 INIT_VPORT_Q_COUNTER(resp_local_length_error),
100 INIT_VPORT_Q_COUNTER(resp_cqe_error),
101 INIT_VPORT_Q_COUNTER(req_cqe_error),
102 INIT_VPORT_Q_COUNTER(req_remote_invalid_request),
103 INIT_VPORT_Q_COUNTER(req_remote_access_errors),
104 INIT_VPORT_Q_COUNTER(resp_remote_access_errors),
105 INIT_VPORT_Q_COUNTER(resp_cqe_flush_error),
106 INIT_VPORT_Q_COUNTER(req_cqe_flush_error),
107 INIT_VPORT_Q_COUNTER(req_transport_retries_exceeded),
108 INIT_VPORT_Q_COUNTER(req_rnr_retries_exceeded),
109 };
110
111 static const struct mlx5_ib_counter vport_roce_accl_cnts[] = {
112 INIT_VPORT_Q_COUNTER(roce_adp_retrans),
113 INIT_VPORT_Q_COUNTER(roce_adp_retrans_to),
114 INIT_VPORT_Q_COUNTER(roce_slow_restart),
115 INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps),
116 INIT_VPORT_Q_COUNTER(roce_slow_restart_trans),
117 };
118
119 #define INIT_EXT_PPCNT_COUNTER(_name) \
120 { .name = #_name, .offset = \
121 MLX5_BYTE_OFF(ppcnt_reg, \
122 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)}
123
124 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = {
125 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated),
126 };
127
128 #define INIT_OP_COUNTER(_name, _type) \
129 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type}
130
131 static const struct mlx5_ib_counter basic_op_cnts[] = {
132 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS),
133 };
134
135 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = {
136 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS),
137 };
138
139 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = {
140 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS),
141 };
142
143 static const struct mlx5_ib_counter packets_op_cnts[] = {
144 INIT_OP_COUNTER(rdma_tx_packets, RDMA_TX_PACKETS),
145 INIT_OP_COUNTER(rdma_tx_bytes, RDMA_TX_BYTES),
146 INIT_OP_COUNTER(rdma_rx_packets, RDMA_RX_PACKETS),
147 INIT_OP_COUNTER(rdma_rx_bytes, RDMA_RX_BYTES),
148 };
149
mlx5_ib_read_counters(struct ib_counters * counters,struct ib_counters_read_attr * read_attr,struct uverbs_attr_bundle * attrs)150 static int mlx5_ib_read_counters(struct ib_counters *counters,
151 struct ib_counters_read_attr *read_attr,
152 struct uverbs_attr_bundle *attrs)
153 {
154 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
155 struct mlx5_read_counters_attr mread_attr = {};
156 struct mlx5_ib_flow_counters_desc *desc;
157 int ret, i;
158
159 mutex_lock(&mcounters->mcntrs_mutex);
160 if (mcounters->cntrs_max_index > read_attr->ncounters) {
161 ret = -EINVAL;
162 goto err_bound;
163 }
164
165 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64),
166 GFP_KERNEL);
167 if (!mread_attr.out) {
168 ret = -ENOMEM;
169 goto err_bound;
170 }
171
172 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl;
173 mread_attr.flags = read_attr->flags;
174 ret = mcounters->read_counters(counters->device, &mread_attr);
175 if (ret)
176 goto err_read;
177
178 /* do the pass over the counters data array to assign according to the
179 * descriptions and indexing pairs
180 */
181 desc = mcounters->counters_data;
182 for (i = 0; i < mcounters->ncounters; i++)
183 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description];
184
185 err_read:
186 kfree(mread_attr.out);
187 err_bound:
188 mutex_unlock(&mcounters->mcntrs_mutex);
189 return ret;
190 }
191
mlx5_ib_destroy_counters(struct ib_counters * counters)192 static int mlx5_ib_destroy_counters(struct ib_counters *counters)
193 {
194 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
195
196 mlx5_ib_counters_clear_description(counters);
197 if (mcounters->hw_cntrs_hndl)
198 mlx5_fc_destroy(to_mdev(counters->device)->mdev,
199 mcounters->hw_cntrs_hndl);
200 return 0;
201 }
202
mlx5_ib_create_counters(struct ib_counters * counters,struct uverbs_attr_bundle * attrs)203 static int mlx5_ib_create_counters(struct ib_counters *counters,
204 struct uverbs_attr_bundle *attrs)
205 {
206 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
207
208 mutex_init(&mcounters->mcntrs_mutex);
209 return 0;
210 }
211
vport_qcounters_supported(struct mlx5_ib_dev * dev)212 static bool vport_qcounters_supported(struct mlx5_ib_dev *dev)
213 {
214 return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) &&
215 MLX5_CAP_GEN(dev->mdev, q_counter_aggregation);
216 }
217
get_counters(struct mlx5_ib_dev * dev,u32 port_num)218 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev,
219 u32 port_num)
220 {
221 if ((is_mdev_switchdev_mode(dev->mdev) &&
222 !vport_qcounters_supported(dev)) || !port_num)
223 return &dev->port[0].cnts;
224
225 return is_mdev_switchdev_mode(dev->mdev) ?
226 &dev->port[1].cnts : &dev->port[port_num - 1].cnts;
227 }
228
229 /**
230 * mlx5_ib_get_counters_id - Returns counters id to use for device+port
231 * @dev: Pointer to mlx5 IB device
232 * @port_num: Zero based port number
233 *
234 * mlx5_ib_get_counters_id() Returns counters set id to use for given
235 * device port combination in switchdev and non switchdev mode of the
236 * parent device.
237 */
mlx5_ib_get_counters_id(struct mlx5_ib_dev * dev,u32 port_num)238 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num)
239 {
240 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1);
241
242 return cnts->set_id;
243 }
244
do_alloc_stats(const struct mlx5_ib_counters * cnts)245 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts)
246 {
247 struct rdma_hw_stats *stats;
248 u32 num_hw_counters;
249 int i;
250
251 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
252 cnts->num_ext_ppcnt_counters;
253 stats = rdma_alloc_hw_stats_struct(cnts->descs,
254 num_hw_counters +
255 cnts->num_op_counters,
256 RDMA_HW_STATS_DEFAULT_LIFESPAN);
257 if (!stats)
258 return NULL;
259
260 for (i = 0; i < cnts->num_op_counters; i++)
261 set_bit(num_hw_counters + i, stats->is_disabled);
262
263 return stats;
264 }
265
266 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_device_stats(struct ib_device * ibdev)267 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev)
268 {
269 struct mlx5_ib_dev *dev = to_mdev(ibdev);
270 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts;
271
272 return do_alloc_stats(cnts);
273 }
274
275 static struct rdma_hw_stats *
mlx5_ib_alloc_hw_port_stats(struct ib_device * ibdev,u32 port_num)276 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num)
277 {
278 struct mlx5_ib_dev *dev = to_mdev(ibdev);
279 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
280
281 return do_alloc_stats(cnts);
282 }
283
mlx5_ib_query_q_counters(struct mlx5_core_dev * mdev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats,u16 set_id)284 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev,
285 const struct mlx5_ib_counters *cnts,
286 struct rdma_hw_stats *stats,
287 u16 set_id)
288 {
289 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
290 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
291 __be32 val;
292 int ret, i;
293
294 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
295 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id);
296 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
297 if (ret)
298 return ret;
299
300 for (i = 0; i < cnts->num_q_counters; i++) {
301 val = *(__be32 *)((void *)out + cnts->offsets[i]);
302 stats->value[i] = (u64)be32_to_cpu(val);
303 }
304
305 return 0;
306 }
307
mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev * dev,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)308 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev,
309 const struct mlx5_ib_counters *cnts,
310 struct rdma_hw_stats *stats)
311 {
312 int offset = cnts->num_q_counters + cnts->num_cong_counters;
313 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {};
314 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
315 int ret, i;
316 void *out;
317
318 out = kvzalloc(sz, GFP_KERNEL);
319 if (!out)
320 return -ENOMEM;
321
322 MLX5_SET(ppcnt_reg, in, local_port, 1);
323 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
324 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT,
325 0, 0);
326 if (ret)
327 goto free;
328
329 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++)
330 stats->value[i + offset] =
331 be64_to_cpup((__be64 *)(out +
332 cnts->offsets[i + offset]));
333 free:
334 kvfree(out);
335 return ret;
336 }
337
mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev * dev,u32 port_num,const struct mlx5_ib_counters * cnts,struct rdma_hw_stats * stats)338 static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev,
339 u32 port_num,
340 const struct mlx5_ib_counters *cnts,
341 struct rdma_hw_stats *stats)
342
343 {
344 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {};
345 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {};
346 struct mlx5_core_dev *mdev;
347 __be32 val;
348 int ret, i;
349
350 if (!dev->port[port_num].rep ||
351 dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK)
352 return 0;
353
354 mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw);
355 if (!mdev)
356 return -EOPNOTSUPP;
357
358 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER);
359 MLX5_SET(query_q_counter_in, in, other_vport, 1);
360 MLX5_SET(query_q_counter_in, in, vport_number,
361 dev->port[port_num].rep->vport);
362 MLX5_SET(query_q_counter_in, in, aggregate, 1);
363 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out);
364 if (ret)
365 return ret;
366
367 for (i = 0; i < cnts->num_q_counters; i++) {
368 val = *(__be32 *)((void *)out + cnts->offsets[i]);
369 stats->value[i] = (u64)be32_to_cpu(val);
370 }
371
372 return 0;
373 }
374
do_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)375 static int do_get_hw_stats(struct ib_device *ibdev,
376 struct rdma_hw_stats *stats,
377 u32 port_num, int index)
378 {
379 struct mlx5_ib_dev *dev = to_mdev(ibdev);
380 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num);
381 struct mlx5_core_dev *mdev;
382 int ret, num_counters;
383
384 if (!stats)
385 return -EINVAL;
386
387 num_counters = cnts->num_q_counters +
388 cnts->num_cong_counters +
389 cnts->num_ext_ppcnt_counters;
390
391 if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0)
392 ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts,
393 stats);
394 else
395 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats,
396 cnts->set_id);
397 if (ret)
398 return ret;
399
400 /* We don't expose device counters over Vports */
401 if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0)
402 goto done;
403
404 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
405 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats);
406 if (ret)
407 return ret;
408 }
409
410 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
411 if (!port_num)
412 port_num = 1;
413 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL);
414 if (!mdev) {
415 /* If port is not affiliated yet, its in down state
416 * which doesn't have any counters yet, so it would be
417 * zero. So no need to read from the HCA.
418 */
419 goto done;
420 }
421 ret = mlx5_lag_query_cong_counters(dev->mdev,
422 stats->value +
423 cnts->num_q_counters,
424 cnts->num_cong_counters,
425 cnts->offsets +
426 cnts->num_q_counters);
427
428 mlx5_ib_put_native_port_mdev(dev, port_num);
429 if (ret)
430 return ret;
431 }
432
433 done:
434 return num_counters;
435 }
436
is_rdma_bytes_counter(u32 type)437 static bool is_rdma_bytes_counter(u32 type)
438 {
439 if (type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES ||
440 type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES ||
441 type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP ||
442 type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP)
443 return true;
444
445 return false;
446 }
447
do_per_qp_get_op_stat(struct rdma_counter * counter)448 static int do_per_qp_get_op_stat(struct rdma_counter *counter)
449 {
450 struct mlx5_ib_dev *dev = to_mdev(counter->device);
451 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
452 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
453 int i, ret, index, num_hw_counters;
454 u64 packets = 0, bytes = 0;
455
456 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP;
457 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) {
458 if (!mcounter->fc[i])
459 continue;
460
461 ret = mlx5_fc_query(dev->mdev, mcounter->fc[i],
462 &packets, &bytes);
463 if (ret)
464 return ret;
465
466 num_hw_counters = cnts->num_q_counters +
467 cnts->num_cong_counters +
468 cnts->num_ext_ppcnt_counters;
469
470 index = i - MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP +
471 num_hw_counters;
472
473 if (is_rdma_bytes_counter(i))
474 counter->stats->value[index] = bytes;
475 else
476 counter->stats->value[index] = packets;
477
478 clear_bit(index, counter->stats->is_disabled);
479 }
480 return 0;
481 }
482
do_get_op_stat(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)483 static int do_get_op_stat(struct ib_device *ibdev,
484 struct rdma_hw_stats *stats,
485 u32 port_num, int index)
486 {
487 struct mlx5_ib_dev *dev = to_mdev(ibdev);
488 const struct mlx5_ib_counters *cnts;
489 const struct mlx5_ib_op_fc *opfcs;
490 u64 packets, bytes;
491 u32 type;
492 int ret;
493
494 cnts = get_counters(dev, port_num);
495
496 opfcs = cnts->opfcs;
497 type = *(u32 *)cnts->descs[index].priv;
498 if (type >= MLX5_IB_OPCOUNTER_MAX)
499 return -EINVAL;
500
501 if (!opfcs[type].fc)
502 goto out;
503
504 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc,
505 &packets, &bytes);
506 if (ret)
507 return ret;
508
509 if (is_rdma_bytes_counter(type))
510 stats->value[index] = bytes;
511 else
512 stats->value[index] = packets;
513 out:
514 return index;
515 }
516
do_get_op_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num)517 static int do_get_op_stats(struct ib_device *ibdev,
518 struct rdma_hw_stats *stats,
519 u32 port_num)
520 {
521 struct mlx5_ib_dev *dev = to_mdev(ibdev);
522 const struct mlx5_ib_counters *cnts;
523 int index, ret, num_hw_counters;
524
525 cnts = get_counters(dev, port_num);
526 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
527 cnts->num_ext_ppcnt_counters;
528 for (index = num_hw_counters;
529 index < (num_hw_counters + cnts->num_op_counters); index++) {
530 ret = do_get_op_stat(ibdev, stats, port_num, index);
531 if (ret != index)
532 return ret;
533 }
534
535 return cnts->num_op_counters;
536 }
537
mlx5_ib_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u32 port_num,int index)538 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
539 struct rdma_hw_stats *stats,
540 u32 port_num, int index)
541 {
542 int num_counters, num_hw_counters, num_op_counters;
543 struct mlx5_ib_dev *dev = to_mdev(ibdev);
544 const struct mlx5_ib_counters *cnts;
545
546 cnts = get_counters(dev, port_num);
547 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
548 cnts->num_ext_ppcnt_counters;
549 num_counters = num_hw_counters + cnts->num_op_counters;
550
551 if (index < 0 || index > num_counters)
552 return -EINVAL;
553 else if (index > 0 && index < num_hw_counters)
554 return do_get_hw_stats(ibdev, stats, port_num, index);
555 else if (index >= num_hw_counters && index < num_counters)
556 return do_get_op_stat(ibdev, stats, port_num, index);
557
558 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index);
559 if (num_hw_counters < 0)
560 return num_hw_counters;
561
562 num_op_counters = do_get_op_stats(ibdev, stats, port_num);
563 if (num_op_counters < 0)
564 return num_op_counters;
565
566 return num_hw_counters + num_op_counters;
567 }
568
569 static struct rdma_hw_stats *
mlx5_ib_counter_alloc_stats(struct rdma_counter * counter)570 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter)
571 {
572 struct mlx5_ib_dev *dev = to_mdev(counter->device);
573 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
574
575 return do_alloc_stats(cnts);
576 }
577
mlx5_ib_counter_update_stats(struct rdma_counter * counter)578 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter)
579 {
580 struct mlx5_ib_dev *dev = to_mdev(counter->device);
581 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port);
582 int ret;
583
584 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, counter->stats,
585 counter->id);
586 if (ret)
587 return ret;
588
589 if (!counter->mode.bind_opcnt)
590 return 0;
591
592 return do_per_qp_get_op_stat(counter);
593 }
594
mlx5_ib_counter_dealloc(struct rdma_counter * counter)595 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter)
596 {
597 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
598 struct mlx5_ib_dev *dev = to_mdev(counter->device);
599 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
600
601 if (!counter->id)
602 return 0;
603
604 WARN_ON(!xa_empty(&mcounter->qpn_opfc_xa));
605 mlx5r_fs_destroy_fcs(dev, counter);
606 MLX5_SET(dealloc_q_counter_in, in, opcode,
607 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
608 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id);
609 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
610 }
611
mlx5_ib_counter_bind_qp(struct rdma_counter * counter,struct ib_qp * qp,u32 port)612 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter,
613 struct ib_qp *qp, u32 port)
614 {
615 struct mlx5_ib_dev *dev = to_mdev(qp->device);
616 bool new = false;
617 int err;
618
619 if (!counter->id) {
620 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
621 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
622
623 MLX5_SET(alloc_q_counter_in, in, opcode,
624 MLX5_CMD_OP_ALLOC_Q_COUNTER);
625 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID);
626 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
627 if (err)
628 return err;
629 counter->id =
630 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
631 new = true;
632 }
633
634 err = mlx5_ib_qp_set_counter(qp, counter);
635 if (err)
636 goto fail_set_counter;
637
638 err = mlx5r_fs_bind_op_fc(qp, counter, port);
639 if (err)
640 goto fail_bind_op_fc;
641
642 return 0;
643
644 fail_bind_op_fc:
645 mlx5_ib_qp_set_counter(qp, NULL);
646 fail_set_counter:
647 if (new) {
648 mlx5_ib_counter_dealloc(counter);
649 counter->id = 0;
650 }
651
652 return err;
653 }
654
mlx5_ib_counter_unbind_qp(struct ib_qp * qp,u32 port)655 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp, u32 port)
656 {
657 struct rdma_counter *counter = qp->counter;
658 int err;
659
660 mlx5r_fs_unbind_op_fc(qp, counter);
661
662 err = mlx5_ib_qp_set_counter(qp, NULL);
663 if (err)
664 goto fail_set_counter;
665
666 return 0;
667
668 fail_set_counter:
669 mlx5r_fs_bind_op_fc(qp, counter, port);
670 return err;
671 }
672
mlx5_ib_fill_counters(struct mlx5_ib_dev * dev,struct rdma_stat_desc * descs,size_t * offsets,u32 port_num)673 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev,
674 struct rdma_stat_desc *descs, size_t *offsets,
675 u32 port_num)
676 {
677 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
678 port_num != MLX5_VPORT_PF;
679 const struct mlx5_ib_counter *names;
680 int j = 0, i, size;
681
682 names = is_vport ? vport_basic_q_cnts : basic_q_cnts;
683 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
684 ARRAY_SIZE(basic_q_cnts);
685 for (i = 0; i < size; i++, j++) {
686 descs[j].name = names[i].name;
687 offsets[j] = names[i].offset;
688 }
689
690 names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts;
691 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
692 ARRAY_SIZE(out_of_seq_q_cnts);
693 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) {
694 for (i = 0; i < size; i++, j++) {
695 descs[j].name = names[i].name;
696 offsets[j] = names[i].offset;
697 }
698 }
699
700 names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts;
701 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
702 ARRAY_SIZE(retrans_q_cnts);
703 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
704 for (i = 0; i < size; i++, j++) {
705 descs[j].name = names[i].name;
706 offsets[j] = names[i].offset;
707 }
708 }
709
710 names = is_vport ? vport_extended_err_cnts : extended_err_cnts;
711 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
712 ARRAY_SIZE(extended_err_cnts);
713 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) {
714 for (i = 0; i < size; i++, j++) {
715 descs[j].name = names[i].name;
716 offsets[j] = names[i].offset;
717 }
718 }
719
720 names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts;
721 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
722 ARRAY_SIZE(roce_accl_cnts);
723 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) {
724 for (i = 0; i < size; i++, j++) {
725 descs[j].name = names[i].name;
726 offsets[j] = names[i].offset;
727 }
728 }
729
730 if (is_vport)
731 return;
732
733 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
734 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) {
735 descs[j].name = cong_cnts[i].name;
736 offsets[j] = cong_cnts[i].offset;
737 }
738 }
739
740 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
741 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) {
742 descs[j].name = ext_ppcnt_cnts[i].name;
743 offsets[j] = ext_ppcnt_cnts[i].offset;
744 }
745 }
746
747 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) {
748 descs[j].name = basic_op_cnts[i].name;
749 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
750 descs[j].priv = &basic_op_cnts[i].type;
751 }
752
753 if (MLX5_CAP_FLOWTABLE(dev->mdev,
754 ft_field_support_2_nic_receive_rdma.bth_opcode)) {
755 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) {
756 descs[j].name = rdmarx_cnp_op_cnts[i].name;
757 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
758 descs[j].priv = &rdmarx_cnp_op_cnts[i].type;
759 }
760 }
761
762 if (MLX5_CAP_FLOWTABLE(dev->mdev,
763 ft_field_support_2_nic_transmit_rdma.bth_opcode)) {
764 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) {
765 descs[j].name = rdmatx_cnp_op_cnts[i].name;
766 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
767 descs[j].priv = &rdmatx_cnp_op_cnts[i].type;
768 }
769 }
770
771 for (i = 0; i < ARRAY_SIZE(packets_op_cnts); i++, j++) {
772 descs[j].name = packets_op_cnts[i].name;
773 descs[j].flags |= IB_STAT_FLAG_OPTIONAL;
774 descs[j].priv = &packets_op_cnts[i].type;
775 }
776 }
777
778
__mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev,struct mlx5_ib_counters * cnts,u32 port_num)779 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev,
780 struct mlx5_ib_counters *cnts, u32 port_num)
781 {
782 bool is_vport = is_mdev_switchdev_mode(dev->mdev) &&
783 port_num != MLX5_VPORT_PF;
784 u32 num_counters, num_op_counters = 0, size;
785
786 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) :
787 ARRAY_SIZE(basic_q_cnts);
788 num_counters = size;
789
790 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) :
791 ARRAY_SIZE(out_of_seq_q_cnts);
792 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt))
793 num_counters += size;
794
795 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) :
796 ARRAY_SIZE(retrans_q_cnts);
797 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters))
798 num_counters += size;
799
800 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) :
801 ARRAY_SIZE(extended_err_cnts);
802 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters))
803 num_counters += size;
804
805 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) :
806 ARRAY_SIZE(roce_accl_cnts);
807 if (MLX5_CAP_GEN(dev->mdev, roce_accl))
808 num_counters += size;
809
810 cnts->num_q_counters = num_counters;
811
812 if (is_vport)
813 goto skip_non_qcounters;
814
815 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) {
816 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts);
817 num_counters += ARRAY_SIZE(cong_cnts);
818 }
819 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) {
820 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts);
821 num_counters += ARRAY_SIZE(ext_ppcnt_cnts);
822 }
823
824 num_op_counters = ARRAY_SIZE(basic_op_cnts);
825
826 num_op_counters += ARRAY_SIZE(packets_op_cnts);
827
828 if (MLX5_CAP_FLOWTABLE(dev->mdev,
829 ft_field_support_2_nic_receive_rdma.bth_opcode))
830 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts);
831
832 if (MLX5_CAP_FLOWTABLE(dev->mdev,
833 ft_field_support_2_nic_transmit_rdma.bth_opcode))
834 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts);
835
836 skip_non_qcounters:
837 cnts->num_op_counters = num_op_counters;
838 num_counters += num_op_counters;
839 cnts->descs = kcalloc(num_counters,
840 sizeof(struct rdma_stat_desc), GFP_KERNEL);
841 if (!cnts->descs)
842 return -ENOMEM;
843
844 cnts->offsets = kcalloc(num_counters,
845 sizeof(*cnts->offsets), GFP_KERNEL);
846 if (!cnts->offsets)
847 goto err;
848
849 return 0;
850
851 err:
852 kfree(cnts->descs);
853 cnts->descs = NULL;
854 return -ENOMEM;
855 }
856
857 /*
858 * Checks if the given flow counter type should be sharing the same flow counter
859 * with another type and if it should, checks if that other type flow counter
860 * was already created, if both conditions are met return true and the counter
861 * else return false.
862 */
mlx5r_is_opfc_shared_and_in_use(struct mlx5_ib_op_fc * opfcs,u32 type,struct mlx5_ib_op_fc ** opfc)863 bool mlx5r_is_opfc_shared_and_in_use(struct mlx5_ib_op_fc *opfcs, u32 type,
864 struct mlx5_ib_op_fc **opfc)
865 {
866 u32 shared_fc_type;
867
868 switch (type) {
869 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS:
870 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES;
871 break;
872 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES:
873 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS;
874 break;
875 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS:
876 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES;
877 break;
878 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES:
879 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS;
880 break;
881 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP:
882 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP;
883 break;
884 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP:
885 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP;
886 break;
887 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP:
888 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP;
889 break;
890 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP:
891 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP;
892 break;
893 default:
894 return false;
895 }
896
897 *opfc = &opfcs[shared_fc_type];
898 if (!(*opfc)->fc)
899 return false;
900
901 return true;
902 }
903
mlx5_ib_dealloc_counters(struct mlx5_ib_dev * dev)904 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev)
905 {
906 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
907 int num_cnt_ports = dev->num_ports;
908 struct mlx5_ib_op_fc *in_use_opfc;
909 int i, j;
910
911 if (is_mdev_switchdev_mode(dev->mdev))
912 num_cnt_ports = min(2, num_cnt_ports);
913
914 MLX5_SET(dealloc_q_counter_in, in, opcode,
915 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
916
917 for (i = 0; i < num_cnt_ports; i++) {
918 if (dev->port[i].cnts.set_id) {
919 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
920 dev->port[i].cnts.set_id);
921 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in);
922 }
923 kfree(dev->port[i].cnts.descs);
924 kfree(dev->port[i].cnts.offsets);
925
926 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) {
927 if (!dev->port[i].cnts.opfcs[j].fc)
928 continue;
929
930 if (mlx5r_is_opfc_shared_and_in_use(
931 dev->port[i].cnts.opfcs, j, &in_use_opfc))
932 goto skip;
933
934 mlx5_ib_fs_remove_op_fc(dev,
935 &dev->port[i].cnts.opfcs[j], j);
936 mlx5_fc_destroy(dev->mdev,
937 dev->port[i].cnts.opfcs[j].fc);
938 skip:
939 dev->port[i].cnts.opfcs[j].fc = NULL;
940 }
941 }
942 }
943
mlx5_ib_alloc_counters(struct mlx5_ib_dev * dev)944 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev)
945 {
946 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
947 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
948 int num_cnt_ports = dev->num_ports;
949 int err = 0;
950 int i;
951 bool is_shared;
952
953 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
954 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0;
955
956 /*
957 * In switchdev we need to allocate two ports, one that is used for
958 * the device Q_counters and it is essentially the real Q_counters of
959 * this device, while the other is used as a helper for PF to be able to
960 * query all other vports.
961 */
962 if (is_mdev_switchdev_mode(dev->mdev))
963 num_cnt_ports = min(2, num_cnt_ports);
964
965 for (i = 0; i < num_cnt_ports; i++) {
966 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i);
967 if (err)
968 goto err_alloc;
969
970 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs,
971 dev->port[i].cnts.offsets, i);
972
973 MLX5_SET(alloc_q_counter_in, in, uid,
974 is_shared ? MLX5_SHARED_RESOURCE_UID : 0);
975
976 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out);
977 if (err) {
978 mlx5_ib_warn(dev,
979 "couldn't allocate queue counter for port %d, err %d\n",
980 i + 1, err);
981 goto err_alloc;
982 }
983
984 dev->port[i].cnts.set_id =
985 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
986 }
987 return 0;
988
989 err_alloc:
990 mlx5_ib_dealloc_counters(dev);
991 return err;
992 }
993
read_flow_counters(struct ib_device * ibdev,struct mlx5_read_counters_attr * read_attr)994 static int read_flow_counters(struct ib_device *ibdev,
995 struct mlx5_read_counters_attr *read_attr)
996 {
997 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl;
998 struct mlx5_ib_dev *dev = to_mdev(ibdev);
999
1000 return mlx5_fc_query(dev->mdev, fc,
1001 &read_attr->out[IB_COUNTER_PACKETS],
1002 &read_attr->out[IB_COUNTER_BYTES]);
1003 }
1004
1005 /* flow counters currently expose two counters packets and bytes */
1006 #define FLOW_COUNTERS_NUM 2
counters_set_description(struct ib_counters * counters,enum mlx5_ib_counters_type counters_type,struct mlx5_ib_flow_counters_desc * desc_data,u32 ncounters)1007 static int counters_set_description(
1008 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type,
1009 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters)
1010 {
1011 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters);
1012 u32 cntrs_max_index = 0;
1013 int i;
1014
1015 if (counters_type != MLX5_IB_COUNTERS_FLOW)
1016 return -EINVAL;
1017
1018 /* init the fields for the object */
1019 mcounters->type = counters_type;
1020 mcounters->read_counters = read_flow_counters;
1021 mcounters->counters_num = FLOW_COUNTERS_NUM;
1022 mcounters->ncounters = ncounters;
1023 /* each counter entry have both description and index pair */
1024 for (i = 0; i < ncounters; i++) {
1025 if (desc_data[i].description > IB_COUNTER_BYTES)
1026 return -EINVAL;
1027
1028 if (cntrs_max_index <= desc_data[i].index)
1029 cntrs_max_index = desc_data[i].index + 1;
1030 }
1031
1032 mutex_lock(&mcounters->mcntrs_mutex);
1033 mcounters->counters_data = desc_data;
1034 mcounters->cntrs_max_index = cntrs_max_index;
1035 mutex_unlock(&mcounters->mcntrs_mutex);
1036
1037 return 0;
1038 }
1039
1040 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2))
mlx5_ib_flow_counters_set_data(struct ib_counters * ibcounters,struct mlx5_ib_create_flow * ucmd)1041 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters,
1042 struct mlx5_ib_create_flow *ucmd)
1043 {
1044 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters);
1045 struct mlx5_ib_flow_counters_data *cntrs_data = NULL;
1046 struct mlx5_ib_flow_counters_desc *desc_data = NULL;
1047 bool hw_hndl = false;
1048 int ret = 0;
1049
1050 if (ucmd && ucmd->ncounters_data != 0) {
1051 cntrs_data = ucmd->data;
1052 if (cntrs_data->ncounters > MAX_COUNTERS_NUM)
1053 return -EINVAL;
1054
1055 desc_data = kcalloc(cntrs_data->ncounters,
1056 sizeof(*desc_data),
1057 GFP_KERNEL);
1058 if (!desc_data)
1059 return -ENOMEM;
1060
1061 if (copy_from_user(desc_data,
1062 u64_to_user_ptr(cntrs_data->counters_data),
1063 sizeof(*desc_data) * cntrs_data->ncounters)) {
1064 ret = -EFAULT;
1065 goto free;
1066 }
1067 }
1068
1069 if (!mcounters->hw_cntrs_hndl) {
1070 mcounters->hw_cntrs_hndl = mlx5_fc_create(
1071 to_mdev(ibcounters->device)->mdev, false);
1072 if (IS_ERR(mcounters->hw_cntrs_hndl)) {
1073 ret = PTR_ERR(mcounters->hw_cntrs_hndl);
1074 goto free;
1075 }
1076 hw_hndl = true;
1077 }
1078
1079 if (desc_data) {
1080 /* counters already bound to at least one flow */
1081 if (mcounters->cntrs_max_index) {
1082 ret = -EINVAL;
1083 goto free_hndl;
1084 }
1085
1086 ret = counters_set_description(ibcounters,
1087 MLX5_IB_COUNTERS_FLOW,
1088 desc_data,
1089 cntrs_data->ncounters);
1090 if (ret)
1091 goto free_hndl;
1092
1093 } else if (!mcounters->cntrs_max_index) {
1094 /* counters not bound yet, must have udata passed */
1095 ret = -EINVAL;
1096 goto free_hndl;
1097 }
1098
1099 return 0;
1100
1101 free_hndl:
1102 if (hw_hndl) {
1103 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev,
1104 mcounters->hw_cntrs_hndl);
1105 mcounters->hw_cntrs_hndl = NULL;
1106 }
1107 free:
1108 kfree(desc_data);
1109 return ret;
1110 }
1111
mlx5_ib_counters_clear_description(struct ib_counters * counters)1112 void mlx5_ib_counters_clear_description(struct ib_counters *counters)
1113 {
1114 struct mlx5_ib_mcounters *mcounters;
1115
1116 if (!counters || atomic_read(&counters->usecnt) != 1)
1117 return;
1118
1119 mcounters = to_mcounters(counters);
1120
1121 mutex_lock(&mcounters->mcntrs_mutex);
1122 kfree(mcounters->counters_data);
1123 mcounters->counters_data = NULL;
1124 mcounters->cntrs_max_index = 0;
1125 mutex_unlock(&mcounters->mcntrs_mutex);
1126 }
1127
mlx5_ib_modify_stat(struct ib_device * device,u32 port,unsigned int index,bool enable)1128 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port,
1129 unsigned int index, bool enable)
1130 {
1131 struct mlx5_ib_dev *dev = to_mdev(device);
1132 struct mlx5_ib_op_fc *opfc, *in_use_opfc;
1133 struct mlx5_ib_counters *cnts;
1134 u32 num_hw_counters, type;
1135 int ret;
1136
1137 cnts = &dev->port[port - 1].cnts;
1138 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters +
1139 cnts->num_ext_ppcnt_counters;
1140 if (index < num_hw_counters ||
1141 index >= (num_hw_counters + cnts->num_op_counters))
1142 return -EINVAL;
1143
1144 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL))
1145 return -EINVAL;
1146
1147 type = *(u32 *)cnts->descs[index].priv;
1148 if (type >= MLX5_IB_OPCOUNTER_MAX)
1149 return -EINVAL;
1150
1151 opfc = &cnts->opfcs[type];
1152
1153 if (enable) {
1154 if (opfc->fc)
1155 return -EEXIST;
1156
1157 if (mlx5r_is_opfc_shared_and_in_use(cnts->opfcs, type,
1158 &in_use_opfc)) {
1159 opfc->fc = in_use_opfc->fc;
1160 opfc->rule[0] = in_use_opfc->rule[0];
1161 return 0;
1162 }
1163
1164 opfc->fc = mlx5_fc_create(dev->mdev, false);
1165 if (IS_ERR(opfc->fc))
1166 return PTR_ERR(opfc->fc);
1167
1168 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type);
1169 if (ret) {
1170 mlx5_fc_destroy(dev->mdev, opfc->fc);
1171 opfc->fc = NULL;
1172 }
1173 return ret;
1174 }
1175
1176 if (!opfc->fc)
1177 return -EINVAL;
1178
1179 if (mlx5r_is_opfc_shared_and_in_use(cnts->opfcs, type, &in_use_opfc))
1180 goto out;
1181
1182 mlx5_ib_fs_remove_op_fc(dev, opfc, type);
1183 mlx5_fc_destroy(dev->mdev, opfc->fc);
1184 out:
1185 opfc->fc = NULL;
1186 return 0;
1187 }
1188
mlx5_ib_counter_init(struct rdma_counter * counter)1189 static void mlx5_ib_counter_init(struct rdma_counter *counter)
1190 {
1191 struct mlx5_rdma_counter *mcounter = to_mcounter(counter);
1192
1193 xa_init(&mcounter->qpn_opfc_xa);
1194 }
1195
1196 static const struct ib_device_ops hw_stats_ops = {
1197 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1198 .get_hw_stats = mlx5_ib_get_hw_stats,
1199 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1200 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1201 .counter_dealloc = mlx5_ib_counter_dealloc,
1202 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1203 .counter_update_stats = mlx5_ib_counter_update_stats,
1204 .modify_hw_stat = mlx5_ib_modify_stat,
1205 .counter_init = mlx5_ib_counter_init,
1206
1207 INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter),
1208 };
1209
1210 static const struct ib_device_ops hw_switchdev_vport_op = {
1211 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats,
1212 };
1213
1214 static const struct ib_device_ops hw_switchdev_stats_ops = {
1215 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats,
1216 .get_hw_stats = mlx5_ib_get_hw_stats,
1217 .counter_bind_qp = mlx5_ib_counter_bind_qp,
1218 .counter_unbind_qp = mlx5_ib_counter_unbind_qp,
1219 .counter_dealloc = mlx5_ib_counter_dealloc,
1220 .counter_alloc_stats = mlx5_ib_counter_alloc_stats,
1221 .counter_update_stats = mlx5_ib_counter_update_stats,
1222 .counter_init = mlx5_ib_counter_init,
1223
1224 INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter),
1225 };
1226
1227 static const struct ib_device_ops counters_ops = {
1228 .create_counters = mlx5_ib_create_counters,
1229 .destroy_counters = mlx5_ib_destroy_counters,
1230 .read_counters = mlx5_ib_read_counters,
1231
1232 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
1233 };
1234
mlx5_ib_counters_init(struct mlx5_ib_dev * dev)1235 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev)
1236 {
1237 ib_set_device_ops(&dev->ib_dev, &counters_ops);
1238
1239 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1240 return 0;
1241
1242 if (is_mdev_switchdev_mode(dev->mdev)) {
1243 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops);
1244 if (vport_qcounters_supported(dev))
1245 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op);
1246 } else
1247 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops);
1248 return mlx5_ib_alloc_counters(dev);
1249 }
1250
mlx5_ib_counters_cleanup(struct mlx5_ib_dev * dev)1251 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev)
1252 {
1253 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt))
1254 return;
1255
1256 mlx5_ib_dealloc_counters(dev);
1257 }
1258