1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include "mlx5_ib.h" 7 #include <linux/mlx5/eswitch.h> 8 #include <linux/mlx5/vport.h> 9 #include "counters.h" 10 #include "ib_rep.h" 11 #include "qp.h" 12 13 struct mlx5_ib_counter { 14 const char *name; 15 size_t offset; 16 u32 type; 17 }; 18 19 #define INIT_Q_COUNTER(_name) \ 20 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} 21 22 #define INIT_VPORT_Q_COUNTER(_name) \ 23 { .name = "vport_" #_name, .offset = \ 24 MLX5_BYTE_OFF(query_q_counter_out, _name)} 25 26 static const struct mlx5_ib_counter basic_q_cnts[] = { 27 INIT_Q_COUNTER(rx_write_requests), 28 INIT_Q_COUNTER(rx_read_requests), 29 INIT_Q_COUNTER(rx_atomic_requests), 30 INIT_Q_COUNTER(rx_dct_connect), 31 INIT_Q_COUNTER(out_of_buffer), 32 }; 33 34 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { 35 INIT_Q_COUNTER(out_of_sequence), 36 }; 37 38 static const struct mlx5_ib_counter retrans_q_cnts[] = { 39 INIT_Q_COUNTER(duplicate_request), 40 INIT_Q_COUNTER(rnr_nak_retry_err), 41 INIT_Q_COUNTER(packet_seq_err), 42 INIT_Q_COUNTER(implied_nak_seq_err), 43 INIT_Q_COUNTER(local_ack_timeout_err), 44 }; 45 46 static const struct mlx5_ib_counter vport_basic_q_cnts[] = { 47 INIT_VPORT_Q_COUNTER(rx_write_requests), 48 INIT_VPORT_Q_COUNTER(rx_read_requests), 49 INIT_VPORT_Q_COUNTER(rx_atomic_requests), 50 INIT_VPORT_Q_COUNTER(rx_dct_connect), 51 INIT_VPORT_Q_COUNTER(out_of_buffer), 52 }; 53 54 static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = { 55 INIT_VPORT_Q_COUNTER(out_of_sequence), 56 }; 57 58 static const struct mlx5_ib_counter vport_retrans_q_cnts[] = { 59 INIT_VPORT_Q_COUNTER(duplicate_request), 60 INIT_VPORT_Q_COUNTER(rnr_nak_retry_err), 61 INIT_VPORT_Q_COUNTER(packet_seq_err), 62 INIT_VPORT_Q_COUNTER(implied_nak_seq_err), 63 INIT_VPORT_Q_COUNTER(local_ack_timeout_err), 64 }; 65 66 #define INIT_CONG_COUNTER(_name) \ 67 { .name = #_name, .offset = \ 68 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} 69 70 static const struct mlx5_ib_counter cong_cnts[] = { 71 INIT_CONG_COUNTER(rp_cnp_ignored), 72 INIT_CONG_COUNTER(rp_cnp_handled), 73 INIT_CONG_COUNTER(np_ecn_marked_roce_packets), 74 INIT_CONG_COUNTER(np_cnp_sent), 75 }; 76 77 static const struct mlx5_ib_counter extended_err_cnts[] = { 78 INIT_Q_COUNTER(resp_local_length_error), 79 INIT_Q_COUNTER(resp_cqe_error), 80 INIT_Q_COUNTER(req_cqe_error), 81 INIT_Q_COUNTER(req_remote_invalid_request), 82 INIT_Q_COUNTER(req_remote_access_errors), 83 INIT_Q_COUNTER(resp_remote_access_errors), 84 INIT_Q_COUNTER(resp_cqe_flush_error), 85 INIT_Q_COUNTER(req_cqe_flush_error), 86 INIT_Q_COUNTER(req_transport_retries_exceeded), 87 INIT_Q_COUNTER(req_rnr_retries_exceeded), 88 }; 89 90 static const struct mlx5_ib_counter roce_accl_cnts[] = { 91 INIT_Q_COUNTER(roce_adp_retrans), 92 INIT_Q_COUNTER(roce_adp_retrans_to), 93 INIT_Q_COUNTER(roce_slow_restart), 94 INIT_Q_COUNTER(roce_slow_restart_cnps), 95 INIT_Q_COUNTER(roce_slow_restart_trans), 96 }; 97 98 static const struct mlx5_ib_counter vport_extended_err_cnts[] = { 99 INIT_VPORT_Q_COUNTER(resp_local_length_error), 100 INIT_VPORT_Q_COUNTER(resp_cqe_error), 101 INIT_VPORT_Q_COUNTER(req_cqe_error), 102 INIT_VPORT_Q_COUNTER(req_remote_invalid_request), 103 INIT_VPORT_Q_COUNTER(req_remote_access_errors), 104 INIT_VPORT_Q_COUNTER(resp_remote_access_errors), 105 INIT_VPORT_Q_COUNTER(resp_cqe_flush_error), 106 INIT_VPORT_Q_COUNTER(req_cqe_flush_error), 107 INIT_VPORT_Q_COUNTER(req_transport_retries_exceeded), 108 INIT_VPORT_Q_COUNTER(req_rnr_retries_exceeded), 109 }; 110 111 static const struct mlx5_ib_counter vport_roce_accl_cnts[] = { 112 INIT_VPORT_Q_COUNTER(roce_adp_retrans), 113 INIT_VPORT_Q_COUNTER(roce_adp_retrans_to), 114 INIT_VPORT_Q_COUNTER(roce_slow_restart), 115 INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps), 116 INIT_VPORT_Q_COUNTER(roce_slow_restart_trans), 117 }; 118 119 #define INIT_EXT_PPCNT_COUNTER(_name) \ 120 { .name = #_name, .offset = \ 121 MLX5_BYTE_OFF(ppcnt_reg, \ 122 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} 123 124 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { 125 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), 126 }; 127 128 #define INIT_OP_COUNTER(_name, _type) \ 129 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type} 130 131 static const struct mlx5_ib_counter basic_op_cnts[] = { 132 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS), 133 }; 134 135 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = { 136 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS), 137 }; 138 139 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = { 140 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS), 141 }; 142 143 static const struct mlx5_ib_counter packets_op_cnts[] = { 144 INIT_OP_COUNTER(rdma_tx_packets, RDMA_TX_PACKETS), 145 INIT_OP_COUNTER(rdma_tx_bytes, RDMA_TX_BYTES), 146 INIT_OP_COUNTER(rdma_rx_packets, RDMA_RX_PACKETS), 147 INIT_OP_COUNTER(rdma_rx_bytes, RDMA_RX_BYTES), 148 }; 149 150 static int mlx5_ib_read_counters(struct ib_counters *counters, 151 struct ib_counters_read_attr *read_attr, 152 struct uverbs_attr_bundle *attrs) 153 { 154 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 155 struct mlx5_read_counters_attr mread_attr = {}; 156 struct mlx5_ib_flow_counters_desc *desc; 157 int ret, i; 158 159 mutex_lock(&mcounters->mcntrs_mutex); 160 if (mcounters->cntrs_max_index > read_attr->ncounters) { 161 ret = -EINVAL; 162 goto err_bound; 163 } 164 165 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), 166 GFP_KERNEL); 167 if (!mread_attr.out) { 168 ret = -ENOMEM; 169 goto err_bound; 170 } 171 172 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; 173 mread_attr.flags = read_attr->flags; 174 ret = mcounters->read_counters(counters->device, &mread_attr); 175 if (ret) 176 goto err_read; 177 178 /* do the pass over the counters data array to assign according to the 179 * descriptions and indexing pairs 180 */ 181 desc = mcounters->counters_data; 182 for (i = 0; i < mcounters->ncounters; i++) 183 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; 184 185 err_read: 186 kfree(mread_attr.out); 187 err_bound: 188 mutex_unlock(&mcounters->mcntrs_mutex); 189 return ret; 190 } 191 192 static int mlx5_ib_destroy_counters(struct ib_counters *counters) 193 { 194 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 195 196 mlx5_ib_counters_clear_description(counters); 197 if (mcounters->hw_cntrs_hndl) 198 mlx5_fc_destroy(to_mdev(counters->device)->mdev, 199 mcounters->hw_cntrs_hndl); 200 return 0; 201 } 202 203 static int mlx5_ib_create_counters(struct ib_counters *counters, 204 struct uverbs_attr_bundle *attrs) 205 { 206 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 207 208 mutex_init(&mcounters->mcntrs_mutex); 209 return 0; 210 } 211 212 static bool vport_qcounters_supported(struct mlx5_ib_dev *dev) 213 { 214 return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) && 215 MLX5_CAP_GEN(dev->mdev, q_counter_aggregation); 216 } 217 218 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, 219 u32 port_num) 220 { 221 if ((is_mdev_switchdev_mode(dev->mdev) && 222 !vport_qcounters_supported(dev)) || !port_num) 223 return &dev->port[0].cnts; 224 225 return is_mdev_switchdev_mode(dev->mdev) ? 226 &dev->port[1].cnts : &dev->port[port_num - 1].cnts; 227 } 228 229 /** 230 * mlx5_ib_get_counters_id - Returns counters id to use for device+port 231 * @dev: Pointer to mlx5 IB device 232 * @port_num: Zero based port number 233 * 234 * mlx5_ib_get_counters_id() Returns counters set id to use for given 235 * device port combination in switchdev and non switchdev mode of the 236 * parent device. 237 */ 238 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num) 239 { 240 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1); 241 242 return cnts->set_id; 243 } 244 245 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts) 246 { 247 struct rdma_hw_stats *stats; 248 u32 num_hw_counters; 249 int i; 250 251 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 252 cnts->num_ext_ppcnt_counters; 253 stats = rdma_alloc_hw_stats_struct(cnts->descs, 254 num_hw_counters + 255 cnts->num_op_counters, 256 RDMA_HW_STATS_DEFAULT_LIFESPAN); 257 if (!stats) 258 return NULL; 259 260 for (i = 0; i < cnts->num_op_counters; i++) 261 set_bit(num_hw_counters + i, stats->is_disabled); 262 263 return stats; 264 } 265 266 static struct rdma_hw_stats * 267 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev) 268 { 269 struct mlx5_ib_dev *dev = to_mdev(ibdev); 270 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts; 271 272 return do_alloc_stats(cnts); 273 } 274 275 static struct rdma_hw_stats * 276 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num) 277 { 278 struct mlx5_ib_dev *dev = to_mdev(ibdev); 279 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 280 281 return do_alloc_stats(cnts); 282 } 283 284 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, 285 const struct mlx5_ib_counters *cnts, 286 struct rdma_hw_stats *stats, 287 u16 set_id) 288 { 289 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 290 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 291 __be32 val; 292 int ret, i; 293 294 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 295 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); 296 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 297 if (ret) 298 return ret; 299 300 for (i = 0; i < cnts->num_q_counters; i++) { 301 val = *(__be32 *)((void *)out + cnts->offsets[i]); 302 stats->value[i] = (u64)be32_to_cpu(val); 303 } 304 305 return 0; 306 } 307 308 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, 309 const struct mlx5_ib_counters *cnts, 310 struct rdma_hw_stats *stats) 311 { 312 int offset = cnts->num_q_counters + cnts->num_cong_counters; 313 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; 314 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); 315 int ret, i; 316 void *out; 317 318 out = kvzalloc(sz, GFP_KERNEL); 319 if (!out) 320 return -ENOMEM; 321 322 MLX5_SET(ppcnt_reg, in, local_port, 1); 323 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); 324 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, 325 0, 0); 326 if (ret) 327 goto free; 328 329 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) 330 stats->value[i + offset] = 331 be64_to_cpup((__be64 *)(out + 332 cnts->offsets[i + offset])); 333 free: 334 kvfree(out); 335 return ret; 336 } 337 338 static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev, 339 u32 port_num, 340 const struct mlx5_ib_counters *cnts, 341 struct rdma_hw_stats *stats) 342 343 { 344 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 345 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 346 struct mlx5_core_dev *mdev; 347 __be32 val; 348 int ret, i; 349 350 if (!dev->port[port_num].rep || 351 dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK) 352 return 0; 353 354 mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw); 355 if (!mdev) 356 return -EOPNOTSUPP; 357 358 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 359 MLX5_SET(query_q_counter_in, in, other_vport, 1); 360 MLX5_SET(query_q_counter_in, in, vport_number, 361 dev->port[port_num].rep->vport); 362 MLX5_SET(query_q_counter_in, in, aggregate, 1); 363 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 364 if (ret) 365 return ret; 366 367 for (i = 0; i < cnts->num_q_counters; i++) { 368 val = *(__be32 *)((void *)out + cnts->offsets[i]); 369 stats->value[i] = (u64)be32_to_cpu(val); 370 } 371 372 return 0; 373 } 374 375 static int do_get_hw_stats(struct ib_device *ibdev, 376 struct rdma_hw_stats *stats, 377 u32 port_num, int index) 378 { 379 struct mlx5_ib_dev *dev = to_mdev(ibdev); 380 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 381 struct mlx5_core_dev *mdev; 382 int ret, num_counters; 383 384 if (!stats) 385 return -EINVAL; 386 387 num_counters = cnts->num_q_counters + 388 cnts->num_cong_counters + 389 cnts->num_ext_ppcnt_counters; 390 391 if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0) 392 ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts, 393 stats); 394 else 395 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, 396 cnts->set_id); 397 if (ret) 398 return ret; 399 400 /* We don't expose device counters over Vports */ 401 if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0) 402 goto done; 403 404 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 405 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); 406 if (ret) 407 return ret; 408 } 409 410 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 411 if (!port_num) 412 port_num = 1; 413 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL); 414 if (!mdev) { 415 /* If port is not affiliated yet, its in down state 416 * which doesn't have any counters yet, so it would be 417 * zero. So no need to read from the HCA. 418 */ 419 goto done; 420 } 421 ret = mlx5_lag_query_cong_counters(dev->mdev, 422 stats->value + 423 cnts->num_q_counters, 424 cnts->num_cong_counters, 425 cnts->offsets + 426 cnts->num_q_counters); 427 428 mlx5_ib_put_native_port_mdev(dev, port_num); 429 if (ret) 430 return ret; 431 } 432 433 done: 434 return num_counters; 435 } 436 437 static bool is_rdma_bytes_counter(u32 type) 438 { 439 if (type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES || 440 type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES || 441 type == MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP || 442 type == MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP) 443 return true; 444 445 return false; 446 } 447 448 static int do_per_qp_get_op_stat(struct rdma_counter *counter) 449 { 450 struct mlx5_ib_dev *dev = to_mdev(counter->device); 451 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port); 452 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 453 int i, ret, index, num_hw_counters; 454 u64 packets = 0, bytes = 0; 455 456 for (i = MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP; 457 i <= MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; i++) { 458 if (!mcounter->fc[i]) 459 continue; 460 461 ret = mlx5_fc_query(dev->mdev, mcounter->fc[i], 462 &packets, &bytes); 463 if (ret) 464 return ret; 465 466 num_hw_counters = cnts->num_q_counters + 467 cnts->num_cong_counters + 468 cnts->num_ext_ppcnt_counters; 469 470 index = i - MLX5_IB_OPCOUNTER_CC_RX_CE_PKTS_PER_QP + 471 num_hw_counters; 472 473 if (is_rdma_bytes_counter(i)) 474 counter->stats->value[index] = bytes; 475 else 476 counter->stats->value[index] = packets; 477 478 clear_bit(index, counter->stats->is_disabled); 479 } 480 return 0; 481 } 482 483 static int do_get_op_stat(struct ib_device *ibdev, 484 struct rdma_hw_stats *stats, 485 u32 port_num, int index) 486 { 487 struct mlx5_ib_dev *dev = to_mdev(ibdev); 488 const struct mlx5_ib_counters *cnts; 489 const struct mlx5_ib_op_fc *opfcs; 490 u64 packets, bytes; 491 u32 type; 492 int ret; 493 494 cnts = get_counters(dev, port_num); 495 496 opfcs = cnts->opfcs; 497 type = *(u32 *)cnts->descs[index].priv; 498 if (type >= MLX5_IB_OPCOUNTER_MAX) 499 return -EINVAL; 500 501 if (!opfcs[type].fc) 502 goto out; 503 504 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc, 505 &packets, &bytes); 506 if (ret) 507 return ret; 508 509 if (is_rdma_bytes_counter(type)) 510 stats->value[index] = bytes; 511 else 512 stats->value[index] = packets; 513 out: 514 return index; 515 } 516 517 static int do_get_op_stats(struct ib_device *ibdev, 518 struct rdma_hw_stats *stats, 519 u32 port_num) 520 { 521 struct mlx5_ib_dev *dev = to_mdev(ibdev); 522 const struct mlx5_ib_counters *cnts; 523 int index, ret, num_hw_counters; 524 525 cnts = get_counters(dev, port_num); 526 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 527 cnts->num_ext_ppcnt_counters; 528 for (index = num_hw_counters; 529 index < (num_hw_counters + cnts->num_op_counters); index++) { 530 ret = do_get_op_stat(ibdev, stats, port_num, index); 531 if (ret != index) 532 return ret; 533 } 534 535 return cnts->num_op_counters; 536 } 537 538 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, 539 struct rdma_hw_stats *stats, 540 u32 port_num, int index) 541 { 542 int num_counters, num_hw_counters, num_op_counters; 543 struct mlx5_ib_dev *dev = to_mdev(ibdev); 544 const struct mlx5_ib_counters *cnts; 545 546 cnts = get_counters(dev, port_num); 547 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 548 cnts->num_ext_ppcnt_counters; 549 num_counters = num_hw_counters + cnts->num_op_counters; 550 551 if (index < 0 || index > num_counters) 552 return -EINVAL; 553 else if (index > 0 && index < num_hw_counters) 554 return do_get_hw_stats(ibdev, stats, port_num, index); 555 else if (index >= num_hw_counters && index < num_counters) 556 return do_get_op_stat(ibdev, stats, port_num, index); 557 558 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index); 559 if (num_hw_counters < 0) 560 return num_hw_counters; 561 562 num_op_counters = do_get_op_stats(ibdev, stats, port_num); 563 if (num_op_counters < 0) 564 return num_op_counters; 565 566 return num_hw_counters + num_op_counters; 567 } 568 569 static struct rdma_hw_stats * 570 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) 571 { 572 struct mlx5_ib_dev *dev = to_mdev(counter->device); 573 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port); 574 575 return do_alloc_stats(cnts); 576 } 577 578 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) 579 { 580 struct mlx5_ib_dev *dev = to_mdev(counter->device); 581 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port); 582 int ret; 583 584 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, counter->stats, 585 counter->id); 586 if (ret) 587 return ret; 588 589 if (!counter->mode.bind_opcnt) 590 return 0; 591 592 return do_per_qp_get_op_stat(counter); 593 } 594 595 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) 596 { 597 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 598 struct mlx5_ib_dev *dev = to_mdev(counter->device); 599 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 600 601 if (!counter->id) 602 return 0; 603 604 WARN_ON(!xa_empty(&mcounter->qpn_opfc_xa)); 605 mlx5r_fs_destroy_fcs(dev, counter); 606 MLX5_SET(dealloc_q_counter_in, in, opcode, 607 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 608 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); 609 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 610 } 611 612 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, 613 struct ib_qp *qp, u32 port) 614 { 615 struct mlx5_ib_dev *dev = to_mdev(qp->device); 616 bool new = false; 617 int err; 618 619 if (!counter->id) { 620 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 621 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 622 623 MLX5_SET(alloc_q_counter_in, in, opcode, 624 MLX5_CMD_OP_ALLOC_Q_COUNTER); 625 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); 626 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 627 if (err) 628 return err; 629 counter->id = 630 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 631 new = true; 632 } 633 634 err = mlx5_ib_qp_set_counter(qp, counter); 635 if (err) 636 goto fail_set_counter; 637 638 err = mlx5r_fs_bind_op_fc(qp, counter, port); 639 if (err) 640 goto fail_bind_op_fc; 641 642 return 0; 643 644 fail_bind_op_fc: 645 mlx5_ib_qp_set_counter(qp, NULL); 646 fail_set_counter: 647 if (new) { 648 mlx5_ib_counter_dealloc(counter); 649 counter->id = 0; 650 } 651 652 return err; 653 } 654 655 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp, u32 port) 656 { 657 struct rdma_counter *counter = qp->counter; 658 int err; 659 660 mlx5r_fs_unbind_op_fc(qp, counter); 661 662 err = mlx5_ib_qp_set_counter(qp, NULL); 663 if (err) 664 goto fail_set_counter; 665 666 return 0; 667 668 fail_set_counter: 669 mlx5r_fs_bind_op_fc(qp, counter, port); 670 return err; 671 } 672 673 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, 674 struct rdma_stat_desc *descs, size_t *offsets, 675 u32 port_num) 676 { 677 bool is_vport = is_mdev_switchdev_mode(dev->mdev) && 678 port_num != MLX5_VPORT_PF; 679 const struct mlx5_ib_counter *names; 680 int j = 0, i, size; 681 682 names = is_vport ? vport_basic_q_cnts : basic_q_cnts; 683 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) : 684 ARRAY_SIZE(basic_q_cnts); 685 for (i = 0; i < size; i++, j++) { 686 descs[j].name = names[i].name; 687 offsets[j] = names[i].offset; 688 } 689 690 names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts; 691 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) : 692 ARRAY_SIZE(out_of_seq_q_cnts); 693 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { 694 for (i = 0; i < size; i++, j++) { 695 descs[j].name = names[i].name; 696 offsets[j] = names[i].offset; 697 } 698 } 699 700 names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts; 701 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) : 702 ARRAY_SIZE(retrans_q_cnts); 703 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 704 for (i = 0; i < size; i++, j++) { 705 descs[j].name = names[i].name; 706 offsets[j] = names[i].offset; 707 } 708 } 709 710 names = is_vport ? vport_extended_err_cnts : extended_err_cnts; 711 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) : 712 ARRAY_SIZE(extended_err_cnts); 713 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { 714 for (i = 0; i < size; i++, j++) { 715 descs[j].name = names[i].name; 716 offsets[j] = names[i].offset; 717 } 718 } 719 720 names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts; 721 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) : 722 ARRAY_SIZE(roce_accl_cnts); 723 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { 724 for (i = 0; i < size; i++, j++) { 725 descs[j].name = names[i].name; 726 offsets[j] = names[i].offset; 727 } 728 } 729 730 if (is_vport) 731 return; 732 733 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 734 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { 735 descs[j].name = cong_cnts[i].name; 736 offsets[j] = cong_cnts[i].offset; 737 } 738 } 739 740 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 741 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { 742 descs[j].name = ext_ppcnt_cnts[i].name; 743 offsets[j] = ext_ppcnt_cnts[i].offset; 744 } 745 } 746 747 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) { 748 descs[j].name = basic_op_cnts[i].name; 749 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 750 descs[j].priv = &basic_op_cnts[i].type; 751 } 752 753 if (MLX5_CAP_FLOWTABLE(dev->mdev, 754 ft_field_support_2_nic_receive_rdma.bth_opcode)) { 755 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) { 756 descs[j].name = rdmarx_cnp_op_cnts[i].name; 757 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 758 descs[j].priv = &rdmarx_cnp_op_cnts[i].type; 759 } 760 } 761 762 if (MLX5_CAP_FLOWTABLE(dev->mdev, 763 ft_field_support_2_nic_transmit_rdma.bth_opcode)) { 764 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) { 765 descs[j].name = rdmatx_cnp_op_cnts[i].name; 766 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 767 descs[j].priv = &rdmatx_cnp_op_cnts[i].type; 768 } 769 } 770 771 for (i = 0; i < ARRAY_SIZE(packets_op_cnts); i++, j++) { 772 descs[j].name = packets_op_cnts[i].name; 773 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 774 descs[j].priv = &packets_op_cnts[i].type; 775 } 776 } 777 778 779 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, 780 struct mlx5_ib_counters *cnts, u32 port_num) 781 { 782 bool is_vport = is_mdev_switchdev_mode(dev->mdev) && 783 port_num != MLX5_VPORT_PF; 784 u32 num_counters, num_op_counters = 0, size; 785 786 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) : 787 ARRAY_SIZE(basic_q_cnts); 788 num_counters = size; 789 790 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) : 791 ARRAY_SIZE(out_of_seq_q_cnts); 792 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) 793 num_counters += size; 794 795 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) : 796 ARRAY_SIZE(retrans_q_cnts); 797 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 798 num_counters += size; 799 800 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) : 801 ARRAY_SIZE(extended_err_cnts); 802 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) 803 num_counters += size; 804 805 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) : 806 ARRAY_SIZE(roce_accl_cnts); 807 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) 808 num_counters += size; 809 810 cnts->num_q_counters = num_counters; 811 812 if (is_vport) 813 goto skip_non_qcounters; 814 815 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 816 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); 817 num_counters += ARRAY_SIZE(cong_cnts); 818 } 819 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 820 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); 821 num_counters += ARRAY_SIZE(ext_ppcnt_cnts); 822 } 823 824 num_op_counters = ARRAY_SIZE(basic_op_cnts); 825 826 num_op_counters += ARRAY_SIZE(packets_op_cnts); 827 828 if (MLX5_CAP_FLOWTABLE(dev->mdev, 829 ft_field_support_2_nic_receive_rdma.bth_opcode)) 830 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts); 831 832 if (MLX5_CAP_FLOWTABLE(dev->mdev, 833 ft_field_support_2_nic_transmit_rdma.bth_opcode)) 834 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts); 835 836 skip_non_qcounters: 837 cnts->num_op_counters = num_op_counters; 838 num_counters += num_op_counters; 839 cnts->descs = kcalloc(num_counters, 840 sizeof(struct rdma_stat_desc), GFP_KERNEL); 841 if (!cnts->descs) 842 return -ENOMEM; 843 844 cnts->offsets = kcalloc(num_counters, 845 sizeof(*cnts->offsets), GFP_KERNEL); 846 if (!cnts->offsets) 847 goto err; 848 849 return 0; 850 851 err: 852 kfree(cnts->descs); 853 cnts->descs = NULL; 854 return -ENOMEM; 855 } 856 857 /* 858 * Checks if the given flow counter type should be sharing the same flow counter 859 * with another type and if it should, checks if that other type flow counter 860 * was already created, if both conditions are met return true and the counter 861 * else return false. 862 */ 863 bool mlx5r_is_opfc_shared_and_in_use(struct mlx5_ib_op_fc *opfcs, u32 type, 864 struct mlx5_ib_op_fc **opfc) 865 { 866 u32 shared_fc_type; 867 868 switch (type) { 869 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS: 870 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES; 871 break; 872 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES: 873 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS; 874 break; 875 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS: 876 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES; 877 break; 878 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES: 879 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS; 880 break; 881 case MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP: 882 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP; 883 break; 884 case MLX5_IB_OPCOUNTER_RDMA_TX_BYTES_PER_QP: 885 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_TX_PACKETS_PER_QP; 886 break; 887 case MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP: 888 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP; 889 break; 890 case MLX5_IB_OPCOUNTER_RDMA_RX_BYTES_PER_QP: 891 shared_fc_type = MLX5_IB_OPCOUNTER_RDMA_RX_PACKETS_PER_QP; 892 break; 893 default: 894 return false; 895 } 896 897 *opfc = &opfcs[shared_fc_type]; 898 if (!(*opfc)->fc) 899 return false; 900 901 return true; 902 } 903 904 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) 905 { 906 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 907 int num_cnt_ports = dev->num_ports; 908 struct mlx5_ib_op_fc *in_use_opfc; 909 int i, j; 910 911 if (is_mdev_switchdev_mode(dev->mdev)) 912 num_cnt_ports = min(2, num_cnt_ports); 913 914 MLX5_SET(dealloc_q_counter_in, in, opcode, 915 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 916 917 for (i = 0; i < num_cnt_ports; i++) { 918 if (dev->port[i].cnts.set_id) { 919 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 920 dev->port[i].cnts.set_id); 921 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 922 } 923 kfree(dev->port[i].cnts.descs); 924 kfree(dev->port[i].cnts.offsets); 925 926 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) { 927 if (!dev->port[i].cnts.opfcs[j].fc) 928 continue; 929 930 if (mlx5r_is_opfc_shared_and_in_use( 931 dev->port[i].cnts.opfcs, j, &in_use_opfc)) 932 goto skip; 933 934 mlx5_ib_fs_remove_op_fc(dev, 935 &dev->port[i].cnts.opfcs[j], j); 936 mlx5_fc_destroy(dev->mdev, 937 dev->port[i].cnts.opfcs[j].fc); 938 skip: 939 dev->port[i].cnts.opfcs[j].fc = NULL; 940 } 941 } 942 } 943 944 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) 945 { 946 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 947 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 948 int num_cnt_ports = dev->num_ports; 949 int err = 0; 950 int i; 951 bool is_shared; 952 953 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 954 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; 955 956 /* 957 * In switchdev we need to allocate two ports, one that is used for 958 * the device Q_counters and it is essentially the real Q_counters of 959 * this device, while the other is used as a helper for PF to be able to 960 * query all other vports. 961 */ 962 if (is_mdev_switchdev_mode(dev->mdev)) 963 num_cnt_ports = min(2, num_cnt_ports); 964 965 for (i = 0; i < num_cnt_ports; i++) { 966 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i); 967 if (err) 968 goto err_alloc; 969 970 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs, 971 dev->port[i].cnts.offsets, i); 972 973 MLX5_SET(alloc_q_counter_in, in, uid, 974 is_shared ? MLX5_SHARED_RESOURCE_UID : 0); 975 976 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 977 if (err) { 978 mlx5_ib_warn(dev, 979 "couldn't allocate queue counter for port %d, err %d\n", 980 i + 1, err); 981 goto err_alloc; 982 } 983 984 dev->port[i].cnts.set_id = 985 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 986 } 987 return 0; 988 989 err_alloc: 990 mlx5_ib_dealloc_counters(dev); 991 return err; 992 } 993 994 static int read_flow_counters(struct ib_device *ibdev, 995 struct mlx5_read_counters_attr *read_attr) 996 { 997 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; 998 struct mlx5_ib_dev *dev = to_mdev(ibdev); 999 1000 return mlx5_fc_query(dev->mdev, fc, 1001 &read_attr->out[IB_COUNTER_PACKETS], 1002 &read_attr->out[IB_COUNTER_BYTES]); 1003 } 1004 1005 /* flow counters currently expose two counters packets and bytes */ 1006 #define FLOW_COUNTERS_NUM 2 1007 static int counters_set_description( 1008 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, 1009 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) 1010 { 1011 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 1012 u32 cntrs_max_index = 0; 1013 int i; 1014 1015 if (counters_type != MLX5_IB_COUNTERS_FLOW) 1016 return -EINVAL; 1017 1018 /* init the fields for the object */ 1019 mcounters->type = counters_type; 1020 mcounters->read_counters = read_flow_counters; 1021 mcounters->counters_num = FLOW_COUNTERS_NUM; 1022 mcounters->ncounters = ncounters; 1023 /* each counter entry have both description and index pair */ 1024 for (i = 0; i < ncounters; i++) { 1025 if (desc_data[i].description > IB_COUNTER_BYTES) 1026 return -EINVAL; 1027 1028 if (cntrs_max_index <= desc_data[i].index) 1029 cntrs_max_index = desc_data[i].index + 1; 1030 } 1031 1032 mutex_lock(&mcounters->mcntrs_mutex); 1033 mcounters->counters_data = desc_data; 1034 mcounters->cntrs_max_index = cntrs_max_index; 1035 mutex_unlock(&mcounters->mcntrs_mutex); 1036 1037 return 0; 1038 } 1039 1040 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) 1041 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, 1042 struct mlx5_ib_create_flow *ucmd) 1043 { 1044 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); 1045 struct mlx5_ib_flow_counters_data *cntrs_data = NULL; 1046 struct mlx5_ib_flow_counters_desc *desc_data = NULL; 1047 bool hw_hndl = false; 1048 int ret = 0; 1049 1050 if (ucmd && ucmd->ncounters_data != 0) { 1051 cntrs_data = ucmd->data; 1052 if (cntrs_data->ncounters > MAX_COUNTERS_NUM) 1053 return -EINVAL; 1054 1055 desc_data = kcalloc(cntrs_data->ncounters, 1056 sizeof(*desc_data), 1057 GFP_KERNEL); 1058 if (!desc_data) 1059 return -ENOMEM; 1060 1061 if (copy_from_user(desc_data, 1062 u64_to_user_ptr(cntrs_data->counters_data), 1063 sizeof(*desc_data) * cntrs_data->ncounters)) { 1064 ret = -EFAULT; 1065 goto free; 1066 } 1067 } 1068 1069 if (!mcounters->hw_cntrs_hndl) { 1070 mcounters->hw_cntrs_hndl = mlx5_fc_create( 1071 to_mdev(ibcounters->device)->mdev, false); 1072 if (IS_ERR(mcounters->hw_cntrs_hndl)) { 1073 ret = PTR_ERR(mcounters->hw_cntrs_hndl); 1074 goto free; 1075 } 1076 hw_hndl = true; 1077 } 1078 1079 if (desc_data) { 1080 /* counters already bound to at least one flow */ 1081 if (mcounters->cntrs_max_index) { 1082 ret = -EINVAL; 1083 goto free_hndl; 1084 } 1085 1086 ret = counters_set_description(ibcounters, 1087 MLX5_IB_COUNTERS_FLOW, 1088 desc_data, 1089 cntrs_data->ncounters); 1090 if (ret) 1091 goto free_hndl; 1092 1093 } else if (!mcounters->cntrs_max_index) { 1094 /* counters not bound yet, must have udata passed */ 1095 ret = -EINVAL; 1096 goto free_hndl; 1097 } 1098 1099 return 0; 1100 1101 free_hndl: 1102 if (hw_hndl) { 1103 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, 1104 mcounters->hw_cntrs_hndl); 1105 mcounters->hw_cntrs_hndl = NULL; 1106 } 1107 free: 1108 kfree(desc_data); 1109 return ret; 1110 } 1111 1112 void mlx5_ib_counters_clear_description(struct ib_counters *counters) 1113 { 1114 struct mlx5_ib_mcounters *mcounters; 1115 1116 if (!counters || atomic_read(&counters->usecnt) != 1) 1117 return; 1118 1119 mcounters = to_mcounters(counters); 1120 1121 mutex_lock(&mcounters->mcntrs_mutex); 1122 kfree(mcounters->counters_data); 1123 mcounters->counters_data = NULL; 1124 mcounters->cntrs_max_index = 0; 1125 mutex_unlock(&mcounters->mcntrs_mutex); 1126 } 1127 1128 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port, 1129 unsigned int index, bool enable) 1130 { 1131 struct mlx5_ib_dev *dev = to_mdev(device); 1132 struct mlx5_ib_op_fc *opfc, *in_use_opfc; 1133 struct mlx5_ib_counters *cnts; 1134 u32 num_hw_counters, type; 1135 int ret; 1136 1137 cnts = &dev->port[port - 1].cnts; 1138 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 1139 cnts->num_ext_ppcnt_counters; 1140 if (index < num_hw_counters || 1141 index >= (num_hw_counters + cnts->num_op_counters)) 1142 return -EINVAL; 1143 1144 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 1145 return -EINVAL; 1146 1147 type = *(u32 *)cnts->descs[index].priv; 1148 if (type >= MLX5_IB_OPCOUNTER_MAX) 1149 return -EINVAL; 1150 1151 opfc = &cnts->opfcs[type]; 1152 1153 if (enable) { 1154 if (opfc->fc) 1155 return -EEXIST; 1156 1157 if (mlx5r_is_opfc_shared_and_in_use(cnts->opfcs, type, 1158 &in_use_opfc)) { 1159 opfc->fc = in_use_opfc->fc; 1160 opfc->rule[0] = in_use_opfc->rule[0]; 1161 return 0; 1162 } 1163 1164 opfc->fc = mlx5_fc_create(dev->mdev, false); 1165 if (IS_ERR(opfc->fc)) 1166 return PTR_ERR(opfc->fc); 1167 1168 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type); 1169 if (ret) { 1170 mlx5_fc_destroy(dev->mdev, opfc->fc); 1171 opfc->fc = NULL; 1172 } 1173 return ret; 1174 } 1175 1176 if (!opfc->fc) 1177 return -EINVAL; 1178 1179 if (mlx5r_is_opfc_shared_and_in_use(cnts->opfcs, type, &in_use_opfc)) 1180 goto out; 1181 1182 mlx5_ib_fs_remove_op_fc(dev, opfc, type); 1183 mlx5_fc_destroy(dev->mdev, opfc->fc); 1184 out: 1185 opfc->fc = NULL; 1186 return 0; 1187 } 1188 1189 static void mlx5_ib_counter_init(struct rdma_counter *counter) 1190 { 1191 struct mlx5_rdma_counter *mcounter = to_mcounter(counter); 1192 1193 xa_init(&mcounter->qpn_opfc_xa); 1194 } 1195 1196 static const struct ib_device_ops hw_stats_ops = { 1197 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats, 1198 .get_hw_stats = mlx5_ib_get_hw_stats, 1199 .counter_bind_qp = mlx5_ib_counter_bind_qp, 1200 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 1201 .counter_dealloc = mlx5_ib_counter_dealloc, 1202 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 1203 .counter_update_stats = mlx5_ib_counter_update_stats, 1204 .modify_hw_stat = mlx5_ib_modify_stat, 1205 .counter_init = mlx5_ib_counter_init, 1206 1207 INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter), 1208 }; 1209 1210 static const struct ib_device_ops hw_switchdev_vport_op = { 1211 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats, 1212 }; 1213 1214 static const struct ib_device_ops hw_switchdev_stats_ops = { 1215 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats, 1216 .get_hw_stats = mlx5_ib_get_hw_stats, 1217 .counter_bind_qp = mlx5_ib_counter_bind_qp, 1218 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 1219 .counter_dealloc = mlx5_ib_counter_dealloc, 1220 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 1221 .counter_update_stats = mlx5_ib_counter_update_stats, 1222 .counter_init = mlx5_ib_counter_init, 1223 1224 INIT_RDMA_OBJ_SIZE(rdma_counter, mlx5_rdma_counter, rdma_counter), 1225 }; 1226 1227 static const struct ib_device_ops counters_ops = { 1228 .create_counters = mlx5_ib_create_counters, 1229 .destroy_counters = mlx5_ib_destroy_counters, 1230 .read_counters = mlx5_ib_read_counters, 1231 1232 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), 1233 }; 1234 1235 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) 1236 { 1237 ib_set_device_ops(&dev->ib_dev, &counters_ops); 1238 1239 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 1240 return 0; 1241 1242 if (is_mdev_switchdev_mode(dev->mdev)) { 1243 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops); 1244 if (vport_qcounters_supported(dev)) 1245 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op); 1246 } else 1247 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); 1248 return mlx5_ib_alloc_counters(dev); 1249 } 1250 1251 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) 1252 { 1253 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 1254 return; 1255 1256 mlx5_ib_dealloc_counters(dev); 1257 } 1258