1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved. 4 */ 5 6 #include "mlx5_ib.h" 7 #include <linux/mlx5/eswitch.h> 8 #include <linux/mlx5/vport.h> 9 #include "counters.h" 10 #include "ib_rep.h" 11 #include "qp.h" 12 13 struct mlx5_ib_counter { 14 const char *name; 15 size_t offset; 16 u32 type; 17 }; 18 19 #define INIT_Q_COUNTER(_name) \ 20 { .name = #_name, .offset = MLX5_BYTE_OFF(query_q_counter_out, _name)} 21 22 #define INIT_VPORT_Q_COUNTER(_name) \ 23 { .name = "vport_" #_name, .offset = \ 24 MLX5_BYTE_OFF(query_q_counter_out, _name)} 25 26 static const struct mlx5_ib_counter basic_q_cnts[] = { 27 INIT_Q_COUNTER(rx_write_requests), 28 INIT_Q_COUNTER(rx_read_requests), 29 INIT_Q_COUNTER(rx_atomic_requests), 30 INIT_Q_COUNTER(rx_dct_connect), 31 INIT_Q_COUNTER(out_of_buffer), 32 }; 33 34 static const struct mlx5_ib_counter out_of_seq_q_cnts[] = { 35 INIT_Q_COUNTER(out_of_sequence), 36 }; 37 38 static const struct mlx5_ib_counter retrans_q_cnts[] = { 39 INIT_Q_COUNTER(duplicate_request), 40 INIT_Q_COUNTER(rnr_nak_retry_err), 41 INIT_Q_COUNTER(packet_seq_err), 42 INIT_Q_COUNTER(implied_nak_seq_err), 43 INIT_Q_COUNTER(local_ack_timeout_err), 44 }; 45 46 static const struct mlx5_ib_counter vport_basic_q_cnts[] = { 47 INIT_VPORT_Q_COUNTER(rx_write_requests), 48 INIT_VPORT_Q_COUNTER(rx_read_requests), 49 INIT_VPORT_Q_COUNTER(rx_atomic_requests), 50 INIT_VPORT_Q_COUNTER(rx_dct_connect), 51 INIT_VPORT_Q_COUNTER(out_of_buffer), 52 }; 53 54 static const struct mlx5_ib_counter vport_out_of_seq_q_cnts[] = { 55 INIT_VPORT_Q_COUNTER(out_of_sequence), 56 }; 57 58 static const struct mlx5_ib_counter vport_retrans_q_cnts[] = { 59 INIT_VPORT_Q_COUNTER(duplicate_request), 60 INIT_VPORT_Q_COUNTER(rnr_nak_retry_err), 61 INIT_VPORT_Q_COUNTER(packet_seq_err), 62 INIT_VPORT_Q_COUNTER(implied_nak_seq_err), 63 INIT_VPORT_Q_COUNTER(local_ack_timeout_err), 64 }; 65 66 #define INIT_CONG_COUNTER(_name) \ 67 { .name = #_name, .offset = \ 68 MLX5_BYTE_OFF(query_cong_statistics_out, _name ## _high)} 69 70 static const struct mlx5_ib_counter cong_cnts[] = { 71 INIT_CONG_COUNTER(rp_cnp_ignored), 72 INIT_CONG_COUNTER(rp_cnp_handled), 73 INIT_CONG_COUNTER(np_ecn_marked_roce_packets), 74 INIT_CONG_COUNTER(np_cnp_sent), 75 }; 76 77 static const struct mlx5_ib_counter extended_err_cnts[] = { 78 INIT_Q_COUNTER(resp_local_length_error), 79 INIT_Q_COUNTER(resp_cqe_error), 80 INIT_Q_COUNTER(req_cqe_error), 81 INIT_Q_COUNTER(req_remote_invalid_request), 82 INIT_Q_COUNTER(req_remote_access_errors), 83 INIT_Q_COUNTER(resp_remote_access_errors), 84 INIT_Q_COUNTER(resp_cqe_flush_error), 85 INIT_Q_COUNTER(req_cqe_flush_error), 86 }; 87 88 static const struct mlx5_ib_counter roce_accl_cnts[] = { 89 INIT_Q_COUNTER(roce_adp_retrans), 90 INIT_Q_COUNTER(roce_adp_retrans_to), 91 INIT_Q_COUNTER(roce_slow_restart), 92 INIT_Q_COUNTER(roce_slow_restart_cnps), 93 INIT_Q_COUNTER(roce_slow_restart_trans), 94 }; 95 96 static const struct mlx5_ib_counter vport_extended_err_cnts[] = { 97 INIT_VPORT_Q_COUNTER(resp_local_length_error), 98 INIT_VPORT_Q_COUNTER(resp_cqe_error), 99 INIT_VPORT_Q_COUNTER(req_cqe_error), 100 INIT_VPORT_Q_COUNTER(req_remote_invalid_request), 101 INIT_VPORT_Q_COUNTER(req_remote_access_errors), 102 INIT_VPORT_Q_COUNTER(resp_remote_access_errors), 103 INIT_VPORT_Q_COUNTER(resp_cqe_flush_error), 104 INIT_VPORT_Q_COUNTER(req_cqe_flush_error), 105 }; 106 107 static const struct mlx5_ib_counter vport_roce_accl_cnts[] = { 108 INIT_VPORT_Q_COUNTER(roce_adp_retrans), 109 INIT_VPORT_Q_COUNTER(roce_adp_retrans_to), 110 INIT_VPORT_Q_COUNTER(roce_slow_restart), 111 INIT_VPORT_Q_COUNTER(roce_slow_restart_cnps), 112 INIT_VPORT_Q_COUNTER(roce_slow_restart_trans), 113 }; 114 115 #define INIT_EXT_PPCNT_COUNTER(_name) \ 116 { .name = #_name, .offset = \ 117 MLX5_BYTE_OFF(ppcnt_reg, \ 118 counter_set.eth_extended_cntrs_grp_data_layout._name##_high)} 119 120 static const struct mlx5_ib_counter ext_ppcnt_cnts[] = { 121 INIT_EXT_PPCNT_COUNTER(rx_icrc_encapsulated), 122 }; 123 124 #define INIT_OP_COUNTER(_name, _type) \ 125 { .name = #_name, .type = MLX5_IB_OPCOUNTER_##_type} 126 127 static const struct mlx5_ib_counter basic_op_cnts[] = { 128 INIT_OP_COUNTER(cc_rx_ce_pkts, CC_RX_CE_PKTS), 129 }; 130 131 static const struct mlx5_ib_counter rdmarx_cnp_op_cnts[] = { 132 INIT_OP_COUNTER(cc_rx_cnp_pkts, CC_RX_CNP_PKTS), 133 }; 134 135 static const struct mlx5_ib_counter rdmatx_cnp_op_cnts[] = { 136 INIT_OP_COUNTER(cc_tx_cnp_pkts, CC_TX_CNP_PKTS), 137 }; 138 139 static int mlx5_ib_read_counters(struct ib_counters *counters, 140 struct ib_counters_read_attr *read_attr, 141 struct uverbs_attr_bundle *attrs) 142 { 143 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 144 struct mlx5_read_counters_attr mread_attr = {}; 145 struct mlx5_ib_flow_counters_desc *desc; 146 int ret, i; 147 148 mutex_lock(&mcounters->mcntrs_mutex); 149 if (mcounters->cntrs_max_index > read_attr->ncounters) { 150 ret = -EINVAL; 151 goto err_bound; 152 } 153 154 mread_attr.out = kcalloc(mcounters->counters_num, sizeof(u64), 155 GFP_KERNEL); 156 if (!mread_attr.out) { 157 ret = -ENOMEM; 158 goto err_bound; 159 } 160 161 mread_attr.hw_cntrs_hndl = mcounters->hw_cntrs_hndl; 162 mread_attr.flags = read_attr->flags; 163 ret = mcounters->read_counters(counters->device, &mread_attr); 164 if (ret) 165 goto err_read; 166 167 /* do the pass over the counters data array to assign according to the 168 * descriptions and indexing pairs 169 */ 170 desc = mcounters->counters_data; 171 for (i = 0; i < mcounters->ncounters; i++) 172 read_attr->counters_buff[desc[i].index] += mread_attr.out[desc[i].description]; 173 174 err_read: 175 kfree(mread_attr.out); 176 err_bound: 177 mutex_unlock(&mcounters->mcntrs_mutex); 178 return ret; 179 } 180 181 static int mlx5_ib_destroy_counters(struct ib_counters *counters) 182 { 183 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 184 185 mlx5_ib_counters_clear_description(counters); 186 if (mcounters->hw_cntrs_hndl) 187 mlx5_fc_destroy(to_mdev(counters->device)->mdev, 188 mcounters->hw_cntrs_hndl); 189 return 0; 190 } 191 192 static int mlx5_ib_create_counters(struct ib_counters *counters, 193 struct uverbs_attr_bundle *attrs) 194 { 195 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 196 197 mutex_init(&mcounters->mcntrs_mutex); 198 return 0; 199 } 200 201 static bool vport_qcounters_supported(struct mlx5_ib_dev *dev) 202 { 203 return MLX5_CAP_GEN(dev->mdev, q_counter_other_vport) && 204 MLX5_CAP_GEN(dev->mdev, q_counter_aggregation); 205 } 206 207 static const struct mlx5_ib_counters *get_counters(struct mlx5_ib_dev *dev, 208 u32 port_num) 209 { 210 if ((is_mdev_switchdev_mode(dev->mdev) && 211 !vport_qcounters_supported(dev)) || !port_num) 212 return &dev->port[0].cnts; 213 214 return is_mdev_switchdev_mode(dev->mdev) ? 215 &dev->port[1].cnts : &dev->port[port_num - 1].cnts; 216 } 217 218 /** 219 * mlx5_ib_get_counters_id - Returns counters id to use for device+port 220 * @dev: Pointer to mlx5 IB device 221 * @port_num: Zero based port number 222 * 223 * mlx5_ib_get_counters_id() Returns counters set id to use for given 224 * device port combination in switchdev and non switchdev mode of the 225 * parent device. 226 */ 227 u16 mlx5_ib_get_counters_id(struct mlx5_ib_dev *dev, u32 port_num) 228 { 229 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num + 1); 230 231 return cnts->set_id; 232 } 233 234 static struct rdma_hw_stats *do_alloc_stats(const struct mlx5_ib_counters *cnts) 235 { 236 struct rdma_hw_stats *stats; 237 u32 num_hw_counters; 238 int i; 239 240 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 241 cnts->num_ext_ppcnt_counters; 242 stats = rdma_alloc_hw_stats_struct(cnts->descs, 243 num_hw_counters + 244 cnts->num_op_counters, 245 RDMA_HW_STATS_DEFAULT_LIFESPAN); 246 if (!stats) 247 return NULL; 248 249 for (i = 0; i < cnts->num_op_counters; i++) 250 set_bit(num_hw_counters + i, stats->is_disabled); 251 252 return stats; 253 } 254 255 static struct rdma_hw_stats * 256 mlx5_ib_alloc_hw_device_stats(struct ib_device *ibdev) 257 { 258 struct mlx5_ib_dev *dev = to_mdev(ibdev); 259 const struct mlx5_ib_counters *cnts = &dev->port[0].cnts; 260 261 return do_alloc_stats(cnts); 262 } 263 264 static struct rdma_hw_stats * 265 mlx5_ib_alloc_hw_port_stats(struct ib_device *ibdev, u32 port_num) 266 { 267 struct mlx5_ib_dev *dev = to_mdev(ibdev); 268 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 269 270 return do_alloc_stats(cnts); 271 } 272 273 static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, 274 const struct mlx5_ib_counters *cnts, 275 struct rdma_hw_stats *stats, 276 u16 set_id) 277 { 278 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 279 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 280 __be32 val; 281 int ret, i; 282 283 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 284 MLX5_SET(query_q_counter_in, in, counter_set_id, set_id); 285 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 286 if (ret) 287 return ret; 288 289 for (i = 0; i < cnts->num_q_counters; i++) { 290 val = *(__be32 *)((void *)out + cnts->offsets[i]); 291 stats->value[i] = (u64)be32_to_cpu(val); 292 } 293 294 return 0; 295 } 296 297 static int mlx5_ib_query_ext_ppcnt_counters(struct mlx5_ib_dev *dev, 298 const struct mlx5_ib_counters *cnts, 299 struct rdma_hw_stats *stats) 300 { 301 int offset = cnts->num_q_counters + cnts->num_cong_counters; 302 u32 in[MLX5_ST_SZ_DW(ppcnt_reg)] = {}; 303 int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); 304 int ret, i; 305 void *out; 306 307 out = kvzalloc(sz, GFP_KERNEL); 308 if (!out) 309 return -ENOMEM; 310 311 MLX5_SET(ppcnt_reg, in, local_port, 1); 312 MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP); 313 ret = mlx5_core_access_reg(dev->mdev, in, sz, out, sz, MLX5_REG_PPCNT, 314 0, 0); 315 if (ret) 316 goto free; 317 318 for (i = 0; i < cnts->num_ext_ppcnt_counters; i++) 319 stats->value[i + offset] = 320 be64_to_cpup((__be64 *)(out + 321 cnts->offsets[i + offset])); 322 free: 323 kvfree(out); 324 return ret; 325 } 326 327 static int mlx5_ib_query_q_counters_vport(struct mlx5_ib_dev *dev, 328 u32 port_num, 329 const struct mlx5_ib_counters *cnts, 330 struct rdma_hw_stats *stats) 331 332 { 333 u32 out[MLX5_ST_SZ_DW(query_q_counter_out)] = {}; 334 u32 in[MLX5_ST_SZ_DW(query_q_counter_in)] = {}; 335 struct mlx5_core_dev *mdev; 336 __be32 val; 337 int ret, i; 338 339 if (!dev->port[port_num].rep || 340 dev->port[port_num].rep->vport == MLX5_VPORT_UPLINK) 341 return 0; 342 343 mdev = mlx5_eswitch_get_core_dev(dev->port[port_num].rep->esw); 344 if (!mdev) 345 return -EOPNOTSUPP; 346 347 MLX5_SET(query_q_counter_in, in, opcode, MLX5_CMD_OP_QUERY_Q_COUNTER); 348 MLX5_SET(query_q_counter_in, in, other_vport, 1); 349 MLX5_SET(query_q_counter_in, in, vport_number, 350 dev->port[port_num].rep->vport); 351 MLX5_SET(query_q_counter_in, in, aggregate, 1); 352 ret = mlx5_cmd_exec_inout(mdev, query_q_counter, in, out); 353 if (ret) 354 return ret; 355 356 for (i = 0; i < cnts->num_q_counters; i++) { 357 val = *(__be32 *)((void *)out + cnts->offsets[i]); 358 stats->value[i] = (u64)be32_to_cpu(val); 359 } 360 361 return 0; 362 } 363 364 static int do_get_hw_stats(struct ib_device *ibdev, 365 struct rdma_hw_stats *stats, 366 u32 port_num, int index) 367 { 368 struct mlx5_ib_dev *dev = to_mdev(ibdev); 369 const struct mlx5_ib_counters *cnts = get_counters(dev, port_num); 370 struct mlx5_core_dev *mdev; 371 int ret, num_counters; 372 373 if (!stats) 374 return -EINVAL; 375 376 num_counters = cnts->num_q_counters + 377 cnts->num_cong_counters + 378 cnts->num_ext_ppcnt_counters; 379 380 if (is_mdev_switchdev_mode(dev->mdev) && dev->is_rep && port_num != 0) 381 ret = mlx5_ib_query_q_counters_vport(dev, port_num - 1, cnts, 382 stats); 383 else 384 ret = mlx5_ib_query_q_counters(dev->mdev, cnts, stats, 385 cnts->set_id); 386 if (ret) 387 return ret; 388 389 /* We don't expose device counters over Vports */ 390 if (is_mdev_switchdev_mode(dev->mdev) && port_num != 0) 391 goto done; 392 393 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 394 ret = mlx5_ib_query_ext_ppcnt_counters(dev, cnts, stats); 395 if (ret) 396 return ret; 397 } 398 399 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 400 if (!port_num) 401 port_num = 1; 402 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, NULL); 403 if (!mdev) { 404 /* If port is not affiliated yet, its in down state 405 * which doesn't have any counters yet, so it would be 406 * zero. So no need to read from the HCA. 407 */ 408 goto done; 409 } 410 ret = mlx5_lag_query_cong_counters(dev->mdev, 411 stats->value + 412 cnts->num_q_counters, 413 cnts->num_cong_counters, 414 cnts->offsets + 415 cnts->num_q_counters); 416 417 mlx5_ib_put_native_port_mdev(dev, port_num); 418 if (ret) 419 return ret; 420 } 421 422 done: 423 return num_counters; 424 } 425 426 static int do_get_op_stat(struct ib_device *ibdev, 427 struct rdma_hw_stats *stats, 428 u32 port_num, int index) 429 { 430 struct mlx5_ib_dev *dev = to_mdev(ibdev); 431 const struct mlx5_ib_counters *cnts; 432 const struct mlx5_ib_op_fc *opfcs; 433 u64 packets = 0, bytes; 434 u32 type; 435 int ret; 436 437 cnts = get_counters(dev, port_num); 438 439 opfcs = cnts->opfcs; 440 type = *(u32 *)cnts->descs[index].priv; 441 if (type >= MLX5_IB_OPCOUNTER_MAX) 442 return -EINVAL; 443 444 if (!opfcs[type].fc) 445 goto out; 446 447 ret = mlx5_fc_query(dev->mdev, opfcs[type].fc, 448 &packets, &bytes); 449 if (ret) 450 return ret; 451 452 out: 453 stats->value[index] = packets; 454 return index; 455 } 456 457 static int do_get_op_stats(struct ib_device *ibdev, 458 struct rdma_hw_stats *stats, 459 u32 port_num) 460 { 461 struct mlx5_ib_dev *dev = to_mdev(ibdev); 462 const struct mlx5_ib_counters *cnts; 463 int index, ret, num_hw_counters; 464 465 cnts = get_counters(dev, port_num); 466 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 467 cnts->num_ext_ppcnt_counters; 468 for (index = num_hw_counters; 469 index < (num_hw_counters + cnts->num_op_counters); index++) { 470 ret = do_get_op_stat(ibdev, stats, port_num, index); 471 if (ret != index) 472 return ret; 473 } 474 475 return cnts->num_op_counters; 476 } 477 478 static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, 479 struct rdma_hw_stats *stats, 480 u32 port_num, int index) 481 { 482 int num_counters, num_hw_counters, num_op_counters; 483 struct mlx5_ib_dev *dev = to_mdev(ibdev); 484 const struct mlx5_ib_counters *cnts; 485 486 cnts = get_counters(dev, port_num); 487 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 488 cnts->num_ext_ppcnt_counters; 489 num_counters = num_hw_counters + cnts->num_op_counters; 490 491 if (index < 0 || index > num_counters) 492 return -EINVAL; 493 else if (index > 0 && index < num_hw_counters) 494 return do_get_hw_stats(ibdev, stats, port_num, index); 495 else if (index >= num_hw_counters && index < num_counters) 496 return do_get_op_stat(ibdev, stats, port_num, index); 497 498 num_hw_counters = do_get_hw_stats(ibdev, stats, port_num, index); 499 if (num_hw_counters < 0) 500 return num_hw_counters; 501 502 num_op_counters = do_get_op_stats(ibdev, stats, port_num); 503 if (num_op_counters < 0) 504 return num_op_counters; 505 506 return num_hw_counters + num_op_counters; 507 } 508 509 static struct rdma_hw_stats * 510 mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) 511 { 512 struct mlx5_ib_dev *dev = to_mdev(counter->device); 513 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port); 514 515 return do_alloc_stats(cnts); 516 } 517 518 static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) 519 { 520 struct mlx5_ib_dev *dev = to_mdev(counter->device); 521 const struct mlx5_ib_counters *cnts = get_counters(dev, counter->port); 522 523 return mlx5_ib_query_q_counters(dev->mdev, cnts, 524 counter->stats, counter->id); 525 } 526 527 static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) 528 { 529 struct mlx5_ib_dev *dev = to_mdev(counter->device); 530 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 531 532 if (!counter->id) 533 return 0; 534 535 MLX5_SET(dealloc_q_counter_in, in, opcode, 536 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 537 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, counter->id); 538 return mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 539 } 540 541 static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, 542 struct ib_qp *qp) 543 { 544 struct mlx5_ib_dev *dev = to_mdev(qp->device); 545 int err; 546 547 if (!counter->id) { 548 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 549 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 550 551 MLX5_SET(alloc_q_counter_in, in, opcode, 552 MLX5_CMD_OP_ALLOC_Q_COUNTER); 553 MLX5_SET(alloc_q_counter_in, in, uid, MLX5_SHARED_RESOURCE_UID); 554 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 555 if (err) 556 return err; 557 counter->id = 558 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 559 } 560 561 err = mlx5_ib_qp_set_counter(qp, counter); 562 if (err) 563 goto fail_set_counter; 564 565 return 0; 566 567 fail_set_counter: 568 mlx5_ib_counter_dealloc(counter); 569 counter->id = 0; 570 571 return err; 572 } 573 574 static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) 575 { 576 return mlx5_ib_qp_set_counter(qp, NULL); 577 } 578 579 static void mlx5_ib_fill_counters(struct mlx5_ib_dev *dev, 580 struct rdma_stat_desc *descs, size_t *offsets, 581 u32 port_num) 582 { 583 bool is_vport = is_mdev_switchdev_mode(dev->mdev) && 584 port_num != MLX5_VPORT_PF; 585 const struct mlx5_ib_counter *names; 586 int j = 0, i, size; 587 588 names = is_vport ? vport_basic_q_cnts : basic_q_cnts; 589 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) : 590 ARRAY_SIZE(basic_q_cnts); 591 for (i = 0; i < size; i++, j++) { 592 descs[j].name = names[i].name; 593 offsets[j] = names[i].offset; 594 } 595 596 names = is_vport ? vport_out_of_seq_q_cnts : out_of_seq_q_cnts; 597 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) : 598 ARRAY_SIZE(out_of_seq_q_cnts); 599 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) { 600 for (i = 0; i < size; i++, j++) { 601 descs[j].name = names[i].name; 602 offsets[j] = names[i].offset; 603 } 604 } 605 606 names = is_vport ? vport_retrans_q_cnts : retrans_q_cnts; 607 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) : 608 ARRAY_SIZE(retrans_q_cnts); 609 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) { 610 for (i = 0; i < size; i++, j++) { 611 descs[j].name = names[i].name; 612 offsets[j] = names[i].offset; 613 } 614 } 615 616 names = is_vport ? vport_extended_err_cnts : extended_err_cnts; 617 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) : 618 ARRAY_SIZE(extended_err_cnts); 619 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) { 620 for (i = 0; i < size; i++, j++) { 621 descs[j].name = names[i].name; 622 offsets[j] = names[i].offset; 623 } 624 } 625 626 names = is_vport ? vport_roce_accl_cnts : roce_accl_cnts; 627 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) : 628 ARRAY_SIZE(roce_accl_cnts); 629 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) { 630 for (i = 0; i < size; i++, j++) { 631 descs[j].name = names[i].name; 632 offsets[j] = names[i].offset; 633 } 634 } 635 636 if (is_vport) 637 return; 638 639 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 640 for (i = 0; i < ARRAY_SIZE(cong_cnts); i++, j++) { 641 descs[j].name = cong_cnts[i].name; 642 offsets[j] = cong_cnts[i].offset; 643 } 644 } 645 646 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 647 for (i = 0; i < ARRAY_SIZE(ext_ppcnt_cnts); i++, j++) { 648 descs[j].name = ext_ppcnt_cnts[i].name; 649 offsets[j] = ext_ppcnt_cnts[i].offset; 650 } 651 } 652 653 for (i = 0; i < ARRAY_SIZE(basic_op_cnts); i++, j++) { 654 descs[j].name = basic_op_cnts[i].name; 655 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 656 descs[j].priv = &basic_op_cnts[i].type; 657 } 658 659 if (MLX5_CAP_FLOWTABLE(dev->mdev, 660 ft_field_support_2_nic_receive_rdma.bth_opcode)) { 661 for (i = 0; i < ARRAY_SIZE(rdmarx_cnp_op_cnts); i++, j++) { 662 descs[j].name = rdmarx_cnp_op_cnts[i].name; 663 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 664 descs[j].priv = &rdmarx_cnp_op_cnts[i].type; 665 } 666 } 667 668 if (MLX5_CAP_FLOWTABLE(dev->mdev, 669 ft_field_support_2_nic_transmit_rdma.bth_opcode)) { 670 for (i = 0; i < ARRAY_SIZE(rdmatx_cnp_op_cnts); i++, j++) { 671 descs[j].name = rdmatx_cnp_op_cnts[i].name; 672 descs[j].flags |= IB_STAT_FLAG_OPTIONAL; 673 descs[j].priv = &rdmatx_cnp_op_cnts[i].type; 674 } 675 } 676 } 677 678 679 static int __mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev, 680 struct mlx5_ib_counters *cnts, u32 port_num) 681 { 682 bool is_vport = is_mdev_switchdev_mode(dev->mdev) && 683 port_num != MLX5_VPORT_PF; 684 u32 num_counters, num_op_counters = 0, size; 685 686 size = is_vport ? ARRAY_SIZE(vport_basic_q_cnts) : 687 ARRAY_SIZE(basic_q_cnts); 688 num_counters = size; 689 690 size = is_vport ? ARRAY_SIZE(vport_out_of_seq_q_cnts) : 691 ARRAY_SIZE(out_of_seq_q_cnts); 692 if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt)) 693 num_counters += size; 694 695 size = is_vport ? ARRAY_SIZE(vport_retrans_q_cnts) : 696 ARRAY_SIZE(retrans_q_cnts); 697 if (MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) 698 num_counters += size; 699 700 size = is_vport ? ARRAY_SIZE(vport_extended_err_cnts) : 701 ARRAY_SIZE(extended_err_cnts); 702 if (MLX5_CAP_GEN(dev->mdev, enhanced_error_q_counters)) 703 num_counters += size; 704 705 size = is_vport ? ARRAY_SIZE(vport_roce_accl_cnts) : 706 ARRAY_SIZE(roce_accl_cnts); 707 if (MLX5_CAP_GEN(dev->mdev, roce_accl)) 708 num_counters += size; 709 710 cnts->num_q_counters = num_counters; 711 712 if (is_vport) 713 goto skip_non_qcounters; 714 715 if (MLX5_CAP_GEN(dev->mdev, cc_query_allowed)) { 716 cnts->num_cong_counters = ARRAY_SIZE(cong_cnts); 717 num_counters += ARRAY_SIZE(cong_cnts); 718 } 719 if (MLX5_CAP_PCAM_FEATURE(dev->mdev, rx_icrc_encapsulated_counter)) { 720 cnts->num_ext_ppcnt_counters = ARRAY_SIZE(ext_ppcnt_cnts); 721 num_counters += ARRAY_SIZE(ext_ppcnt_cnts); 722 } 723 724 num_op_counters = ARRAY_SIZE(basic_op_cnts); 725 726 if (MLX5_CAP_FLOWTABLE(dev->mdev, 727 ft_field_support_2_nic_receive_rdma.bth_opcode)) 728 num_op_counters += ARRAY_SIZE(rdmarx_cnp_op_cnts); 729 730 if (MLX5_CAP_FLOWTABLE(dev->mdev, 731 ft_field_support_2_nic_transmit_rdma.bth_opcode)) 732 num_op_counters += ARRAY_SIZE(rdmatx_cnp_op_cnts); 733 734 skip_non_qcounters: 735 cnts->num_op_counters = num_op_counters; 736 num_counters += num_op_counters; 737 cnts->descs = kcalloc(num_counters, 738 sizeof(struct rdma_stat_desc), GFP_KERNEL); 739 if (!cnts->descs) 740 return -ENOMEM; 741 742 cnts->offsets = kcalloc(num_counters, 743 sizeof(*cnts->offsets), GFP_KERNEL); 744 if (!cnts->offsets) 745 goto err; 746 747 return 0; 748 749 err: 750 kfree(cnts->descs); 751 cnts->descs = NULL; 752 return -ENOMEM; 753 } 754 755 static void mlx5_ib_dealloc_counters(struct mlx5_ib_dev *dev) 756 { 757 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {}; 758 int num_cnt_ports = dev->num_ports; 759 int i, j; 760 761 if (is_mdev_switchdev_mode(dev->mdev)) 762 num_cnt_ports = min(2, num_cnt_ports); 763 764 MLX5_SET(dealloc_q_counter_in, in, opcode, 765 MLX5_CMD_OP_DEALLOC_Q_COUNTER); 766 767 for (i = 0; i < num_cnt_ports; i++) { 768 if (dev->port[i].cnts.set_id) { 769 MLX5_SET(dealloc_q_counter_in, in, counter_set_id, 770 dev->port[i].cnts.set_id); 771 mlx5_cmd_exec_in(dev->mdev, dealloc_q_counter, in); 772 } 773 kfree(dev->port[i].cnts.descs); 774 kfree(dev->port[i].cnts.offsets); 775 776 for (j = 0; j < MLX5_IB_OPCOUNTER_MAX; j++) { 777 if (!dev->port[i].cnts.opfcs[j].fc) 778 continue; 779 780 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) 781 mlx5_ib_fs_remove_op_fc(dev, 782 &dev->port[i].cnts.opfcs[j], j); 783 mlx5_fc_destroy(dev->mdev, 784 dev->port[i].cnts.opfcs[j].fc); 785 dev->port[i].cnts.opfcs[j].fc = NULL; 786 } 787 } 788 } 789 790 static int mlx5_ib_alloc_counters(struct mlx5_ib_dev *dev) 791 { 792 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {}; 793 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {}; 794 int num_cnt_ports = dev->num_ports; 795 int err = 0; 796 int i; 797 bool is_shared; 798 799 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER); 800 is_shared = MLX5_CAP_GEN(dev->mdev, log_max_uctx) != 0; 801 802 /* 803 * In switchdev we need to allocate two ports, one that is used for 804 * the device Q_counters and it is essentially the real Q_counters of 805 * this device, while the other is used as a helper for PF to be able to 806 * query all other vports. 807 */ 808 if (is_mdev_switchdev_mode(dev->mdev)) 809 num_cnt_ports = min(2, num_cnt_ports); 810 811 for (i = 0; i < num_cnt_ports; i++) { 812 err = __mlx5_ib_alloc_counters(dev, &dev->port[i].cnts, i); 813 if (err) 814 goto err_alloc; 815 816 mlx5_ib_fill_counters(dev, dev->port[i].cnts.descs, 817 dev->port[i].cnts.offsets, i); 818 819 MLX5_SET(alloc_q_counter_in, in, uid, 820 is_shared ? MLX5_SHARED_RESOURCE_UID : 0); 821 822 err = mlx5_cmd_exec_inout(dev->mdev, alloc_q_counter, in, out); 823 if (err) { 824 mlx5_ib_warn(dev, 825 "couldn't allocate queue counter for port %d, err %d\n", 826 i + 1, err); 827 goto err_alloc; 828 } 829 830 dev->port[i].cnts.set_id = 831 MLX5_GET(alloc_q_counter_out, out, counter_set_id); 832 } 833 return 0; 834 835 err_alloc: 836 mlx5_ib_dealloc_counters(dev); 837 return err; 838 } 839 840 static int read_flow_counters(struct ib_device *ibdev, 841 struct mlx5_read_counters_attr *read_attr) 842 { 843 struct mlx5_fc *fc = read_attr->hw_cntrs_hndl; 844 struct mlx5_ib_dev *dev = to_mdev(ibdev); 845 846 return mlx5_fc_query(dev->mdev, fc, 847 &read_attr->out[IB_COUNTER_PACKETS], 848 &read_attr->out[IB_COUNTER_BYTES]); 849 } 850 851 /* flow counters currently expose two counters packets and bytes */ 852 #define FLOW_COUNTERS_NUM 2 853 static int counters_set_description( 854 struct ib_counters *counters, enum mlx5_ib_counters_type counters_type, 855 struct mlx5_ib_flow_counters_desc *desc_data, u32 ncounters) 856 { 857 struct mlx5_ib_mcounters *mcounters = to_mcounters(counters); 858 u32 cntrs_max_index = 0; 859 int i; 860 861 if (counters_type != MLX5_IB_COUNTERS_FLOW) 862 return -EINVAL; 863 864 /* init the fields for the object */ 865 mcounters->type = counters_type; 866 mcounters->read_counters = read_flow_counters; 867 mcounters->counters_num = FLOW_COUNTERS_NUM; 868 mcounters->ncounters = ncounters; 869 /* each counter entry have both description and index pair */ 870 for (i = 0; i < ncounters; i++) { 871 if (desc_data[i].description > IB_COUNTER_BYTES) 872 return -EINVAL; 873 874 if (cntrs_max_index <= desc_data[i].index) 875 cntrs_max_index = desc_data[i].index + 1; 876 } 877 878 mutex_lock(&mcounters->mcntrs_mutex); 879 mcounters->counters_data = desc_data; 880 mcounters->cntrs_max_index = cntrs_max_index; 881 mutex_unlock(&mcounters->mcntrs_mutex); 882 883 return 0; 884 } 885 886 #define MAX_COUNTERS_NUM (USHRT_MAX / (sizeof(u32) * 2)) 887 int mlx5_ib_flow_counters_set_data(struct ib_counters *ibcounters, 888 struct mlx5_ib_create_flow *ucmd) 889 { 890 struct mlx5_ib_mcounters *mcounters = to_mcounters(ibcounters); 891 struct mlx5_ib_flow_counters_data *cntrs_data = NULL; 892 struct mlx5_ib_flow_counters_desc *desc_data = NULL; 893 bool hw_hndl = false; 894 int ret = 0; 895 896 if (ucmd && ucmd->ncounters_data != 0) { 897 cntrs_data = ucmd->data; 898 if (cntrs_data->ncounters > MAX_COUNTERS_NUM) 899 return -EINVAL; 900 901 desc_data = kcalloc(cntrs_data->ncounters, 902 sizeof(*desc_data), 903 GFP_KERNEL); 904 if (!desc_data) 905 return -ENOMEM; 906 907 if (copy_from_user(desc_data, 908 u64_to_user_ptr(cntrs_data->counters_data), 909 sizeof(*desc_data) * cntrs_data->ncounters)) { 910 ret = -EFAULT; 911 goto free; 912 } 913 } 914 915 if (!mcounters->hw_cntrs_hndl) { 916 mcounters->hw_cntrs_hndl = mlx5_fc_create( 917 to_mdev(ibcounters->device)->mdev, false); 918 if (IS_ERR(mcounters->hw_cntrs_hndl)) { 919 ret = PTR_ERR(mcounters->hw_cntrs_hndl); 920 goto free; 921 } 922 hw_hndl = true; 923 } 924 925 if (desc_data) { 926 /* counters already bound to at least one flow */ 927 if (mcounters->cntrs_max_index) { 928 ret = -EINVAL; 929 goto free_hndl; 930 } 931 932 ret = counters_set_description(ibcounters, 933 MLX5_IB_COUNTERS_FLOW, 934 desc_data, 935 cntrs_data->ncounters); 936 if (ret) 937 goto free_hndl; 938 939 } else if (!mcounters->cntrs_max_index) { 940 /* counters not bound yet, must have udata passed */ 941 ret = -EINVAL; 942 goto free_hndl; 943 } 944 945 return 0; 946 947 free_hndl: 948 if (hw_hndl) { 949 mlx5_fc_destroy(to_mdev(ibcounters->device)->mdev, 950 mcounters->hw_cntrs_hndl); 951 mcounters->hw_cntrs_hndl = NULL; 952 } 953 free: 954 kfree(desc_data); 955 return ret; 956 } 957 958 void mlx5_ib_counters_clear_description(struct ib_counters *counters) 959 { 960 struct mlx5_ib_mcounters *mcounters; 961 962 if (!counters || atomic_read(&counters->usecnt) != 1) 963 return; 964 965 mcounters = to_mcounters(counters); 966 967 mutex_lock(&mcounters->mcntrs_mutex); 968 kfree(mcounters->counters_data); 969 mcounters->counters_data = NULL; 970 mcounters->cntrs_max_index = 0; 971 mutex_unlock(&mcounters->mcntrs_mutex); 972 } 973 974 static int mlx5_ib_modify_stat(struct ib_device *device, u32 port, 975 unsigned int index, bool enable) 976 { 977 struct mlx5_ib_dev *dev = to_mdev(device); 978 struct mlx5_ib_counters *cnts; 979 struct mlx5_ib_op_fc *opfc; 980 u32 num_hw_counters, type; 981 int ret; 982 983 cnts = &dev->port[port - 1].cnts; 984 num_hw_counters = cnts->num_q_counters + cnts->num_cong_counters + 985 cnts->num_ext_ppcnt_counters; 986 if (index < num_hw_counters || 987 index >= (num_hw_counters + cnts->num_op_counters)) 988 return -EINVAL; 989 990 if (!(cnts->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 991 return -EINVAL; 992 993 type = *(u32 *)cnts->descs[index].priv; 994 if (type >= MLX5_IB_OPCOUNTER_MAX) 995 return -EINVAL; 996 997 opfc = &cnts->opfcs[type]; 998 999 if (enable) { 1000 if (opfc->fc) 1001 return -EEXIST; 1002 1003 opfc->fc = mlx5_fc_create(dev->mdev, false); 1004 if (IS_ERR(opfc->fc)) 1005 return PTR_ERR(opfc->fc); 1006 1007 ret = mlx5_ib_fs_add_op_fc(dev, port, opfc, type); 1008 if (ret) { 1009 mlx5_fc_destroy(dev->mdev, opfc->fc); 1010 opfc->fc = NULL; 1011 } 1012 return ret; 1013 } 1014 1015 if (!opfc->fc) 1016 return -EINVAL; 1017 1018 mlx5_ib_fs_remove_op_fc(dev, opfc, type); 1019 mlx5_fc_destroy(dev->mdev, opfc->fc); 1020 opfc->fc = NULL; 1021 return 0; 1022 } 1023 1024 static const struct ib_device_ops hw_stats_ops = { 1025 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats, 1026 .get_hw_stats = mlx5_ib_get_hw_stats, 1027 .counter_bind_qp = mlx5_ib_counter_bind_qp, 1028 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 1029 .counter_dealloc = mlx5_ib_counter_dealloc, 1030 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 1031 .counter_update_stats = mlx5_ib_counter_update_stats, 1032 .modify_hw_stat = IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) ? 1033 mlx5_ib_modify_stat : NULL, 1034 }; 1035 1036 static const struct ib_device_ops hw_switchdev_vport_op = { 1037 .alloc_hw_port_stats = mlx5_ib_alloc_hw_port_stats, 1038 }; 1039 1040 static const struct ib_device_ops hw_switchdev_stats_ops = { 1041 .alloc_hw_device_stats = mlx5_ib_alloc_hw_device_stats, 1042 .get_hw_stats = mlx5_ib_get_hw_stats, 1043 .counter_bind_qp = mlx5_ib_counter_bind_qp, 1044 .counter_unbind_qp = mlx5_ib_counter_unbind_qp, 1045 .counter_dealloc = mlx5_ib_counter_dealloc, 1046 .counter_alloc_stats = mlx5_ib_counter_alloc_stats, 1047 .counter_update_stats = mlx5_ib_counter_update_stats, 1048 }; 1049 1050 static const struct ib_device_ops counters_ops = { 1051 .create_counters = mlx5_ib_create_counters, 1052 .destroy_counters = mlx5_ib_destroy_counters, 1053 .read_counters = mlx5_ib_read_counters, 1054 1055 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs), 1056 }; 1057 1058 int mlx5_ib_counters_init(struct mlx5_ib_dev *dev) 1059 { 1060 ib_set_device_ops(&dev->ib_dev, &counters_ops); 1061 1062 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 1063 return 0; 1064 1065 if (is_mdev_switchdev_mode(dev->mdev)) { 1066 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_stats_ops); 1067 if (vport_qcounters_supported(dev)) 1068 ib_set_device_ops(&dev->ib_dev, &hw_switchdev_vport_op); 1069 } else 1070 ib_set_device_ops(&dev->ib_dev, &hw_stats_ops); 1071 return mlx5_ib_alloc_counters(dev); 1072 } 1073 1074 void mlx5_ib_counters_cleanup(struct mlx5_ib_dev *dev) 1075 { 1076 if (!MLX5_CAP_GEN(dev->mdev, max_qp_cnt)) 1077 return; 1078 1079 mlx5_ib_dealloc_counters(dev); 1080 } 1081