1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2019 Mellanox Technologies. All rights reserved. 4 */ 5 #include <rdma/ib_verbs.h> 6 #include <rdma/rdma_counter.h> 7 8 #include "core_priv.h" 9 #include "restrack.h" 10 11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID) 12 13 static int __counter_set_mode(struct rdma_port_counter *port_counter, 14 enum rdma_nl_counter_mode new_mode, 15 enum rdma_nl_counter_mask new_mask, 16 bool bind_opcnt) 17 { 18 if (new_mode == RDMA_COUNTER_MODE_AUTO) { 19 if (new_mask & (~ALL_AUTO_MODE_MASKS)) 20 return -EINVAL; 21 if (port_counter->num_counters) 22 return -EBUSY; 23 } 24 25 port_counter->mode.mode = new_mode; 26 port_counter->mode.mask = new_mask; 27 port_counter->mode.bind_opcnt = bind_opcnt; 28 return 0; 29 } 30 31 /* 32 * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode 33 * 34 * @dev: Device to operate 35 * @port: Port to use 36 * @mask: Mask to configure 37 * @extack: Message to the user 38 * 39 * Return 0 on success. If counter mode wasn't changed then it is considered 40 * as success as well. 41 * Return -EBUSY when changing to auto mode while there are bounded counters. 42 * 43 */ 44 int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port, 45 enum rdma_nl_counter_mask mask, 46 bool bind_opcnt, 47 struct netlink_ext_ack *extack) 48 { 49 struct rdma_port_counter *port_counter; 50 enum rdma_nl_counter_mode mode; 51 int ret; 52 53 port_counter = &dev->port_data[port].port_counter; 54 if (!port_counter->hstats) 55 return -EOPNOTSUPP; 56 57 mutex_lock(&port_counter->lock); 58 if (mask) 59 mode = RDMA_COUNTER_MODE_AUTO; 60 else 61 mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL : 62 RDMA_COUNTER_MODE_NONE; 63 64 if (port_counter->mode.mode == mode && 65 port_counter->mode.mask == mask && 66 port_counter->mode.bind_opcnt == bind_opcnt) { 67 ret = 0; 68 goto out; 69 } 70 71 ret = __counter_set_mode(port_counter, mode, mask, bind_opcnt); 72 73 out: 74 mutex_unlock(&port_counter->lock); 75 if (ret == -EBUSY) 76 NL_SET_ERR_MSG( 77 extack, 78 "Modifying auto mode is not allowed when there is a bound QP"); 79 return ret; 80 } 81 82 static void auto_mode_init_counter(struct rdma_counter *counter, 83 const struct ib_qp *qp, 84 enum rdma_nl_counter_mask new_mask) 85 { 86 struct auto_mode_param *param = &counter->mode.param; 87 88 counter->mode.mode = RDMA_COUNTER_MODE_AUTO; 89 counter->mode.mask = new_mask; 90 91 if (new_mask & RDMA_COUNTER_MASK_QP_TYPE) 92 param->qp_type = qp->qp_type; 93 } 94 95 static int __rdma_counter_bind_qp(struct rdma_counter *counter, 96 struct ib_qp *qp, u32 port) 97 { 98 int ret; 99 100 if (qp->counter) 101 return -EINVAL; 102 103 if (!qp->device->ops.counter_bind_qp) 104 return -EOPNOTSUPP; 105 106 mutex_lock(&counter->lock); 107 ret = qp->device->ops.counter_bind_qp(counter, qp, port); 108 mutex_unlock(&counter->lock); 109 110 return ret; 111 } 112 113 int rdma_counter_modify(struct ib_device *dev, u32 port, 114 unsigned int index, bool enable) 115 { 116 struct rdma_hw_stats *stats; 117 int ret = 0; 118 119 if (!dev->ops.modify_hw_stat) 120 return -EOPNOTSUPP; 121 122 stats = ib_get_hw_stats_port(dev, port); 123 if (!stats || index >= stats->num_counters || 124 !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 125 return -EINVAL; 126 127 mutex_lock(&stats->lock); 128 129 if (enable != test_bit(index, stats->is_disabled)) 130 goto out; 131 132 ret = dev->ops.modify_hw_stat(dev, port, index, enable); 133 if (ret) 134 goto out; 135 136 if (enable) 137 clear_bit(index, stats->is_disabled); 138 else 139 set_bit(index, stats->is_disabled); 140 out: 141 mutex_unlock(&stats->lock); 142 return ret; 143 } 144 145 static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, 146 struct ib_qp *qp, 147 enum rdma_nl_counter_mode mode, 148 bool bind_opcnt) 149 { 150 struct rdma_port_counter *port_counter; 151 struct rdma_counter *counter; 152 int ret; 153 154 if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats) 155 return NULL; 156 157 counter = rdma_zalloc_drv_obj(dev, rdma_counter); 158 if (!counter) 159 return NULL; 160 161 counter->device = dev; 162 counter->port = port; 163 164 dev->ops.counter_init(counter); 165 166 rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER); 167 counter->stats = dev->ops.counter_alloc_stats(counter); 168 if (!counter->stats) 169 goto err_stats; 170 171 port_counter = &dev->port_data[port].port_counter; 172 mutex_lock(&port_counter->lock); 173 switch (mode) { 174 case RDMA_COUNTER_MODE_MANUAL: 175 ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL, 176 0, bind_opcnt); 177 if (ret) { 178 mutex_unlock(&port_counter->lock); 179 goto err_mode; 180 } 181 break; 182 case RDMA_COUNTER_MODE_AUTO: 183 auto_mode_init_counter(counter, qp, port_counter->mode.mask); 184 break; 185 default: 186 ret = -EOPNOTSUPP; 187 mutex_unlock(&port_counter->lock); 188 goto err_mode; 189 } 190 191 port_counter->num_counters++; 192 mutex_unlock(&port_counter->lock); 193 194 counter->mode.mode = mode; 195 counter->mode.bind_opcnt = bind_opcnt; 196 kref_init(&counter->kref); 197 mutex_init(&counter->lock); 198 199 ret = __rdma_counter_bind_qp(counter, qp, port); 200 if (ret) 201 goto err_bind; 202 203 rdma_restrack_parent_name(&counter->res, &qp->res); 204 rdma_restrack_add(&counter->res); 205 return counter; 206 207 err_bind: 208 mutex_lock(&port_counter->lock); 209 port_counter->num_counters--; 210 if (!port_counter->num_counters && 211 port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL) 212 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, 213 false); 214 mutex_unlock(&port_counter->lock); 215 err_mode: 216 rdma_free_hw_stats_struct(counter->stats); 217 err_stats: 218 rdma_restrack_put(&counter->res); 219 kfree(counter); 220 return NULL; 221 } 222 223 static void rdma_counter_free(struct rdma_counter *counter) 224 { 225 struct rdma_port_counter *port_counter; 226 227 port_counter = &counter->device->port_data[counter->port].port_counter; 228 mutex_lock(&port_counter->lock); 229 port_counter->num_counters--; 230 if (!port_counter->num_counters && 231 (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL)) 232 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, 233 false); 234 235 mutex_unlock(&port_counter->lock); 236 237 rdma_free_hw_stats_struct(counter->stats); 238 kfree(counter); 239 } 240 241 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, 242 enum rdma_nl_counter_mask auto_mask) 243 { 244 struct auto_mode_param *param = &counter->mode.param; 245 bool match = true; 246 247 if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE) 248 match &= (param->qp_type == qp->qp_type); 249 250 if (auto_mask & RDMA_COUNTER_MASK_PID) 251 match &= (task_pid_nr(counter->res.task) == 252 task_pid_nr(qp->res.task)); 253 254 return match; 255 } 256 257 static int __rdma_counter_unbind_qp(struct ib_qp *qp, u32 port) 258 { 259 struct rdma_counter *counter = qp->counter; 260 int ret; 261 262 if (!qp->device->ops.counter_unbind_qp) 263 return -EOPNOTSUPP; 264 265 mutex_lock(&counter->lock); 266 ret = qp->device->ops.counter_unbind_qp(qp, port); 267 mutex_unlock(&counter->lock); 268 269 return ret; 270 } 271 272 static void counter_history_stat_update(struct rdma_counter *counter) 273 { 274 struct ib_device *dev = counter->device; 275 struct rdma_port_counter *port_counter; 276 int i; 277 278 port_counter = &dev->port_data[counter->port].port_counter; 279 if (!port_counter->hstats) 280 return; 281 282 rdma_counter_query_stats(counter); 283 284 for (i = 0; i < counter->stats->num_counters; i++) 285 port_counter->hstats->value[i] += counter->stats->value[i]; 286 } 287 288 /* 289 * rdma_get_counter_auto_mode - Find the counter that @qp should be bound 290 * with in auto mode 291 * 292 * Return: The counter (with ref-count increased) if found 293 */ 294 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp, 295 u32 port) 296 { 297 struct rdma_port_counter *port_counter; 298 struct rdma_counter *counter = NULL; 299 struct ib_device *dev = qp->device; 300 struct rdma_restrack_entry *res; 301 struct rdma_restrack_root *rt; 302 unsigned long id = 0; 303 304 port_counter = &dev->port_data[port].port_counter; 305 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 306 xa_lock(&rt->xa); 307 xa_for_each(&rt->xa, id, res) { 308 counter = container_of(res, struct rdma_counter, res); 309 if ((counter->device != qp->device) || (counter->port != port)) 310 goto next; 311 312 if (auto_mode_match(qp, counter, port_counter->mode.mask)) 313 break; 314 next: 315 counter = NULL; 316 } 317 318 if (counter && !kref_get_unless_zero(&counter->kref)) 319 counter = NULL; 320 321 xa_unlock(&rt->xa); 322 return counter; 323 } 324 325 static void counter_release(struct kref *kref) 326 { 327 struct rdma_counter *counter; 328 329 counter = container_of(kref, struct rdma_counter, kref); 330 counter_history_stat_update(counter); 331 rdma_restrack_del(&counter->res); 332 counter->device->ops.counter_dealloc(counter); 333 rdma_counter_free(counter); 334 } 335 336 /* 337 * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on 338 * the auto-mode rule 339 */ 340 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port) 341 { 342 struct rdma_port_counter *port_counter; 343 struct ib_device *dev = qp->device; 344 struct rdma_counter *counter; 345 int ret; 346 347 if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res)) 348 return 0; 349 350 if (!rdma_is_port_valid(dev, port)) 351 return -EINVAL; 352 353 port_counter = &dev->port_data[port].port_counter; 354 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) 355 return 0; 356 357 counter = rdma_get_counter_auto_mode(qp, port); 358 if (counter) { 359 ret = __rdma_counter_bind_qp(counter, qp, port); 360 if (ret) { 361 kref_put(&counter->kref, counter_release); 362 return ret; 363 } 364 } else { 365 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO, 366 port_counter->mode.bind_opcnt); 367 if (!counter) 368 return -ENOMEM; 369 } 370 371 return 0; 372 } 373 374 /* 375 * rdma_counter_unbind_qp - Unbind a qp from a counter 376 * @force: 377 * true - Decrease the counter ref-count anyway (e.g., qp destroy) 378 */ 379 int rdma_counter_unbind_qp(struct ib_qp *qp, u32 port, bool force) 380 { 381 struct rdma_counter *counter = qp->counter; 382 int ret; 383 384 if (!counter) 385 return -EINVAL; 386 387 ret = __rdma_counter_unbind_qp(qp, port); 388 if (ret && !force) 389 return ret; 390 391 kref_put(&counter->kref, counter_release); 392 return 0; 393 } 394 395 int rdma_counter_query_stats(struct rdma_counter *counter) 396 { 397 struct ib_device *dev = counter->device; 398 int ret; 399 400 if (!dev->ops.counter_update_stats) 401 return -EINVAL; 402 403 mutex_lock(&counter->lock); 404 ret = dev->ops.counter_update_stats(counter); 405 mutex_unlock(&counter->lock); 406 407 return ret; 408 } 409 410 static u64 get_running_counters_hwstat_sum(struct ib_device *dev, 411 u32 port, u32 index) 412 { 413 struct rdma_restrack_entry *res; 414 struct rdma_restrack_root *rt; 415 struct rdma_counter *counter; 416 unsigned long id = 0; 417 u64 sum = 0; 418 419 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 420 xa_lock(&rt->xa); 421 xa_for_each(&rt->xa, id, res) { 422 if (!rdma_restrack_get(res)) 423 continue; 424 425 xa_unlock(&rt->xa); 426 427 counter = container_of(res, struct rdma_counter, res); 428 if ((counter->device != dev) || (counter->port != port) || 429 rdma_counter_query_stats(counter)) 430 goto next; 431 432 sum += counter->stats->value[index]; 433 434 next: 435 xa_lock(&rt->xa); 436 rdma_restrack_put(res); 437 } 438 439 xa_unlock(&rt->xa); 440 return sum; 441 } 442 443 /* 444 * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a 445 * specific port, including the running ones and history data 446 */ 447 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index) 448 { 449 struct rdma_port_counter *port_counter; 450 u64 sum; 451 452 port_counter = &dev->port_data[port].port_counter; 453 if (!port_counter->hstats) 454 return 0; 455 456 sum = get_running_counters_hwstat_sum(dev, port, index); 457 sum += port_counter->hstats->value[index]; 458 459 return sum; 460 } 461 462 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num) 463 { 464 struct rdma_restrack_entry *res = NULL; 465 struct ib_qp *qp = NULL; 466 467 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num); 468 if (IS_ERR(res)) 469 return NULL; 470 471 qp = container_of(res, struct ib_qp, res); 472 if (qp->qp_type == IB_QPT_RAW_PACKET && !rdma_dev_has_raw_cap(dev)) 473 goto err; 474 475 return qp; 476 477 err: 478 rdma_restrack_put(res); 479 return NULL; 480 } 481 482 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, 483 u32 counter_id) 484 { 485 struct rdma_restrack_entry *res; 486 struct rdma_counter *counter; 487 488 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id); 489 if (IS_ERR(res)) 490 return NULL; 491 492 counter = container_of(res, struct rdma_counter, res); 493 if (!kref_get_unless_zero(&counter->kref)) 494 counter = NULL; 495 rdma_restrack_put(res); 496 497 return counter; 498 } 499 500 /* 501 * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id 502 */ 503 int rdma_counter_bind_qpn(struct ib_device *dev, u32 port, 504 u32 qp_num, u32 counter_id) 505 { 506 struct rdma_port_counter *port_counter; 507 struct rdma_counter *counter; 508 struct ib_qp *qp; 509 int ret; 510 511 port_counter = &dev->port_data[port].port_counter; 512 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 513 return -EINVAL; 514 515 qp = rdma_counter_get_qp(dev, qp_num); 516 if (!qp) 517 return -ENOENT; 518 519 counter = rdma_get_counter_by_id(dev, counter_id); 520 if (!counter) { 521 ret = -ENOENT; 522 goto err; 523 } 524 525 if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) { 526 ret = -EINVAL; 527 goto err_task; 528 } 529 530 if ((counter->device != qp->device) || (counter->port != qp->port)) { 531 ret = -EINVAL; 532 goto err_task; 533 } 534 535 ret = __rdma_counter_bind_qp(counter, qp, port); 536 if (ret) 537 goto err_task; 538 539 rdma_restrack_put(&qp->res); 540 return 0; 541 542 err_task: 543 kref_put(&counter->kref, counter_release); 544 err: 545 rdma_restrack_put(&qp->res); 546 return ret; 547 } 548 549 /* 550 * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it 551 * The id of new counter is returned in @counter_id 552 */ 553 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u32 port, 554 u32 qp_num, u32 *counter_id) 555 { 556 struct rdma_port_counter *port_counter; 557 struct rdma_counter *counter; 558 struct ib_qp *qp; 559 int ret; 560 561 if (!rdma_is_port_valid(dev, port)) 562 return -EINVAL; 563 564 port_counter = &dev->port_data[port].port_counter; 565 if (!port_counter->hstats) 566 return -EOPNOTSUPP; 567 568 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 569 return -EINVAL; 570 571 qp = rdma_counter_get_qp(dev, qp_num); 572 if (!qp) 573 return -ENOENT; 574 575 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 576 ret = -EINVAL; 577 goto err; 578 } 579 580 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL, true); 581 if (!counter) { 582 ret = -ENOMEM; 583 goto err; 584 } 585 586 if (counter_id) 587 *counter_id = counter->id; 588 589 rdma_restrack_put(&qp->res); 590 return 0; 591 592 err: 593 rdma_restrack_put(&qp->res); 594 return ret; 595 } 596 597 /* 598 * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter 599 */ 600 int rdma_counter_unbind_qpn(struct ib_device *dev, u32 port, 601 u32 qp_num, u32 counter_id) 602 { 603 struct rdma_port_counter *port_counter; 604 struct ib_qp *qp; 605 int ret; 606 607 if (!rdma_is_port_valid(dev, port)) 608 return -EINVAL; 609 610 qp = rdma_counter_get_qp(dev, qp_num); 611 if (!qp) 612 return -ENOENT; 613 614 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 615 ret = -EINVAL; 616 goto out; 617 } 618 619 port_counter = &dev->port_data[port].port_counter; 620 if (!qp->counter || qp->counter->id != counter_id || 621 port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) { 622 ret = -EINVAL; 623 goto out; 624 } 625 626 ret = rdma_counter_unbind_qp(qp, port, false); 627 628 out: 629 rdma_restrack_put(&qp->res); 630 return ret; 631 } 632 633 int rdma_counter_get_mode(struct ib_device *dev, u32 port, 634 enum rdma_nl_counter_mode *mode, 635 enum rdma_nl_counter_mask *mask, 636 bool *opcnt) 637 { 638 struct rdma_port_counter *port_counter; 639 640 port_counter = &dev->port_data[port].port_counter; 641 *mode = port_counter->mode.mode; 642 *mask = port_counter->mode.mask; 643 *opcnt = port_counter->mode.bind_opcnt; 644 645 return 0; 646 } 647 648 void rdma_counter_init(struct ib_device *dev) 649 { 650 struct rdma_port_counter *port_counter; 651 u32 port, i; 652 653 if (!dev->port_data) 654 return; 655 656 rdma_for_each_port(dev, port) { 657 port_counter = &dev->port_data[port].port_counter; 658 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE; 659 mutex_init(&port_counter->lock); 660 661 if (!dev->ops.alloc_hw_port_stats) 662 continue; 663 664 port_counter->hstats = dev->ops.alloc_hw_port_stats(dev, port); 665 if (!port_counter->hstats) 666 goto fail; 667 } 668 669 return; 670 671 fail: 672 for (i = port; i >= rdma_start_port(dev); i--) { 673 port_counter = &dev->port_data[i].port_counter; 674 rdma_free_hw_stats_struct(port_counter->hstats); 675 port_counter->hstats = NULL; 676 mutex_destroy(&port_counter->lock); 677 } 678 } 679 680 void rdma_counter_release(struct ib_device *dev) 681 { 682 struct rdma_port_counter *port_counter; 683 u32 port; 684 685 rdma_for_each_port(dev, port) { 686 port_counter = &dev->port_data[port].port_counter; 687 rdma_free_hw_stats_struct(port_counter->hstats); 688 mutex_destroy(&port_counter->lock); 689 } 690 } 691