1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* 3 * Copyright (c) 2019 Mellanox Technologies. All rights reserved. 4 */ 5 #include <rdma/ib_verbs.h> 6 #include <rdma/rdma_counter.h> 7 8 #include "core_priv.h" 9 #include "restrack.h" 10 11 #define ALL_AUTO_MODE_MASKS (RDMA_COUNTER_MASK_QP_TYPE | RDMA_COUNTER_MASK_PID) 12 13 static int __counter_set_mode(struct rdma_port_counter *port_counter, 14 enum rdma_nl_counter_mode new_mode, 15 enum rdma_nl_counter_mask new_mask, 16 bool bind_opcnt) 17 { 18 if (new_mode == RDMA_COUNTER_MODE_AUTO) { 19 if (new_mask & (~ALL_AUTO_MODE_MASKS)) 20 return -EINVAL; 21 if (port_counter->num_counters) 22 return -EBUSY; 23 } 24 25 port_counter->mode.mode = new_mode; 26 port_counter->mode.mask = new_mask; 27 port_counter->mode.bind_opcnt = bind_opcnt; 28 return 0; 29 } 30 31 /* 32 * rdma_counter_set_auto_mode() - Turn on/off per-port auto mode 33 * 34 * @dev: Device to operate 35 * @port: Port to use 36 * @mask: Mask to configure 37 * @extack: Message to the user 38 * 39 * Return 0 on success. If counter mode wasn't changed then it is considered 40 * as success as well. 41 * Return -EBUSY when changing to auto mode while there are bounded counters. 42 * 43 */ 44 int rdma_counter_set_auto_mode(struct ib_device *dev, u32 port, 45 enum rdma_nl_counter_mask mask, 46 bool bind_opcnt, 47 struct netlink_ext_ack *extack) 48 { 49 struct rdma_port_counter *port_counter; 50 enum rdma_nl_counter_mode mode; 51 int ret; 52 53 port_counter = &dev->port_data[port].port_counter; 54 if (!port_counter->hstats) 55 return -EOPNOTSUPP; 56 57 mutex_lock(&port_counter->lock); 58 if (mask) 59 mode = RDMA_COUNTER_MODE_AUTO; 60 else 61 mode = (port_counter->num_counters) ? RDMA_COUNTER_MODE_MANUAL : 62 RDMA_COUNTER_MODE_NONE; 63 64 if (port_counter->mode.mode == mode && 65 port_counter->mode.mask == mask && 66 port_counter->mode.bind_opcnt == bind_opcnt) { 67 ret = 0; 68 goto out; 69 } 70 71 ret = __counter_set_mode(port_counter, mode, mask, bind_opcnt); 72 73 out: 74 mutex_unlock(&port_counter->lock); 75 if (ret == -EBUSY) 76 NL_SET_ERR_MSG( 77 extack, 78 "Modifying auto mode is not allowed when there is a bound QP"); 79 return ret; 80 } 81 82 static void auto_mode_init_counter(struct rdma_counter *counter, 83 const struct ib_qp *qp, 84 enum rdma_nl_counter_mask new_mask) 85 { 86 struct auto_mode_param *param = &counter->mode.param; 87 88 counter->mode.mode = RDMA_COUNTER_MODE_AUTO; 89 counter->mode.mask = new_mask; 90 91 if (new_mask & RDMA_COUNTER_MASK_QP_TYPE) 92 param->qp_type = qp->qp_type; 93 } 94 95 static int __rdma_counter_bind_qp(struct rdma_counter *counter, 96 struct ib_qp *qp, u32 port) 97 { 98 int ret; 99 100 if (qp->counter) 101 return -EINVAL; 102 103 if (!qp->device->ops.counter_bind_qp) 104 return -EOPNOTSUPP; 105 106 mutex_lock(&counter->lock); 107 ret = qp->device->ops.counter_bind_qp(counter, qp, port); 108 mutex_unlock(&counter->lock); 109 110 return ret; 111 } 112 113 int rdma_counter_modify(struct ib_device *dev, u32 port, 114 unsigned int index, bool enable) 115 { 116 struct rdma_hw_stats *stats; 117 int ret = 0; 118 119 if (!dev->ops.modify_hw_stat) 120 return -EOPNOTSUPP; 121 122 stats = ib_get_hw_stats_port(dev, port); 123 if (!stats || index >= stats->num_counters || 124 !(stats->descs[index].flags & IB_STAT_FLAG_OPTIONAL)) 125 return -EINVAL; 126 127 mutex_lock(&stats->lock); 128 129 if (enable != test_bit(index, stats->is_disabled)) 130 goto out; 131 132 ret = dev->ops.modify_hw_stat(dev, port, index, enable); 133 if (ret) 134 goto out; 135 136 if (enable) 137 clear_bit(index, stats->is_disabled); 138 else 139 set_bit(index, stats->is_disabled); 140 out: 141 mutex_unlock(&stats->lock); 142 return ret; 143 } 144 145 static struct rdma_counter *alloc_and_bind(struct ib_device *dev, u32 port, 146 struct ib_qp *qp, 147 enum rdma_nl_counter_mode mode, 148 bool bind_opcnt) 149 { 150 struct rdma_port_counter *port_counter; 151 struct rdma_counter *counter; 152 int ret; 153 154 if (!dev->ops.counter_dealloc || !dev->ops.counter_alloc_stats) 155 return NULL; 156 157 counter = rdma_zalloc_drv_obj(dev, rdma_counter); 158 if (!counter) 159 return NULL; 160 161 counter->device = dev; 162 counter->port = port; 163 164 dev->ops.counter_init(counter); 165 166 rdma_restrack_new(&counter->res, RDMA_RESTRACK_COUNTER); 167 counter->stats = dev->ops.counter_alloc_stats(counter); 168 if (!counter->stats) 169 goto err_stats; 170 171 port_counter = &dev->port_data[port].port_counter; 172 mutex_lock(&port_counter->lock); 173 switch (mode) { 174 case RDMA_COUNTER_MODE_MANUAL: 175 ret = __counter_set_mode(port_counter, RDMA_COUNTER_MODE_MANUAL, 176 0, bind_opcnt); 177 if (ret) { 178 mutex_unlock(&port_counter->lock); 179 goto err_mode; 180 } 181 break; 182 case RDMA_COUNTER_MODE_AUTO: 183 auto_mode_init_counter(counter, qp, port_counter->mode.mask); 184 break; 185 default: 186 ret = -EOPNOTSUPP; 187 mutex_unlock(&port_counter->lock); 188 goto err_mode; 189 } 190 191 port_counter->num_counters++; 192 mutex_unlock(&port_counter->lock); 193 194 counter->mode.mode = mode; 195 counter->mode.bind_opcnt = bind_opcnt; 196 kref_init(&counter->kref); 197 mutex_init(&counter->lock); 198 199 ret = __rdma_counter_bind_qp(counter, qp, port); 200 if (ret) 201 goto err_bind; 202 203 rdma_restrack_parent_name(&counter->res, &qp->res); 204 rdma_restrack_add(&counter->res); 205 return counter; 206 207 err_bind: 208 mutex_lock(&port_counter->lock); 209 port_counter->num_counters--; 210 if (!port_counter->num_counters && 211 port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL) 212 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, 213 false); 214 mutex_unlock(&port_counter->lock); 215 err_mode: 216 rdma_free_hw_stats_struct(counter->stats); 217 err_stats: 218 rdma_restrack_put(&counter->res); 219 kfree(counter); 220 return NULL; 221 } 222 223 static void rdma_counter_free(struct rdma_counter *counter) 224 { 225 struct rdma_port_counter *port_counter; 226 227 port_counter = &counter->device->port_data[counter->port].port_counter; 228 mutex_lock(&port_counter->lock); 229 port_counter->num_counters--; 230 if (!port_counter->num_counters && 231 (port_counter->mode.mode == RDMA_COUNTER_MODE_MANUAL)) 232 __counter_set_mode(port_counter, RDMA_COUNTER_MODE_NONE, 0, 233 false); 234 235 mutex_unlock(&port_counter->lock); 236 237 rdma_restrack_del(&counter->res); 238 rdma_free_hw_stats_struct(counter->stats); 239 kfree(counter); 240 } 241 242 static bool auto_mode_match(struct ib_qp *qp, struct rdma_counter *counter, 243 enum rdma_nl_counter_mask auto_mask) 244 { 245 struct auto_mode_param *param = &counter->mode.param; 246 bool match = true; 247 248 if (auto_mask & RDMA_COUNTER_MASK_QP_TYPE) 249 match &= (param->qp_type == qp->qp_type); 250 251 if (auto_mask & RDMA_COUNTER_MASK_PID) 252 match &= (task_pid_nr(counter->res.task) == 253 task_pid_nr(qp->res.task)); 254 255 return match; 256 } 257 258 static int __rdma_counter_unbind_qp(struct ib_qp *qp, u32 port) 259 { 260 struct rdma_counter *counter = qp->counter; 261 int ret; 262 263 if (!qp->device->ops.counter_unbind_qp) 264 return -EOPNOTSUPP; 265 266 mutex_lock(&counter->lock); 267 ret = qp->device->ops.counter_unbind_qp(qp, port); 268 mutex_unlock(&counter->lock); 269 270 return ret; 271 } 272 273 static void counter_history_stat_update(struct rdma_counter *counter) 274 { 275 struct ib_device *dev = counter->device; 276 struct rdma_port_counter *port_counter; 277 int i; 278 279 port_counter = &dev->port_data[counter->port].port_counter; 280 if (!port_counter->hstats) 281 return; 282 283 rdma_counter_query_stats(counter); 284 285 for (i = 0; i < counter->stats->num_counters; i++) 286 port_counter->hstats->value[i] += counter->stats->value[i]; 287 } 288 289 /* 290 * rdma_get_counter_auto_mode - Find the counter that @qp should be bound 291 * with in auto mode 292 * 293 * Return: The counter (with ref-count increased) if found 294 */ 295 static struct rdma_counter *rdma_get_counter_auto_mode(struct ib_qp *qp, 296 u32 port) 297 { 298 struct rdma_port_counter *port_counter; 299 struct rdma_counter *counter = NULL; 300 struct ib_device *dev = qp->device; 301 struct rdma_restrack_entry *res; 302 struct rdma_restrack_root *rt; 303 unsigned long id = 0; 304 305 port_counter = &dev->port_data[port].port_counter; 306 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 307 xa_lock(&rt->xa); 308 xa_for_each(&rt->xa, id, res) { 309 counter = container_of(res, struct rdma_counter, res); 310 if ((counter->device != qp->device) || (counter->port != port)) 311 goto next; 312 313 if (auto_mode_match(qp, counter, port_counter->mode.mask)) 314 break; 315 next: 316 counter = NULL; 317 } 318 319 if (counter && !kref_get_unless_zero(&counter->kref)) 320 counter = NULL; 321 322 xa_unlock(&rt->xa); 323 return counter; 324 } 325 326 static void counter_release(struct kref *kref) 327 { 328 struct rdma_counter *counter; 329 330 counter = container_of(kref, struct rdma_counter, kref); 331 counter_history_stat_update(counter); 332 counter->device->ops.counter_dealloc(counter); 333 rdma_counter_free(counter); 334 } 335 336 /* 337 * rdma_counter_bind_qp_auto - Check and bind the QP to a counter base on 338 * the auto-mode rule 339 */ 340 int rdma_counter_bind_qp_auto(struct ib_qp *qp, u32 port) 341 { 342 struct rdma_port_counter *port_counter; 343 struct ib_device *dev = qp->device; 344 struct rdma_counter *counter; 345 int ret; 346 347 if (!rdma_restrack_is_tracked(&qp->res) || rdma_is_kernel_res(&qp->res)) 348 return 0; 349 350 if (!rdma_is_port_valid(dev, port)) 351 return -EINVAL; 352 353 port_counter = &dev->port_data[port].port_counter; 354 if (port_counter->mode.mode != RDMA_COUNTER_MODE_AUTO) 355 return 0; 356 357 counter = rdma_get_counter_auto_mode(qp, port); 358 if (counter) { 359 ret = __rdma_counter_bind_qp(counter, qp, port); 360 if (ret) { 361 kref_put(&counter->kref, counter_release); 362 return ret; 363 } 364 } else { 365 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_AUTO, 366 port_counter->mode.bind_opcnt); 367 if (!counter) 368 return -ENOMEM; 369 } 370 371 return 0; 372 } 373 374 /* 375 * rdma_counter_unbind_qp - Unbind a qp from a counter 376 * @force: 377 * true - Decrease the counter ref-count anyway (e.g., qp destroy) 378 */ 379 int rdma_counter_unbind_qp(struct ib_qp *qp, u32 port, bool force) 380 { 381 struct rdma_counter *counter = qp->counter; 382 int ret; 383 384 if (!counter) 385 return -EINVAL; 386 387 ret = __rdma_counter_unbind_qp(qp, port); 388 if (ret && !force) 389 return ret; 390 391 kref_put(&counter->kref, counter_release); 392 return 0; 393 } 394 395 int rdma_counter_query_stats(struct rdma_counter *counter) 396 { 397 struct ib_device *dev = counter->device; 398 int ret; 399 400 if (!dev->ops.counter_update_stats) 401 return -EINVAL; 402 403 mutex_lock(&counter->lock); 404 ret = dev->ops.counter_update_stats(counter); 405 mutex_unlock(&counter->lock); 406 407 return ret; 408 } 409 410 static u64 get_running_counters_hwstat_sum(struct ib_device *dev, 411 u32 port, u32 index) 412 { 413 struct rdma_restrack_entry *res; 414 struct rdma_restrack_root *rt; 415 struct rdma_counter *counter; 416 unsigned long id = 0; 417 u64 sum = 0; 418 419 rt = &dev->res[RDMA_RESTRACK_COUNTER]; 420 xa_lock(&rt->xa); 421 xa_for_each(&rt->xa, id, res) { 422 if (!rdma_restrack_get(res)) 423 continue; 424 425 xa_unlock(&rt->xa); 426 427 counter = container_of(res, struct rdma_counter, res); 428 if ((counter->device != dev) || (counter->port != port) || 429 rdma_counter_query_stats(counter)) 430 goto next; 431 432 sum += counter->stats->value[index]; 433 434 next: 435 xa_lock(&rt->xa); 436 rdma_restrack_put(res); 437 } 438 439 xa_unlock(&rt->xa); 440 return sum; 441 } 442 443 /* 444 * rdma_counter_get_hwstat_value() - Get the sum value of all counters on a 445 * specific port, including the running ones and history data 446 */ 447 u64 rdma_counter_get_hwstat_value(struct ib_device *dev, u32 port, u32 index) 448 { 449 struct rdma_port_counter *port_counter; 450 u64 sum; 451 452 port_counter = &dev->port_data[port].port_counter; 453 if (!port_counter->hstats) 454 return 0; 455 456 sum = get_running_counters_hwstat_sum(dev, port, index); 457 sum += port_counter->hstats->value[index]; 458 459 return sum; 460 } 461 462 static struct ib_qp *rdma_counter_get_qp(struct ib_device *dev, u32 qp_num) 463 { 464 struct rdma_restrack_entry *res = NULL; 465 struct ib_qp *qp = NULL; 466 467 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_QP, qp_num); 468 if (IS_ERR(res)) 469 return NULL; 470 471 qp = container_of(res, struct ib_qp, res); 472 if (qp->qp_type == IB_QPT_RAW_PACKET && !rdma_dev_has_raw_cap(dev)) 473 goto err; 474 475 return qp; 476 477 err: 478 rdma_restrack_put(res); 479 return NULL; 480 } 481 482 static struct rdma_counter *rdma_get_counter_by_id(struct ib_device *dev, 483 u32 counter_id) 484 { 485 struct rdma_restrack_entry *res; 486 struct rdma_counter *counter; 487 488 res = rdma_restrack_get_byid(dev, RDMA_RESTRACK_COUNTER, counter_id); 489 if (IS_ERR(res)) 490 return NULL; 491 492 counter = container_of(res, struct rdma_counter, res); 493 kref_get(&counter->kref); 494 rdma_restrack_put(res); 495 496 return counter; 497 } 498 499 /* 500 * rdma_counter_bind_qpn() - Bind QP @qp_num to counter @counter_id 501 */ 502 int rdma_counter_bind_qpn(struct ib_device *dev, u32 port, 503 u32 qp_num, u32 counter_id) 504 { 505 struct rdma_port_counter *port_counter; 506 struct rdma_counter *counter; 507 struct ib_qp *qp; 508 int ret; 509 510 port_counter = &dev->port_data[port].port_counter; 511 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 512 return -EINVAL; 513 514 qp = rdma_counter_get_qp(dev, qp_num); 515 if (!qp) 516 return -ENOENT; 517 518 counter = rdma_get_counter_by_id(dev, counter_id); 519 if (!counter) { 520 ret = -ENOENT; 521 goto err; 522 } 523 524 if (rdma_is_kernel_res(&counter->res) != rdma_is_kernel_res(&qp->res)) { 525 ret = -EINVAL; 526 goto err_task; 527 } 528 529 if ((counter->device != qp->device) || (counter->port != qp->port)) { 530 ret = -EINVAL; 531 goto err_task; 532 } 533 534 ret = __rdma_counter_bind_qp(counter, qp, port); 535 if (ret) 536 goto err_task; 537 538 rdma_restrack_put(&qp->res); 539 return 0; 540 541 err_task: 542 kref_put(&counter->kref, counter_release); 543 err: 544 rdma_restrack_put(&qp->res); 545 return ret; 546 } 547 548 /* 549 * rdma_counter_bind_qpn_alloc() - Alloc a counter and bind QP @qp_num to it 550 * The id of new counter is returned in @counter_id 551 */ 552 int rdma_counter_bind_qpn_alloc(struct ib_device *dev, u32 port, 553 u32 qp_num, u32 *counter_id) 554 { 555 struct rdma_port_counter *port_counter; 556 struct rdma_counter *counter; 557 struct ib_qp *qp; 558 int ret; 559 560 if (!rdma_is_port_valid(dev, port)) 561 return -EINVAL; 562 563 port_counter = &dev->port_data[port].port_counter; 564 if (!port_counter->hstats) 565 return -EOPNOTSUPP; 566 567 if (port_counter->mode.mode == RDMA_COUNTER_MODE_AUTO) 568 return -EINVAL; 569 570 qp = rdma_counter_get_qp(dev, qp_num); 571 if (!qp) 572 return -ENOENT; 573 574 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 575 ret = -EINVAL; 576 goto err; 577 } 578 579 counter = alloc_and_bind(dev, port, qp, RDMA_COUNTER_MODE_MANUAL, true); 580 if (!counter) { 581 ret = -ENOMEM; 582 goto err; 583 } 584 585 if (counter_id) 586 *counter_id = counter->id; 587 588 rdma_restrack_put(&qp->res); 589 return 0; 590 591 err: 592 rdma_restrack_put(&qp->res); 593 return ret; 594 } 595 596 /* 597 * rdma_counter_unbind_qpn() - Unbind QP @qp_num from a counter 598 */ 599 int rdma_counter_unbind_qpn(struct ib_device *dev, u32 port, 600 u32 qp_num, u32 counter_id) 601 { 602 struct rdma_port_counter *port_counter; 603 struct ib_qp *qp; 604 int ret; 605 606 if (!rdma_is_port_valid(dev, port)) 607 return -EINVAL; 608 609 qp = rdma_counter_get_qp(dev, qp_num); 610 if (!qp) 611 return -ENOENT; 612 613 if (rdma_is_port_valid(dev, qp->port) && (qp->port != port)) { 614 ret = -EINVAL; 615 goto out; 616 } 617 618 port_counter = &dev->port_data[port].port_counter; 619 if (!qp->counter || qp->counter->id != counter_id || 620 port_counter->mode.mode != RDMA_COUNTER_MODE_MANUAL) { 621 ret = -EINVAL; 622 goto out; 623 } 624 625 ret = rdma_counter_unbind_qp(qp, port, false); 626 627 out: 628 rdma_restrack_put(&qp->res); 629 return ret; 630 } 631 632 int rdma_counter_get_mode(struct ib_device *dev, u32 port, 633 enum rdma_nl_counter_mode *mode, 634 enum rdma_nl_counter_mask *mask, 635 bool *opcnt) 636 { 637 struct rdma_port_counter *port_counter; 638 639 port_counter = &dev->port_data[port].port_counter; 640 *mode = port_counter->mode.mode; 641 *mask = port_counter->mode.mask; 642 *opcnt = port_counter->mode.bind_opcnt; 643 644 return 0; 645 } 646 647 void rdma_counter_init(struct ib_device *dev) 648 { 649 struct rdma_port_counter *port_counter; 650 u32 port, i; 651 652 if (!dev->port_data) 653 return; 654 655 rdma_for_each_port(dev, port) { 656 port_counter = &dev->port_data[port].port_counter; 657 port_counter->mode.mode = RDMA_COUNTER_MODE_NONE; 658 mutex_init(&port_counter->lock); 659 660 if (!dev->ops.alloc_hw_port_stats) 661 continue; 662 663 port_counter->hstats = dev->ops.alloc_hw_port_stats(dev, port); 664 if (!port_counter->hstats) 665 goto fail; 666 } 667 668 return; 669 670 fail: 671 for (i = port; i >= rdma_start_port(dev); i--) { 672 port_counter = &dev->port_data[i].port_counter; 673 rdma_free_hw_stats_struct(port_counter->hstats); 674 port_counter->hstats = NULL; 675 mutex_destroy(&port_counter->lock); 676 } 677 } 678 679 void rdma_counter_release(struct ib_device *dev) 680 { 681 struct rdma_port_counter *port_counter; 682 u32 port; 683 684 rdma_for_each_port(dev, port) { 685 port_counter = &dev->port_data[port].port_counter; 686 rdma_free_hw_stats_struct(port_counter->hstats); 687 mutex_destroy(&port_counter->lock); 688 } 689 } 690