1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2024 NVIDIA Corporation & Affiliates */ 3 4 #include "internal.h" 5 6 static u16 hws_bwc_gen_queue_idx(struct mlx5hws_context *ctx) 7 { 8 /* assign random queue */ 9 return get_random_u8() % mlx5hws_bwc_queues(ctx); 10 } 11 12 static u16 13 hws_bwc_get_burst_th(struct mlx5hws_context *ctx, u16 queue_id) 14 { 15 return min(ctx->send_queue[queue_id].num_entries / 2, 16 MLX5HWS_BWC_MATCHER_REHASH_BURST_TH); 17 } 18 19 static struct mutex * 20 hws_bwc_get_queue_lock(struct mlx5hws_context *ctx, u16 idx) 21 { 22 return &ctx->bwc_send_queue_locks[idx]; 23 } 24 25 static void hws_bwc_lock_all_queues(struct mlx5hws_context *ctx) 26 { 27 u16 bwc_queues = mlx5hws_bwc_queues(ctx); 28 struct mutex *queue_lock; /* Protect the queue */ 29 int i; 30 31 for (i = 0; i < bwc_queues; i++) { 32 queue_lock = hws_bwc_get_queue_lock(ctx, i); 33 mutex_lock(queue_lock); 34 } 35 } 36 37 static void hws_bwc_unlock_all_queues(struct mlx5hws_context *ctx) 38 { 39 u16 bwc_queues = mlx5hws_bwc_queues(ctx); 40 struct mutex *queue_lock; /* Protect the queue */ 41 int i = bwc_queues; 42 43 while (i--) { 44 queue_lock = hws_bwc_get_queue_lock(ctx, i); 45 mutex_unlock(queue_lock); 46 } 47 } 48 49 static void hws_bwc_matcher_init_attr(struct mlx5hws_bwc_matcher *bwc_matcher, 50 u32 priority, 51 u8 size_log_rx, u8 size_log_tx, 52 struct mlx5hws_matcher_attr *attr) 53 { 54 memset(attr, 0, sizeof(*attr)); 55 56 attr->priority = priority; 57 attr->optimize_using_rule_idx = 0; 58 attr->mode = MLX5HWS_MATCHER_RESOURCE_MODE_RULE; 59 attr->optimize_flow_src = MLX5HWS_MATCHER_FLOW_SRC_ANY; 60 attr->insert_mode = MLX5HWS_MATCHER_INSERT_BY_HASH; 61 attr->distribute_mode = MLX5HWS_MATCHER_DISTRIBUTE_BY_HASH; 62 attr->size[MLX5HWS_MATCHER_SIZE_TYPE_RX].rule.num_log = size_log_rx; 63 attr->size[MLX5HWS_MATCHER_SIZE_TYPE_TX].rule.num_log = size_log_tx; 64 attr->resizable = true; 65 attr->max_num_of_at_attach = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; 66 } 67 68 static int 69 hws_bwc_matcher_move_all_simple(struct mlx5hws_bwc_matcher *bwc_matcher) 70 { 71 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 72 struct mlx5hws_matcher *matcher = bwc_matcher->matcher; 73 int drain_error = 0, move_error = 0, poll_error = 0; 74 u16 bwc_queues = mlx5hws_bwc_queues(ctx); 75 struct mlx5hws_rule_attr rule_attr; 76 struct mlx5hws_bwc_rule *bwc_rule; 77 struct mlx5hws_send_engine *queue; 78 struct list_head *rules_list; 79 u32 pending_rules; 80 int i, ret = 0; 81 bool drain; 82 83 mlx5hws_bwc_rule_fill_attr(bwc_matcher, 0, 0, &rule_attr); 84 85 for (i = 0; i < bwc_queues; i++) { 86 if (list_empty(&bwc_matcher->rules[i])) 87 continue; 88 89 pending_rules = 0; 90 rule_attr.queue_id = mlx5hws_bwc_get_queue_id(ctx, i); 91 rules_list = &bwc_matcher->rules[i]; 92 93 list_for_each_entry(bwc_rule, rules_list, list_node) { 94 ret = mlx5hws_matcher_resize_rule_move(matcher, 95 bwc_rule->rule, 96 &rule_attr); 97 if (unlikely(ret)) { 98 if (!move_error) { 99 mlx5hws_err(ctx, 100 "Moving BWC rule: move failed (%d), attempting to move rest of the rules\n", 101 ret); 102 move_error = ret; 103 } 104 /* Rule wasn't queued, no need to poll */ 105 continue; 106 } 107 108 pending_rules++; 109 drain = pending_rules >= 110 hws_bwc_get_burst_th(ctx, rule_attr.queue_id); 111 ret = mlx5hws_bwc_queue_poll(ctx, 112 rule_attr.queue_id, 113 &pending_rules, 114 drain); 115 if (unlikely(ret)) { 116 if (ret == -ETIMEDOUT) { 117 mlx5hws_err(ctx, 118 "Moving BWC rule: timeout polling for completions (%d), aborting rehash\n", 119 ret); 120 return ret; 121 } 122 if (!poll_error) { 123 mlx5hws_err(ctx, 124 "Moving BWC rule: polling for completions failed (%d), attempting to move rest of the rules\n", 125 ret); 126 poll_error = ret; 127 } 128 } 129 } 130 131 if (pending_rules) { 132 queue = &ctx->send_queue[rule_attr.queue_id]; 133 mlx5hws_send_engine_flush_queue(queue); 134 ret = mlx5hws_bwc_queue_poll(ctx, 135 rule_attr.queue_id, 136 &pending_rules, 137 true); 138 if (unlikely(ret)) { 139 if (ret == -ETIMEDOUT) { 140 mlx5hws_err(ctx, 141 "Moving bwc rule: timeout draining completions (%d), aborting rehash\n", 142 ret); 143 return ret; 144 } 145 if (!drain_error) { 146 mlx5hws_err(ctx, 147 "Moving bwc rule: drain failed (%d), attempting to move rest of the rules\n", 148 ret); 149 drain_error = ret; 150 } 151 } 152 } 153 } 154 155 /* Return the first error that happened */ 156 if (unlikely(move_error)) 157 return move_error; 158 if (unlikely(poll_error)) 159 return poll_error; 160 if (unlikely(drain_error)) 161 return drain_error; 162 163 return ret; 164 } 165 166 static int hws_bwc_matcher_move_all(struct mlx5hws_bwc_matcher *bwc_matcher) 167 { 168 switch (bwc_matcher->matcher_type) { 169 case MLX5HWS_BWC_MATCHER_SIMPLE: 170 return hws_bwc_matcher_move_all_simple(bwc_matcher); 171 case MLX5HWS_BWC_MATCHER_COMPLEX_FIRST: 172 return mlx5hws_bwc_matcher_complex_move_first(bwc_matcher); 173 case MLX5HWS_BWC_MATCHER_COMPLEX_SUBMATCHER: 174 return mlx5hws_bwc_matcher_complex_move(bwc_matcher); 175 default: 176 return -EINVAL; 177 } 178 } 179 180 static int hws_bwc_matcher_move(struct mlx5hws_bwc_matcher *bwc_matcher) 181 { 182 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 183 struct mlx5hws_matcher_attr matcher_attr = {0}; 184 struct mlx5hws_matcher *old_matcher; 185 struct mlx5hws_matcher *new_matcher; 186 int ret; 187 188 hws_bwc_matcher_init_attr(bwc_matcher, 189 bwc_matcher->priority, 190 bwc_matcher->rx_size.size_log, 191 bwc_matcher->tx_size.size_log, 192 &matcher_attr); 193 194 old_matcher = bwc_matcher->matcher; 195 new_matcher = mlx5hws_matcher_create(old_matcher->tbl, 196 &bwc_matcher->mt, 1, 197 bwc_matcher->at, 198 bwc_matcher->num_of_at, 199 &matcher_attr); 200 if (!new_matcher) { 201 mlx5hws_err(ctx, "Rehash error: matcher creation failed\n"); 202 return -ENOMEM; 203 } 204 205 ret = mlx5hws_matcher_resize_set_target(old_matcher, new_matcher); 206 if (ret) { 207 mlx5hws_err(ctx, "Rehash error: failed setting resize target\n"); 208 return ret; 209 } 210 211 ret = hws_bwc_matcher_move_all(bwc_matcher); 212 if (ret) 213 mlx5hws_err(ctx, "Rehash error: moving rules failed, attempting to remove the old matcher\n"); 214 215 /* Error during rehash can't be rolled back. 216 * The best option here is to allow the rehash to complete and remove 217 * the old matcher - can't leave the matcher in the 'in_resize' state. 218 */ 219 220 bwc_matcher->matcher = new_matcher; 221 mlx5hws_matcher_destroy(old_matcher); 222 223 return ret; 224 } 225 226 int mlx5hws_bwc_matcher_create_simple(struct mlx5hws_bwc_matcher *bwc_matcher, 227 struct mlx5hws_table *table, 228 u32 priority, 229 u8 match_criteria_enable, 230 struct mlx5hws_match_parameters *mask, 231 enum mlx5hws_action_type action_types[]) 232 { 233 enum mlx5hws_action_type init_action_types[1] = { MLX5HWS_ACTION_TYP_LAST }; 234 struct mlx5hws_context *ctx = table->ctx; 235 u16 bwc_queues = mlx5hws_bwc_queues(ctx); 236 struct mlx5hws_matcher_attr attr = {0}; 237 int i; 238 239 bwc_matcher->rules = kzalloc_objs(*bwc_matcher->rules, bwc_queues); 240 if (!bwc_matcher->rules) 241 goto err; 242 243 for (i = 0; i < bwc_queues; i++) 244 INIT_LIST_HEAD(&bwc_matcher->rules[i]); 245 246 hws_bwc_matcher_init_attr(bwc_matcher, 247 priority, 248 bwc_matcher->rx_size.size_log, 249 bwc_matcher->tx_size.size_log, 250 &attr); 251 252 bwc_matcher->matcher_type = MLX5HWS_BWC_MATCHER_SIMPLE; 253 bwc_matcher->priority = priority; 254 255 bwc_matcher->size_of_at_array = MLX5HWS_BWC_MATCHER_ATTACH_AT_NUM; 256 bwc_matcher->at = kzalloc_objs(*bwc_matcher->at, 257 bwc_matcher->size_of_at_array); 258 if (!bwc_matcher->at) 259 goto free_bwc_matcher_rules; 260 261 /* create dummy action template */ 262 bwc_matcher->at[0] = 263 mlx5hws_action_template_create(action_types ? 264 action_types : init_action_types); 265 if (!bwc_matcher->at[0]) { 266 mlx5hws_err(table->ctx, "BWC matcher: failed creating action template\n"); 267 goto free_bwc_matcher_at_array; 268 } 269 270 bwc_matcher->num_of_at = 1; 271 272 bwc_matcher->mt = mlx5hws_match_template_create(ctx, 273 mask->match_buf, 274 mask->match_sz, 275 match_criteria_enable); 276 if (!bwc_matcher->mt) { 277 mlx5hws_err(table->ctx, "BWC matcher: failed creating match template\n"); 278 goto free_at; 279 } 280 281 bwc_matcher->matcher = mlx5hws_matcher_create(table, 282 &bwc_matcher->mt, 1, 283 &bwc_matcher->at[0], 284 bwc_matcher->num_of_at, 285 &attr); 286 if (!bwc_matcher->matcher) { 287 mlx5hws_err(table->ctx, "BWC matcher: failed creating HWS matcher\n"); 288 goto free_mt; 289 } 290 291 return 0; 292 293 free_mt: 294 mlx5hws_match_template_destroy(bwc_matcher->mt); 295 free_at: 296 mlx5hws_action_template_destroy(bwc_matcher->at[0]); 297 free_bwc_matcher_at_array: 298 kfree(bwc_matcher->at); 299 free_bwc_matcher_rules: 300 kfree(bwc_matcher->rules); 301 err: 302 return -EINVAL; 303 } 304 305 static void 306 hws_bwc_matcher_init_size_rxtx(struct mlx5hws_bwc_matcher_size *size) 307 { 308 size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG; 309 atomic_set(&size->num_of_rules, 0); 310 atomic_set(&size->rehash_required, false); 311 } 312 313 static void hws_bwc_matcher_init_size(struct mlx5hws_bwc_matcher *bwc_matcher) 314 { 315 hws_bwc_matcher_init_size_rxtx(&bwc_matcher->rx_size); 316 hws_bwc_matcher_init_size_rxtx(&bwc_matcher->tx_size); 317 } 318 319 struct mlx5hws_bwc_matcher * 320 mlx5hws_bwc_matcher_create(struct mlx5hws_table *table, 321 u32 priority, 322 u8 match_criteria_enable, 323 struct mlx5hws_match_parameters *mask) 324 { 325 struct mlx5hws_bwc_matcher *bwc_matcher; 326 bool is_complex; 327 int ret; 328 329 if (!mlx5hws_context_bwc_supported(table->ctx)) { 330 mlx5hws_err(table->ctx, 331 "BWC matcher: context created w/o BWC API compatibility\n"); 332 return NULL; 333 } 334 335 bwc_matcher = kzalloc_obj(*bwc_matcher); 336 if (!bwc_matcher) 337 return NULL; 338 339 hws_bwc_matcher_init_size(bwc_matcher); 340 341 /* Check if the required match params can be all matched 342 * in single STE, otherwise complex matcher is needed. 343 */ 344 345 is_complex = mlx5hws_bwc_match_params_is_complex(table->ctx, match_criteria_enable, mask); 346 if (is_complex) 347 ret = mlx5hws_bwc_matcher_create_complex(bwc_matcher, 348 table, 349 priority, 350 match_criteria_enable, 351 mask); 352 else 353 ret = mlx5hws_bwc_matcher_create_simple(bwc_matcher, 354 table, 355 priority, 356 match_criteria_enable, 357 mask, 358 NULL); 359 if (ret) 360 goto free_bwc_matcher; 361 362 return bwc_matcher; 363 364 free_bwc_matcher: 365 kfree(bwc_matcher); 366 367 return NULL; 368 } 369 370 int mlx5hws_bwc_matcher_destroy_simple(struct mlx5hws_bwc_matcher *bwc_matcher) 371 { 372 int i; 373 374 mlx5hws_matcher_destroy(bwc_matcher->matcher); 375 bwc_matcher->matcher = NULL; 376 377 for (i = 0; i < bwc_matcher->num_of_at; i++) 378 mlx5hws_action_template_destroy(bwc_matcher->at[i]); 379 kfree(bwc_matcher->at); 380 381 mlx5hws_match_template_destroy(bwc_matcher->mt); 382 kfree(bwc_matcher->rules); 383 384 return 0; 385 } 386 387 int mlx5hws_bwc_matcher_destroy(struct mlx5hws_bwc_matcher *bwc_matcher) 388 { 389 u32 rx_rules = atomic_read(&bwc_matcher->rx_size.num_of_rules); 390 u32 tx_rules = atomic_read(&bwc_matcher->tx_size.num_of_rules); 391 392 if (rx_rules || tx_rules) 393 mlx5hws_err(bwc_matcher->matcher->tbl->ctx, 394 "BWC matcher destroy: matcher still has %u RX and %u TX rules\n", 395 rx_rules, tx_rules); 396 397 if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) 398 mlx5hws_bwc_matcher_destroy_complex(bwc_matcher); 399 else 400 mlx5hws_bwc_matcher_destroy_simple(bwc_matcher); 401 402 kfree(bwc_matcher); 403 return 0; 404 } 405 406 int mlx5hws_bwc_queue_poll(struct mlx5hws_context *ctx, 407 u16 queue_id, 408 u32 *pending_rules, 409 bool drain) 410 { 411 unsigned long timeout = jiffies + 412 secs_to_jiffies(MLX5HWS_BWC_POLLING_TIMEOUT); 413 struct mlx5hws_flow_op_result comp[MLX5HWS_BWC_MATCHER_REHASH_BURST_TH]; 414 u16 burst_th = hws_bwc_get_burst_th(ctx, queue_id); 415 bool got_comp = *pending_rules >= burst_th; 416 bool queue_full; 417 int err = 0; 418 int ret; 419 int i; 420 421 /* Check if there are any completions at all */ 422 if (!got_comp && !drain) 423 return 0; 424 425 if (unlikely(ctx->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) { 426 /* If the device is down for any reason (e.g. FLR), the HW will 427 * no longer generate completions. 428 * Note that ETIMEDOUT is returned here because the BWC layer 429 * already has a special handling for timeouts - it breaks the 430 * rehash / resize / shrink loops to avoid chain of timeouts. 431 */ 432 mlx5_core_warn_once(ctx->mdev, 433 "BWC poll: device is down, polling for completion aborted\n"); 434 return -ETIMEDOUT; 435 } 436 437 queue_full = mlx5hws_send_engine_full(&ctx->send_queue[queue_id]); 438 while (queue_full || ((got_comp || drain) && *pending_rules)) { 439 ret = mlx5hws_send_queue_poll(ctx, queue_id, comp, burst_th); 440 if (unlikely(ret < 0)) { 441 mlx5hws_err(ctx, "BWC poll error: polling queue %d returned %d\n", 442 queue_id, ret); 443 return -EINVAL; 444 } 445 446 if (ret) { 447 (*pending_rules) -= ret; 448 for (i = 0; i < ret; i++) { 449 if (unlikely(comp[i].status != MLX5HWS_FLOW_OP_SUCCESS)) { 450 mlx5hws_err(ctx, 451 "BWC poll error: polling queue %d returned completion with error\n", 452 queue_id); 453 err = -EINVAL; 454 } 455 } 456 queue_full = false; 457 } 458 459 got_comp = !!ret; 460 461 if (unlikely(!got_comp && time_after(jiffies, timeout))) { 462 mlx5hws_err(ctx, "BWC poll error: polling queue %d - TIMEOUT\n", queue_id); 463 return -ETIMEDOUT; 464 } 465 } 466 467 return err; 468 } 469 470 void 471 mlx5hws_bwc_rule_fill_attr(struct mlx5hws_bwc_matcher *bwc_matcher, 472 u16 bwc_queue_idx, 473 u32 flow_source, 474 struct mlx5hws_rule_attr *rule_attr) 475 { 476 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 477 478 /* no use of INSERT_BY_INDEX in bwc rule */ 479 rule_attr->rule_idx = 0; 480 481 /* notify HW at each rule insertion/deletion */ 482 rule_attr->burst = 0; 483 484 /* We don't need user data, but the API requires it to exist */ 485 rule_attr->user_data = (void *)0xFACADE; 486 487 rule_attr->queue_id = mlx5hws_bwc_get_queue_id(ctx, bwc_queue_idx); 488 rule_attr->flow_source = flow_source; 489 } 490 491 struct mlx5hws_bwc_rule * 492 mlx5hws_bwc_rule_alloc(struct mlx5hws_bwc_matcher *bwc_matcher) 493 { 494 struct mlx5hws_bwc_rule *bwc_rule; 495 496 bwc_rule = kzalloc_obj(*bwc_rule); 497 if (unlikely(!bwc_rule)) 498 goto out_err; 499 500 bwc_rule->rule = kzalloc_obj(*bwc_rule->rule); 501 if (unlikely(!bwc_rule->rule)) 502 goto free_rule; 503 504 bwc_rule->bwc_matcher = bwc_matcher; 505 return bwc_rule; 506 507 free_rule: 508 kfree(bwc_rule); 509 out_err: 510 return NULL; 511 } 512 513 void mlx5hws_bwc_rule_free(struct mlx5hws_bwc_rule *bwc_rule) 514 { 515 if (likely(bwc_rule->rule)) 516 kfree(bwc_rule->rule); 517 kfree(bwc_rule); 518 } 519 520 static void hws_bwc_rule_list_add(struct mlx5hws_bwc_rule *bwc_rule, u16 idx) 521 { 522 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 523 524 bwc_rule->bwc_queue_idx = idx; 525 list_add(&bwc_rule->list_node, &bwc_matcher->rules[idx]); 526 } 527 528 static void hws_bwc_rule_list_remove(struct mlx5hws_bwc_rule *bwc_rule) 529 { 530 list_del_init(&bwc_rule->list_node); 531 } 532 533 static int 534 hws_bwc_rule_destroy_hws_async(struct mlx5hws_bwc_rule *bwc_rule, 535 struct mlx5hws_rule_attr *attr) 536 { 537 return mlx5hws_rule_destroy(bwc_rule->rule, attr); 538 } 539 540 static int 541 hws_bwc_rule_destroy_hws_sync(struct mlx5hws_bwc_rule *bwc_rule, 542 struct mlx5hws_rule_attr *rule_attr) 543 { 544 struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx; 545 u32 expected_completions = 1; 546 int ret; 547 548 ret = hws_bwc_rule_destroy_hws_async(bwc_rule, rule_attr); 549 if (unlikely(ret)) 550 return ret; 551 552 ret = mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id, 553 &expected_completions, true); 554 if (unlikely(ret)) 555 return ret; 556 557 if (unlikely(bwc_rule->rule->status != MLX5HWS_RULE_STATUS_DELETED && 558 bwc_rule->rule->status != MLX5HWS_RULE_STATUS_DELETING)) { 559 mlx5hws_err(ctx, "Failed destroying BWC rule: rule status %d\n", 560 bwc_rule->rule->status); 561 return -EINVAL; 562 } 563 564 return 0; 565 } 566 567 static void hws_bwc_rule_cnt_dec(struct mlx5hws_bwc_rule *bwc_rule) 568 { 569 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 570 571 if (!bwc_rule->skip_rx) 572 atomic_dec(&bwc_matcher->rx_size.num_of_rules); 573 if (!bwc_rule->skip_tx) 574 atomic_dec(&bwc_matcher->tx_size.num_of_rules); 575 } 576 577 static int 578 hws_bwc_matcher_rehash_shrink(struct mlx5hws_bwc_matcher *bwc_matcher) 579 { 580 struct mlx5hws_bwc_matcher_size *rx_size = &bwc_matcher->rx_size; 581 struct mlx5hws_bwc_matcher_size *tx_size = &bwc_matcher->tx_size; 582 583 /* It is possible that another thread has added a rule. 584 * Need to check again if we really need rehash/shrink. 585 */ 586 if (atomic_read(&rx_size->num_of_rules) || 587 atomic_read(&tx_size->num_of_rules)) 588 return 0; 589 590 /* If the current matcher RX/TX size is already at its initial size. */ 591 if (rx_size->size_log == MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG && 592 tx_size->size_log == MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG) 593 return 0; 594 595 /* Now we've done all the checking - do the shrinking: 596 * - reset match RTC size to the initial size 597 * - create new matcher 598 * - move the rules, which will not do anything as the matcher is empty 599 * - destroy the old matcher 600 */ 601 602 rx_size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG; 603 tx_size->size_log = MLX5HWS_BWC_MATCHER_INIT_SIZE_LOG; 604 605 return hws_bwc_matcher_move(bwc_matcher); 606 } 607 608 static int hws_bwc_rule_cnt_dec_with_shrink(struct mlx5hws_bwc_rule *bwc_rule, 609 u16 bwc_queue_idx) 610 { 611 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 612 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 613 struct mutex *queue_lock; /* Protect the queue */ 614 int ret; 615 616 hws_bwc_rule_cnt_dec(bwc_rule); 617 618 if (atomic_read(&bwc_matcher->rx_size.num_of_rules) || 619 atomic_read(&bwc_matcher->tx_size.num_of_rules)) 620 return 0; 621 622 /* Matcher has no more rules - shrink it to save ICM. */ 623 624 queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx); 625 mutex_unlock(queue_lock); 626 627 hws_bwc_lock_all_queues(ctx); 628 ret = hws_bwc_matcher_rehash_shrink(bwc_matcher); 629 hws_bwc_unlock_all_queues(ctx); 630 631 mutex_lock(queue_lock); 632 633 if (unlikely(ret)) 634 mlx5hws_err(ctx, 635 "BWC rule deletion: shrinking empty matcher failed (%d)\n", 636 ret); 637 638 return ret; 639 } 640 641 int mlx5hws_bwc_rule_destroy_simple(struct mlx5hws_bwc_rule *bwc_rule) 642 { 643 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 644 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 645 u16 idx = bwc_rule->bwc_queue_idx; 646 struct mlx5hws_rule_attr attr; 647 struct mutex *queue_lock; /* Protect the queue */ 648 int ret; 649 650 mlx5hws_bwc_rule_fill_attr(bwc_matcher, idx, 0, &attr); 651 652 queue_lock = hws_bwc_get_queue_lock(ctx, idx); 653 654 mutex_lock(queue_lock); 655 656 ret = hws_bwc_rule_destroy_hws_sync(bwc_rule, &attr); 657 hws_bwc_rule_list_remove(bwc_rule); 658 hws_bwc_rule_cnt_dec_with_shrink(bwc_rule, idx); 659 660 mutex_unlock(queue_lock); 661 662 return ret; 663 } 664 665 int mlx5hws_bwc_rule_destroy(struct mlx5hws_bwc_rule *bwc_rule) 666 { 667 bool is_complex = bwc_rule->bwc_matcher->matcher_type == 668 MLX5HWS_BWC_MATCHER_COMPLEX_FIRST; 669 int ret = 0; 670 671 if (is_complex) 672 ret = mlx5hws_bwc_rule_destroy_complex(bwc_rule); 673 else 674 ret = mlx5hws_bwc_rule_destroy_simple(bwc_rule); 675 676 mlx5hws_bwc_rule_free(bwc_rule); 677 return ret; 678 } 679 680 static int 681 hws_bwc_rule_create_async(struct mlx5hws_bwc_rule *bwc_rule, 682 u32 *match_param, 683 u8 at_idx, 684 struct mlx5hws_rule_action rule_actions[], 685 struct mlx5hws_rule_attr *rule_attr) 686 { 687 return mlx5hws_rule_create(bwc_rule->bwc_matcher->matcher, 688 0, /* only one match template supported */ 689 match_param, 690 at_idx, 691 rule_actions, 692 rule_attr, 693 bwc_rule->rule); 694 } 695 696 static int 697 hws_bwc_rule_create_sync(struct mlx5hws_bwc_rule *bwc_rule, 698 u32 *match_param, 699 u8 at_idx, 700 struct mlx5hws_rule_action rule_actions[], 701 struct mlx5hws_rule_attr *rule_attr) 702 703 { 704 struct mlx5hws_context *ctx = bwc_rule->bwc_matcher->matcher->tbl->ctx; 705 u32 expected_completions = 1; 706 int ret; 707 708 ret = hws_bwc_rule_create_async(bwc_rule, match_param, 709 at_idx, rule_actions, 710 rule_attr); 711 if (unlikely(ret)) 712 return ret; 713 714 return mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id, 715 &expected_completions, true); 716 } 717 718 static int 719 hws_bwc_rule_update_sync(struct mlx5hws_bwc_rule *bwc_rule, 720 u8 at_idx, 721 struct mlx5hws_rule_action rule_actions[], 722 struct mlx5hws_rule_attr *rule_attr) 723 { 724 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 725 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 726 u32 expected_completions = 1; 727 int ret; 728 729 ret = mlx5hws_rule_action_update(bwc_rule->rule, 730 at_idx, 731 rule_actions, 732 rule_attr); 733 if (unlikely(ret)) 734 return ret; 735 736 ret = mlx5hws_bwc_queue_poll(ctx, rule_attr->queue_id, 737 &expected_completions, true); 738 if (unlikely(ret)) 739 mlx5hws_err(ctx, "Failed updating BWC rule (%d)\n", ret); 740 741 return ret; 742 } 743 744 static bool 745 hws_bwc_matcher_size_maxed_out(struct mlx5hws_bwc_matcher *bwc_matcher, 746 struct mlx5hws_bwc_matcher_size *size) 747 { 748 struct mlx5hws_cmd_query_caps *caps = bwc_matcher->matcher->tbl->ctx->caps; 749 750 /* check the match RTC size */ 751 return (size->size_log + MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH + 752 MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP) > 753 (caps->ste_alloc_log_max - 1); 754 } 755 756 static bool 757 hws_bwc_matcher_rehash_size_needed(struct mlx5hws_bwc_matcher *bwc_matcher, 758 struct mlx5hws_bwc_matcher_size *size, 759 u32 num_of_rules) 760 { 761 if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size))) 762 return false; 763 764 if (unlikely((num_of_rules * 100 / MLX5HWS_BWC_MATCHER_REHASH_PERCENT_TH) >= 765 (1UL << size->size_log))) 766 return true; 767 768 return false; 769 } 770 771 static void 772 hws_bwc_rule_actions_to_action_types(struct mlx5hws_rule_action rule_actions[], 773 enum mlx5hws_action_type action_types[]) 774 { 775 int i = 0; 776 777 for (i = 0; 778 rule_actions[i].action && (rule_actions[i].action->type != MLX5HWS_ACTION_TYP_LAST); 779 i++) { 780 action_types[i] = (enum mlx5hws_action_type)rule_actions[i].action->type; 781 } 782 783 action_types[i] = MLX5HWS_ACTION_TYP_LAST; 784 } 785 786 static int 787 hws_bwc_matcher_extend_at(struct mlx5hws_bwc_matcher *bwc_matcher, 788 struct mlx5hws_rule_action rule_actions[]) 789 { 790 enum mlx5hws_action_type action_types[MLX5HWS_BWC_MAX_ACTS]; 791 void *p; 792 793 if (unlikely(bwc_matcher->num_of_at >= bwc_matcher->size_of_at_array)) { 794 if (bwc_matcher->size_of_at_array >= MLX5HWS_MATCHER_MAX_AT) 795 return -ENOMEM; 796 bwc_matcher->size_of_at_array *= 2; 797 p = krealloc(bwc_matcher->at, 798 bwc_matcher->size_of_at_array * 799 sizeof(*bwc_matcher->at), 800 __GFP_ZERO | GFP_KERNEL); 801 if (!p) { 802 bwc_matcher->size_of_at_array /= 2; 803 return -ENOMEM; 804 } 805 806 bwc_matcher->at = p; 807 } 808 809 hws_bwc_rule_actions_to_action_types(rule_actions, action_types); 810 811 bwc_matcher->at[bwc_matcher->num_of_at] = 812 mlx5hws_action_template_create(action_types); 813 814 if (unlikely(!bwc_matcher->at[bwc_matcher->num_of_at])) 815 return -ENOMEM; 816 817 bwc_matcher->num_of_at++; 818 return 0; 819 } 820 821 static int 822 hws_bwc_matcher_extend_size(struct mlx5hws_bwc_matcher *bwc_matcher, 823 struct mlx5hws_bwc_matcher_size *size) 824 { 825 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 826 struct mlx5hws_cmd_query_caps *caps = ctx->caps; 827 828 if (unlikely(hws_bwc_matcher_size_maxed_out(bwc_matcher, size))) { 829 mlx5hws_err(ctx, "Can't resize matcher: depth exceeds limit %d\n", 830 caps->rtc_log_depth_max); 831 return -ENOMEM; 832 } 833 834 size->size_log = min(size->size_log + MLX5HWS_BWC_MATCHER_SIZE_LOG_STEP, 835 caps->ste_alloc_log_max - 836 MLX5HWS_MATCHER_ASSURED_MAIN_TBL_DEPTH); 837 838 return 0; 839 } 840 841 static int 842 hws_bwc_matcher_find_at(struct mlx5hws_bwc_matcher *bwc_matcher, 843 struct mlx5hws_rule_action rule_actions[]) 844 { 845 enum mlx5hws_action_type *action_type_arr; 846 int i, j; 847 848 /* start from index 1 - first action template is a dummy */ 849 for (i = 1; i < bwc_matcher->num_of_at; i++) { 850 j = 0; 851 action_type_arr = bwc_matcher->at[i]->action_type_arr; 852 853 while (rule_actions[j].action && 854 rule_actions[j].action->type != MLX5HWS_ACTION_TYP_LAST) { 855 if (action_type_arr[j] != rule_actions[j].action->type) 856 break; 857 j++; 858 } 859 860 if (action_type_arr[j] == MLX5HWS_ACTION_TYP_LAST && 861 (!rule_actions[j].action || 862 rule_actions[j].action->type == MLX5HWS_ACTION_TYP_LAST)) 863 return i; 864 } 865 866 return -1; 867 } 868 869 static int 870 hws_bwc_matcher_rehash_size(struct mlx5hws_bwc_matcher *bwc_matcher) 871 { 872 bool need_rx_rehash, need_tx_rehash; 873 int ret; 874 875 need_rx_rehash = atomic_read(&bwc_matcher->rx_size.rehash_required); 876 need_tx_rehash = atomic_read(&bwc_matcher->tx_size.rehash_required); 877 878 /* It is possible that another rule has already performed rehash. 879 * Need to check again if we really need rehash. 880 */ 881 if (!need_rx_rehash && !need_tx_rehash) 882 return 0; 883 884 /* If the current matcher RX/TX size is already at its max size, 885 * it can't be rehashed. 886 */ 887 if (need_rx_rehash && 888 hws_bwc_matcher_size_maxed_out(bwc_matcher, 889 &bwc_matcher->rx_size)) { 890 atomic_set(&bwc_matcher->rx_size.rehash_required, false); 891 need_rx_rehash = false; 892 } 893 if (need_tx_rehash && 894 hws_bwc_matcher_size_maxed_out(bwc_matcher, 895 &bwc_matcher->tx_size)) { 896 atomic_set(&bwc_matcher->tx_size.rehash_required, false); 897 need_tx_rehash = false; 898 } 899 900 /* If both RX and TX rehash flags are now off, it means that whatever 901 * we wanted to rehash is now at its max size - no rehash can be done. 902 * Return and try adding the rule again - perhaps there was some change. 903 */ 904 if (!need_rx_rehash && !need_tx_rehash) 905 return 0; 906 907 /* Now we're done all the checking - do the rehash: 908 * - extend match RTC size 909 * - create new matcher 910 * - move all the rules to the new matcher 911 * - destroy the old matcher 912 */ 913 atomic_set(&bwc_matcher->rx_size.rehash_required, false); 914 atomic_set(&bwc_matcher->tx_size.rehash_required, false); 915 916 if (need_rx_rehash) { 917 ret = hws_bwc_matcher_extend_size(bwc_matcher, 918 &bwc_matcher->rx_size); 919 if (ret) 920 return ret; 921 } 922 923 if (need_tx_rehash) { 924 ret = hws_bwc_matcher_extend_size(bwc_matcher, 925 &bwc_matcher->tx_size); 926 if (ret) 927 return ret; 928 } 929 930 return hws_bwc_matcher_move(bwc_matcher); 931 } 932 933 static int hws_bwc_rule_get_at_idx(struct mlx5hws_bwc_rule *bwc_rule, 934 struct mlx5hws_rule_action rule_actions[], 935 u16 bwc_queue_idx) 936 { 937 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 938 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 939 struct mutex *queue_lock; /* Protect the queue */ 940 int at_idx, ret; 941 942 /* check if rehash needed due to missing action template */ 943 at_idx = hws_bwc_matcher_find_at(bwc_matcher, rule_actions); 944 if (likely(at_idx >= 0)) 945 return at_idx; 946 947 /* we need to extend BWC matcher action templates array */ 948 queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx); 949 mutex_unlock(queue_lock); 950 hws_bwc_lock_all_queues(ctx); 951 952 /* check again - perhaps other thread already did extend_at */ 953 at_idx = hws_bwc_matcher_find_at(bwc_matcher, rule_actions); 954 if (at_idx >= 0) 955 goto out; 956 957 ret = hws_bwc_matcher_extend_at(bwc_matcher, rule_actions); 958 if (unlikely(ret)) { 959 mlx5hws_err(ctx, "BWC rule: failed extending AT (%d)", ret); 960 at_idx = -EINVAL; 961 goto out; 962 } 963 964 /* action templates array was extended, we need the last idx */ 965 at_idx = bwc_matcher->num_of_at - 1; 966 ret = mlx5hws_matcher_attach_at(bwc_matcher->matcher, 967 bwc_matcher->at[at_idx]); 968 if (unlikely(ret)) { 969 mlx5hws_err(ctx, "BWC rule: failed attaching new AT (%d)", ret); 970 at_idx = -EINVAL; 971 goto out; 972 } 973 974 out: 975 hws_bwc_unlock_all_queues(ctx); 976 mutex_lock(queue_lock); 977 return at_idx; 978 } 979 980 static void hws_bwc_rule_cnt_inc_rxtx(struct mlx5hws_bwc_rule *bwc_rule, 981 struct mlx5hws_bwc_matcher_size *size) 982 { 983 u32 num_of_rules = atomic_inc_return(&size->num_of_rules); 984 985 if (unlikely(hws_bwc_matcher_rehash_size_needed(bwc_rule->bwc_matcher, 986 size, num_of_rules))) 987 atomic_set(&size->rehash_required, true); 988 } 989 990 static void hws_bwc_rule_cnt_inc(struct mlx5hws_bwc_rule *bwc_rule) 991 { 992 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 993 994 if (!bwc_rule->skip_rx) 995 hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->rx_size); 996 if (!bwc_rule->skip_tx) 997 hws_bwc_rule_cnt_inc_rxtx(bwc_rule, &bwc_matcher->tx_size); 998 } 999 1000 static int hws_bwc_rule_cnt_inc_with_rehash(struct mlx5hws_bwc_rule *bwc_rule, 1001 u16 bwc_queue_idx) 1002 { 1003 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 1004 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 1005 struct mutex *queue_lock; /* Protect the queue */ 1006 int ret; 1007 1008 hws_bwc_rule_cnt_inc(bwc_rule); 1009 1010 if (!atomic_read(&bwc_matcher->rx_size.rehash_required) && 1011 !atomic_read(&bwc_matcher->tx_size.rehash_required)) 1012 return 0; 1013 1014 queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx); 1015 mutex_unlock(queue_lock); 1016 1017 hws_bwc_lock_all_queues(ctx); 1018 ret = hws_bwc_matcher_rehash_size(bwc_matcher); 1019 hws_bwc_unlock_all_queues(ctx); 1020 1021 mutex_lock(queue_lock); 1022 1023 if (likely(!ret)) 1024 return 0; 1025 1026 /* Failed to rehash. Print a diagnostic and rollback the counters. */ 1027 mlx5hws_err(ctx, 1028 "BWC rule insertion: rehash to sizes [%d, %d] failed (%d)\n", 1029 bwc_matcher->rx_size.size_log, 1030 bwc_matcher->tx_size.size_log, ret); 1031 hws_bwc_rule_cnt_dec(bwc_rule); 1032 1033 return ret; 1034 } 1035 1036 int mlx5hws_bwc_rule_create_simple(struct mlx5hws_bwc_rule *bwc_rule, 1037 u32 *match_param, 1038 struct mlx5hws_rule_action rule_actions[], 1039 u32 flow_source, 1040 u16 bwc_queue_idx) 1041 { 1042 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 1043 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 1044 struct mlx5hws_rule_attr rule_attr; 1045 struct mutex *queue_lock; /* Protect the queue */ 1046 int ret = 0; 1047 int at_idx; 1048 1049 mlx5hws_bwc_rule_fill_attr(bwc_matcher, bwc_queue_idx, flow_source, &rule_attr); 1050 1051 queue_lock = hws_bwc_get_queue_lock(ctx, bwc_queue_idx); 1052 1053 mutex_lock(queue_lock); 1054 1055 at_idx = hws_bwc_rule_get_at_idx(bwc_rule, rule_actions, bwc_queue_idx); 1056 if (unlikely(at_idx < 0)) { 1057 mutex_unlock(queue_lock); 1058 mlx5hws_err(ctx, "BWC rule create: failed getting AT (%d)", 1059 ret); 1060 return -EINVAL; 1061 } 1062 1063 ret = hws_bwc_rule_cnt_inc_with_rehash(bwc_rule, bwc_queue_idx); 1064 if (unlikely(ret)) { 1065 mutex_unlock(queue_lock); 1066 return ret; 1067 } 1068 1069 ret = hws_bwc_rule_create_sync(bwc_rule, 1070 match_param, 1071 at_idx, 1072 rule_actions, 1073 &rule_attr); 1074 if (likely(!ret)) { 1075 hws_bwc_rule_list_add(bwc_rule, bwc_queue_idx); 1076 mutex_unlock(queue_lock); 1077 return 0; /* rule inserted successfully */ 1078 } 1079 1080 /* Rule insertion could fail due to queue being full, timeout, or 1081 * matcher in resize. In such cases, no point in trying to rehash. 1082 */ 1083 if (ret == -EBUSY || ret == -ETIMEDOUT || ret == -EAGAIN) { 1084 mutex_unlock(queue_lock); 1085 mlx5hws_err(ctx, 1086 "BWC rule insertion failed - %s (%d)\n", 1087 ret == -EBUSY ? "queue is full" : 1088 ret == -ETIMEDOUT ? "timeout" : 1089 ret == -EAGAIN ? "matcher in resize" : "N/A", 1090 ret); 1091 hws_bwc_rule_cnt_dec(bwc_rule); 1092 return ret; 1093 } 1094 1095 /* At this point the rule wasn't added. 1096 * It could be because there was collision, or some other problem. 1097 * Try rehash by size and insert rule again - last chance. 1098 */ 1099 if (!bwc_rule->skip_rx) 1100 atomic_set(&bwc_matcher->rx_size.rehash_required, true); 1101 if (!bwc_rule->skip_tx) 1102 atomic_set(&bwc_matcher->tx_size.rehash_required, true); 1103 1104 mutex_unlock(queue_lock); 1105 1106 hws_bwc_lock_all_queues(ctx); 1107 ret = hws_bwc_matcher_rehash_size(bwc_matcher); 1108 hws_bwc_unlock_all_queues(ctx); 1109 1110 if (ret) { 1111 mlx5hws_err(ctx, "BWC rule insertion: rehash failed (%d)\n", ret); 1112 hws_bwc_rule_cnt_dec(bwc_rule); 1113 return ret; 1114 } 1115 1116 /* Rehash done, but we still have that pesky rule to add */ 1117 mutex_lock(queue_lock); 1118 1119 ret = hws_bwc_rule_create_sync(bwc_rule, 1120 match_param, 1121 at_idx, 1122 rule_actions, 1123 &rule_attr); 1124 1125 if (unlikely(ret)) { 1126 mutex_unlock(queue_lock); 1127 mlx5hws_err(ctx, "BWC rule insertion failed (%d)\n", ret); 1128 hws_bwc_rule_cnt_dec(bwc_rule); 1129 return ret; 1130 } 1131 1132 hws_bwc_rule_list_add(bwc_rule, bwc_queue_idx); 1133 mutex_unlock(queue_lock); 1134 1135 return 0; 1136 } 1137 1138 struct mlx5hws_bwc_rule * 1139 mlx5hws_bwc_rule_create(struct mlx5hws_bwc_matcher *bwc_matcher, 1140 struct mlx5hws_match_parameters *params, 1141 u32 flow_source, 1142 struct mlx5hws_rule_action rule_actions[]) 1143 { 1144 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 1145 struct mlx5hws_bwc_rule *bwc_rule; 1146 u16 bwc_queue_idx; 1147 int ret; 1148 1149 if (unlikely(!mlx5hws_context_bwc_supported(ctx))) { 1150 mlx5hws_err(ctx, "BWC rule: Context created w/o BWC API compatibility\n"); 1151 return NULL; 1152 } 1153 1154 bwc_rule = mlx5hws_bwc_rule_alloc(bwc_matcher); 1155 if (unlikely(!bwc_rule)) 1156 return NULL; 1157 1158 bwc_rule->flow_source = flow_source; 1159 mlx5hws_rule_skip(bwc_matcher->matcher, flow_source, 1160 &bwc_rule->skip_rx, &bwc_rule->skip_tx); 1161 1162 bwc_queue_idx = hws_bwc_gen_queue_idx(ctx); 1163 1164 if (bwc_matcher->matcher_type == MLX5HWS_BWC_MATCHER_COMPLEX_FIRST) 1165 ret = mlx5hws_bwc_rule_create_complex(bwc_rule, 1166 params, 1167 flow_source, 1168 rule_actions, 1169 bwc_queue_idx); 1170 else 1171 ret = mlx5hws_bwc_rule_create_simple(bwc_rule, 1172 params->match_buf, 1173 rule_actions, 1174 flow_source, 1175 bwc_queue_idx); 1176 if (unlikely(ret)) { 1177 mlx5hws_bwc_rule_free(bwc_rule); 1178 return NULL; 1179 } 1180 1181 return bwc_rule; 1182 } 1183 1184 static int 1185 hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule, 1186 struct mlx5hws_rule_action rule_actions[]) 1187 { 1188 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 1189 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 1190 struct mlx5hws_rule_attr rule_attr; 1191 struct mutex *queue_lock; /* Protect the queue */ 1192 int at_idx, ret; 1193 u16 idx; 1194 1195 idx = bwc_rule->bwc_queue_idx; 1196 1197 mlx5hws_bwc_rule_fill_attr(bwc_matcher, idx, bwc_rule->flow_source, 1198 &rule_attr); 1199 queue_lock = hws_bwc_get_queue_lock(ctx, idx); 1200 1201 mutex_lock(queue_lock); 1202 1203 at_idx = hws_bwc_rule_get_at_idx(bwc_rule, rule_actions, idx); 1204 if (unlikely(at_idx < 0)) { 1205 mutex_unlock(queue_lock); 1206 mlx5hws_err(ctx, "BWC rule update: failed getting AT\n"); 1207 return -EINVAL; 1208 } 1209 1210 ret = hws_bwc_rule_update_sync(bwc_rule, 1211 at_idx, 1212 rule_actions, 1213 &rule_attr); 1214 mutex_unlock(queue_lock); 1215 1216 if (unlikely(ret)) 1217 mlx5hws_err(ctx, "BWC rule: update failed (%d)\n", ret); 1218 1219 return ret; 1220 } 1221 1222 int mlx5hws_bwc_rule_action_update(struct mlx5hws_bwc_rule *bwc_rule, 1223 struct mlx5hws_rule_action rule_actions[]) 1224 { 1225 struct mlx5hws_bwc_matcher *bwc_matcher = bwc_rule->bwc_matcher; 1226 struct mlx5hws_context *ctx = bwc_matcher->matcher->tbl->ctx; 1227 1228 if (unlikely(!mlx5hws_context_bwc_supported(ctx))) { 1229 mlx5hws_err(ctx, "BWC rule: Context created w/o BWC API compatibility\n"); 1230 return -EINVAL; 1231 } 1232 1233 /* For complex rules, the update should happen on the last subrule. */ 1234 while (bwc_rule->next_subrule) 1235 bwc_rule = bwc_rule->next_subrule; 1236 1237 return hws_bwc_rule_action_update(bwc_rule, rule_actions); 1238 } 1239