1 /*- 2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3 * 4 * Copyright (c) 2017 - 2026 Intel Corporation 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenFabrics.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include "osdep.h" 36 #include "irdma_hmc.h" 37 #include "irdma_defs.h" 38 #include "irdma_type.h" 39 #include "irdma_protos.h" 40 41 #include "irdma_ws.h" 42 43 /** 44 * irdma_alloc_node - Allocate a WS node and init 45 * @vsi: vsi pointer 46 * @user_pri: user priority 47 * @node_type: Type of node, leaf or parent 48 * @parent: parent node pointer 49 */ 50 static struct irdma_ws_node * 51 irdma_alloc_node(struct irdma_sc_vsi *vsi, 52 u8 user_pri, 53 enum irdma_ws_node_type node_type, 54 struct irdma_ws_node *parent) 55 { 56 struct irdma_virt_mem ws_mem; 57 struct irdma_ws_node *node; 58 u16 node_index = 0; 59 60 ws_mem.size = sizeof(*node); 61 ws_mem.va = kzalloc(ws_mem.size, GFP_KERNEL); 62 if (!ws_mem.va) 63 return NULL; 64 65 if (parent) { 66 node_index = irdma_alloc_ws_node_id(vsi->dev); 67 if (node_index == IRDMA_WS_NODE_INVALID) { 68 kfree(ws_mem.va); 69 return NULL; 70 } 71 } 72 73 node = ws_mem.va; 74 node->index = node_index; 75 node->vsi_index = vsi->vsi_idx; 76 INIT_LIST_HEAD(&node->child_list_head); 77 if (node_type == WS_NODE_TYPE_LEAF) { 78 node->type_leaf = true; 79 node->traffic_class = vsi->qos[user_pri].traffic_class; 80 node->user_pri = user_pri; 81 node->rel_bw = vsi->qos[user_pri].rel_bw; 82 if (!node->rel_bw) 83 node->rel_bw = 1; 84 85 node->prio_type = IRDMA_PRIO_WEIGHTED_RR; 86 } else { 87 node->rel_bw = 1; 88 node->prio_type = IRDMA_PRIO_WEIGHTED_RR; 89 node->enable = true; 90 } 91 92 node->parent = parent; 93 94 return node; 95 } 96 97 /** 98 * irdma_free_node - Free a WS node 99 * @vsi: VSI stricture of device 100 * @node: Pointer to node to free 101 */ 102 static void 103 irdma_free_node(struct irdma_sc_vsi *vsi, 104 struct irdma_ws_node *node) 105 { 106 struct irdma_virt_mem ws_mem; 107 108 if (node->index) 109 irdma_free_ws_node_id(vsi->dev, node->index); 110 111 ws_mem.va = node; 112 ws_mem.size = sizeof(*node); 113 kfree(ws_mem.va); 114 } 115 116 /** 117 * irdma_ws_cqp_cmd - Post CQP work scheduler node cmd 118 * @vsi: vsi pointer 119 * @node: pointer to node 120 * @cmd: add, remove or modify 121 * @qs_handle: Pointer to store the qs_handle for a leaf node 122 */ 123 static int 124 irdma_ws_cqp_cmd(struct irdma_sc_vsi *vsi, 125 struct irdma_ws_node *node, u8 cmd, u16 *qs_handle) 126 { 127 struct irdma_ws_node_info node_info = {0}; 128 129 node_info.id = node->index; 130 node_info.vsi = node->vsi_index; 131 if (node->parent) 132 node_info.parent_id = node->parent->index; 133 else 134 node_info.parent_id = node_info.id; 135 136 node_info.weight = node->rel_bw; 137 node_info.tc = node->traffic_class; 138 node_info.prio_type = node->prio_type; 139 node_info.type_leaf = node->type_leaf; 140 node_info.enable = node->enable; 141 if (irdma_cqp_ws_node_cmd(vsi->dev, cmd, &node_info)) { 142 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, "CQP WS CMD failed\n"); 143 return -ENOMEM; 144 } 145 146 if (node->type_leaf && cmd == IRDMA_OP_WS_ADD_NODE && qs_handle) 147 *qs_handle = node_info.qs_handle; 148 149 return 0; 150 } 151 152 /** 153 * ws_find_node - Find SC WS node based on VSI id or TC 154 * @parent: parent node of First VSI or TC node 155 * @match_val: value to match 156 * @type: match type VSI/TC 157 */ 158 static struct irdma_ws_node * 159 ws_find_node(struct irdma_ws_node *parent, 160 u16 match_val, 161 enum irdma_ws_match_type type) 162 { 163 struct irdma_ws_node *node; 164 165 switch (type) { 166 case WS_MATCH_TYPE_VSI: 167 list_for_each_entry(node, &parent->child_list_head, siblings) { 168 if (node->vsi_index == match_val) 169 return node; 170 } 171 break; 172 case WS_MATCH_TYPE_TC: 173 list_for_each_entry(node, &parent->child_list_head, siblings) { 174 if (node->traffic_class == match_val) 175 return node; 176 } 177 break; 178 default: 179 break; 180 } 181 182 return NULL; 183 } 184 185 /** 186 * irdma_ws_in_use - Checks to see if a leaf node is in use 187 * @vsi: vsi pointer 188 * @user_pri: user priority 189 */ 190 static bool 191 irdma_ws_in_use(struct irdma_sc_vsi *vsi, u8 user_pri) 192 { 193 int i; 194 195 if (!list_empty(&vsi->qos[user_pri].qplist)) 196 return true; 197 198 /* 199 * Check if the qs handle associated with the given user priority is in use by any other user priority. If so, 200 * nothing left to do 201 */ 202 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 203 if (vsi->qos[i].qs_handle == vsi->qos[user_pri].qs_handle && 204 !list_empty(&vsi->qos[i].qplist)) 205 return true; 206 } 207 208 return false; 209 } 210 211 /** 212 * irdma_remove_leaf - Remove leaf node unconditionally 213 * @vsi: vsi pointer 214 * @user_pri: user priority 215 */ 216 static void 217 irdma_remove_leaf(struct irdma_sc_vsi *vsi, u8 user_pri) 218 { 219 struct irdma_ws_node *ws_tree_root, *vsi_node, *tc_node; 220 u16 qs_handle; 221 int i; 222 223 qs_handle = vsi->qos[user_pri].qs_handle; 224 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 225 if (vsi->qos[i].qs_handle == qs_handle) 226 vsi->qos[i].valid = false; 227 } 228 229 ws_tree_root = vsi->dev->ws_tree_root; 230 if (!ws_tree_root) 231 return; 232 233 vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx, 234 WS_MATCH_TYPE_VSI); 235 if (!vsi_node) 236 return; 237 238 tc_node = ws_find_node(vsi_node, 239 vsi->qos[user_pri].traffic_class, 240 WS_MATCH_TYPE_TC); 241 if (!tc_node) 242 return; 243 244 list_del(&tc_node->siblings); 245 irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE, NULL); 246 247 vsi->unregister_qset(vsi, tc_node); 248 irdma_free_node(vsi, tc_node); 249 /* Check if VSI node can be freed */ 250 if (list_empty(&vsi_node->child_list_head)) { 251 irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE, NULL); 252 list_del(&vsi_node->siblings); 253 irdma_free_node(vsi, vsi_node); 254 /* Free head node there are no remaining VSI nodes */ 255 if (list_empty(&ws_tree_root->child_list_head)) { 256 irdma_ws_cqp_cmd(vsi, ws_tree_root, 257 IRDMA_OP_WS_DELETE_NODE, NULL); 258 irdma_free_node(vsi, ws_tree_root); 259 vsi->dev->ws_tree_root = NULL; 260 } 261 } 262 } 263 264 static int 265 irdma_enable_leaf(struct irdma_sc_vsi *vsi, 266 struct irdma_ws_node *tc_node) 267 { 268 int ret; 269 270 ret = vsi->register_qset(vsi, tc_node); 271 if (ret) 272 return ret; 273 274 tc_node->enable = true; 275 ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_MODIFY_NODE, NULL); 276 if (ret) 277 goto enable_err; 278 return 0; 279 280 enable_err: 281 vsi->unregister_qset(vsi, tc_node); 282 283 return ret; 284 } 285 286 static struct irdma_ws_node * 287 irdma_add_leaf_node(struct irdma_sc_vsi *vsi, 288 struct irdma_ws_node *vsi_node, 289 u8 user_pri, u16 traffic_class) 290 { 291 struct irdma_ws_node *tc_node = 292 irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_LEAF, vsi_node); 293 int i, ret = 0; 294 295 if (!tc_node) 296 return NULL; 297 ret = irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_ADD_NODE, &tc_node->qs_handle); 298 if (ret) { 299 irdma_free_node(vsi, tc_node); 300 return NULL; 301 } 302 vsi->qos[tc_node->user_pri].qs_handle = tc_node->qs_handle; 303 304 list_add(&tc_node->siblings, &vsi_node->child_list_head); 305 306 ret = irdma_enable_leaf(vsi, tc_node); 307 if (ret) 308 goto reg_err; 309 310 /* 311 * Iterate through other UPs and update the QS handle if they have a matching traffic class. 312 */ 313 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) { 314 if (vsi->qos[i].traffic_class == traffic_class) { 315 vsi->qos[i].qs_handle = tc_node->qs_handle; 316 vsi->qos[i].l2_sched_node_id = 317 tc_node->l2_sched_node_id; 318 vsi->qos[i].valid = true; 319 } 320 } 321 return tc_node; 322 323 reg_err: 324 irdma_ws_cqp_cmd(vsi, tc_node, IRDMA_OP_WS_DELETE_NODE, NULL); 325 list_del(&tc_node->siblings); 326 irdma_free_node(vsi, tc_node); 327 328 return NULL; 329 } 330 331 /** 332 * irdma_ws_add - Build work scheduler tree, set RDMA qs_handle 333 * @vsi: vsi pointer 334 * @user_pri: user priority 335 */ 336 int 337 irdma_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri) 338 { 339 struct irdma_ws_node *ws_tree_root; 340 struct irdma_ws_node *vsi_node; 341 struct irdma_ws_node *tc_node; 342 u16 traffic_class; 343 int ret = 0; 344 345 mutex_lock(&vsi->dev->ws_mutex); 346 if (vsi->tc_change_pending) { 347 ret = -EBUSY; 348 goto exit; 349 } 350 351 if (vsi->qos[user_pri].valid) 352 goto exit; 353 354 ws_tree_root = vsi->dev->ws_tree_root; 355 if (!ws_tree_root) { 356 ws_tree_root = irdma_alloc_node(vsi, user_pri, 357 WS_NODE_TYPE_PARENT, NULL); 358 if (!ws_tree_root) { 359 ret = -ENOMEM; 360 goto exit; 361 } 362 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, 363 "Creating root node = %d\n", ws_tree_root->index); 364 365 ret = irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_ADD_NODE, 366 NULL); 367 if (ret) { 368 irdma_free_node(vsi, ws_tree_root); 369 goto exit; 370 } 371 372 vsi->dev->ws_tree_root = ws_tree_root; 373 } 374 375 /* Find a second tier node that matches the VSI */ 376 vsi_node = ws_find_node(ws_tree_root, vsi->vsi_idx, 377 WS_MATCH_TYPE_VSI); 378 379 /* If VSI node doesn't exist, add one */ 380 if (!vsi_node) { 381 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, 382 "Node not found matching VSI %d\n", vsi->vsi_idx); 383 vsi_node = irdma_alloc_node(vsi, user_pri, WS_NODE_TYPE_PARENT, 384 ws_tree_root); 385 if (!vsi_node) { 386 ret = -ENOMEM; 387 goto vsi_add_err; 388 } 389 390 ret = irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_ADD_NODE, 391 NULL); 392 if (ret) { 393 irdma_free_node(vsi, vsi_node); 394 goto vsi_add_err; 395 } 396 397 list_add(&vsi_node->siblings, &ws_tree_root->child_list_head); 398 } 399 400 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, 401 "Using node %d which represents VSI %d\n", vsi_node->index, 402 vsi->vsi_idx); 403 traffic_class = vsi->qos[user_pri].traffic_class; 404 tc_node = ws_find_node(vsi_node, traffic_class, 405 WS_MATCH_TYPE_TC); 406 if (!tc_node) { 407 /* Add leaf node */ 408 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, 409 "Node not found matching VSI %d and TC %d\n", 410 vsi->vsi_idx, traffic_class); 411 tc_node = irdma_add_leaf_node(vsi, vsi_node, user_pri, 412 traffic_class); 413 if (!tc_node) { 414 ret = -ENOMEM; 415 goto leaf_add_err; 416 } 417 } 418 irdma_debug(vsi->dev, IRDMA_DEBUG_WS, 419 "Using node %d which represents VSI %d TC %d\n", 420 tc_node->index, vsi->vsi_idx, traffic_class); 421 goto exit; 422 423 leaf_add_err: 424 if (list_empty(&vsi_node->child_list_head)) { 425 if (irdma_ws_cqp_cmd(vsi, vsi_node, IRDMA_OP_WS_DELETE_NODE, 426 NULL)) 427 goto exit; 428 list_del(&vsi_node->siblings); 429 irdma_free_node(vsi, vsi_node); 430 } 431 432 vsi_add_err: 433 /* Free head node there are no remaining VSI nodes */ 434 if (list_empty(&ws_tree_root->child_list_head)) { 435 irdma_ws_cqp_cmd(vsi, ws_tree_root, IRDMA_OP_WS_DELETE_NODE, 436 NULL); 437 vsi->dev->ws_tree_root = NULL; 438 irdma_free_node(vsi, ws_tree_root); 439 } 440 441 exit: 442 mutex_unlock(&vsi->dev->ws_mutex); 443 return ret; 444 } 445 446 /** 447 * irdma_ws_remove - Free WS scheduler node, update WS tree 448 * @vsi: vsi pointer 449 * @user_pri: user priority 450 */ 451 void 452 irdma_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri) 453 { 454 mutex_lock(&vsi->qos[user_pri].qos_mutex); 455 mutex_lock(&vsi->dev->ws_mutex); 456 if (irdma_ws_in_use(vsi, user_pri)) 457 goto exit; 458 irdma_remove_leaf(vsi, user_pri); 459 exit: 460 mutex_unlock(&vsi->dev->ws_mutex); 461 mutex_unlock(&vsi->qos[user_pri].qos_mutex); 462 } 463 464 /** 465 * irdma_ws_reset - Reset entire WS tree 466 * @vsi: vsi pointer 467 */ 468 void 469 irdma_ws_reset(struct irdma_sc_vsi *vsi) 470 { 471 u8 i; 472 473 mutex_lock(&vsi->dev->ws_mutex); 474 for (i = 0; i < IRDMA_MAX_USER_PRIORITY; ++i) 475 irdma_remove_leaf(vsi, i); 476 mutex_unlock(&vsi->dev->ws_mutex); 477 } 478