1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/slab.h> 13 #include <linux/string.h> 14 #include "qed.h" 15 #include "qed_iscsi.h" 16 #include "qed_ll2.h" 17 #include "qed_ooo.h" 18 #include "qed_cxt.h" 19 20 static struct qed_ooo_archipelago 21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn, 22 struct qed_ooo_info 23 *p_ooo_info, 24 u32 cid) 25 { 26 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 27 struct qed_ooo_archipelago *p_archipelago; 28 29 if (idx >= p_ooo_info->max_num_archipelagos) 30 return NULL; 31 32 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 33 34 if (list_empty(&p_archipelago->isles_list)) 35 return NULL; 36 37 return p_archipelago; 38 } 39 40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn, 41 struct qed_ooo_info *p_ooo_info, 42 u32 cid, u8 isle) 43 { 44 struct qed_ooo_archipelago *p_archipelago = NULL; 45 struct qed_ooo_isle *p_isle = NULL; 46 u8 the_num_of_isle = 1; 47 48 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 49 if (!p_archipelago) { 50 DP_NOTICE(p_hwfn, 51 "Connection %d is not found in OOO list\n", cid); 52 return NULL; 53 } 54 55 list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) { 56 if (the_num_of_isle == isle) 57 return p_isle; 58 the_num_of_isle++; 59 } 60 61 return NULL; 62 } 63 64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn, 65 struct qed_ooo_info *p_ooo_info, 66 struct ooo_opaque *p_cqe) 67 { 68 struct qed_ooo_history *p_history = &p_ooo_info->ooo_history; 69 70 if (p_history->head_idx == p_history->num_of_cqes) 71 p_history->head_idx = 0; 72 p_history->p_cqes[p_history->head_idx] = *p_cqe; 73 p_history->head_idx++; 74 } 75 76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn) 77 { 78 u16 max_num_archipelagos = 0, cid_base; 79 struct qed_ooo_info *p_ooo_info; 80 enum protocol_type proto; 81 u16 max_num_isles = 0; 82 u32 i; 83 84 switch (p_hwfn->hw_info.personality) { 85 case QED_PCI_ISCSI: 86 proto = PROTOCOLID_ISCSI; 87 break; 88 case QED_PCI_ETH_RDMA: 89 case QED_PCI_ETH_IWARP: 90 proto = PROTOCOLID_IWARP; 91 break; 92 default: 93 DP_NOTICE(p_hwfn, 94 "Failed to allocate qed_ooo_info: unknown personality\n"); 95 return -EINVAL; 96 } 97 98 max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto, 99 NULL); 100 max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos; 101 cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto); 102 103 if (!max_num_archipelagos) { 104 DP_NOTICE(p_hwfn, 105 "Failed to allocate qed_ooo_info: unknown amount of connections\n"); 106 return -EINVAL; 107 } 108 109 p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL); 110 if (!p_ooo_info) 111 return -ENOMEM; 112 113 p_ooo_info->cid_base = cid_base; 114 p_ooo_info->max_num_archipelagos = max_num_archipelagos; 115 116 INIT_LIST_HEAD(&p_ooo_info->free_buffers_list); 117 INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list); 118 INIT_LIST_HEAD(&p_ooo_info->free_isles_list); 119 120 p_ooo_info->p_isles_mem = kcalloc(max_num_isles, 121 sizeof(struct qed_ooo_isle), 122 GFP_KERNEL); 123 if (!p_ooo_info->p_isles_mem) 124 goto no_isles_mem; 125 126 for (i = 0; i < max_num_isles; i++) { 127 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list); 128 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry, 129 &p_ooo_info->free_isles_list); 130 } 131 132 p_ooo_info->p_archipelagos_mem = 133 kcalloc(max_num_archipelagos, 134 sizeof(struct qed_ooo_archipelago), 135 GFP_KERNEL); 136 if (!p_ooo_info->p_archipelagos_mem) 137 goto no_archipelagos_mem; 138 139 for (i = 0; i < max_num_archipelagos; i++) 140 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list); 141 142 p_ooo_info->ooo_history.p_cqes = 143 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES, 144 sizeof(struct ooo_opaque), 145 GFP_KERNEL); 146 if (!p_ooo_info->ooo_history.p_cqes) 147 goto no_history_mem; 148 149 p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES; 150 151 p_hwfn->p_ooo_info = p_ooo_info; 152 return 0; 153 154 no_history_mem: 155 kfree(p_ooo_info->p_archipelagos_mem); 156 no_archipelagos_mem: 157 kfree(p_ooo_info->p_isles_mem); 158 no_isles_mem: 159 kfree(p_ooo_info); 160 return -ENOMEM; 161 } 162 163 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn, 164 struct qed_ooo_info *p_ooo_info, u32 cid) 165 { 166 struct qed_ooo_archipelago *p_archipelago; 167 struct qed_ooo_buffer *p_buffer; 168 struct qed_ooo_isle *p_isle; 169 170 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 171 if (!p_archipelago) 172 return; 173 174 while (!list_empty(&p_archipelago->isles_list)) { 175 p_isle = list_first_entry(&p_archipelago->isles_list, 176 struct qed_ooo_isle, list_entry); 177 178 list_del(&p_isle->list_entry); 179 180 while (!list_empty(&p_isle->buffers_list)) { 181 p_buffer = list_first_entry(&p_isle->buffers_list, 182 struct qed_ooo_buffer, 183 list_entry); 184 185 if (!p_buffer) 186 break; 187 188 list_move_tail(&p_buffer->list_entry, 189 &p_ooo_info->free_buffers_list); 190 } 191 list_add_tail(&p_isle->list_entry, 192 &p_ooo_info->free_isles_list); 193 } 194 } 195 196 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn, 197 struct qed_ooo_info *p_ooo_info) 198 { 199 struct qed_ooo_archipelago *p_archipelago; 200 struct qed_ooo_buffer *p_buffer; 201 struct qed_ooo_isle *p_isle; 202 u32 i; 203 204 for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) { 205 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]); 206 207 while (!list_empty(&p_archipelago->isles_list)) { 208 p_isle = list_first_entry(&p_archipelago->isles_list, 209 struct qed_ooo_isle, 210 list_entry); 211 212 list_del(&p_isle->list_entry); 213 214 while (!list_empty(&p_isle->buffers_list)) { 215 p_buffer = 216 list_first_entry(&p_isle->buffers_list, 217 struct qed_ooo_buffer, 218 list_entry); 219 220 if (!p_buffer) 221 break; 222 223 list_move_tail(&p_buffer->list_entry, 224 &p_ooo_info->free_buffers_list); 225 } 226 list_add_tail(&p_isle->list_entry, 227 &p_ooo_info->free_isles_list); 228 } 229 } 230 if (!list_empty(&p_ooo_info->ready_buffers_list)) 231 list_splice_tail_init(&p_ooo_info->ready_buffers_list, 232 &p_ooo_info->free_buffers_list); 233 } 234 235 void qed_ooo_setup(struct qed_hwfn *p_hwfn) 236 { 237 qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info); 238 memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0, 239 p_hwfn->p_ooo_info->ooo_history.num_of_cqes * 240 sizeof(struct ooo_opaque)); 241 p_hwfn->p_ooo_info->ooo_history.head_idx = 0; 242 } 243 244 void qed_ooo_free(struct qed_hwfn *p_hwfn) 245 { 246 struct qed_ooo_info *p_ooo_info = p_hwfn->p_ooo_info; 247 struct qed_ooo_buffer *p_buffer; 248 249 if (!p_ooo_info) 250 return; 251 252 qed_ooo_release_all_isles(p_hwfn, p_ooo_info); 253 while (!list_empty(&p_ooo_info->free_buffers_list)) { 254 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 255 struct qed_ooo_buffer, list_entry); 256 257 if (!p_buffer) 258 break; 259 260 list_del(&p_buffer->list_entry); 261 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 262 p_buffer->rx_buffer_size, 263 p_buffer->rx_buffer_virt_addr, 264 p_buffer->rx_buffer_phys_addr); 265 kfree(p_buffer); 266 } 267 268 kfree(p_ooo_info->p_isles_mem); 269 kfree(p_ooo_info->p_archipelagos_mem); 270 kfree(p_ooo_info->ooo_history.p_cqes); 271 kfree(p_ooo_info); 272 p_hwfn->p_ooo_info = NULL; 273 } 274 275 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn, 276 struct qed_ooo_info *p_ooo_info, 277 struct qed_ooo_buffer *p_buffer) 278 { 279 list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list); 280 } 281 282 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn, 283 struct qed_ooo_info *p_ooo_info) 284 { 285 struct qed_ooo_buffer *p_buffer = NULL; 286 287 if (!list_empty(&p_ooo_info->free_buffers_list)) { 288 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 289 struct qed_ooo_buffer, list_entry); 290 291 list_del(&p_buffer->list_entry); 292 } 293 294 return p_buffer; 295 } 296 297 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn, 298 struct qed_ooo_info *p_ooo_info, 299 struct qed_ooo_buffer *p_buffer, u8 on_tail) 300 { 301 if (on_tail) 302 list_add_tail(&p_buffer->list_entry, 303 &p_ooo_info->ready_buffers_list); 304 else 305 list_add(&p_buffer->list_entry, 306 &p_ooo_info->ready_buffers_list); 307 } 308 309 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn, 310 struct qed_ooo_info *p_ooo_info) 311 { 312 struct qed_ooo_buffer *p_buffer = NULL; 313 314 if (!list_empty(&p_ooo_info->ready_buffers_list)) { 315 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list, 316 struct qed_ooo_buffer, list_entry); 317 318 list_del(&p_buffer->list_entry); 319 } 320 321 return p_buffer; 322 } 323 324 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn, 325 struct qed_ooo_info *p_ooo_info, 326 u32 cid, u8 drop_isle, u8 drop_size) 327 { 328 struct qed_ooo_isle *p_isle = NULL; 329 u8 isle_idx; 330 331 for (isle_idx = 0; isle_idx < drop_size; isle_idx++) { 332 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle); 333 if (!p_isle) { 334 DP_NOTICE(p_hwfn, 335 "Isle %d is not found(cid %d)\n", 336 drop_isle, cid); 337 return; 338 } 339 if (list_empty(&p_isle->buffers_list)) 340 DP_NOTICE(p_hwfn, 341 "Isle %d is empty(cid %d)\n", drop_isle, cid); 342 else 343 list_splice_tail_init(&p_isle->buffers_list, 344 &p_ooo_info->free_buffers_list); 345 346 list_del(&p_isle->list_entry); 347 p_ooo_info->cur_isles_number--; 348 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list); 349 } 350 } 351 352 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn, 353 struct qed_ooo_info *p_ooo_info, 354 u32 cid, u8 ooo_isle, 355 struct qed_ooo_buffer *p_buffer) 356 { 357 struct qed_ooo_archipelago *p_archipelago = NULL; 358 struct qed_ooo_isle *p_prev_isle = NULL; 359 struct qed_ooo_isle *p_isle = NULL; 360 361 if (ooo_isle > 1) { 362 p_prev_isle = qed_ooo_seek_isle(p_hwfn, 363 p_ooo_info, cid, ooo_isle - 1); 364 if (!p_prev_isle) { 365 DP_NOTICE(p_hwfn, 366 "Isle %d is not found(cid %d)\n", 367 ooo_isle - 1, cid); 368 return; 369 } 370 } 371 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 372 if (!p_archipelago && (ooo_isle != 1)) { 373 DP_NOTICE(p_hwfn, 374 "Connection %d is not found in OOO list\n", cid); 375 return; 376 } 377 378 if (!list_empty(&p_ooo_info->free_isles_list)) { 379 p_isle = list_first_entry(&p_ooo_info->free_isles_list, 380 struct qed_ooo_isle, list_entry); 381 382 list_del(&p_isle->list_entry); 383 if (!list_empty(&p_isle->buffers_list)) { 384 DP_NOTICE(p_hwfn, "Free isle is not empty\n"); 385 INIT_LIST_HEAD(&p_isle->buffers_list); 386 } 387 } else { 388 DP_NOTICE(p_hwfn, "No more free isles\n"); 389 return; 390 } 391 392 if (!p_archipelago) { 393 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 394 395 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 396 } 397 398 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 399 p_ooo_info->cur_isles_number++; 400 p_ooo_info->gen_isles_number++; 401 402 if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number) 403 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number; 404 405 if (!p_prev_isle) 406 list_add(&p_isle->list_entry, &p_archipelago->isles_list); 407 else 408 list_add(&p_isle->list_entry, &p_prev_isle->list_entry); 409 } 410 411 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn, 412 struct qed_ooo_info *p_ooo_info, 413 u32 cid, 414 u8 ooo_isle, 415 struct qed_ooo_buffer *p_buffer, u8 buffer_side) 416 { 417 struct qed_ooo_isle *p_isle = NULL; 418 419 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle); 420 if (!p_isle) { 421 DP_NOTICE(p_hwfn, 422 "Isle %d is not found(cid %d)\n", ooo_isle, cid); 423 return; 424 } 425 426 if (buffer_side == QED_OOO_LEFT_BUF) 427 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 428 else 429 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list); 430 } 431 432 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn, 433 struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle) 434 { 435 struct qed_ooo_isle *p_right_isle = NULL; 436 struct qed_ooo_isle *p_left_isle = NULL; 437 438 p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 439 left_isle + 1); 440 if (!p_right_isle) { 441 DP_NOTICE(p_hwfn, 442 "Right isle %d is not found(cid %d)\n", 443 left_isle + 1, cid); 444 return; 445 } 446 447 list_del(&p_right_isle->list_entry); 448 p_ooo_info->cur_isles_number--; 449 if (left_isle) { 450 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 451 left_isle); 452 if (!p_left_isle) { 453 DP_NOTICE(p_hwfn, 454 "Left isle %d is not found(cid %d)\n", 455 left_isle, cid); 456 return; 457 } 458 list_splice_tail_init(&p_right_isle->buffers_list, 459 &p_left_isle->buffers_list); 460 } else { 461 list_splice_tail_init(&p_right_isle->buffers_list, 462 &p_ooo_info->ready_buffers_list); 463 } 464 list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list); 465 } 466