1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/slab.h> 13 #include <linux/string.h> 14 #include "qed.h" 15 #include "qed_iscsi.h" 16 #include "qed_ll2.h" 17 #include "qed_ooo.h" 18 #include "qed_cxt.h" 19 #include "qed_nvmetcp.h" 20 static struct qed_ooo_archipelago 21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn, 22 struct qed_ooo_info 23 *p_ooo_info, 24 u32 cid) 25 { 26 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 27 struct qed_ooo_archipelago *p_archipelago; 28 29 if (unlikely(idx >= p_ooo_info->max_num_archipelagos)) 30 return NULL; 31 32 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 33 34 if (unlikely(list_empty(&p_archipelago->isles_list))) 35 return NULL; 36 37 return p_archipelago; 38 } 39 40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn, 41 struct qed_ooo_info *p_ooo_info, 42 u32 cid, u8 isle) 43 { 44 struct qed_ooo_archipelago *p_archipelago = NULL; 45 struct qed_ooo_isle *p_isle = NULL; 46 u8 the_num_of_isle = 1; 47 48 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 49 if (unlikely(!p_archipelago)) { 50 DP_NOTICE(p_hwfn, 51 "Connection %d is not found in OOO list\n", cid); 52 return NULL; 53 } 54 55 list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) { 56 if (the_num_of_isle == isle) 57 return p_isle; 58 the_num_of_isle++; 59 } 60 61 return NULL; 62 } 63 64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn, 65 struct qed_ooo_info *p_ooo_info, 66 struct ooo_opaque *p_cqe) 67 { 68 struct qed_ooo_history *p_history = &p_ooo_info->ooo_history; 69 70 if (p_history->head_idx == p_history->num_of_cqes) 71 p_history->head_idx = 0; 72 p_history->p_cqes[p_history->head_idx] = *p_cqe; 73 p_history->head_idx++; 74 } 75 76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn) 77 { 78 u16 max_num_archipelagos = 0, cid_base; 79 struct qed_ooo_info *p_ooo_info; 80 enum protocol_type proto; 81 u16 max_num_isles = 0; 82 u32 i; 83 84 switch (p_hwfn->hw_info.personality) { 85 case QED_PCI_ISCSI: 86 case QED_PCI_NVMETCP: 87 proto = PROTOCOLID_TCP_ULP; 88 break; 89 case QED_PCI_ETH_RDMA: 90 case QED_PCI_ETH_IWARP: 91 proto = PROTOCOLID_IWARP; 92 break; 93 default: 94 DP_NOTICE(p_hwfn, 95 "Failed to allocate qed_ooo_info: unknown personality\n"); 96 return -EINVAL; 97 } 98 99 max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto, 100 NULL); 101 max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos; 102 cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto); 103 104 if (!max_num_archipelagos) { 105 DP_NOTICE(p_hwfn, 106 "Failed to allocate qed_ooo_info: unknown amount of connections\n"); 107 return -EINVAL; 108 } 109 110 p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL); 111 if (!p_ooo_info) 112 return -ENOMEM; 113 114 p_ooo_info->cid_base = cid_base; 115 p_ooo_info->max_num_archipelagos = max_num_archipelagos; 116 117 INIT_LIST_HEAD(&p_ooo_info->free_buffers_list); 118 INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list); 119 INIT_LIST_HEAD(&p_ooo_info->free_isles_list); 120 121 p_ooo_info->p_isles_mem = kcalloc(max_num_isles, 122 sizeof(struct qed_ooo_isle), 123 GFP_KERNEL); 124 if (!p_ooo_info->p_isles_mem) 125 goto no_isles_mem; 126 127 for (i = 0; i < max_num_isles; i++) { 128 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list); 129 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry, 130 &p_ooo_info->free_isles_list); 131 } 132 133 p_ooo_info->p_archipelagos_mem = 134 kcalloc(max_num_archipelagos, 135 sizeof(struct qed_ooo_archipelago), 136 GFP_KERNEL); 137 if (!p_ooo_info->p_archipelagos_mem) 138 goto no_archipelagos_mem; 139 140 for (i = 0; i < max_num_archipelagos; i++) 141 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list); 142 143 p_ooo_info->ooo_history.p_cqes = 144 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES, 145 sizeof(struct ooo_opaque), 146 GFP_KERNEL); 147 if (!p_ooo_info->ooo_history.p_cqes) 148 goto no_history_mem; 149 150 p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES; 151 152 p_hwfn->p_ooo_info = p_ooo_info; 153 return 0; 154 155 no_history_mem: 156 kfree(p_ooo_info->p_archipelagos_mem); 157 no_archipelagos_mem: 158 kfree(p_ooo_info->p_isles_mem); 159 no_isles_mem: 160 kfree(p_ooo_info); 161 return -ENOMEM; 162 } 163 164 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn, 165 struct qed_ooo_info *p_ooo_info, u32 cid) 166 { 167 struct qed_ooo_archipelago *p_archipelago; 168 struct qed_ooo_buffer *p_buffer; 169 struct qed_ooo_isle *p_isle; 170 171 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 172 if (!p_archipelago) 173 return; 174 175 while (!list_empty(&p_archipelago->isles_list)) { 176 p_isle = list_first_entry(&p_archipelago->isles_list, 177 struct qed_ooo_isle, list_entry); 178 179 list_del(&p_isle->list_entry); 180 181 while (!list_empty(&p_isle->buffers_list)) { 182 p_buffer = list_first_entry(&p_isle->buffers_list, 183 struct qed_ooo_buffer, 184 list_entry); 185 186 if (!p_buffer) 187 break; 188 189 list_move_tail(&p_buffer->list_entry, 190 &p_ooo_info->free_buffers_list); 191 } 192 list_add_tail(&p_isle->list_entry, 193 &p_ooo_info->free_isles_list); 194 } 195 } 196 197 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn, 198 struct qed_ooo_info *p_ooo_info) 199 { 200 struct qed_ooo_archipelago *p_archipelago; 201 struct qed_ooo_buffer *p_buffer; 202 struct qed_ooo_isle *p_isle; 203 u32 i; 204 205 for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) { 206 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]); 207 208 while (!list_empty(&p_archipelago->isles_list)) { 209 p_isle = list_first_entry(&p_archipelago->isles_list, 210 struct qed_ooo_isle, 211 list_entry); 212 213 list_del(&p_isle->list_entry); 214 215 while (!list_empty(&p_isle->buffers_list)) { 216 p_buffer = 217 list_first_entry(&p_isle->buffers_list, 218 struct qed_ooo_buffer, 219 list_entry); 220 221 if (!p_buffer) 222 break; 223 224 list_move_tail(&p_buffer->list_entry, 225 &p_ooo_info->free_buffers_list); 226 } 227 list_add_tail(&p_isle->list_entry, 228 &p_ooo_info->free_isles_list); 229 } 230 } 231 if (!list_empty(&p_ooo_info->ready_buffers_list)) 232 list_splice_tail_init(&p_ooo_info->ready_buffers_list, 233 &p_ooo_info->free_buffers_list); 234 } 235 236 void qed_ooo_setup(struct qed_hwfn *p_hwfn) 237 { 238 qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info); 239 memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0, 240 p_hwfn->p_ooo_info->ooo_history.num_of_cqes * 241 sizeof(struct ooo_opaque)); 242 p_hwfn->p_ooo_info->ooo_history.head_idx = 0; 243 } 244 245 void qed_ooo_free(struct qed_hwfn *p_hwfn) 246 { 247 struct qed_ooo_info *p_ooo_info = p_hwfn->p_ooo_info; 248 struct qed_ooo_buffer *p_buffer; 249 250 if (!p_ooo_info) 251 return; 252 253 qed_ooo_release_all_isles(p_hwfn, p_ooo_info); 254 while (!list_empty(&p_ooo_info->free_buffers_list)) { 255 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 256 struct qed_ooo_buffer, list_entry); 257 258 if (!p_buffer) 259 break; 260 261 list_del(&p_buffer->list_entry); 262 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 263 p_buffer->rx_buffer_size, 264 p_buffer->rx_buffer_virt_addr, 265 p_buffer->rx_buffer_phys_addr); 266 kfree(p_buffer); 267 } 268 269 kfree(p_ooo_info->p_isles_mem); 270 kfree(p_ooo_info->p_archipelagos_mem); 271 kfree(p_ooo_info->ooo_history.p_cqes); 272 kfree(p_ooo_info); 273 p_hwfn->p_ooo_info = NULL; 274 } 275 276 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn, 277 struct qed_ooo_info *p_ooo_info, 278 struct qed_ooo_buffer *p_buffer) 279 { 280 list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list); 281 } 282 283 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn, 284 struct qed_ooo_info *p_ooo_info) 285 { 286 struct qed_ooo_buffer *p_buffer = NULL; 287 288 if (!list_empty(&p_ooo_info->free_buffers_list)) { 289 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 290 struct qed_ooo_buffer, list_entry); 291 292 list_del(&p_buffer->list_entry); 293 } 294 295 return p_buffer; 296 } 297 298 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn, 299 struct qed_ooo_info *p_ooo_info, 300 struct qed_ooo_buffer *p_buffer, u8 on_tail) 301 { 302 if (on_tail) 303 list_add_tail(&p_buffer->list_entry, 304 &p_ooo_info->ready_buffers_list); 305 else 306 list_add(&p_buffer->list_entry, 307 &p_ooo_info->ready_buffers_list); 308 } 309 310 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn, 311 struct qed_ooo_info *p_ooo_info) 312 { 313 struct qed_ooo_buffer *p_buffer = NULL; 314 315 if (!list_empty(&p_ooo_info->ready_buffers_list)) { 316 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list, 317 struct qed_ooo_buffer, list_entry); 318 319 list_del(&p_buffer->list_entry); 320 } 321 322 return p_buffer; 323 } 324 325 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn, 326 struct qed_ooo_info *p_ooo_info, 327 u32 cid, u8 drop_isle, u8 drop_size) 328 { 329 struct qed_ooo_isle *p_isle = NULL; 330 u8 isle_idx; 331 332 for (isle_idx = 0; isle_idx < drop_size; isle_idx++) { 333 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle); 334 if (!p_isle) { 335 DP_NOTICE(p_hwfn, 336 "Isle %d is not found(cid %d)\n", 337 drop_isle, cid); 338 return; 339 } 340 if (list_empty(&p_isle->buffers_list)) 341 DP_NOTICE(p_hwfn, 342 "Isle %d is empty(cid %d)\n", drop_isle, cid); 343 else 344 list_splice_tail_init(&p_isle->buffers_list, 345 &p_ooo_info->free_buffers_list); 346 347 list_del(&p_isle->list_entry); 348 p_ooo_info->cur_isles_number--; 349 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list); 350 } 351 } 352 353 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn, 354 struct qed_ooo_info *p_ooo_info, 355 u32 cid, u8 ooo_isle, 356 struct qed_ooo_buffer *p_buffer) 357 { 358 struct qed_ooo_archipelago *p_archipelago = NULL; 359 struct qed_ooo_isle *p_prev_isle = NULL; 360 struct qed_ooo_isle *p_isle = NULL; 361 362 if (ooo_isle > 1) { 363 p_prev_isle = qed_ooo_seek_isle(p_hwfn, 364 p_ooo_info, cid, ooo_isle - 1); 365 if (unlikely(!p_prev_isle)) { 366 DP_NOTICE(p_hwfn, 367 "Isle %d is not found(cid %d)\n", 368 ooo_isle - 1, cid); 369 return; 370 } 371 } 372 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 373 if (unlikely(!p_archipelago && ooo_isle != 1)) { 374 DP_NOTICE(p_hwfn, 375 "Connection %d is not found in OOO list\n", cid); 376 return; 377 } 378 379 if (!list_empty(&p_ooo_info->free_isles_list)) { 380 p_isle = list_first_entry(&p_ooo_info->free_isles_list, 381 struct qed_ooo_isle, list_entry); 382 383 list_del(&p_isle->list_entry); 384 if (unlikely(!list_empty(&p_isle->buffers_list))) { 385 DP_NOTICE(p_hwfn, "Free isle is not empty\n"); 386 INIT_LIST_HEAD(&p_isle->buffers_list); 387 } 388 } else { 389 DP_NOTICE(p_hwfn, "No more free isles\n"); 390 return; 391 } 392 393 if (!p_archipelago) { 394 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 395 396 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 397 } 398 399 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 400 p_ooo_info->cur_isles_number++; 401 p_ooo_info->gen_isles_number++; 402 403 if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number) 404 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number; 405 406 if (!p_prev_isle) 407 list_add(&p_isle->list_entry, &p_archipelago->isles_list); 408 else 409 list_add(&p_isle->list_entry, &p_prev_isle->list_entry); 410 } 411 412 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn, 413 struct qed_ooo_info *p_ooo_info, 414 u32 cid, 415 u8 ooo_isle, 416 struct qed_ooo_buffer *p_buffer, u8 buffer_side) 417 { 418 struct qed_ooo_isle *p_isle = NULL; 419 420 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle); 421 if (unlikely(!p_isle)) { 422 DP_NOTICE(p_hwfn, 423 "Isle %d is not found(cid %d)\n", ooo_isle, cid); 424 return; 425 } 426 427 if (unlikely(buffer_side == QED_OOO_LEFT_BUF)) 428 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 429 else 430 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list); 431 } 432 433 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn, 434 struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle) 435 { 436 struct qed_ooo_isle *p_right_isle = NULL; 437 struct qed_ooo_isle *p_left_isle = NULL; 438 439 p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 440 left_isle + 1); 441 if (unlikely(!p_right_isle)) { 442 DP_NOTICE(p_hwfn, 443 "Right isle %d is not found(cid %d)\n", 444 left_isle + 1, cid); 445 return; 446 } 447 448 list_del(&p_right_isle->list_entry); 449 p_ooo_info->cur_isles_number--; 450 if (left_isle) { 451 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 452 left_isle); 453 if (unlikely(!p_left_isle)) { 454 DP_NOTICE(p_hwfn, 455 "Left isle %d is not found(cid %d)\n", 456 left_isle, cid); 457 return; 458 } 459 list_splice_tail_init(&p_right_isle->buffers_list, 460 &p_left_isle->buffers_list); 461 } else { 462 list_splice_tail_init(&p_right_isle->buffers_list, 463 &p_ooo_info->ready_buffers_list); 464 } 465 list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list); 466 } 467