1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) 2 /* QLogic qed NIC Driver 3 * Copyright (c) 2015-2017 QLogic Corporation 4 * Copyright (c) 2019-2020 Marvell International Ltd. 5 */ 6 7 #include <linux/types.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/kernel.h> 10 #include <linux/list.h> 11 #include <linux/pci.h> 12 #include <linux/slab.h> 13 #include <linux/string.h> 14 #include "qed.h" 15 #include "qed_iscsi.h" 16 #include "qed_ll2.h" 17 #include "qed_ooo.h" 18 #include "qed_cxt.h" 19 #include "qed_nvmetcp.h" 20 static struct qed_ooo_archipelago 21 *qed_ooo_seek_archipelago(struct qed_hwfn *p_hwfn, 22 struct qed_ooo_info 23 *p_ooo_info, 24 u32 cid) 25 { 26 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 27 struct qed_ooo_archipelago *p_archipelago; 28 29 if (unlikely(idx >= p_ooo_info->max_num_archipelagos)) 30 return NULL; 31 32 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 33 34 if (unlikely(list_empty(&p_archipelago->isles_list))) 35 return NULL; 36 37 return p_archipelago; 38 } 39 40 static struct qed_ooo_isle *qed_ooo_seek_isle(struct qed_hwfn *p_hwfn, 41 struct qed_ooo_info *p_ooo_info, 42 u32 cid, u8 isle) 43 { 44 struct qed_ooo_archipelago *p_archipelago = NULL; 45 struct qed_ooo_isle *p_isle = NULL; 46 u8 the_num_of_isle = 1; 47 48 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 49 if (unlikely(!p_archipelago)) { 50 DP_NOTICE(p_hwfn, 51 "Connection %d is not found in OOO list\n", cid); 52 return NULL; 53 } 54 55 list_for_each_entry(p_isle, &p_archipelago->isles_list, list_entry) { 56 if (the_num_of_isle == isle) 57 return p_isle; 58 the_num_of_isle++; 59 } 60 61 return NULL; 62 } 63 64 void qed_ooo_save_history_entry(struct qed_hwfn *p_hwfn, 65 struct qed_ooo_info *p_ooo_info, 66 struct ooo_opaque *p_cqe) 67 { 68 struct qed_ooo_history *p_history = &p_ooo_info->ooo_history; 69 70 if (p_history->head_idx == p_history->num_of_cqes) 71 p_history->head_idx = 0; 72 p_history->p_cqes[p_history->head_idx] = *p_cqe; 73 p_history->head_idx++; 74 } 75 76 int qed_ooo_alloc(struct qed_hwfn *p_hwfn) 77 { 78 u16 max_num_archipelagos = 0, cid_base; 79 struct qed_ooo_info *p_ooo_info; 80 enum protocol_type proto; 81 u16 max_num_isles = 0; 82 u32 i; 83 84 switch (p_hwfn->hw_info.personality) { 85 case QED_PCI_ISCSI: 86 case QED_PCI_NVMETCP: 87 proto = PROTOCOLID_TCP_ULP; 88 break; 89 case QED_PCI_ETH_RDMA: 90 case QED_PCI_ETH_IWARP: 91 proto = PROTOCOLID_IWARP; 92 break; 93 default: 94 DP_NOTICE(p_hwfn, 95 "Failed to allocate qed_ooo_info: unknown personality\n"); 96 return -EINVAL; 97 } 98 99 max_num_archipelagos = (u16)qed_cxt_get_proto_cid_count(p_hwfn, proto, 100 NULL); 101 max_num_isles = QED_MAX_NUM_ISLES + max_num_archipelagos; 102 cid_base = (u16)qed_cxt_get_proto_cid_start(p_hwfn, proto); 103 104 if (!max_num_archipelagos) { 105 DP_NOTICE(p_hwfn, 106 "Failed to allocate qed_ooo_info: unknown amount of connections\n"); 107 return -EINVAL; 108 } 109 110 p_ooo_info = kzalloc(sizeof(*p_ooo_info), GFP_KERNEL); 111 if (!p_ooo_info) 112 return -ENOMEM; 113 114 p_ooo_info->cid_base = cid_base; 115 p_ooo_info->max_num_archipelagos = max_num_archipelagos; 116 117 INIT_LIST_HEAD(&p_ooo_info->free_buffers_list); 118 INIT_LIST_HEAD(&p_ooo_info->ready_buffers_list); 119 INIT_LIST_HEAD(&p_ooo_info->free_isles_list); 120 121 p_ooo_info->p_isles_mem = kcalloc(max_num_isles, 122 sizeof(struct qed_ooo_isle), 123 GFP_KERNEL); 124 if (!p_ooo_info->p_isles_mem) 125 goto no_isles_mem; 126 127 for (i = 0; i < max_num_isles; i++) { 128 INIT_LIST_HEAD(&p_ooo_info->p_isles_mem[i].buffers_list); 129 list_add_tail(&p_ooo_info->p_isles_mem[i].list_entry, 130 &p_ooo_info->free_isles_list); 131 } 132 133 p_ooo_info->p_archipelagos_mem = 134 kcalloc(max_num_archipelagos, 135 sizeof(struct qed_ooo_archipelago), 136 GFP_KERNEL); 137 if (!p_ooo_info->p_archipelagos_mem) 138 goto no_archipelagos_mem; 139 140 for (i = 0; i < max_num_archipelagos; i++) 141 INIT_LIST_HEAD(&p_ooo_info->p_archipelagos_mem[i].isles_list); 142 143 p_ooo_info->ooo_history.p_cqes = 144 kcalloc(QED_MAX_NUM_OOO_HISTORY_ENTRIES, 145 sizeof(struct ooo_opaque), 146 GFP_KERNEL); 147 if (!p_ooo_info->ooo_history.p_cqes) 148 goto no_history_mem; 149 150 p_ooo_info->ooo_history.num_of_cqes = QED_MAX_NUM_OOO_HISTORY_ENTRIES; 151 152 p_hwfn->p_ooo_info = p_ooo_info; 153 return 0; 154 155 no_history_mem: 156 kfree(p_ooo_info->p_archipelagos_mem); 157 no_archipelagos_mem: 158 kfree(p_ooo_info->p_isles_mem); 159 no_isles_mem: 160 kfree(p_ooo_info); 161 return -ENOMEM; 162 } 163 164 void qed_ooo_release_connection_isles(struct qed_hwfn *p_hwfn, 165 struct qed_ooo_info *p_ooo_info, u32 cid) 166 { 167 struct qed_ooo_archipelago *p_archipelago; 168 struct qed_ooo_buffer *p_buffer; 169 struct qed_ooo_isle *p_isle; 170 171 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 172 if (!p_archipelago) 173 return; 174 175 while (!list_empty(&p_archipelago->isles_list)) { 176 p_isle = list_first_entry(&p_archipelago->isles_list, 177 struct qed_ooo_isle, list_entry); 178 179 list_del(&p_isle->list_entry); 180 181 while (!list_empty(&p_isle->buffers_list)) { 182 p_buffer = list_first_entry(&p_isle->buffers_list, 183 struct qed_ooo_buffer, 184 list_entry); 185 186 list_move_tail(&p_buffer->list_entry, 187 &p_ooo_info->free_buffers_list); 188 } 189 list_add_tail(&p_isle->list_entry, 190 &p_ooo_info->free_isles_list); 191 } 192 } 193 194 void qed_ooo_release_all_isles(struct qed_hwfn *p_hwfn, 195 struct qed_ooo_info *p_ooo_info) 196 { 197 struct qed_ooo_archipelago *p_archipelago; 198 struct qed_ooo_buffer *p_buffer; 199 struct qed_ooo_isle *p_isle; 200 u32 i; 201 202 for (i = 0; i < p_ooo_info->max_num_archipelagos; i++) { 203 p_archipelago = &(p_ooo_info->p_archipelagos_mem[i]); 204 205 while (!list_empty(&p_archipelago->isles_list)) { 206 p_isle = list_first_entry(&p_archipelago->isles_list, 207 struct qed_ooo_isle, 208 list_entry); 209 210 list_del(&p_isle->list_entry); 211 212 while (!list_empty(&p_isle->buffers_list)) { 213 p_buffer = 214 list_first_entry(&p_isle->buffers_list, 215 struct qed_ooo_buffer, 216 list_entry); 217 218 list_move_tail(&p_buffer->list_entry, 219 &p_ooo_info->free_buffers_list); 220 } 221 list_add_tail(&p_isle->list_entry, 222 &p_ooo_info->free_isles_list); 223 } 224 } 225 if (!list_empty(&p_ooo_info->ready_buffers_list)) 226 list_splice_tail_init(&p_ooo_info->ready_buffers_list, 227 &p_ooo_info->free_buffers_list); 228 } 229 230 void qed_ooo_setup(struct qed_hwfn *p_hwfn) 231 { 232 qed_ooo_release_all_isles(p_hwfn, p_hwfn->p_ooo_info); 233 memset(p_hwfn->p_ooo_info->ooo_history.p_cqes, 0, 234 p_hwfn->p_ooo_info->ooo_history.num_of_cqes * 235 sizeof(struct ooo_opaque)); 236 p_hwfn->p_ooo_info->ooo_history.head_idx = 0; 237 } 238 239 void qed_ooo_free(struct qed_hwfn *p_hwfn) 240 { 241 struct qed_ooo_info *p_ooo_info = p_hwfn->p_ooo_info; 242 struct qed_ooo_buffer *p_buffer; 243 244 if (!p_ooo_info) 245 return; 246 247 qed_ooo_release_all_isles(p_hwfn, p_ooo_info); 248 while (!list_empty(&p_ooo_info->free_buffers_list)) { 249 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 250 struct qed_ooo_buffer, list_entry); 251 252 list_del(&p_buffer->list_entry); 253 dma_free_coherent(&p_hwfn->cdev->pdev->dev, 254 p_buffer->rx_buffer_size, 255 p_buffer->rx_buffer_virt_addr, 256 p_buffer->rx_buffer_phys_addr); 257 kfree(p_buffer); 258 } 259 260 kfree(p_ooo_info->p_isles_mem); 261 kfree(p_ooo_info->p_archipelagos_mem); 262 kfree(p_ooo_info->ooo_history.p_cqes); 263 kfree(p_ooo_info); 264 p_hwfn->p_ooo_info = NULL; 265 } 266 267 void qed_ooo_put_free_buffer(struct qed_hwfn *p_hwfn, 268 struct qed_ooo_info *p_ooo_info, 269 struct qed_ooo_buffer *p_buffer) 270 { 271 list_add_tail(&p_buffer->list_entry, &p_ooo_info->free_buffers_list); 272 } 273 274 struct qed_ooo_buffer *qed_ooo_get_free_buffer(struct qed_hwfn *p_hwfn, 275 struct qed_ooo_info *p_ooo_info) 276 { 277 struct qed_ooo_buffer *p_buffer = NULL; 278 279 if (!list_empty(&p_ooo_info->free_buffers_list)) { 280 p_buffer = list_first_entry(&p_ooo_info->free_buffers_list, 281 struct qed_ooo_buffer, list_entry); 282 283 list_del(&p_buffer->list_entry); 284 } 285 286 return p_buffer; 287 } 288 289 void qed_ooo_put_ready_buffer(struct qed_hwfn *p_hwfn, 290 struct qed_ooo_info *p_ooo_info, 291 struct qed_ooo_buffer *p_buffer, u8 on_tail) 292 { 293 if (on_tail) 294 list_add_tail(&p_buffer->list_entry, 295 &p_ooo_info->ready_buffers_list); 296 else 297 list_add(&p_buffer->list_entry, 298 &p_ooo_info->ready_buffers_list); 299 } 300 301 struct qed_ooo_buffer *qed_ooo_get_ready_buffer(struct qed_hwfn *p_hwfn, 302 struct qed_ooo_info *p_ooo_info) 303 { 304 struct qed_ooo_buffer *p_buffer = NULL; 305 306 if (!list_empty(&p_ooo_info->ready_buffers_list)) { 307 p_buffer = list_first_entry(&p_ooo_info->ready_buffers_list, 308 struct qed_ooo_buffer, list_entry); 309 310 list_del(&p_buffer->list_entry); 311 } 312 313 return p_buffer; 314 } 315 316 void qed_ooo_delete_isles(struct qed_hwfn *p_hwfn, 317 struct qed_ooo_info *p_ooo_info, 318 u32 cid, u8 drop_isle, u8 drop_size) 319 { 320 struct qed_ooo_isle *p_isle = NULL; 321 u8 isle_idx; 322 323 for (isle_idx = 0; isle_idx < drop_size; isle_idx++) { 324 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, drop_isle); 325 if (!p_isle) { 326 DP_NOTICE(p_hwfn, 327 "Isle %d is not found(cid %d)\n", 328 drop_isle, cid); 329 return; 330 } 331 if (list_empty(&p_isle->buffers_list)) 332 DP_NOTICE(p_hwfn, 333 "Isle %d is empty(cid %d)\n", drop_isle, cid); 334 else 335 list_splice_tail_init(&p_isle->buffers_list, 336 &p_ooo_info->free_buffers_list); 337 338 list_del(&p_isle->list_entry); 339 p_ooo_info->cur_isles_number--; 340 list_add(&p_isle->list_entry, &p_ooo_info->free_isles_list); 341 } 342 } 343 344 void qed_ooo_add_new_isle(struct qed_hwfn *p_hwfn, 345 struct qed_ooo_info *p_ooo_info, 346 u32 cid, u8 ooo_isle, 347 struct qed_ooo_buffer *p_buffer) 348 { 349 struct qed_ooo_archipelago *p_archipelago = NULL; 350 struct qed_ooo_isle *p_prev_isle = NULL; 351 struct qed_ooo_isle *p_isle = NULL; 352 353 if (ooo_isle > 1) { 354 p_prev_isle = qed_ooo_seek_isle(p_hwfn, 355 p_ooo_info, cid, ooo_isle - 1); 356 if (unlikely(!p_prev_isle)) { 357 DP_NOTICE(p_hwfn, 358 "Isle %d is not found(cid %d)\n", 359 ooo_isle - 1, cid); 360 return; 361 } 362 } 363 p_archipelago = qed_ooo_seek_archipelago(p_hwfn, p_ooo_info, cid); 364 if (unlikely(!p_archipelago && ooo_isle != 1)) { 365 DP_NOTICE(p_hwfn, 366 "Connection %d is not found in OOO list\n", cid); 367 return; 368 } 369 370 if (!list_empty(&p_ooo_info->free_isles_list)) { 371 p_isle = list_first_entry(&p_ooo_info->free_isles_list, 372 struct qed_ooo_isle, list_entry); 373 374 list_del(&p_isle->list_entry); 375 if (unlikely(!list_empty(&p_isle->buffers_list))) { 376 DP_NOTICE(p_hwfn, "Free isle is not empty\n"); 377 INIT_LIST_HEAD(&p_isle->buffers_list); 378 } 379 } else { 380 DP_NOTICE(p_hwfn, "No more free isles\n"); 381 return; 382 } 383 384 if (!p_archipelago) { 385 u32 idx = (cid & 0xffff) - p_ooo_info->cid_base; 386 387 p_archipelago = &p_ooo_info->p_archipelagos_mem[idx]; 388 } 389 390 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 391 p_ooo_info->cur_isles_number++; 392 p_ooo_info->gen_isles_number++; 393 394 if (p_ooo_info->cur_isles_number > p_ooo_info->max_isles_number) 395 p_ooo_info->max_isles_number = p_ooo_info->cur_isles_number; 396 397 if (!p_prev_isle) 398 list_add(&p_isle->list_entry, &p_archipelago->isles_list); 399 else 400 list_add(&p_isle->list_entry, &p_prev_isle->list_entry); 401 } 402 403 void qed_ooo_add_new_buffer(struct qed_hwfn *p_hwfn, 404 struct qed_ooo_info *p_ooo_info, 405 u32 cid, 406 u8 ooo_isle, 407 struct qed_ooo_buffer *p_buffer, u8 buffer_side) 408 { 409 struct qed_ooo_isle *p_isle = NULL; 410 411 p_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, ooo_isle); 412 if (unlikely(!p_isle)) { 413 DP_NOTICE(p_hwfn, 414 "Isle %d is not found(cid %d)\n", ooo_isle, cid); 415 return; 416 } 417 418 if (unlikely(buffer_side == QED_OOO_LEFT_BUF)) 419 list_add(&p_buffer->list_entry, &p_isle->buffers_list); 420 else 421 list_add_tail(&p_buffer->list_entry, &p_isle->buffers_list); 422 } 423 424 void qed_ooo_join_isles(struct qed_hwfn *p_hwfn, 425 struct qed_ooo_info *p_ooo_info, u32 cid, u8 left_isle) 426 { 427 struct qed_ooo_isle *p_right_isle = NULL; 428 struct qed_ooo_isle *p_left_isle = NULL; 429 430 p_right_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 431 left_isle + 1); 432 if (unlikely(!p_right_isle)) { 433 DP_NOTICE(p_hwfn, 434 "Right isle %d is not found(cid %d)\n", 435 left_isle + 1, cid); 436 return; 437 } 438 439 list_del(&p_right_isle->list_entry); 440 p_ooo_info->cur_isles_number--; 441 if (left_isle) { 442 p_left_isle = qed_ooo_seek_isle(p_hwfn, p_ooo_info, cid, 443 left_isle); 444 if (unlikely(!p_left_isle)) { 445 DP_NOTICE(p_hwfn, 446 "Left isle %d is not found(cid %d)\n", 447 left_isle, cid); 448 return; 449 } 450 list_splice_tail_init(&p_right_isle->buffers_list, 451 &p_left_isle->buffers_list); 452 } else { 453 list_splice_tail_init(&p_right_isle->buffers_list, 454 &p_ooo_info->ready_buffers_list); 455 } 456 list_add_tail(&p_right_isle->list_entry, &p_ooo_info->free_isles_list); 457 } 458