1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/pci.h> 10 #include <linux/device.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/semaphore.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/io.h> 16 #include <linux/err.h> 17 18 #include "hinic_hw_if.h" 19 #include "hinic_hw_eqs.h" 20 #include "hinic_hw_wqe.h" 21 #include "hinic_hw_wq.h" 22 #include "hinic_hw_cmdq.h" 23 #include "hinic_hw_qp_ctxt.h" 24 #include "hinic_hw_qp.h" 25 #include "hinic_hw_io.h" 26 27 #define CI_Q_ADDR_SIZE sizeof(u32) 28 29 #define CI_ADDR(base_addr, q_id) ((base_addr) + \ 30 (q_id) * CI_Q_ADDR_SIZE) 31 32 #define CI_TABLE_SIZE(num_qps) ((num_qps) * CI_Q_ADDR_SIZE) 33 34 #define DB_IDX(db, db_base) \ 35 (((unsigned long)(db) - (unsigned long)(db_base)) / HINIC_DB_PAGE_SIZE) 36 37 enum io_cmd { 38 IO_CMD_MODIFY_QUEUE_CTXT = 0, 39 IO_CMD_CLEAN_QUEUE_CTXT, 40 }; 41 42 static void init_db_area_idx(struct hinic_free_db_area *free_db_area) 43 { 44 int i; 45 46 for (i = 0; i < HINIC_DB_MAX_AREAS; i++) 47 free_db_area->db_idx[i] = i; 48 49 free_db_area->alloc_pos = 0; 50 free_db_area->return_pos = HINIC_DB_MAX_AREAS; 51 52 free_db_area->num_free = HINIC_DB_MAX_AREAS; 53 54 sema_init(&free_db_area->idx_lock, 1); 55 } 56 57 static void __iomem *get_db_area(struct hinic_func_to_io *func_to_io) 58 { 59 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area; 60 int pos, idx; 61 62 down(&free_db_area->idx_lock); 63 64 free_db_area->num_free--; 65 66 if (free_db_area->num_free < 0) { 67 free_db_area->num_free++; 68 up(&free_db_area->idx_lock); 69 return ERR_PTR(-ENOMEM); 70 } 71 72 pos = free_db_area->alloc_pos++; 73 pos &= HINIC_DB_MAX_AREAS - 1; 74 75 idx = free_db_area->db_idx[pos]; 76 77 free_db_area->db_idx[pos] = -1; 78 79 up(&free_db_area->idx_lock); 80 81 return func_to_io->db_base + idx * HINIC_DB_PAGE_SIZE; 82 } 83 84 static void return_db_area(struct hinic_func_to_io *func_to_io, 85 void __iomem *db_base) 86 { 87 struct hinic_free_db_area *free_db_area = &func_to_io->free_db_area; 88 int pos, idx = DB_IDX(db_base, func_to_io->db_base); 89 90 down(&free_db_area->idx_lock); 91 92 pos = free_db_area->return_pos++; 93 pos &= HINIC_DB_MAX_AREAS - 1; 94 95 free_db_area->db_idx[pos] = idx; 96 97 free_db_area->num_free++; 98 99 up(&free_db_area->idx_lock); 100 } 101 102 static int write_sq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn, 103 u16 num_sqs) 104 { 105 struct hinic_hwif *hwif = func_to_io->hwif; 106 struct hinic_sq_ctxt_block *sq_ctxt_block; 107 struct pci_dev *pdev = hwif->pdev; 108 struct hinic_cmdq_buf cmdq_buf; 109 struct hinic_sq_ctxt *sq_ctxt; 110 struct hinic_qp *qp; 111 u64 out_param; 112 int err, i; 113 114 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 115 if (err) { 116 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n"); 117 return err; 118 } 119 120 sq_ctxt_block = cmdq_buf.buf; 121 sq_ctxt = sq_ctxt_block->sq_ctxt; 122 123 hinic_qp_prepare_header(&sq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_SQ, 124 num_sqs, func_to_io->max_qps); 125 for (i = 0; i < num_sqs; i++) { 126 qp = &func_to_io->qps[i]; 127 128 hinic_sq_prepare_ctxt(&sq_ctxt[i], &qp->sq, 129 base_qpn + qp->q_id); 130 } 131 132 cmdq_buf.size = HINIC_SQ_CTXT_SIZE(num_sqs); 133 134 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC, 135 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf, 136 &out_param); 137 if ((err) || (out_param != 0)) { 138 dev_err(&pdev->dev, "Failed to set SQ ctxts\n"); 139 err = -EFAULT; 140 } 141 142 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 143 return err; 144 } 145 146 static int write_rq_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn, 147 u16 num_rqs) 148 { 149 struct hinic_hwif *hwif = func_to_io->hwif; 150 struct hinic_rq_ctxt_block *rq_ctxt_block; 151 struct pci_dev *pdev = hwif->pdev; 152 struct hinic_cmdq_buf cmdq_buf; 153 struct hinic_rq_ctxt *rq_ctxt; 154 struct hinic_qp *qp; 155 u64 out_param; 156 int err, i; 157 158 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 159 if (err) { 160 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n"); 161 return err; 162 } 163 164 rq_ctxt_block = cmdq_buf.buf; 165 rq_ctxt = rq_ctxt_block->rq_ctxt; 166 167 hinic_qp_prepare_header(&rq_ctxt_block->hdr, HINIC_QP_CTXT_TYPE_RQ, 168 num_rqs, func_to_io->max_qps); 169 for (i = 0; i < num_rqs; i++) { 170 qp = &func_to_io->qps[i]; 171 172 hinic_rq_prepare_ctxt(&rq_ctxt[i], &qp->rq, 173 base_qpn + qp->q_id); 174 } 175 176 cmdq_buf.size = HINIC_RQ_CTXT_SIZE(num_rqs); 177 178 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC, 179 IO_CMD_MODIFY_QUEUE_CTXT, &cmdq_buf, 180 &out_param); 181 if ((err) || (out_param != 0)) { 182 dev_err(&pdev->dev, "Failed to set RQ ctxts\n"); 183 err = -EFAULT; 184 } 185 186 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 187 return err; 188 } 189 190 /** 191 * write_qp_ctxts - write the qp ctxt to HW 192 * @func_to_io: func to io channel that holds the IO components 193 * @base_qpn: first qp number 194 * @num_qps: number of qps to write 195 * 196 * Return 0 - Success, negative - Failure 197 **/ 198 static int write_qp_ctxts(struct hinic_func_to_io *func_to_io, u16 base_qpn, 199 u16 num_qps) 200 { 201 return (write_sq_ctxts(func_to_io, base_qpn, num_qps) || 202 write_rq_ctxts(func_to_io, base_qpn, num_qps)); 203 } 204 205 static int hinic_clean_queue_offload_ctxt(struct hinic_func_to_io *func_to_io, 206 enum hinic_qp_ctxt_type ctxt_type) 207 { 208 struct hinic_hwif *hwif = func_to_io->hwif; 209 struct hinic_clean_queue_ctxt *ctxt_block; 210 struct pci_dev *pdev = hwif->pdev; 211 struct hinic_cmdq_buf cmdq_buf; 212 u64 out_param = 0; 213 int err; 214 215 err = hinic_alloc_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 216 if (err) { 217 dev_err(&pdev->dev, "Failed to allocate cmdq buf\n"); 218 return err; 219 } 220 221 ctxt_block = cmdq_buf.buf; 222 ctxt_block->cmdq_hdr.num_queues = func_to_io->max_qps; 223 ctxt_block->cmdq_hdr.queue_type = ctxt_type; 224 ctxt_block->cmdq_hdr.addr_offset = 0; 225 226 /* TSO/LRO ctxt size: 0x0:0B; 0x1:160B; 0x2:200B; 0x3:240B */ 227 ctxt_block->ctxt_size = 0x3; 228 229 hinic_cpu_to_be32(ctxt_block, sizeof(*ctxt_block)); 230 231 cmdq_buf.size = sizeof(*ctxt_block); 232 233 err = hinic_cmdq_direct_resp(&func_to_io->cmdqs, HINIC_MOD_L2NIC, 234 IO_CMD_CLEAN_QUEUE_CTXT, 235 &cmdq_buf, &out_param); 236 237 if (err || out_param) { 238 dev_err(&pdev->dev, "Failed to clean offload ctxts, err: %d, out_param: 0x%llx\n", 239 err, out_param); 240 241 err = -EFAULT; 242 } 243 244 hinic_free_cmdq_buf(&func_to_io->cmdqs, &cmdq_buf); 245 246 return err; 247 } 248 249 static int hinic_clean_qp_offload_ctxt(struct hinic_func_to_io *func_to_io) 250 { 251 /* clean LRO/TSO context space */ 252 return (hinic_clean_queue_offload_ctxt(func_to_io, 253 HINIC_QP_CTXT_TYPE_SQ) || 254 hinic_clean_queue_offload_ctxt(func_to_io, 255 HINIC_QP_CTXT_TYPE_RQ)); 256 } 257 258 /** 259 * init_qp - Initialize a Queue Pair 260 * @func_to_io: func to io channel that holds the IO components 261 * @qp: pointer to the qp to initialize 262 * @q_id: the id of the qp 263 * @sq_msix_entry: msix entry for sq 264 * @rq_msix_entry: msix entry for rq 265 * 266 * Return 0 - Success, negative - Failure 267 **/ 268 static int init_qp(struct hinic_func_to_io *func_to_io, 269 struct hinic_qp *qp, int q_id, 270 struct msix_entry *sq_msix_entry, 271 struct msix_entry *rq_msix_entry) 272 { 273 struct hinic_hwif *hwif = func_to_io->hwif; 274 struct pci_dev *pdev = hwif->pdev; 275 void __iomem *db_base; 276 int err; 277 278 qp->q_id = q_id; 279 280 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->sq_wq[q_id], 281 HINIC_SQ_WQEBB_SIZE, HINIC_SQ_PAGE_SIZE, 282 HINIC_SQ_DEPTH, HINIC_SQ_WQE_MAX_SIZE); 283 if (err) { 284 dev_err(&pdev->dev, "Failed to allocate WQ for SQ\n"); 285 return err; 286 } 287 288 err = hinic_wq_allocate(&func_to_io->wqs, &func_to_io->rq_wq[q_id], 289 HINIC_RQ_WQEBB_SIZE, HINIC_RQ_PAGE_SIZE, 290 HINIC_RQ_DEPTH, HINIC_RQ_WQE_SIZE); 291 if (err) { 292 dev_err(&pdev->dev, "Failed to allocate WQ for RQ\n"); 293 goto err_rq_alloc; 294 } 295 296 db_base = get_db_area(func_to_io); 297 if (IS_ERR(db_base)) { 298 dev_err(&pdev->dev, "Failed to get DB area for SQ\n"); 299 err = PTR_ERR(db_base); 300 goto err_get_db; 301 } 302 303 func_to_io->sq_db[q_id] = db_base; 304 305 err = hinic_init_sq(&qp->sq, hwif, &func_to_io->sq_wq[q_id], 306 sq_msix_entry, 307 CI_ADDR(func_to_io->ci_addr_base, q_id), 308 CI_ADDR(func_to_io->ci_dma_base, q_id), db_base); 309 if (err) { 310 dev_err(&pdev->dev, "Failed to init SQ\n"); 311 goto err_sq_init; 312 } 313 314 err = hinic_init_rq(&qp->rq, hwif, &func_to_io->rq_wq[q_id], 315 rq_msix_entry); 316 if (err) { 317 dev_err(&pdev->dev, "Failed to init RQ\n"); 318 goto err_rq_init; 319 } 320 321 return 0; 322 323 err_rq_init: 324 hinic_clean_sq(&qp->sq); 325 326 err_sq_init: 327 return_db_area(func_to_io, db_base); 328 329 err_get_db: 330 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]); 331 332 err_rq_alloc: 333 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]); 334 return err; 335 } 336 337 /** 338 * destroy_qp - Clean the resources of a Queue Pair 339 * @func_to_io: func to io channel that holds the IO components 340 * @qp: pointer to the qp to clean 341 **/ 342 static void destroy_qp(struct hinic_func_to_io *func_to_io, 343 struct hinic_qp *qp) 344 { 345 int q_id = qp->q_id; 346 347 hinic_clean_rq(&qp->rq); 348 hinic_clean_sq(&qp->sq); 349 350 return_db_area(func_to_io, func_to_io->sq_db[q_id]); 351 352 hinic_wq_free(&func_to_io->wqs, &func_to_io->rq_wq[q_id]); 353 hinic_wq_free(&func_to_io->wqs, &func_to_io->sq_wq[q_id]); 354 } 355 356 /** 357 * hinic_io_create_qps - Create Queue Pairs 358 * @func_to_io: func to io channel that holds the IO components 359 * @base_qpn: base qp number 360 * @num_qps: number queue pairs to create 361 * @sq_msix_entry: msix entries for sq 362 * @rq_msix_entry: msix entries for rq 363 * 364 * Return 0 - Success, negative - Failure 365 **/ 366 int hinic_io_create_qps(struct hinic_func_to_io *func_to_io, 367 u16 base_qpn, int num_qps, 368 struct msix_entry *sq_msix_entries, 369 struct msix_entry *rq_msix_entries) 370 { 371 struct hinic_hwif *hwif = func_to_io->hwif; 372 struct pci_dev *pdev = hwif->pdev; 373 size_t qps_size, wq_size, db_size; 374 void *ci_addr_base; 375 int i, j, err; 376 377 qps_size = num_qps * sizeof(*func_to_io->qps); 378 func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL); 379 if (!func_to_io->qps) 380 return -ENOMEM; 381 382 wq_size = num_qps * sizeof(*func_to_io->sq_wq); 383 func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL); 384 if (!func_to_io->sq_wq) { 385 err = -ENOMEM; 386 goto err_sq_wq; 387 } 388 389 wq_size = num_qps * sizeof(*func_to_io->rq_wq); 390 func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL); 391 if (!func_to_io->rq_wq) { 392 err = -ENOMEM; 393 goto err_rq_wq; 394 } 395 396 db_size = num_qps * sizeof(*func_to_io->sq_db); 397 func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL); 398 if (!func_to_io->sq_db) { 399 err = -ENOMEM; 400 goto err_sq_db; 401 } 402 403 ci_addr_base = dma_alloc_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps), 404 &func_to_io->ci_dma_base, 405 GFP_KERNEL); 406 if (!ci_addr_base) { 407 dev_err(&pdev->dev, "Failed to allocate CI area\n"); 408 err = -ENOMEM; 409 goto err_ci_base; 410 } 411 412 func_to_io->ci_addr_base = ci_addr_base; 413 414 for (i = 0; i < num_qps; i++) { 415 err = init_qp(func_to_io, &func_to_io->qps[i], i, 416 &sq_msix_entries[i], &rq_msix_entries[i]); 417 if (err) { 418 dev_err(&pdev->dev, "Failed to create QP %d\n", i); 419 goto err_init_qp; 420 } 421 } 422 423 err = write_qp_ctxts(func_to_io, base_qpn, num_qps); 424 if (err) { 425 dev_err(&pdev->dev, "Failed to init QP ctxts\n"); 426 goto err_write_qp_ctxts; 427 } 428 429 err = hinic_clean_qp_offload_ctxt(func_to_io); 430 if (err) { 431 dev_err(&pdev->dev, "Failed to clean QP contexts space\n"); 432 goto err_write_qp_ctxts; 433 } 434 435 return 0; 436 437 err_write_qp_ctxts: 438 err_init_qp: 439 for (j = 0; j < i; j++) 440 destroy_qp(func_to_io, &func_to_io->qps[j]); 441 442 dma_free_coherent(&pdev->dev, CI_TABLE_SIZE(num_qps), 443 func_to_io->ci_addr_base, func_to_io->ci_dma_base); 444 445 err_ci_base: 446 devm_kfree(&pdev->dev, func_to_io->sq_db); 447 448 err_sq_db: 449 devm_kfree(&pdev->dev, func_to_io->rq_wq); 450 451 err_rq_wq: 452 devm_kfree(&pdev->dev, func_to_io->sq_wq); 453 454 err_sq_wq: 455 devm_kfree(&pdev->dev, func_to_io->qps); 456 return err; 457 } 458 459 /** 460 * hinic_io_destroy_qps - Destroy the IO Queue Pairs 461 * @func_to_io: func to io channel that holds the IO components 462 * @num_qps: number queue pairs to destroy 463 **/ 464 void hinic_io_destroy_qps(struct hinic_func_to_io *func_to_io, int num_qps) 465 { 466 struct hinic_hwif *hwif = func_to_io->hwif; 467 struct pci_dev *pdev = hwif->pdev; 468 size_t ci_table_size; 469 int i; 470 471 ci_table_size = CI_TABLE_SIZE(num_qps); 472 473 for (i = 0; i < num_qps; i++) 474 destroy_qp(func_to_io, &func_to_io->qps[i]); 475 476 dma_free_coherent(&pdev->dev, ci_table_size, func_to_io->ci_addr_base, 477 func_to_io->ci_dma_base); 478 479 devm_kfree(&pdev->dev, func_to_io->sq_db); 480 481 devm_kfree(&pdev->dev, func_to_io->rq_wq); 482 devm_kfree(&pdev->dev, func_to_io->sq_wq); 483 484 devm_kfree(&pdev->dev, func_to_io->qps); 485 } 486 487 /** 488 * hinic_io_init - Initialize the IO components 489 * @func_to_io: func to io channel that holds the IO components 490 * @hwif: HW interface for accessing IO 491 * @max_qps: maximum QPs in HW 492 * @num_ceqs: number completion event queues 493 * @ceq_msix_entries: msix entries for ceqs 494 * 495 * Return 0 - Success, negative - Failure 496 **/ 497 int hinic_io_init(struct hinic_func_to_io *func_to_io, 498 struct hinic_hwif *hwif, u16 max_qps, int num_ceqs, 499 struct msix_entry *ceq_msix_entries) 500 { 501 struct pci_dev *pdev = hwif->pdev; 502 enum hinic_cmdq_type cmdq, type; 503 void __iomem *db_area; 504 int err; 505 506 func_to_io->hwif = hwif; 507 func_to_io->qps = NULL; 508 func_to_io->max_qps = max_qps; 509 510 err = hinic_ceqs_init(&func_to_io->ceqs, hwif, num_ceqs, 511 HINIC_DEFAULT_CEQ_LEN, HINIC_EQ_PAGE_SIZE, 512 ceq_msix_entries); 513 if (err) { 514 dev_err(&pdev->dev, "Failed to init CEQs\n"); 515 return err; 516 } 517 518 err = hinic_wqs_alloc(&func_to_io->wqs, 2 * max_qps, hwif); 519 if (err) { 520 dev_err(&pdev->dev, "Failed to allocate WQS for IO\n"); 521 goto err_wqs_alloc; 522 } 523 524 func_to_io->db_base = pci_ioremap_bar(pdev, HINIC_PCI_DB_BAR); 525 if (!func_to_io->db_base) { 526 dev_err(&pdev->dev, "Failed to remap IO DB area\n"); 527 err = -ENOMEM; 528 goto err_db_ioremap; 529 } 530 531 init_db_area_idx(&func_to_io->free_db_area); 532 533 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) { 534 db_area = get_db_area(func_to_io); 535 if (IS_ERR(db_area)) { 536 dev_err(&pdev->dev, "Failed to get cmdq db area\n"); 537 err = PTR_ERR(db_area); 538 goto err_db_area; 539 } 540 541 func_to_io->cmdq_db_area[cmdq] = db_area; 542 } 543 544 err = hinic_init_cmdqs(&func_to_io->cmdqs, hwif, 545 func_to_io->cmdq_db_area); 546 if (err) { 547 dev_err(&pdev->dev, "Failed to initialize cmdqs\n"); 548 goto err_init_cmdqs; 549 } 550 551 return 0; 552 553 err_init_cmdqs: 554 err_db_area: 555 for (type = HINIC_CMDQ_SYNC; type < cmdq; type++) 556 return_db_area(func_to_io, func_to_io->cmdq_db_area[type]); 557 558 iounmap(func_to_io->db_base); 559 560 err_db_ioremap: 561 hinic_wqs_free(&func_to_io->wqs); 562 563 err_wqs_alloc: 564 hinic_ceqs_free(&func_to_io->ceqs); 565 return err; 566 } 567 568 /** 569 * hinic_io_free - Free the IO components 570 * @func_to_io: func to io channel that holds the IO components 571 **/ 572 void hinic_io_free(struct hinic_func_to_io *func_to_io) 573 { 574 enum hinic_cmdq_type cmdq; 575 576 hinic_free_cmdqs(&func_to_io->cmdqs); 577 578 for (cmdq = HINIC_CMDQ_SYNC; cmdq < HINIC_MAX_CMDQ_TYPES; cmdq++) 579 return_db_area(func_to_io, func_to_io->cmdq_db_area[cmdq]); 580 581 iounmap(func_to_io->db_base); 582 hinic_wqs_free(&func_to_io->wqs); 583 hinic_ceqs_free(&func_to_io->ceqs); 584 } 585