1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Huawei HiNIC PCI Express Linux driver 4 * Copyright(c) 2017 Huawei Technologies Co., Ltd 5 */ 6 7 #include <linux/kernel.h> 8 #include <linux/types.h> 9 #include <linux/errno.h> 10 #include <linux/pci.h> 11 #include <linux/device.h> 12 #include <linux/slab.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/bitops.h> 15 #include <linux/err.h> 16 #include <linux/jiffies.h> 17 #include <linux/delay.h> 18 #include <linux/log2.h> 19 #include <linux/semaphore.h> 20 #include <asm/byteorder.h> 21 #include <asm/barrier.h> 22 23 #include "hinic_hw_csr.h" 24 #include "hinic_hw_if.h" 25 #include "hinic_hw_api_cmd.h" 26 27 #define API_CHAIN_NUM_CELLS 32 28 29 #define API_CMD_CELL_SIZE_SHIFT 6 30 #define API_CMD_CELL_SIZE_MIN (BIT(API_CMD_CELL_SIZE_SHIFT)) 31 32 #define API_CMD_CELL_SIZE(cell_size) \ 33 (((cell_size) >= API_CMD_CELL_SIZE_MIN) ? \ 34 (1 << (fls(cell_size - 1))) : API_CMD_CELL_SIZE_MIN) 35 36 #define API_CMD_CELL_SIZE_VAL(size) \ 37 ilog2((size) >> API_CMD_CELL_SIZE_SHIFT) 38 39 #define API_CMD_BUF_SIZE 2048 40 41 /* Sizes of the members in hinic_api_cmd_cell */ 42 #define API_CMD_CELL_DESC_SIZE 8 43 #define API_CMD_CELL_DATA_ADDR_SIZE 8 44 45 #define API_CMD_CELL_ALIGNMENT 8 46 47 #define API_CMD_TIMEOUT 1000 48 49 #define MASKED_IDX(chain, idx) ((idx) & ((chain)->num_cells - 1)) 50 51 #define SIZE_8BYTES(size) (ALIGN((size), 8) >> 3) 52 #define SIZE_4BYTES(size) (ALIGN((size), 4) >> 2) 53 54 #define RD_DMA_ATTR_DEFAULT 0 55 #define WR_DMA_ATTR_DEFAULT 0 56 57 enum api_cmd_data_format { 58 SGE_DATA = 1, /* cell data is passed by hw address */ 59 }; 60 61 enum api_cmd_type { 62 API_CMD_WRITE = 0, 63 }; 64 65 enum api_cmd_bypass { 66 NO_BYPASS = 0, 67 BYPASS = 1, 68 }; 69 70 enum api_cmd_xor_chk_level { 71 XOR_CHK_DIS = 0, 72 73 XOR_CHK_ALL = 3, 74 }; 75 76 static u8 xor_chksum_set(void *data) 77 { 78 int idx; 79 u8 *val, checksum = 0; 80 81 val = data; 82 83 for (idx = 0; idx < 7; idx++) 84 checksum ^= val[idx]; 85 86 return checksum; 87 } 88 89 static void set_prod_idx(struct hinic_api_cmd_chain *chain) 90 { 91 enum hinic_api_cmd_chain_type chain_type = chain->chain_type; 92 struct hinic_hwif *hwif = chain->hwif; 93 u32 addr, prod_idx; 94 95 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain_type); 96 prod_idx = hinic_hwif_read_reg(hwif, addr); 97 98 prod_idx = HINIC_API_CMD_PI_CLEAR(prod_idx, IDX); 99 100 prod_idx |= HINIC_API_CMD_PI_SET(chain->prod_idx, IDX); 101 102 hinic_hwif_write_reg(hwif, addr, prod_idx); 103 } 104 105 static u32 get_hw_cons_idx(struct hinic_api_cmd_chain *chain) 106 { 107 u32 addr, val; 108 109 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); 110 val = hinic_hwif_read_reg(chain->hwif, addr); 111 112 return HINIC_API_CMD_STATUS_GET(val, CONS_IDX); 113 } 114 115 static void dump_api_chain_reg(struct hinic_api_cmd_chain *chain) 116 { 117 u32 addr, val; 118 119 addr = HINIC_CSR_API_CMD_STATUS_ADDR(chain->chain_type); 120 val = hinic_hwif_read_reg(chain->hwif, addr); 121 122 dev_err(&chain->hwif->pdev->dev, "Chain type: 0x%x, cpld error: 0x%x, check error: 0x%x, current fsm: 0x%x\n", 123 chain->chain_type, HINIC_API_CMD_STATUS_GET(val, CPLD_ERR), 124 HINIC_API_CMD_STATUS_GET(val, CHKSUM_ERR), 125 HINIC_API_CMD_STATUS_GET(val, FSM)); 126 127 dev_err(&chain->hwif->pdev->dev, "Chain hw current ci: 0x%x\n", 128 HINIC_API_CMD_STATUS_GET(val, CONS_IDX)); 129 130 addr = HINIC_CSR_API_CMD_CHAIN_PI_ADDR(chain->chain_type); 131 val = hinic_hwif_read_reg(chain->hwif, addr); 132 dev_err(&chain->hwif->pdev->dev, "Chain hw current pi: 0x%x\n", val); 133 } 134 135 /** 136 * chain_busy - check if the chain is still processing last requests 137 * @chain: chain to check 138 * 139 * Return 0 - Success, negative - Failure 140 **/ 141 static int chain_busy(struct hinic_api_cmd_chain *chain) 142 { 143 struct hinic_hwif *hwif = chain->hwif; 144 struct pci_dev *pdev = hwif->pdev; 145 u32 prod_idx; 146 147 switch (chain->chain_type) { 148 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 149 chain->cons_idx = get_hw_cons_idx(chain); 150 prod_idx = chain->prod_idx; 151 152 /* check for a space for a new command */ 153 if (chain->cons_idx == MASKED_IDX(chain, prod_idx + 1)) { 154 dev_err(&pdev->dev, "API CMD chain %d is busy, cons_idx: %d, prod_idx: %d\n", 155 chain->chain_type, chain->cons_idx, 156 chain->prod_idx); 157 dump_api_chain_reg(chain); 158 return -EBUSY; 159 } 160 break; 161 162 default: 163 dev_err(&pdev->dev, "Unknown API CMD Chain type\n"); 164 break; 165 } 166 167 return 0; 168 } 169 170 /** 171 * get_cell_data_size - get the data size of a specific cell type 172 * @type: chain type 173 * 174 * Return the data(Desc + Address) size in the cell 175 **/ 176 static u8 get_cell_data_size(enum hinic_api_cmd_chain_type type) 177 { 178 u8 cell_data_size = 0; 179 180 switch (type) { 181 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 182 cell_data_size = ALIGN(API_CMD_CELL_DESC_SIZE + 183 API_CMD_CELL_DATA_ADDR_SIZE, 184 API_CMD_CELL_ALIGNMENT); 185 break; 186 default: 187 break; 188 } 189 190 return cell_data_size; 191 } 192 193 /** 194 * prepare_cell_ctrl - prepare the ctrl of the cell for the command 195 * @cell_ctrl: the control of the cell to set the control value into it 196 * @data_size: the size of the data in the cell 197 **/ 198 static void prepare_cell_ctrl(u64 *cell_ctrl, u16 data_size) 199 { 200 u8 chksum; 201 u64 ctrl; 202 203 ctrl = HINIC_API_CMD_CELL_CTRL_SET(SIZE_8BYTES(data_size), DATA_SZ) | 204 HINIC_API_CMD_CELL_CTRL_SET(RD_DMA_ATTR_DEFAULT, RD_DMA_ATTR) | 205 HINIC_API_CMD_CELL_CTRL_SET(WR_DMA_ATTR_DEFAULT, WR_DMA_ATTR); 206 207 chksum = xor_chksum_set(&ctrl); 208 209 ctrl |= HINIC_API_CMD_CELL_CTRL_SET(chksum, XOR_CHKSUM); 210 211 /* The data in the HW should be in Big Endian Format */ 212 *cell_ctrl = cpu_to_be64(ctrl); 213 } 214 215 /** 216 * prepare_api_cmd - prepare API CMD command 217 * @chain: chain for the command 218 * @dest: destination node on the card that will receive the command 219 * @cmd: command data 220 * @cmd_size: the command size 221 **/ 222 static void prepare_api_cmd(struct hinic_api_cmd_chain *chain, 223 enum hinic_node_id dest, 224 void *cmd, u16 cmd_size) 225 { 226 struct hinic_api_cmd_cell *cell = chain->curr_node; 227 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 228 struct hinic_hwif *hwif = chain->hwif; 229 struct pci_dev *pdev = hwif->pdev; 230 231 cell_ctxt = &chain->cell_ctxt[chain->prod_idx]; 232 233 switch (chain->chain_type) { 234 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 235 cell->desc = HINIC_API_CMD_DESC_SET(SGE_DATA, API_TYPE) | 236 HINIC_API_CMD_DESC_SET(API_CMD_WRITE, RD_WR) | 237 HINIC_API_CMD_DESC_SET(NO_BYPASS, MGMT_BYPASS); 238 break; 239 240 default: 241 dev_err(&pdev->dev, "unknown Chain type\n"); 242 return; 243 } 244 245 cell->desc |= HINIC_API_CMD_DESC_SET(dest, DEST) | 246 HINIC_API_CMD_DESC_SET(SIZE_4BYTES(cmd_size), SIZE); 247 248 cell->desc |= HINIC_API_CMD_DESC_SET(xor_chksum_set(&cell->desc), 249 XOR_CHKSUM); 250 251 /* The data in the HW should be in Big Endian Format */ 252 cell->desc = cpu_to_be64(cell->desc); 253 254 memcpy(cell_ctxt->api_cmd_vaddr, cmd, cmd_size); 255 } 256 257 /** 258 * prepare_cell - prepare cell ctrl and cmd in the current cell 259 * @chain: chain for the command 260 * @dest: destination node on the card that will receive the command 261 * @cmd: command data 262 * @cmd_size: the command size 263 * 264 * Return 0 - Success, negative - Failure 265 **/ 266 static void prepare_cell(struct hinic_api_cmd_chain *chain, 267 enum hinic_node_id dest, 268 void *cmd, u16 cmd_size) 269 { 270 struct hinic_api_cmd_cell *curr_node = chain->curr_node; 271 u16 data_size = get_cell_data_size(chain->chain_type); 272 273 prepare_cell_ctrl(&curr_node->ctrl, data_size); 274 prepare_api_cmd(chain, dest, cmd, cmd_size); 275 } 276 277 static inline void cmd_chain_prod_idx_inc(struct hinic_api_cmd_chain *chain) 278 { 279 chain->prod_idx = MASKED_IDX(chain, chain->prod_idx + 1); 280 } 281 282 /** 283 * api_cmd_status_update - update the status in the chain struct 284 * @chain: chain to update 285 **/ 286 static void api_cmd_status_update(struct hinic_api_cmd_chain *chain) 287 { 288 enum hinic_api_cmd_chain_type chain_type; 289 struct hinic_api_cmd_status *wb_status; 290 struct hinic_hwif *hwif = chain->hwif; 291 struct pci_dev *pdev = hwif->pdev; 292 u64 status_header; 293 u32 status; 294 295 wb_status = chain->wb_status; 296 status_header = be64_to_cpu(wb_status->header); 297 298 status = be32_to_cpu(wb_status->status); 299 if (HINIC_API_CMD_STATUS_GET(status, CHKSUM_ERR)) { 300 dev_err(&pdev->dev, "API CMD status: Xor check error\n"); 301 return; 302 } 303 304 chain_type = HINIC_API_CMD_STATUS_HEADER_GET(status_header, CHAIN_ID); 305 if (chain_type >= HINIC_API_CMD_MAX) { 306 dev_err(&pdev->dev, "unknown API CMD Chain %d\n", chain_type); 307 return; 308 } 309 310 chain->cons_idx = HINIC_API_CMD_STATUS_GET(status, CONS_IDX); 311 } 312 313 /** 314 * wait_for_status_poll - wait for write to api cmd command to complete 315 * @chain: the chain of the command 316 * 317 * Return 0 - Success, negative - Failure 318 **/ 319 static int wait_for_status_poll(struct hinic_api_cmd_chain *chain) 320 { 321 int err = -ETIMEDOUT; 322 unsigned long end; 323 324 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 325 do { 326 api_cmd_status_update(chain); 327 328 /* wait for CI to be updated - sign for completion */ 329 if (chain->cons_idx == chain->prod_idx) { 330 err = 0; 331 break; 332 } 333 334 msleep(20); 335 } while (time_before(jiffies, end)); 336 337 return err; 338 } 339 340 /** 341 * wait_for_api_cmd_completion - wait for command to complete 342 * @chain: chain for the command 343 * 344 * Return 0 - Success, negative - Failure 345 **/ 346 static int wait_for_api_cmd_completion(struct hinic_api_cmd_chain *chain) 347 { 348 struct hinic_hwif *hwif = chain->hwif; 349 struct pci_dev *pdev = hwif->pdev; 350 int err; 351 352 switch (chain->chain_type) { 353 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 354 err = wait_for_status_poll(chain); 355 if (err) { 356 dev_err(&pdev->dev, "API CMD Poll status timeout\n"); 357 dump_api_chain_reg(chain); 358 break; 359 } 360 break; 361 362 default: 363 dev_err(&pdev->dev, "unknown API CMD Chain type\n"); 364 err = -EINVAL; 365 break; 366 } 367 368 return err; 369 } 370 371 /** 372 * api_cmd - API CMD command 373 * @chain: chain for the command 374 * @dest: destination node on the card that will receive the command 375 * @cmd: command data 376 * @cmd_size: the command size 377 * 378 * Return 0 - Success, negative - Failure 379 **/ 380 static int api_cmd(struct hinic_api_cmd_chain *chain, 381 enum hinic_node_id dest, u8 *cmd, u16 cmd_size) 382 { 383 struct hinic_api_cmd_cell_ctxt *ctxt; 384 int err; 385 386 down(&chain->sem); 387 if (chain_busy(chain)) { 388 up(&chain->sem); 389 return -EBUSY; 390 } 391 392 prepare_cell(chain, dest, cmd, cmd_size); 393 cmd_chain_prod_idx_inc(chain); 394 395 wmb(); /* inc pi before issue the command */ 396 397 set_prod_idx(chain); /* issue the command */ 398 399 ctxt = &chain->cell_ctxt[chain->prod_idx]; 400 401 chain->curr_node = ctxt->cell_vaddr; 402 403 err = wait_for_api_cmd_completion(chain); 404 405 up(&chain->sem); 406 return err; 407 } 408 409 /** 410 * hinic_api_cmd_write - Write API CMD command 411 * @chain: chain for write command 412 * @dest: destination node on the card that will receive the command 413 * @cmd: command data 414 * @size: the command size 415 * 416 * Return 0 - Success, negative - Failure 417 **/ 418 int hinic_api_cmd_write(struct hinic_api_cmd_chain *chain, 419 enum hinic_node_id dest, u8 *cmd, u16 size) 420 { 421 /* Verify the chain type */ 422 if (chain->chain_type == HINIC_API_CMD_WRITE_TO_MGMT_CPU) 423 return api_cmd(chain, dest, cmd, size); 424 425 return -EINVAL; 426 } 427 428 /** 429 * api_cmd_hw_restart - restart the chain in the HW 430 * @chain: the API CMD specific chain to restart 431 * 432 * Return 0 - Success, negative - Failure 433 **/ 434 static int api_cmd_hw_restart(struct hinic_api_cmd_chain *chain) 435 { 436 struct hinic_hwif *hwif = chain->hwif; 437 int err = -ETIMEDOUT; 438 unsigned long end; 439 u32 reg_addr, val; 440 441 /* Read Modify Write */ 442 reg_addr = HINIC_CSR_API_CMD_CHAIN_REQ_ADDR(chain->chain_type); 443 val = hinic_hwif_read_reg(hwif, reg_addr); 444 445 val = HINIC_API_CMD_CHAIN_REQ_CLEAR(val, RESTART); 446 val |= HINIC_API_CMD_CHAIN_REQ_SET(1, RESTART); 447 448 hinic_hwif_write_reg(hwif, reg_addr, val); 449 450 end = jiffies + msecs_to_jiffies(API_CMD_TIMEOUT); 451 do { 452 val = hinic_hwif_read_reg(hwif, reg_addr); 453 454 if (!HINIC_API_CMD_CHAIN_REQ_GET(val, RESTART)) { 455 err = 0; 456 break; 457 } 458 459 msleep(20); 460 } while (time_before(jiffies, end)); 461 462 return err; 463 } 464 465 /** 466 * api_cmd_ctrl_init - set the control register of a chain 467 * @chain: the API CMD specific chain to set control register for 468 **/ 469 static void api_cmd_ctrl_init(struct hinic_api_cmd_chain *chain) 470 { 471 struct hinic_hwif *hwif = chain->hwif; 472 u32 addr, ctrl; 473 u16 cell_size; 474 475 /* Read Modify Write */ 476 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 477 478 cell_size = API_CMD_CELL_SIZE_VAL(chain->cell_size); 479 480 ctrl = hinic_hwif_read_reg(hwif, addr); 481 482 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 483 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 484 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 485 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 486 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 487 488 ctrl |= HINIC_API_CMD_CHAIN_CTRL_SET(1, XOR_ERR) | 489 HINIC_API_CMD_CHAIN_CTRL_SET(XOR_CHK_ALL, XOR_CHK_EN) | 490 HINIC_API_CMD_CHAIN_CTRL_SET(cell_size, CELL_SIZE); 491 492 hinic_hwif_write_reg(hwif, addr, ctrl); 493 } 494 495 /** 496 * api_cmd_set_status_addr - set the status address of a chain in the HW 497 * @chain: the API CMD specific chain to set in HW status address for 498 **/ 499 static void api_cmd_set_status_addr(struct hinic_api_cmd_chain *chain) 500 { 501 struct hinic_hwif *hwif = chain->hwif; 502 u32 addr, val; 503 504 addr = HINIC_CSR_API_CMD_STATUS_HI_ADDR(chain->chain_type); 505 val = upper_32_bits(chain->wb_status_paddr); 506 hinic_hwif_write_reg(hwif, addr, val); 507 508 addr = HINIC_CSR_API_CMD_STATUS_LO_ADDR(chain->chain_type); 509 val = lower_32_bits(chain->wb_status_paddr); 510 hinic_hwif_write_reg(hwif, addr, val); 511 } 512 513 /** 514 * api_cmd_set_num_cells - set the number cells of a chain in the HW 515 * @chain: the API CMD specific chain to set in HW the number of cells for 516 **/ 517 static void api_cmd_set_num_cells(struct hinic_api_cmd_chain *chain) 518 { 519 struct hinic_hwif *hwif = chain->hwif; 520 u32 addr, val; 521 522 addr = HINIC_CSR_API_CMD_CHAIN_NUM_CELLS_ADDR(chain->chain_type); 523 val = chain->num_cells; 524 hinic_hwif_write_reg(hwif, addr, val); 525 } 526 527 /** 528 * api_cmd_head_init - set the head of a chain in the HW 529 * @chain: the API CMD specific chain to set in HW the head for 530 **/ 531 static void api_cmd_head_init(struct hinic_api_cmd_chain *chain) 532 { 533 struct hinic_hwif *hwif = chain->hwif; 534 u32 addr, val; 535 536 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_HI_ADDR(chain->chain_type); 537 val = upper_32_bits(chain->head_cell_paddr); 538 hinic_hwif_write_reg(hwif, addr, val); 539 540 addr = HINIC_CSR_API_CMD_CHAIN_HEAD_LO_ADDR(chain->chain_type); 541 val = lower_32_bits(chain->head_cell_paddr); 542 hinic_hwif_write_reg(hwif, addr, val); 543 } 544 545 /** 546 * api_cmd_chain_hw_clean - clean the HW 547 * @chain: the API CMD specific chain 548 **/ 549 static void api_cmd_chain_hw_clean(struct hinic_api_cmd_chain *chain) 550 { 551 struct hinic_hwif *hwif = chain->hwif; 552 u32 addr, ctrl; 553 554 addr = HINIC_CSR_API_CMD_CHAIN_CTRL_ADDR(chain->chain_type); 555 556 ctrl = hinic_hwif_read_reg(hwif, addr); 557 ctrl = HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, RESTART_WB_STAT) & 558 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_ERR) & 559 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, AEQE_EN) & 560 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, XOR_CHK_EN) & 561 HINIC_API_CMD_CHAIN_CTRL_CLEAR(ctrl, CELL_SIZE); 562 563 hinic_hwif_write_reg(hwif, addr, ctrl); 564 } 565 566 /** 567 * api_cmd_chain_hw_init - initialize the chain in the HW 568 * @chain: the API CMD specific chain to initialize in HW 569 * 570 * Return 0 - Success, negative - Failure 571 **/ 572 static int api_cmd_chain_hw_init(struct hinic_api_cmd_chain *chain) 573 { 574 struct hinic_hwif *hwif = chain->hwif; 575 struct pci_dev *pdev = hwif->pdev; 576 int err; 577 578 api_cmd_chain_hw_clean(chain); 579 580 api_cmd_set_status_addr(chain); 581 582 err = api_cmd_hw_restart(chain); 583 if (err) { 584 dev_err(&pdev->dev, "Failed to restart API CMD HW\n"); 585 return err; 586 } 587 588 api_cmd_ctrl_init(chain); 589 api_cmd_set_num_cells(chain); 590 api_cmd_head_init(chain); 591 return 0; 592 } 593 594 /** 595 * free_cmd_buf - free the dma buffer of API CMD command 596 * @chain: the API CMD specific chain of the cmd 597 * @cell_idx: the cell index of the cmd 598 **/ 599 static void free_cmd_buf(struct hinic_api_cmd_chain *chain, int cell_idx) 600 { 601 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 602 struct hinic_hwif *hwif = chain->hwif; 603 struct pci_dev *pdev = hwif->pdev; 604 605 cell_ctxt = &chain->cell_ctxt[cell_idx]; 606 607 dma_free_coherent(&pdev->dev, API_CMD_BUF_SIZE, 608 cell_ctxt->api_cmd_vaddr, 609 cell_ctxt->api_cmd_paddr); 610 } 611 612 /** 613 * alloc_cmd_buf - allocate a dma buffer for API CMD command 614 * @chain: the API CMD specific chain for the cmd 615 * @cell: the cell in the HW for the cmd 616 * @cell_idx: the index of the cell 617 * 618 * Return 0 - Success, negative - Failure 619 **/ 620 static int alloc_cmd_buf(struct hinic_api_cmd_chain *chain, 621 struct hinic_api_cmd_cell *cell, int cell_idx) 622 { 623 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 624 struct hinic_hwif *hwif = chain->hwif; 625 struct pci_dev *pdev = hwif->pdev; 626 dma_addr_t cmd_paddr; 627 u8 *cmd_vaddr; 628 int err = 0; 629 630 cmd_vaddr = dma_alloc_coherent(&pdev->dev, API_CMD_BUF_SIZE, 631 &cmd_paddr, GFP_KERNEL); 632 if (!cmd_vaddr) 633 return -ENOMEM; 634 635 cell_ctxt = &chain->cell_ctxt[cell_idx]; 636 637 cell_ctxt->api_cmd_vaddr = cmd_vaddr; 638 cell_ctxt->api_cmd_paddr = cmd_paddr; 639 640 /* set the cmd DMA address in the cell */ 641 switch (chain->chain_type) { 642 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 643 /* The data in the HW should be in Big Endian Format */ 644 cell->write.hw_cmd_paddr = cpu_to_be64(cmd_paddr); 645 break; 646 647 default: 648 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 649 free_cmd_buf(chain, cell_idx); 650 err = -EINVAL; 651 break; 652 } 653 654 return err; 655 } 656 657 /** 658 * api_cmd_create_cell - create API CMD cell for specific chain 659 * @chain: the API CMD specific chain to create its cell 660 * @cell_idx: the index of the cell to create 661 * @pre_node: previous cell 662 * @node_vaddr: the returned virt addr of the cell 663 * 664 * Return 0 - Success, negative - Failure 665 **/ 666 static int api_cmd_create_cell(struct hinic_api_cmd_chain *chain, 667 int cell_idx, 668 struct hinic_api_cmd_cell *pre_node, 669 struct hinic_api_cmd_cell **node_vaddr) 670 { 671 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 672 struct hinic_hwif *hwif = chain->hwif; 673 struct pci_dev *pdev = hwif->pdev; 674 struct hinic_api_cmd_cell *node; 675 dma_addr_t node_paddr; 676 int err; 677 678 node = dma_alloc_coherent(&pdev->dev, chain->cell_size, &node_paddr, 679 GFP_KERNEL); 680 if (!node) 681 return -ENOMEM; 682 683 node->read.hw_wb_resp_paddr = 0; 684 685 cell_ctxt = &chain->cell_ctxt[cell_idx]; 686 cell_ctxt->cell_vaddr = node; 687 cell_ctxt->cell_paddr = node_paddr; 688 689 if (!pre_node) { 690 chain->head_cell_paddr = node_paddr; 691 chain->head_node = node; 692 } else { 693 /* The data in the HW should be in Big Endian Format */ 694 pre_node->next_cell_paddr = cpu_to_be64(node_paddr); 695 } 696 697 switch (chain->chain_type) { 698 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 699 err = alloc_cmd_buf(chain, node, cell_idx); 700 if (err) { 701 dev_err(&pdev->dev, "Failed to allocate cmd buffer\n"); 702 goto err_alloc_cmd_buf; 703 } 704 break; 705 706 default: 707 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 708 err = -EINVAL; 709 goto err_alloc_cmd_buf; 710 } 711 712 *node_vaddr = node; 713 return 0; 714 715 err_alloc_cmd_buf: 716 dma_free_coherent(&pdev->dev, chain->cell_size, node, node_paddr); 717 return err; 718 } 719 720 /** 721 * api_cmd_destroy_cell - destroy API CMD cell of specific chain 722 * @chain: the API CMD specific chain to destroy its cell 723 * @cell_idx: the cell to destroy 724 **/ 725 static void api_cmd_destroy_cell(struct hinic_api_cmd_chain *chain, 726 int cell_idx) 727 { 728 struct hinic_api_cmd_cell_ctxt *cell_ctxt; 729 struct hinic_hwif *hwif = chain->hwif; 730 struct pci_dev *pdev = hwif->pdev; 731 struct hinic_api_cmd_cell *node; 732 dma_addr_t node_paddr; 733 size_t node_size; 734 735 cell_ctxt = &chain->cell_ctxt[cell_idx]; 736 737 node = cell_ctxt->cell_vaddr; 738 node_paddr = cell_ctxt->cell_paddr; 739 node_size = chain->cell_size; 740 741 if (cell_ctxt->api_cmd_vaddr) { 742 switch (chain->chain_type) { 743 case HINIC_API_CMD_WRITE_TO_MGMT_CPU: 744 free_cmd_buf(chain, cell_idx); 745 break; 746 default: 747 dev_err(&pdev->dev, "Unsupported API CMD chain type\n"); 748 break; 749 } 750 751 dma_free_coherent(&pdev->dev, node_size, node, 752 node_paddr); 753 } 754 } 755 756 /** 757 * api_cmd_destroy_cells - destroy API CMD cells of specific chain 758 * @chain: the API CMD specific chain to destroy its cells 759 * @num_cells: number of cells to destroy 760 **/ 761 static void api_cmd_destroy_cells(struct hinic_api_cmd_chain *chain, 762 int num_cells) 763 { 764 int cell_idx; 765 766 for (cell_idx = 0; cell_idx < num_cells; cell_idx++) 767 api_cmd_destroy_cell(chain, cell_idx); 768 } 769 770 /** 771 * api_cmd_create_cells - create API CMD cells for specific chain 772 * @chain: the API CMD specific chain 773 * 774 * Return 0 - Success, negative - Failure 775 **/ 776 static int api_cmd_create_cells(struct hinic_api_cmd_chain *chain) 777 { 778 struct hinic_api_cmd_cell *node = NULL, *pre_node = NULL; 779 struct hinic_hwif *hwif = chain->hwif; 780 struct pci_dev *pdev = hwif->pdev; 781 int err, cell_idx; 782 783 for (cell_idx = 0; cell_idx < chain->num_cells; cell_idx++) { 784 err = api_cmd_create_cell(chain, cell_idx, pre_node, &node); 785 if (err) { 786 dev_err(&pdev->dev, "Failed to create API CMD cell\n"); 787 goto err_create_cell; 788 } 789 790 pre_node = node; 791 } 792 793 /* set the Final node to point on the start */ 794 node->next_cell_paddr = cpu_to_be64(chain->head_cell_paddr); 795 796 /* set the current node to be the head */ 797 chain->curr_node = chain->head_node; 798 return 0; 799 800 err_create_cell: 801 api_cmd_destroy_cells(chain, cell_idx); 802 return err; 803 } 804 805 /** 806 * api_chain_init - initialize API CMD specific chain 807 * @chain: the API CMD specific chain to initialize 808 * @attr: attributes to set in the chain 809 * 810 * Return 0 - Success, negative - Failure 811 **/ 812 static int api_chain_init(struct hinic_api_cmd_chain *chain, 813 struct hinic_api_cmd_chain_attr *attr) 814 { 815 struct hinic_hwif *hwif = attr->hwif; 816 struct pci_dev *pdev = hwif->pdev; 817 818 chain->hwif = hwif; 819 chain->chain_type = attr->chain_type; 820 chain->num_cells = attr->num_cells; 821 chain->cell_size = attr->cell_size; 822 823 chain->prod_idx = 0; 824 chain->cons_idx = 0; 825 826 sema_init(&chain->sem, 1); 827 828 chain->cell_ctxt = devm_kcalloc(&pdev->dev, chain->num_cells, 829 sizeof(*chain->cell_ctxt), GFP_KERNEL); 830 if (!chain->cell_ctxt) 831 return -ENOMEM; 832 833 chain->wb_status = dma_alloc_coherent(&pdev->dev, 834 sizeof(*chain->wb_status), 835 &chain->wb_status_paddr, 836 GFP_KERNEL); 837 if (!chain->wb_status) { 838 dev_err(&pdev->dev, "Failed to allocate DMA wb status\n"); 839 return -ENOMEM; 840 } 841 842 return 0; 843 } 844 845 /** 846 * api_chain_free - free API CMD specific chain 847 * @chain: the API CMD specific chain to free 848 **/ 849 static void api_chain_free(struct hinic_api_cmd_chain *chain) 850 { 851 struct hinic_hwif *hwif = chain->hwif; 852 struct pci_dev *pdev = hwif->pdev; 853 854 dma_free_coherent(&pdev->dev, sizeof(*chain->wb_status), 855 chain->wb_status, chain->wb_status_paddr); 856 } 857 858 /** 859 * api_cmd_create_chain - create API CMD specific chain 860 * @attr: attributes to set the chain 861 * 862 * Return the created chain 863 **/ 864 static struct hinic_api_cmd_chain * 865 api_cmd_create_chain(struct hinic_api_cmd_chain_attr *attr) 866 { 867 struct hinic_hwif *hwif = attr->hwif; 868 struct pci_dev *pdev = hwif->pdev; 869 struct hinic_api_cmd_chain *chain; 870 int err; 871 872 if (attr->num_cells & (attr->num_cells - 1)) { 873 dev_err(&pdev->dev, "Invalid number of cells, must be power of 2\n"); 874 return ERR_PTR(-EINVAL); 875 } 876 877 chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); 878 if (!chain) 879 return ERR_PTR(-ENOMEM); 880 881 err = api_chain_init(chain, attr); 882 if (err) { 883 dev_err(&pdev->dev, "Failed to initialize chain\n"); 884 return ERR_PTR(err); 885 } 886 887 err = api_cmd_create_cells(chain); 888 if (err) { 889 dev_err(&pdev->dev, "Failed to create cells for API CMD chain\n"); 890 goto err_create_cells; 891 } 892 893 err = api_cmd_chain_hw_init(chain); 894 if (err) { 895 dev_err(&pdev->dev, "Failed to initialize chain HW\n"); 896 goto err_chain_hw_init; 897 } 898 899 return chain; 900 901 err_chain_hw_init: 902 api_cmd_destroy_cells(chain, chain->num_cells); 903 904 err_create_cells: 905 api_chain_free(chain); 906 return ERR_PTR(err); 907 } 908 909 /** 910 * api_cmd_destroy_chain - destroy API CMD specific chain 911 * @chain: the API CMD specific chain to destroy 912 **/ 913 static void api_cmd_destroy_chain(struct hinic_api_cmd_chain *chain) 914 { 915 api_cmd_chain_hw_clean(chain); 916 api_cmd_destroy_cells(chain, chain->num_cells); 917 api_chain_free(chain); 918 } 919 920 /** 921 * hinic_api_cmd_init - Initialize all the API CMD chains 922 * @chain: the API CMD chains that are initialized 923 * @hwif: the hardware interface of a pci function device 924 * 925 * Return 0 - Success, negative - Failure 926 **/ 927 int hinic_api_cmd_init(struct hinic_api_cmd_chain **chain, 928 struct hinic_hwif *hwif) 929 { 930 enum hinic_api_cmd_chain_type type, chain_type; 931 struct hinic_api_cmd_chain_attr attr; 932 struct pci_dev *pdev = hwif->pdev; 933 size_t hw_cell_sz; 934 int err; 935 936 hw_cell_sz = sizeof(struct hinic_api_cmd_cell); 937 938 attr.hwif = hwif; 939 attr.num_cells = API_CHAIN_NUM_CELLS; 940 attr.cell_size = API_CMD_CELL_SIZE(hw_cell_sz); 941 942 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 943 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 944 attr.chain_type = chain_type; 945 946 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 947 continue; 948 949 chain[chain_type] = api_cmd_create_chain(&attr); 950 if (IS_ERR(chain[chain_type])) { 951 dev_err(&pdev->dev, "Failed to create chain %d\n", 952 chain_type); 953 err = PTR_ERR(chain[chain_type]); 954 goto err_create_chain; 955 } 956 } 957 958 return 0; 959 960 err_create_chain: 961 type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 962 for ( ; type < chain_type; type++) { 963 if (type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 964 continue; 965 966 api_cmd_destroy_chain(chain[type]); 967 } 968 969 return err; 970 } 971 972 /** 973 * hinic_api_cmd_free - free the API CMD chains 974 * @chain: the API CMD chains that are freed 975 **/ 976 void hinic_api_cmd_free(struct hinic_api_cmd_chain **chain) 977 { 978 enum hinic_api_cmd_chain_type chain_type; 979 980 chain_type = HINIC_API_CMD_WRITE_TO_MGMT_CPU; 981 for ( ; chain_type < HINIC_API_CMD_MAX; chain_type++) { 982 if (chain_type != HINIC_API_CMD_WRITE_TO_MGMT_CPU) 983 continue; 984 985 api_cmd_destroy_chain(chain[chain_type]); 986 } 987 } 988