Lines Matching full:ctrl
84 struct nvmet_pci_epf_ctrl *ctrl; member
129 struct nvmet_pci_epf_ctrl *ctrl; member
209 struct nvmet_pci_epf_ctrl ctrl; member
226 static inline u32 nvmet_pci_epf_bar_read32(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_read32() argument
229 __le32 *bar_reg = ctrl->bar + off; in nvmet_pci_epf_bar_read32()
234 static inline void nvmet_pci_epf_bar_write32(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_write32() argument
237 __le32 *bar_reg = ctrl->bar + off; in nvmet_pci_epf_bar_write32()
242 static inline u64 nvmet_pci_epf_bar_read64(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_read64() argument
245 return (u64)nvmet_pci_epf_bar_read32(ctrl, off) | in nvmet_pci_epf_bar_read64()
246 ((u64)nvmet_pci_epf_bar_read32(ctrl, off + 4) << 32); in nvmet_pci_epf_bar_read64()
249 static inline void nvmet_pci_epf_bar_write64(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_bar_write64() argument
252 nvmet_pci_epf_bar_write32(ctrl, off, val & 0xFFFFFFFF); in nvmet_pci_epf_bar_write64()
253 nvmet_pci_epf_bar_write32(ctrl, off + 4, (val >> 32) & 0xFFFFFFFF); in nvmet_pci_epf_bar_write64()
488 static inline int nvmet_pci_epf_transfer(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_transfer() argument
498 return nvmet_pci_epf_transfer_seg(ctrl->nvme_epf, &seg, dir); in nvmet_pci_epf_transfer()
501 static int nvmet_pci_epf_alloc_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_alloc_irq_vectors() argument
503 ctrl->irq_vectors = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_irq_vectors()
506 if (!ctrl->irq_vectors) in nvmet_pci_epf_alloc_irq_vectors()
509 mutex_init(&ctrl->irq_lock); in nvmet_pci_epf_alloc_irq_vectors()
514 static void nvmet_pci_epf_free_irq_vectors(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_free_irq_vectors() argument
516 if (ctrl->irq_vectors) { in nvmet_pci_epf_free_irq_vectors()
517 mutex_destroy(&ctrl->irq_lock); in nvmet_pci_epf_free_irq_vectors()
518 kfree(ctrl->irq_vectors); in nvmet_pci_epf_free_irq_vectors()
519 ctrl->irq_vectors = NULL; in nvmet_pci_epf_free_irq_vectors()
524 nvmet_pci_epf_find_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector) in nvmet_pci_epf_find_irq_vector() argument
529 lockdep_assert_held(&ctrl->irq_lock); in nvmet_pci_epf_find_irq_vector()
531 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_find_irq_vector()
532 iv = &ctrl->irq_vectors[i]; in nvmet_pci_epf_find_irq_vector()
541 nvmet_pci_epf_add_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, u16 vector) in nvmet_pci_epf_add_irq_vector() argument
546 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_add_irq_vector()
548 iv = nvmet_pci_epf_find_irq_vector(ctrl, vector); in nvmet_pci_epf_add_irq_vector()
554 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_add_irq_vector()
555 iv = &ctrl->irq_vectors[i]; in nvmet_pci_epf_add_irq_vector()
568 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_add_irq_vector()
573 static void nvmet_pci_epf_remove_irq_vector(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_remove_irq_vector() argument
578 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_remove_irq_vector()
580 iv = nvmet_pci_epf_find_irq_vector(ctrl, vector); in nvmet_pci_epf_remove_irq_vector()
589 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_remove_irq_vector()
592 static bool nvmet_pci_epf_should_raise_irq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_should_raise_irq() argument
609 ret = iv->nr_irqs >= ctrl->irq_vector_threshold; in nvmet_pci_epf_should_raise_irq()
617 static void nvmet_pci_epf_raise_irq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_raise_irq() argument
620 struct nvmet_pci_epf *nvme_epf = ctrl->nvme_epf; in nvmet_pci_epf_raise_irq()
628 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_raise_irq()
630 if (!nvmet_pci_epf_should_raise_irq(ctrl, cq, force)) in nvmet_pci_epf_raise_irq()
658 dev_err_ratelimited(ctrl->dev, in nvmet_pci_epf_raise_irq()
663 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_raise_irq()
676 struct nvmet_pci_epf_ctrl *ctrl = sq->ctrl; in nvmet_pci_epf_alloc_iod() local
679 iod = mempool_alloc(&ctrl->iod_pool, GFP_KERNEL); in nvmet_pci_epf_alloc_iod()
686 iod->req.port = ctrl->port; in nvmet_pci_epf_alloc_iod()
687 iod->ctrl = ctrl; in nvmet_pci_epf_alloc_iod()
689 iod->cq = &ctrl->cq[sq->qid]; in nvmet_pci_epf_alloc_iod()
731 mempool_free(iod, &iod->ctrl->iod_pool); in nvmet_pci_epf_free_iod()
736 struct nvmet_pci_epf *nvme_epf = iod->ctrl->nvme_epf; in nvmet_pci_epf_transfer_iod_data()
752 static inline u32 nvmet_pci_epf_prp_ofst(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_prp_ofst() argument
755 return prp & ctrl->mps_mask; in nvmet_pci_epf_prp_ofst()
758 static inline size_t nvmet_pci_epf_prp_size(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_prp_size() argument
761 return ctrl->mps - nvmet_pci_epf_prp_ofst(ctrl, prp); in nvmet_pci_epf_prp_size()
767 static int nvmet_pci_epf_get_prp_list(struct nvmet_pci_epf_ctrl *ctrl, u64 prp, in nvmet_pci_epf_get_prp_list() argument
770 size_t nr_prps = (xfer_len + ctrl->mps_mask) >> ctrl->mps_shift; in nvmet_pci_epf_get_prp_list()
781 length = min(nvmet_pci_epf_prp_size(ctrl, prp), nr_prps << 3); in nvmet_pci_epf_get_prp_list()
782 ret = nvmet_pci_epf_transfer(ctrl, prps, prp, length, DMA_FROM_DEVICE); in nvmet_pci_epf_get_prp_list()
789 static int nvmet_pci_epf_iod_parse_prp_list(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_prp_list() argument
801 prps = kzalloc(ctrl->mps, GFP_KERNEL); in nvmet_pci_epf_iod_parse_prp_list()
815 ofst = nvmet_pci_epf_prp_ofst(ctrl, prp); in nvmet_pci_epf_iod_parse_prp_list()
816 nr_segs = (transfer_len + ofst + ctrl->mps - 1) >> ctrl->mps_shift; in nvmet_pci_epf_iod_parse_prp_list()
825 seg->length = nvmet_pci_epf_prp_size(ctrl, prp); in nvmet_pci_epf_iod_parse_prp_list()
843 nr_prps = nvmet_pci_epf_get_prp_list(ctrl, prp, in nvmet_pci_epf_iod_parse_prp_list()
858 if (xfer_len > ctrl->mps && i == nr_prps - 1) { in nvmet_pci_epf_iod_parse_prp_list()
865 if (nvmet_pci_epf_prp_ofst(ctrl, prp)) in nvmet_pci_epf_iod_parse_prp_list()
880 prp_size = min_t(size_t, ctrl->mps, xfer_len); in nvmet_pci_epf_iod_parse_prp_list()
892 dev_err(ctrl->dev, in nvmet_pci_epf_iod_parse_prp_list()
903 dev_err(ctrl->dev, "PRPs list invalid offset\n"); in nvmet_pci_epf_iod_parse_prp_list()
908 dev_err(ctrl->dev, "PRPs list invalid field\n"); in nvmet_pci_epf_iod_parse_prp_list()
913 dev_err(ctrl->dev, "PRPs list internal error\n"); in nvmet_pci_epf_iod_parse_prp_list()
921 static int nvmet_pci_epf_iod_parse_prp_simple(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_prp_simple() argument
931 prp1_size = nvmet_pci_epf_prp_size(ctrl, prp1); in nvmet_pci_epf_iod_parse_prp_simple()
940 if (nvmet_pci_epf_prp_ofst(ctrl, prp2)) { in nvmet_pci_epf_iod_parse_prp_simple()
973 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_iod_parse_prps() local
978 ofst = nvmet_pci_epf_prp_ofst(ctrl, prp1); in nvmet_pci_epf_iod_parse_prps()
984 if (iod->data_len + ofst <= ctrl->mps * 2) in nvmet_pci_epf_iod_parse_prps()
985 return nvmet_pci_epf_iod_parse_prp_simple(ctrl, iod); in nvmet_pci_epf_iod_parse_prps()
987 return nvmet_pci_epf_iod_parse_prp_list(ctrl, iod); in nvmet_pci_epf_iod_parse_prps()
995 nvmet_pci_epf_get_sgl_segment(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_get_sgl_segment() argument
1007 ret = nvmet_pci_epf_transfer(ctrl, buf, le64_to_cpu(desc->addr), length, in nvmet_pci_epf_get_sgl_segment()
1035 static int nvmet_pci_epf_iod_parse_sgl_segments(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_iod_parse_sgl_segments() argument
1055 sgls = nvmet_pci_epf_get_sgl_segment(ctrl, &seg, &nr_sgls); in nvmet_pci_epf_iod_parse_sgl_segments()
1097 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_iod_parse_sgls() local
1109 return nvmet_pci_epf_iod_parse_sgl_segments(ctrl, iod); in nvmet_pci_epf_iod_parse_sgls()
1114 struct nvmet_pci_epf_ctrl *ctrl = iod->ctrl; in nvmet_pci_epf_alloc_iod_data_buf() local
1120 if (iod->data_len > ctrl->mdts) { in nvmet_pci_epf_alloc_iod_data_buf()
1176 dev_err(iod->ctrl->dev, in nvmet_pci_epf_complete_iod()
1222 nvmet_pci_epf_find_port(struct nvmet_pci_epf_ctrl *ctrl, __le16 portid) in nvmet_pci_epf_find_port() argument
1259 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_get_mdts() local
1262 return ilog2(ctrl->mdts) - page_shift; in nvmet_pci_epf_get_mdts()
1268 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_create_cq() local
1269 struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid]; in nvmet_pci_epf_create_cq()
1287 nvmet_pci_epf_bar_write32(ctrl, cq->db, 0); in nvmet_pci_epf_create_cq()
1292 cq->qes = ctrl->io_cqes; in nvmet_pci_epf_create_cq()
1296 cq->iv = nvmet_pci_epf_add_irq_vector(ctrl, vector); in nvmet_pci_epf_create_cq()
1310 ret = nvmet_pci_epf_mem_map(ctrl->nvme_epf, cq->pci_addr, cq->pci_size, in nvmet_pci_epf_create_cq()
1313 dev_err(ctrl->dev, "Failed to map CQ %u (err=%d)\n", in nvmet_pci_epf_create_cq()
1319 dev_err(ctrl->dev, "Invalid partial mapping of queue %u\n", in nvmet_pci_epf_create_cq()
1327 dev_dbg(ctrl->dev, in nvmet_pci_epf_create_cq()
1331 dev_dbg(ctrl->dev, in nvmet_pci_epf_create_cq()
1338 nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); in nvmet_pci_epf_create_cq()
1343 nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); in nvmet_pci_epf_create_cq()
1349 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_delete_cq() local
1350 struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid]; in nvmet_pci_epf_delete_cq()
1358 nvmet_pci_epf_remove_irq_vector(ctrl, cq->vector); in nvmet_pci_epf_delete_cq()
1359 nvmet_pci_epf_mem_unmap(ctrl->nvme_epf, &cq->pci_map); in nvmet_pci_epf_delete_cq()
1368 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_create_sq() local
1369 struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid]; in nvmet_pci_epf_create_sq()
1370 struct nvmet_pci_epf_queue *cq = &ctrl->cq[cqid]; in nvmet_pci_epf_create_sq()
1386 nvmet_pci_epf_bar_write32(ctrl, sq->db, 0); in nvmet_pci_epf_create_sq()
1390 sq->qes = ctrl->io_sqes; in nvmet_pci_epf_create_sq()
1401 dev_err(ctrl->dev, "Failed to create SQ %d work queue\n", sqid); in nvmet_pci_epf_create_sq()
1408 dev_dbg(ctrl->dev, "SQ[%u]: %u entries of %zu B\n", in nvmet_pci_epf_create_sq()
1420 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_delete_sq() local
1421 struct nvmet_pci_epf_queue *sq = &ctrl->sq[sqid]; in nvmet_pci_epf_delete_sq()
1431 if (sq->nvme_sq.ctrl) in nvmet_pci_epf_delete_sq()
1440 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_get_feat() local
1450 if (!ctrl->sq_ab) in nvmet_pci_epf_get_feat()
1453 arb->ab = ilog2(ctrl->sq_ab); in nvmet_pci_epf_get_feat()
1458 irqc->thr = ctrl->irq_vector_threshold; in nvmet_pci_epf_get_feat()
1464 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_get_feat()
1465 iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv); in nvmet_pci_epf_get_feat()
1472 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_get_feat()
1483 struct nvmet_pci_epf_ctrl *ctrl = tctrl->drvdata; in nvmet_pci_epf_set_feat() local
1494 ctrl->sq_ab = 0; in nvmet_pci_epf_set_feat()
1496 ctrl->sq_ab = 1 << arb->ab; in nvmet_pci_epf_set_feat()
1505 ctrl->irq_vector_threshold = irqc->thr + 1; in nvmet_pci_epf_set_feat()
1510 mutex_lock(&ctrl->irq_lock); in nvmet_pci_epf_set_feat()
1511 iv = nvmet_pci_epf_find_irq_vector(ctrl, irqcfg->iv); in nvmet_pci_epf_set_feat()
1518 mutex_unlock(&ctrl->irq_lock); in nvmet_pci_epf_set_feat()
1543 static void nvmet_pci_epf_init_queue(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_init_queue() argument
1549 queue = &ctrl->sq[qid]; in nvmet_pci_epf_init_queue()
1551 queue = &ctrl->cq[qid]; in nvmet_pci_epf_init_queue()
1554 queue->ctrl = ctrl; in nvmet_pci_epf_init_queue()
1560 static int nvmet_pci_epf_alloc_queues(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_alloc_queues() argument
1564 ctrl->sq = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_queues()
1566 if (!ctrl->sq) in nvmet_pci_epf_alloc_queues()
1569 ctrl->cq = kcalloc(ctrl->nr_queues, in nvmet_pci_epf_alloc_queues()
1571 if (!ctrl->cq) { in nvmet_pci_epf_alloc_queues()
1572 kfree(ctrl->sq); in nvmet_pci_epf_alloc_queues()
1573 ctrl->sq = NULL; in nvmet_pci_epf_alloc_queues()
1577 for (qid = 0; qid < ctrl->nr_queues; qid++) { in nvmet_pci_epf_alloc_queues()
1578 nvmet_pci_epf_init_queue(ctrl, qid, true); in nvmet_pci_epf_alloc_queues()
1579 nvmet_pci_epf_init_queue(ctrl, qid, false); in nvmet_pci_epf_alloc_queues()
1585 static void nvmet_pci_epf_free_queues(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_free_queues() argument
1587 kfree(ctrl->sq); in nvmet_pci_epf_free_queues()
1588 ctrl->sq = NULL; in nvmet_pci_epf_free_queues()
1589 kfree(ctrl->cq); in nvmet_pci_epf_free_queues()
1590 ctrl->cq = NULL; in nvmet_pci_epf_free_queues()
1600 if (!iod->ctrl->link_up) { in nvmet_pci_epf_exec_iod_work()
1664 static int nvmet_pci_epf_process_sq(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_process_sq() argument
1671 sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db); in nvmet_pci_epf_process_sq()
1672 while (head != sq->tail && (!ctrl->sq_ab || n < ctrl->sq_ab)) { in nvmet_pci_epf_process_sq()
1678 ret = nvmet_pci_epf_transfer(ctrl, &iod->cmd, in nvmet_pci_epf_process_sq()
1687 dev_dbg(ctrl->dev, "SQ[%u]: head %u, tail %u, command %s\n", in nvmet_pci_epf_process_sq()
1699 sq->tail = nvmet_pci_epf_bar_read32(ctrl, sq->db); in nvmet_pci_epf_process_sq()
1707 struct nvmet_pci_epf_ctrl *ctrl = in nvmet_pci_epf_poll_sqs_work() local
1714 while (ctrl->link_up && ctrl->enabled) { in nvmet_pci_epf_poll_sqs_work()
1717 for (i = 0; i < ctrl->nr_queues; i++) { in nvmet_pci_epf_poll_sqs_work()
1718 sq = &ctrl->sq[i]; in nvmet_pci_epf_poll_sqs_work()
1721 if (nvmet_pci_epf_process_sq(ctrl, sq)) in nvmet_pci_epf_poll_sqs_work()
1752 schedule_delayed_work(&ctrl->poll_sqs, NVMET_PCI_EPF_SQ_POLL_INTERVAL); in nvmet_pci_epf_poll_sqs_work()
1759 struct nvmet_pci_epf_ctrl *ctrl = cq->ctrl; in nvmet_pci_epf_cq_work() local
1765 while (test_bit(NVMET_PCI_EPF_Q_LIVE, &cq->flags) && ctrl->link_up) { in nvmet_pci_epf_cq_work()
1768 cq->head = nvmet_pci_epf_bar_read32(ctrl, cq->db); in nvmet_pci_epf_cq_work()
1797 dev_dbg(ctrl->dev, in nvmet_pci_epf_cq_work()
1815 nvmet_pci_epf_raise_irq(ctrl, cq, false); in nvmet_pci_epf_cq_work()
1825 nvmet_pci_epf_raise_irq(ctrl, cq, true); in nvmet_pci_epf_cq_work()
1832 static void nvmet_pci_epf_clear_ctrl_config(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_clear_ctrl_config() argument
1834 struct nvmet_ctrl *tctrl = ctrl->tctrl; in nvmet_pci_epf_clear_ctrl_config()
1838 ctrl->csts = 0; in nvmet_pci_epf_clear_ctrl_config()
1839 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts); in nvmet_pci_epf_clear_ctrl_config()
1843 ctrl->cc = 0; in nvmet_pci_epf_clear_ctrl_config()
1844 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc); in nvmet_pci_epf_clear_ctrl_config()
1847 static int nvmet_pci_epf_enable_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_enable_ctrl() argument
1853 if (ctrl->enabled) in nvmet_pci_epf_enable_ctrl()
1856 dev_info(ctrl->dev, "Enabling controller\n"); in nvmet_pci_epf_enable_ctrl()
1858 ctrl->mps_shift = nvmet_cc_mps(ctrl->cc) + 12; in nvmet_pci_epf_enable_ctrl()
1859 ctrl->mps = 1UL << ctrl->mps_shift; in nvmet_pci_epf_enable_ctrl()
1860 ctrl->mps_mask = ctrl->mps - 1; in nvmet_pci_epf_enable_ctrl()
1862 ctrl->io_sqes = 1UL << nvmet_cc_iosqes(ctrl->cc); in nvmet_pci_epf_enable_ctrl()
1863 if (ctrl->io_sqes < sizeof(struct nvme_command)) { in nvmet_pci_epf_enable_ctrl()
1864 dev_err(ctrl->dev, "Unsupported I/O SQES %zu (need %zu)\n", in nvmet_pci_epf_enable_ctrl()
1865 ctrl->io_sqes, sizeof(struct nvme_command)); in nvmet_pci_epf_enable_ctrl()
1869 ctrl->io_cqes = 1UL << nvmet_cc_iocqes(ctrl->cc); in nvmet_pci_epf_enable_ctrl()
1870 if (ctrl->io_cqes < sizeof(struct nvme_completion)) { in nvmet_pci_epf_enable_ctrl()
1871 dev_err(ctrl->dev, "Unsupported I/O CQES %zu (need %zu)\n", in nvmet_pci_epf_enable_ctrl()
1872 ctrl->io_cqes, sizeof(struct nvme_completion)); in nvmet_pci_epf_enable_ctrl()
1877 aqa = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_AQA); in nvmet_pci_epf_enable_ctrl()
1878 asq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ASQ); in nvmet_pci_epf_enable_ctrl()
1879 acq = nvmet_pci_epf_bar_read64(ctrl, NVME_REG_ACQ); in nvmet_pci_epf_enable_ctrl()
1883 status = nvmet_pci_epf_create_cq(ctrl->tctrl, 0, in nvmet_pci_epf_enable_ctrl()
1887 dev_err(ctrl->dev, "Failed to create admin completion queue\n"); in nvmet_pci_epf_enable_ctrl()
1893 status = nvmet_pci_epf_create_sq(ctrl->tctrl, 0, 0, in nvmet_pci_epf_enable_ctrl()
1896 dev_err(ctrl->dev, "Failed to create admin submission queue\n"); in nvmet_pci_epf_enable_ctrl()
1897 nvmet_pci_epf_delete_cq(ctrl->tctrl, 0); in nvmet_pci_epf_enable_ctrl()
1901 ctrl->sq_ab = NVMET_PCI_EPF_SQ_AB; in nvmet_pci_epf_enable_ctrl()
1902 ctrl->irq_vector_threshold = NVMET_PCI_EPF_IV_THRESHOLD; in nvmet_pci_epf_enable_ctrl()
1903 ctrl->enabled = true; in nvmet_pci_epf_enable_ctrl()
1904 ctrl->csts = NVME_CSTS_RDY; in nvmet_pci_epf_enable_ctrl()
1907 schedule_delayed_work(&ctrl->poll_sqs, 0); in nvmet_pci_epf_enable_ctrl()
1912 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_enable_ctrl()
1916 static void nvmet_pci_epf_disable_ctrl(struct nvmet_pci_epf_ctrl *ctrl, in nvmet_pci_epf_disable_ctrl() argument
1921 if (!ctrl->enabled) in nvmet_pci_epf_disable_ctrl()
1924 dev_info(ctrl->dev, "%s controller\n", in nvmet_pci_epf_disable_ctrl()
1927 ctrl->enabled = false; in nvmet_pci_epf_disable_ctrl()
1928 cancel_delayed_work_sync(&ctrl->poll_sqs); in nvmet_pci_epf_disable_ctrl()
1931 for (qid = 1; qid < ctrl->nr_queues; qid++) in nvmet_pci_epf_disable_ctrl()
1932 nvmet_pci_epf_delete_sq(ctrl->tctrl, qid); in nvmet_pci_epf_disable_ctrl()
1934 for (qid = 1; qid < ctrl->nr_queues; qid++) in nvmet_pci_epf_disable_ctrl()
1935 nvmet_pci_epf_delete_cq(ctrl->tctrl, qid); in nvmet_pci_epf_disable_ctrl()
1938 nvmet_pci_epf_delete_sq(ctrl->tctrl, 0); in nvmet_pci_epf_disable_ctrl()
1939 nvmet_pci_epf_delete_cq(ctrl->tctrl, 0); in nvmet_pci_epf_disable_ctrl()
1941 ctrl->csts &= ~NVME_CSTS_RDY; in nvmet_pci_epf_disable_ctrl()
1943 ctrl->csts |= NVME_CSTS_SHST_CMPLT; in nvmet_pci_epf_disable_ctrl()
1944 ctrl->cc &= ~NVME_CC_ENABLE; in nvmet_pci_epf_disable_ctrl()
1945 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CC, ctrl->cc); in nvmet_pci_epf_disable_ctrl()
1951 struct nvmet_pci_epf_ctrl *ctrl = in nvmet_pci_epf_poll_cc_work() local
1956 if (!ctrl->tctrl) in nvmet_pci_epf_poll_cc_work()
1959 old_cc = ctrl->cc; in nvmet_pci_epf_poll_cc_work()
1960 new_cc = nvmet_pci_epf_bar_read32(ctrl, NVME_REG_CC); in nvmet_pci_epf_poll_cc_work()
1964 ctrl->cc = new_cc; in nvmet_pci_epf_poll_cc_work()
1967 ret = nvmet_pci_epf_enable_ctrl(ctrl); in nvmet_pci_epf_poll_cc_work()
1973 nvmet_pci_epf_disable_ctrl(ctrl, false); in nvmet_pci_epf_poll_cc_work()
1976 nvmet_pci_epf_disable_ctrl(ctrl, true); in nvmet_pci_epf_poll_cc_work()
1979 ctrl->csts &= ~NVME_CSTS_SHST_CMPLT; in nvmet_pci_epf_poll_cc_work()
1981 nvmet_update_cc(ctrl->tctrl, ctrl->cc); in nvmet_pci_epf_poll_cc_work()
1982 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_CSTS, ctrl->csts); in nvmet_pci_epf_poll_cc_work()
1985 schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL); in nvmet_pci_epf_poll_cc_work()
1988 static void nvmet_pci_epf_init_bar(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_init_bar() argument
1990 struct nvmet_ctrl *tctrl = ctrl->tctrl; in nvmet_pci_epf_init_bar()
1992 ctrl->bar = ctrl->nvme_epf->reg_bar; in nvmet_pci_epf_init_bar()
1995 ctrl->cap = tctrl->cap; in nvmet_pci_epf_init_bar()
1998 ctrl->cap |= 0x1ULL << 16; in nvmet_pci_epf_init_bar()
2001 ctrl->cap &= ~GENMASK_ULL(35, 32); in nvmet_pci_epf_init_bar()
2004 ctrl->cap &= ~(0x1ULL << 36); in nvmet_pci_epf_init_bar()
2007 ctrl->cap &= ~(0x1ULL << 45); in nvmet_pci_epf_init_bar()
2010 ctrl->cap &= ~(0x1ULL << 56); in nvmet_pci_epf_init_bar()
2013 ctrl->cap &= ~(0x1ULL << 57); in nvmet_pci_epf_init_bar()
2015 nvmet_pci_epf_bar_write64(ctrl, NVME_REG_CAP, ctrl->cap); in nvmet_pci_epf_init_bar()
2016 nvmet_pci_epf_bar_write32(ctrl, NVME_REG_VS, tctrl->subsys->ver); in nvmet_pci_epf_init_bar()
2018 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_init_bar()
2024 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_create_ctrl() local
2030 memset(ctrl, 0, sizeof(*ctrl)); in nvmet_pci_epf_create_ctrl()
2031 ctrl->dev = &nvme_epf->epf->dev; in nvmet_pci_epf_create_ctrl()
2032 mutex_init(&ctrl->irq_lock); in nvmet_pci_epf_create_ctrl()
2033 ctrl->nvme_epf = nvme_epf; in nvmet_pci_epf_create_ctrl()
2034 ctrl->mdts = nvme_epf->mdts_kb * SZ_1K; in nvmet_pci_epf_create_ctrl()
2035 INIT_DELAYED_WORK(&ctrl->poll_cc, nvmet_pci_epf_poll_cc_work); in nvmet_pci_epf_create_ctrl()
2036 INIT_DELAYED_WORK(&ctrl->poll_sqs, nvmet_pci_epf_poll_sqs_work); in nvmet_pci_epf_create_ctrl()
2038 ret = mempool_init_kmalloc_pool(&ctrl->iod_pool, in nvmet_pci_epf_create_ctrl()
2042 dev_err(ctrl->dev, "Failed to initialize IOD mempool\n"); in nvmet_pci_epf_create_ctrl()
2046 ctrl->port = nvmet_pci_epf_find_port(ctrl, nvme_epf->portid); in nvmet_pci_epf_create_ctrl()
2047 if (!ctrl->port) { in nvmet_pci_epf_create_ctrl()
2048 dev_err(ctrl->dev, "Port not found\n"); in nvmet_pci_epf_create_ctrl()
2057 args.port = ctrl->port; in nvmet_pci_epf_create_ctrl()
2064 ctrl->tctrl = nvmet_alloc_ctrl(&args); in nvmet_pci_epf_create_ctrl()
2065 if (!ctrl->tctrl) { in nvmet_pci_epf_create_ctrl()
2066 dev_err(ctrl->dev, "Failed to create target controller\n"); in nvmet_pci_epf_create_ctrl()
2070 ctrl->tctrl->drvdata = ctrl; in nvmet_pci_epf_create_ctrl()
2073 if (ctrl->tctrl->pi_support) { in nvmet_pci_epf_create_ctrl()
2074 dev_err(ctrl->dev, in nvmet_pci_epf_create_ctrl()
2081 ctrl->nr_queues = min(ctrl->tctrl->subsys->max_qid + 1, max_nr_queues); in nvmet_pci_epf_create_ctrl()
2082 ret = nvmet_pci_epf_alloc_queues(ctrl); in nvmet_pci_epf_create_ctrl()
2090 ret = nvmet_pci_epf_alloc_irq_vectors(ctrl); in nvmet_pci_epf_create_ctrl()
2094 dev_info(ctrl->dev, in nvmet_pci_epf_create_ctrl()
2095 "New PCI ctrl \"%s\", %u I/O queues, mdts %u B\n", in nvmet_pci_epf_create_ctrl()
2096 ctrl->tctrl->subsys->subsysnqn, ctrl->nr_queues - 1, in nvmet_pci_epf_create_ctrl()
2097 ctrl->mdts); in nvmet_pci_epf_create_ctrl()
2100 nvmet_pci_epf_init_bar(ctrl); in nvmet_pci_epf_create_ctrl()
2105 nvmet_pci_epf_free_queues(ctrl); in nvmet_pci_epf_create_ctrl()
2107 nvmet_ctrl_put(ctrl->tctrl); in nvmet_pci_epf_create_ctrl()
2108 ctrl->tctrl = NULL; in nvmet_pci_epf_create_ctrl()
2110 mempool_exit(&ctrl->iod_pool); in nvmet_pci_epf_create_ctrl()
2114 static void nvmet_pci_epf_start_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_start_ctrl() argument
2117 dev_info(ctrl->dev, "PCI link up\n"); in nvmet_pci_epf_start_ctrl()
2118 ctrl->link_up = true; in nvmet_pci_epf_start_ctrl()
2120 schedule_delayed_work(&ctrl->poll_cc, NVMET_PCI_EPF_CC_POLL_INTERVAL); in nvmet_pci_epf_start_ctrl()
2123 static void nvmet_pci_epf_stop_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_stop_ctrl() argument
2125 dev_info(ctrl->dev, "PCI link down\n"); in nvmet_pci_epf_stop_ctrl()
2126 ctrl->link_up = false; in nvmet_pci_epf_stop_ctrl()
2128 cancel_delayed_work_sync(&ctrl->poll_cc); in nvmet_pci_epf_stop_ctrl()
2130 nvmet_pci_epf_disable_ctrl(ctrl, false); in nvmet_pci_epf_stop_ctrl()
2131 nvmet_pci_epf_clear_ctrl_config(ctrl); in nvmet_pci_epf_stop_ctrl()
2134 static void nvmet_pci_epf_destroy_ctrl(struct nvmet_pci_epf_ctrl *ctrl) in nvmet_pci_epf_destroy_ctrl() argument
2136 if (!ctrl->tctrl) in nvmet_pci_epf_destroy_ctrl()
2139 dev_info(ctrl->dev, "Destroying PCI ctrl \"%s\"\n", in nvmet_pci_epf_destroy_ctrl()
2140 ctrl->tctrl->subsys->subsysnqn); in nvmet_pci_epf_destroy_ctrl()
2142 nvmet_pci_epf_stop_ctrl(ctrl); in nvmet_pci_epf_destroy_ctrl()
2144 nvmet_pci_epf_free_queues(ctrl); in nvmet_pci_epf_destroy_ctrl()
2145 nvmet_pci_epf_free_irq_vectors(ctrl); in nvmet_pci_epf_destroy_ctrl()
2147 nvmet_ctrl_put(ctrl->tctrl); in nvmet_pci_epf_destroy_ctrl()
2148 ctrl->tctrl = NULL; in nvmet_pci_epf_destroy_ctrl()
2150 mempool_exit(&ctrl->iod_pool); in nvmet_pci_epf_destroy_ctrl()
2287 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_epc_init() local
2329 epf->header->vendorid = ctrl->tctrl->subsys->vendor_id; in nvmet_pci_epf_epc_init()
2330 epf->header->subsys_vendor_id = ctrl->tctrl->subsys->subsys_vendor_id; in nvmet_pci_epf_epc_init()
2355 nvmet_pci_epf_start_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_epc_init()
2362 nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_epc_init()
2369 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_epc_deinit() local
2371 nvmet_pci_epf_destroy_ctrl(ctrl); in nvmet_pci_epf_epc_deinit()
2380 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_link_up() local
2382 nvmet_pci_epf_start_ctrl(ctrl); in nvmet_pci_epf_link_up()
2390 struct nvmet_pci_epf_ctrl *ctrl = &nvme_epf->ctrl; in nvmet_pci_epf_link_down() local
2392 nvmet_pci_epf_stop_ctrl(ctrl); in nvmet_pci_epf_link_down()
2435 nvmet_pci_epf_destroy_ctrl(&nvme_epf->ctrl); in nvmet_pci_epf_unbind()
2497 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_portid_store()
2529 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_subsysnqn_store()
2558 if (nvme_epf->ctrl.tctrl) in nvmet_pci_epf_mdts_kb_store()