Lines Matching +full:num +full:- +full:vectors
1 // SPDX-License-Identifier: GPL-2.0-only
5 * Copyright (C) 2022-2023 SolidRun
7 * Author: Alvaro Karsz <alvaro.karsz@solid-run.com>
42 if (likely(snet->cb.callback)) in snet_cfg_irq_hndlr()
43 return snet->cb.callback(snet->cb.private); in snet_cfg_irq_hndlr()
52 if (likely(vq->cb.callback)) in snet_vq_irq_hndlr()
53 return vq->cb.callback(vq->cb.private); in snet_vq_irq_hndlr()
60 struct psnet *psnet = snet->psnet; in snet_free_irqs()
66 pdev = snet->pdev->physfn; in snet_free_irqs()
68 pdev = snet->pdev; in snet_free_irqs()
71 if (snet->cfg_irq != -1) { in snet_free_irqs()
72 devm_free_irq(&pdev->dev, snet->cfg_irq, snet); in snet_free_irqs()
73 snet->cfg_irq = -1; in snet_free_irqs()
76 for (i = 0; i < snet->cfg->vq_num; i++) { in snet_free_irqs()
77 if (snet->vqs[i] && snet->vqs[i]->irq != -1) { in snet_free_irqs()
78 devm_free_irq(&pdev->dev, snet->vqs[i]->irq, snet->vqs[i]); in snet_free_irqs()
79 snet->vqs[i]->irq = -1; in snet_free_irqs()
83 /* IRQ vectors are freed when the pci remove callback is called */ in snet_free_irqs()
91 snet->vqs[idx]->desc_area = desc_area; in snet_set_vq_address()
92 snet->vqs[idx]->driver_area = driver_area; in snet_set_vq_address()
93 snet->vqs[idx]->device_area = device_area; in snet_set_vq_address()
98 static void snet_set_vq_num(struct vdpa_device *vdev, u16 idx, u32 num) in snet_set_vq_num() argument
101 /* save num in vqueue */ in snet_set_vq_num()
102 snet->vqs[idx]->num = num; in snet_set_vq_num()
108 /* not ready - ignore */ in snet_kick_vq()
109 if (unlikely(!snet->vqs[idx]->ready)) in snet_kick_vq()
112 iowrite32(SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); in snet_kick_vq()
120 /* not ready - ignore */ in snet_kick_vq_with_data()
121 if (unlikely(!snet->vqs[idx]->ready)) in snet_kick_vq_with_data()
124 iowrite32((data & 0xFFFF0000) | SNET_KICK_VAL, snet->vqs[idx]->kick_ptr); in snet_kick_vq_with_data()
131 snet->vqs[idx]->cb.callback = cb->callback; in snet_set_vq_cb()
132 snet->vqs[idx]->cb.private = cb->private; in snet_set_vq_cb()
139 snet->vqs[idx]->ready = ready; in snet_set_vq_ready()
146 return snet->vqs[idx]->ready; in snet_get_vq_ready()
152 const struct vdpa_vq_state_packed *p = &state->packed; in snet_vq_state_is_initial()
154 if (p->last_avail_counter == 1 && p->last_used_counter == 1 && in snet_vq_state_is_initial()
155 p->last_avail_idx == 0 && p->last_used_idx == 0) in snet_vq_state_is_initial()
158 const struct vdpa_vq_state_split *s = &state->split; in snet_vq_state_is_initial()
160 if (s->avail_index == 0) in snet_vq_state_is_initial()
173 memcpy(&snet->vqs[idx]->vq_state, state, sizeof(*state)); in snet_set_vq_state()
177 /* Older config - we can't set the VQ state. in snet_set_vq_state()
183 return -EOPNOTSUPP; in snet_set_vq_state()
197 return snet->vqs[idx]->irq; in snet_get_vq_irq()
207 struct pci_dev *pdev = snet->pdev; in snet_reset_dev()
212 if (!snet->status) in snet_reset_dev()
216 if (snet->status & VIRTIO_CONFIG_S_DRIVER_OK) in snet_reset_dev()
220 for (i = 0; i < snet->cfg->vq_num; i++) { in snet_reset_dev()
221 if (!snet->vqs[i]) in snet_reset_dev()
223 snet->vqs[i]->cb.callback = NULL; in snet_reset_dev()
224 snet->vqs[i]->cb.private = NULL; in snet_reset_dev()
225 snet->vqs[i]->desc_area = 0; in snet_reset_dev()
226 snet->vqs[i]->device_area = 0; in snet_reset_dev()
227 snet->vqs[i]->driver_area = 0; in snet_reset_dev()
228 snet->vqs[i]->ready = false; in snet_reset_dev()
232 snet->cb.callback = NULL; in snet_reset_dev()
233 snet->cb.private = NULL; in snet_reset_dev()
237 snet->status = 0; in snet_reset_dev()
238 snet->dpu_ready = false; in snet_reset_dev()
241 SNET_WARN(pdev, "Incomplete reset to SNET[%u] device, err: %d\n", snet->sid, ret); in snet_reset_dev()
243 SNET_DBG(pdev, "Reset SNET[%u] device\n", snet->sid); in snet_reset_dev()
259 return (size_t)snet->cfg->cfg_size; in snet_get_config_size()
266 return snet->cfg->features; in snet_get_features()
273 snet->negotiated_features = snet->cfg->features & features; in snet_set_drv_features()
281 return snet->negotiated_features; in snet_get_drv_features()
288 return (u16)snet->cfg->vq_size; in snet_get_vq_num_max()
295 snet->cb.callback = cb->callback; in snet_set_config_cb()
296 snet->cb.private = cb->private; in snet_set_config_cb()
303 return snet->cfg->virtio_id; in snet_get_device_id()
315 return snet->status; in snet_get_status()
324 if (snet->dpu_ready) in snet_write_conf()
345 off = snet->psnet->cfg.host_cfg_off; in snet_write_conf()
349 snet_write32(snet, off, snet->psnet->negotiated_cfg_ver); in snet_write_conf()
351 snet_write32(snet, off, snet->sid); in snet_write_conf()
353 snet_write32(snet, off, snet->cfg->vq_num); in snet_write_conf()
355 snet_write32(snet, off, snet->cfg_irq_idx); in snet_write_conf()
357 snet_write64(snet, off, snet->negotiated_features); in snet_write_conf()
362 for (i = 0 ; i < snet->cfg->vq_num ; i++) { in snet_write_conf()
363 tmp = (i << 16) | (snet->vqs[i]->num & 0xFFFF); in snet_write_conf()
366 snet_write32(snet, off, snet->vqs[i]->irq_idx); in snet_write_conf()
368 snet_write64(snet, off, snet->vqs[i]->desc_area); in snet_write_conf()
370 snet_write64(snet, off, snet->vqs[i]->device_area); in snet_write_conf()
372 snet_write64(snet, off, snet->vqs[i]->driver_area); in snet_write_conf()
376 snet_write32(snet, off, *(u32 *)&snet->vqs[i]->vq_state); in snet_write_conf()
383 /* Write magic number - data is ready */ in snet_write_conf()
384 snet_write32(snet, snet->psnet->cfg.host_cfg_off, SNET_SIGNATURE); in snet_write_conf()
387 ret = readx_poll_timeout(ioread32, snet->bar + snet->psnet->cfg.host_cfg_off, in snet_write_conf()
390 SNET_ERR(snet->pdev, "Timeout waiting for the DPU to read the config\n"); in snet_write_conf()
395 snet->dpu_ready = true; in snet_write_conf()
405 irq = pci_irq_vector(pdev, snet->cfg_irq_idx); in snet_request_irqs()
406 ret = devm_request_irq(&pdev->dev, irq, snet_cfg_irq_hndlr, 0, in snet_request_irqs()
407 snet->cfg_irq_name, snet); in snet_request_irqs()
412 snet->cfg_irq = irq; in snet_request_irqs()
415 for (i = 0; i < snet->cfg->vq_num; i++) { in snet_request_irqs()
416 irq = pci_irq_vector(pdev, snet->vqs[i]->irq_idx); in snet_request_irqs()
417 ret = devm_request_irq(&pdev->dev, irq, snet_vq_irq_hndlr, 0, in snet_request_irqs()
418 snet->vqs[i]->irq_name, snet->vqs[i]); in snet_request_irqs()
423 snet->vqs[i]->irq = irq; in snet_request_irqs()
431 struct psnet *psnet = snet->psnet; in snet_set_status()
432 struct pci_dev *pdev = snet->pdev; in snet_set_status()
436 if (status == snet->status) in snet_set_status()
440 !(snet->status & VIRTIO_CONFIG_S_DRIVER_OK)) { in snet_set_status()
443 ret = snet_request_irqs(pf_irqs ? pdev->physfn : pdev, snet); in snet_set_status()
449 SNET_INFO(pdev, "Create SNET[%u] device\n", snet->sid); in snet_set_status()
457 snet->status = status; in snet_set_status()
461 snet->status |= VIRTIO_CONFIG_S_FAILED; in snet_set_status()
468 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; in snet_get_config()
473 if (offset + len > snet->cfg->cfg_size) in snet_get_config()
485 void __iomem *cfg_ptr = snet->cfg->virtio_cfg + offset; in snet_set_config()
490 if (offset + len > snet->cfg->cfg_size) in snet_set_config()
505 SNET_ERR(snet->pdev, "SNET[%u] suspend failed, err: %d\n", snet->sid, ret); in snet_suspend()
507 SNET_DBG(snet->pdev, "Suspend SNET[%u] device\n", snet->sid); in snet_suspend()
519 SNET_ERR(snet->pdev, "SNET[%u] resume failed, err: %d\n", snet->sid, ret); in snet_resume()
521 SNET_DBG(snet->pdev, "Resume SNET[%u] device\n", snet->sid); in snet_resume()
573 return -ENODEV; in psnet_open_pf_bar()
576 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "psnet[%s]-bars", pci_name(pdev)); in psnet_open_pf_bar()
578 return -ENOMEM; in psnet_open_pf_bar()
588 psnet->bars[i] = pcim_iomap_table(pdev)[i]; in psnet_open_pf_bar()
599 name = devm_kasprintf(&pdev->dev, GFP_KERNEL, "snet[%s]-bars", pci_name(pdev)); in snet_open_vf_bar()
601 return -ENOMEM; in snet_open_vf_bar()
604 ret = pcim_iomap_regions(pdev, BIT(snet->psnet->cfg.vf_bar), name); in snet_open_vf_bar()
610 snet->bar = pcim_iomap_table(pdev)[snet->psnet->cfg.vf_bar]; in snet_open_vf_bar()
619 if (!cfg->devs) in snet_free_cfg()
623 for (i = 0; i < cfg->devices_num; i++) { in snet_free_cfg()
624 if (!cfg->devs[i]) in snet_free_cfg()
627 kfree(cfg->devs[i]); in snet_free_cfg()
630 kfree(cfg->devs); in snet_free_cfg()
645 if (!psnet->bars[i]) in psnet_detect_bar()
648 if (ioread32(psnet->bars[i] + off) == SNET_SIGNATURE) in psnet_detect_bar()
654 return -ENODEV; in psnet_detect_bar()
662 if (psnet->bars[i] && i != psnet->barno) in psnet_unmap_unused_bars()
673 struct snet_cfg *cfg = &psnet->cfg; in psnet_read_cfg()
688 psnet->barno = barno; in psnet_read_cfg()
694 cfg->key = psnet_read32(psnet, off); in psnet_read_cfg()
696 cfg->cfg_size = psnet_read32(psnet, off); in psnet_read_cfg()
698 cfg->cfg_ver = psnet_read32(psnet, off); in psnet_read_cfg()
703 psnet->negotiated_cfg_ver = min_t(u32, cfg->cfg_ver, SNET_CFG_VERSION); in psnet_read_cfg()
704 SNET_DBG(pdev, "SNET config version %u\n", psnet->negotiated_cfg_ver); in psnet_read_cfg()
706 cfg->vf_num = psnet_read32(psnet, off); in psnet_read_cfg()
708 cfg->vf_bar = psnet_read32(psnet, off); in psnet_read_cfg()
710 cfg->host_cfg_off = psnet_read32(psnet, off); in psnet_read_cfg()
712 cfg->max_size_host_cfg = psnet_read32(psnet, off); in psnet_read_cfg()
714 cfg->virtio_cfg_off = psnet_read32(psnet, off); in psnet_read_cfg()
716 cfg->kick_off = psnet_read32(psnet, off); in psnet_read_cfg()
718 cfg->hwmon_off = psnet_read32(psnet, off); in psnet_read_cfg()
720 cfg->ctrl_off = psnet_read32(psnet, off); in psnet_read_cfg()
722 cfg->flags = psnet_read32(psnet, off); in psnet_read_cfg()
725 off += sizeof(cfg->rsvd); in psnet_read_cfg()
727 cfg->devices_num = psnet_read32(psnet, off); in psnet_read_cfg()
730 cfg->devs = kcalloc(cfg->devices_num, sizeof(void *), GFP_KERNEL); in psnet_read_cfg()
731 if (!cfg->devs) in psnet_read_cfg()
732 return -ENOMEM; in psnet_read_cfg()
735 for (i = 0; i < cfg->devices_num; i++) { in psnet_read_cfg()
736 cfg->devs[i] = kzalloc(sizeof(*cfg->devs[i]), GFP_KERNEL); in psnet_read_cfg()
737 if (!cfg->devs[i]) { in psnet_read_cfg()
739 return -ENOMEM; in psnet_read_cfg()
742 cfg->devs[i]->virtio_id = psnet_read32(psnet, off); in psnet_read_cfg()
744 cfg->devs[i]->vq_num = psnet_read32(psnet, off); in psnet_read_cfg()
746 cfg->devs[i]->vq_size = psnet_read32(psnet, off); in psnet_read_cfg()
748 cfg->devs[i]->vfid = psnet_read32(psnet, off); in psnet_read_cfg()
750 cfg->devs[i]->features = psnet_read64(psnet, off); in psnet_read_cfg()
753 off += sizeof(cfg->devs[i]->rsvd); in psnet_read_cfg()
755 cfg->devs[i]->cfg_size = psnet_read32(psnet, off); in psnet_read_cfg()
759 if (SNET_GENERAL_CFG_LEN + SNET_GENERAL_CFG_VQ_LEN * cfg->devs[i]->vq_num > in psnet_read_cfg()
760 cfg->max_size_host_cfg) { in psnet_read_cfg()
763 return -EINVAL; in psnet_read_cfg()
775 for (i = 0; i < psnet->cfg.devices_num; i++) in psnet_alloc_irq_vector()
776 irq_num += psnet->cfg.devs[i]->vq_num + 1; in psnet_alloc_irq_vector()
780 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); in psnet_alloc_irq_vector()
783 SNET_DBG(pdev, "Allocated %u IRQ vectors from physical function\n", irq_num); in psnet_alloc_irq_vector()
794 irq_num = snet_cfg->vq_num + 1; in snet_alloc_irq_vector()
798 SNET_ERR(pdev, "Failed to allocate IRQ vectors\n"); in snet_alloc_irq_vector()
809 if (!snet->vqs) in snet_free_vqs()
812 for (i = 0 ; i < snet->cfg->vq_num ; i++) { in snet_free_vqs()
813 if (!snet->vqs[i]) in snet_free_vqs()
816 kfree(snet->vqs[i]); in snet_free_vqs()
818 kfree(snet->vqs); in snet_free_vqs()
825 snet->vqs = kcalloc(snet->cfg->vq_num, sizeof(void *), GFP_KERNEL); in snet_build_vqs()
826 if (!snet->vqs) in snet_build_vqs()
827 return -ENOMEM; in snet_build_vqs()
830 for (i = 0; i < snet->cfg->vq_num; i++) { in snet_build_vqs()
831 snet->vqs[i] = kzalloc(sizeof(*snet->vqs[i]), GFP_KERNEL); in snet_build_vqs()
832 if (!snet->vqs[i]) { in snet_build_vqs()
834 return -ENOMEM; in snet_build_vqs()
836 /* Reset IRQ num */ in snet_build_vqs()
837 snet->vqs[i]->irq = -1; in snet_build_vqs()
839 snet->vqs[i]->sid = i; in snet_build_vqs()
840 /* Kick address - every VQ gets 4B */ in snet_build_vqs()
841 snet->vqs[i]->kick_ptr = snet->bar + snet->psnet->cfg.kick_off + in snet_build_vqs()
842 snet->vqs[i]->sid * 4; in snet_build_vqs()
844 iowrite32(0, snet->vqs[i]->kick_ptr); in snet_build_vqs()
853 spin_lock(&psnet->lock); in psnet_get_next_irq_num()
854 irq = psnet->next_irq++; in psnet_get_next_irq_num()
855 spin_unlock(&psnet->lock); in psnet_get_next_irq_num()
862 struct psnet *psnet = snet->psnet; in snet_reserve_irq_idx()
866 snet->cfg_irq_idx = psnet_get_next_irq_num(psnet); in snet_reserve_irq_idx()
867 snprintf(snet->cfg_irq_name, SNET_NAME_SIZE, "snet[%s]-cfg[%d]", in snet_reserve_irq_idx()
868 pci_name(pdev), snet->cfg_irq_idx); in snet_reserve_irq_idx()
870 for (i = 0; i < snet->cfg->vq_num; i++) { in snet_reserve_irq_idx()
872 snet->vqs[i]->irq_idx = psnet_get_next_irq_num(psnet); in snet_reserve_irq_idx()
874 snprintf(snet->vqs[i]->irq_name, SNET_NAME_SIZE, "snet[%s]-vq[%d]", in snet_reserve_irq_idx()
875 pci_name(pdev), snet->vqs[i]->irq_idx); in snet_reserve_irq_idx()
884 for (i = 0; i < cfg->devices_num; i++) { in snet_find_dev_cfg()
885 if (cfg->devs[i]->vfid == vfid) in snet_find_dev_cfg()
886 return cfg->devs[i]; in snet_find_dev_cfg()
908 return -ENOMEM; in snet_vdpa_probe_pf()
911 spin_lock_init(&psnet->lock); in snet_vdpa_probe_pf()
927 * PF MSI-X vectors in snet_vdpa_probe_pf()
937 SNET_DBG(pdev, "Enable %u virtual functions\n", psnet->cfg.vf_num); in snet_vdpa_probe_pf()
938 ret = pci_enable_sriov(pdev, psnet->cfg.vf_num); in snet_vdpa_probe_pf()
940 SNET_ERR(pdev, "Failed to enable SR-IOV\n"); in snet_vdpa_probe_pf()
959 snet_free_cfg(&psnet->cfg); in snet_vdpa_probe_pf()
968 struct pci_dev *pdev_pf = pdev->physfn; in snet_vdpa_probe_vf()
987 dev_cfg = snet_find_dev_cfg(&psnet->cfg, vfid); in snet_vdpa_probe_vf()
990 return -ENODEV; in snet_vdpa_probe_vf()
1004 /* Request for MSI-X IRQs */ in snet_vdpa_probe_vf()
1012 snet = vdpa_alloc_device(struct snet, vdpa, &pdev->dev, &snet_config_ops, 1, 1, NULL, in snet_vdpa_probe_vf()
1016 ret = -ENOMEM; in snet_vdpa_probe_vf()
1021 mutex_init(&snet->ctrl_lock); in snet_vdpa_probe_vf()
1022 spin_lock_init(&snet->ctrl_spinlock); in snet_vdpa_probe_vf()
1025 snet->pdev = pdev; in snet_vdpa_probe_vf()
1026 snet->psnet = psnet; in snet_vdpa_probe_vf()
1027 snet->cfg = dev_cfg; in snet_vdpa_probe_vf()
1028 snet->dpu_ready = false; in snet_vdpa_probe_vf()
1029 snet->sid = vfid; in snet_vdpa_probe_vf()
1031 snet->cfg_irq = -1; in snet_vdpa_probe_vf()
1038 snet->cfg->virtio_cfg = snet->bar + snet->psnet->cfg.virtio_cfg_off; in snet_vdpa_probe_vf()
1057 snet->vdpa.dma_dev = &pdev->dev; in snet_vdpa_probe_vf()
1060 ret = vdpa_register_device(&snet->vdpa, snet->cfg->vq_num); in snet_vdpa_probe_vf()
1071 put_device(&snet->vdpa.dev); in snet_vdpa_probe_vf()
1080 if (pdev->is_virtfn) in snet_vdpa_probe()
1095 snet_free_cfg(&psnet->cfg); in snet_vdpa_remove_pf()
1102 struct psnet *psnet = snet->psnet; in snet_vdpa_remove_vf()
1104 vdpa_unregister_device(&snet->vdpa); in snet_vdpa_remove_vf()
1113 if (pdev->is_virtfn) in snet_vdpa_remove()
1128 .name = "snet-vdpa-driver",
1136 MODULE_AUTHOR("Alvaro Karsz <alvaro.karsz@solid-run.com>");