// SPDX-License-Identifier: GPL-2.0-only /* * Thunderbolt driver - PCI NHI driver * * Copyright (c) 2014 Andreas Noever * Copyright (C) 2018, Intel Corporation */ #include #include #include #include #include #include #include #include #include #include #include #include #include "nhi.h" #include "nhi_regs.h" #include "tb.h" /** * struct tb_nhi_pci - NHI device connected over PCIe * @nhi: NHI device * @msix_ida: Used to allocate MSI-X vectors for rings */ struct tb_nhi_pci { struct tb_nhi nhi; struct ida msix_ida; }; static inline struct tb_nhi_pci *nhi_to_pci(struct tb_nhi *nhi) { return container_of(nhi, struct tb_nhi_pci, nhi); } static void nhi_pci_check_quirks(struct tb_nhi_pci *nhi_pci) { struct tb_nhi *nhi = &nhi_pci->nhi; struct pci_dev *pdev = to_pci_dev(nhi->dev); if (pdev->vendor == PCI_VENDOR_ID_INTEL) { /* * Intel hardware supports auto clear of the interrupt * status register right after interrupt is being * issued. */ nhi->quirks |= QUIRK_AUTO_CLEAR_INT; switch (pdev->device) { case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI: case PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI: /* * Falcon Ridge controller needs the end-to-end * flow control workaround to avoid losing Rx * packets when RING_FLAG_E2E is set. */ nhi->quirks |= QUIRK_E2E; break; } } } static int nhi_pci_check_iommu_pdev(struct pci_dev *pdev, void *data) { if (!pdev->external_facing || !device_iommu_capable(&pdev->dev, IOMMU_CAP_PRE_BOOT_PROTECTION)) return 0; *(bool *)data = true; return 1; /* Stop walking */ } static void nhi_pci_check_iommu(struct tb_nhi_pci *nhi_pci) { struct tb_nhi *nhi = &nhi_pci->nhi; struct pci_dev *pdev = to_pci_dev(nhi->dev); struct pci_bus *bus = pdev->bus; bool port_ok = false; /* * Ideally what we'd do here is grab every PCI device that * represents a tunnelling adapter for this NHI and check their * status directly, but unfortunately USB4 seems to make it * obnoxiously difficult to reliably make any correlation. * * So for now we'll have to bodge it... Hoping that the system * is at least sane enough that an adapter is in the same PCI * segment as its NHI, if we can find *something* on that segment * which meets the requirements for Kernel DMA Protection, we'll * take that to imply that firmware is aware and has (hopefully) * done the right thing in general. We need to know that the PCI * layer has seen the ExternalFacingPort property which will then * inform the IOMMU layer to enforce the complete "untrusted DMA" * flow, but also that the IOMMU driver itself can be trusted not * to have been subverted by a pre-boot DMA attack. */ while (bus->parent) bus = bus->parent; pci_walk_bus(bus, nhi_pci_check_iommu_pdev, &port_ok); nhi->iommu_dma_protection = port_ok; dev_dbg(nhi->dev, "IOMMU DMA protection is %s\n", str_enabled_disabled(port_ok)); } static int nhi_pci_init_msi(struct tb_nhi *nhi) { struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi); struct pci_dev *pdev = to_pci_dev(nhi->dev); struct device *dev = &pdev->dev; int res, irq, nvec; ida_init(&nhi_pci->msix_ida); /* * The NHI has 16 MSI-X vectors or a single MSI. We first try to * get all MSI-X vectors and if we succeed, each ring will have * one MSI-X. If for some reason that does not work out, we * fallback to a single MSI. */ nvec = pci_alloc_irq_vectors(pdev, MSIX_MIN_VECS, MSIX_MAX_VECS, PCI_IRQ_MSIX); if (nvec < 0) { nvec = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); if (nvec < 0) return nvec; INIT_WORK(&nhi->interrupt_work, nhi_interrupt_work); irq = pci_irq_vector(pdev, 0); if (irq < 0) return irq; res = devm_request_irq(&pdev->dev, irq, nhi_msi, IRQF_NO_SUSPEND, "thunderbolt", nhi); if (res) return dev_err_probe(dev, res, "request_irq failed, aborting\n"); } return 0; } static bool nhi_pci_imr_valid(struct pci_dev *pdev) { u8 val; if (!device_property_read_u8(&pdev->dev, "IMR_VALID", &val)) return !!val; return true; } static void nhi_pci_start_dma_port(struct tb_nhi *nhi) { struct pci_dev *pdev = to_pci_dev(nhi->dev); struct pci_dev *root_port; /* * During host router NVM upgrade we should not allow root port to * go into D3cold because some root ports cannot trigger PME * itself. To be on the safe side keep the root port in D0 during * the whole upgrade process. */ root_port = pcie_find_root_port(pdev); if (root_port) pm_runtime_get_noresume(&root_port->dev); } static void nhi_pci_complete_dma_port(struct tb_nhi *nhi) { struct pci_dev *pdev = to_pci_dev(nhi->dev); struct pci_dev *root_port; root_port = pcie_find_root_port(pdev); if (root_port) pm_runtime_put(&root_port->dev); } static int nhi_pci_ring_request_msix(struct tb_ring *ring, bool no_suspend) { struct tb_nhi *nhi = ring->nhi; struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi); struct pci_dev *pdev = to_pci_dev(nhi->dev); unsigned long irqflags; int ret; if (!pdev->msix_enabled) return 0; ret = ida_alloc_max(&nhi_pci->msix_ida, MSIX_MAX_VECS - 1, GFP_KERNEL); if (ret < 0) return ret; ring->vector = ret; ret = pci_irq_vector(pdev, ring->vector); if (ret < 0) goto err_ida_remove; ring->irq = ret; irqflags = no_suspend ? IRQF_NO_SUSPEND : 0; ret = request_irq(ring->irq, ring_msix, irqflags, "thunderbolt", ring); if (ret) goto err_ida_remove; return 0; err_ida_remove: ida_free(&nhi_pci->msix_ida, ring->vector); return ret; } static void nhi_pci_ring_release_msix(struct tb_ring *ring) { struct tb_nhi_pci *nhi_pci = nhi_to_pci(ring->nhi); if (ring->irq <= 0) return; free_irq(ring->irq, ring); ida_free(&nhi_pci->msix_ida, ring->vector); ring->vector = 0; ring->irq = 0; } static void nhi_pci_shutdown(struct tb_nhi *nhi) { struct tb_nhi_pci *nhi_pci = nhi_to_pci(nhi); struct pci_dev *pdev = to_pci_dev(nhi->dev); /* * We have to release the irq before calling flush_work. Otherwise an * already executing IRQ handler could call schedule_work again. */ if (!pdev->msix_enabled) { devm_free_irq(nhi->dev, pdev->irq, nhi); flush_work(&nhi->interrupt_work); } ida_destroy(&nhi_pci->msix_ida); } static bool nhi_pci_is_present(struct tb_nhi *nhi) { return pci_device_is_present(to_pci_dev(nhi->dev)); } static const struct tb_nhi_ops pci_nhi_default_ops = { .pre_nvm_auth = nhi_pci_start_dma_port, .post_nvm_auth = nhi_pci_complete_dma_port, .request_ring_irq = nhi_pci_ring_request_msix, .release_ring_irq = nhi_pci_ring_release_msix, .shutdown = nhi_pci_shutdown, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, }; /* Ice Lake specific NHI operations */ #define ICL_LC_MAILBOX_TIMEOUT 500 /* ms */ static int check_for_device(struct device *dev, void *data) { return tb_is_switch(dev); } static bool icl_nhi_is_device_connected(struct tb_nhi *nhi) { struct tb *tb = dev_get_drvdata(nhi->dev); int ret; ret = device_for_each_child(&tb->root_switch->dev, NULL, check_for_device); return ret > 0; } static int icl_nhi_force_power(struct tb_nhi *nhi, bool power) { struct pci_dev *pdev = to_pci_dev(nhi->dev); u32 vs_cap; /* * The Thunderbolt host controller is present always in Ice Lake * but the firmware may not be loaded and running (depending * whether there is device connected and so on). Each time the * controller is used we need to "Force Power" it first and wait * for the firmware to indicate it is up and running. This "Force * Power" is really not about actually powering on/off the * controller so it is accessible even if "Force Power" is off. * * The actual power management happens inside shared ACPI power * resources using standard ACPI methods. */ pci_read_config_dword(pdev, VS_CAP_22, &vs_cap); if (power) { vs_cap &= ~VS_CAP_22_DMA_DELAY_MASK; vs_cap |= 0x22 << VS_CAP_22_DMA_DELAY_SHIFT; vs_cap |= VS_CAP_22_FORCE_POWER; } else { vs_cap &= ~VS_CAP_22_FORCE_POWER; } pci_write_config_dword(pdev, VS_CAP_22, vs_cap); if (power) { unsigned int retries = 350; u32 val; /* Wait until the firmware tells it is up and running */ do { pci_read_config_dword(pdev, VS_CAP_9, &val); if (val & VS_CAP_9_FW_READY) return 0; usleep_range(3000, 3100); } while (--retries); return -ETIMEDOUT; } return 0; } static void icl_nhi_lc_mailbox_cmd(struct tb_nhi *nhi, enum icl_lc_mailbox_cmd cmd) { struct pci_dev *pdev = to_pci_dev(nhi->dev); u32 data; data = (cmd << VS_CAP_19_CMD_SHIFT) & VS_CAP_19_CMD_MASK; pci_write_config_dword(pdev, VS_CAP_19, data | VS_CAP_19_VALID); } static int icl_nhi_lc_mailbox_cmd_complete(struct tb_nhi *nhi, int timeout) { struct pci_dev *pdev = to_pci_dev(nhi->dev); unsigned long end; u32 data; if (!timeout) goto clear; end = jiffies + msecs_to_jiffies(timeout); do { pci_read_config_dword(pdev, VS_CAP_18, &data); if (data & VS_CAP_18_DONE) goto clear; usleep_range(1000, 1100); } while (time_before(jiffies, end)); return -ETIMEDOUT; clear: /* Clear the valid bit */ pci_write_config_dword(pdev, VS_CAP_19, 0); return 0; } static void icl_nhi_set_ltr(struct tb_nhi *nhi) { struct pci_dev *pdev = to_pci_dev(nhi->dev); u32 max_ltr, ltr; pci_read_config_dword(pdev, VS_CAP_16, &max_ltr); max_ltr &= 0xffff; /* Program the same value for both snoop and no-snoop */ ltr = max_ltr << 16 | max_ltr; pci_write_config_dword(pdev, VS_CAP_15, ltr); } static int icl_nhi_suspend(struct tb_nhi *nhi) { struct tb *tb = dev_get_drvdata(nhi->dev); int ret; if (icl_nhi_is_device_connected(nhi)) return 0; if (tb_switch_is_icm(tb->root_switch)) { /* * If there is no device connected we need to perform * both: a handshake through LC mailbox and force power * down before entering D3. */ icl_nhi_lc_mailbox_cmd(nhi, ICL_LC_PREPARE_FOR_RESET); ret = icl_nhi_lc_mailbox_cmd_complete(nhi, ICL_LC_MAILBOX_TIMEOUT); if (ret) return ret; } return icl_nhi_force_power(nhi, false); } static int icl_nhi_suspend_noirq(struct tb_nhi *nhi, bool wakeup) { struct tb *tb = dev_get_drvdata(nhi->dev); enum icl_lc_mailbox_cmd cmd; if (!pm_suspend_via_firmware()) return icl_nhi_suspend(nhi); if (!tb_switch_is_icm(tb->root_switch)) return 0; cmd = wakeup ? ICL_LC_GO2SX : ICL_LC_GO2SX_NO_WAKE; icl_nhi_lc_mailbox_cmd(nhi, cmd); return icl_nhi_lc_mailbox_cmd_complete(nhi, ICL_LC_MAILBOX_TIMEOUT); } static int icl_nhi_resume(struct tb_nhi *nhi) { int ret; ret = icl_nhi_force_power(nhi, true); if (ret) return ret; icl_nhi_set_ltr(nhi); return 0; } static void icl_nhi_shutdown(struct tb_nhi *nhi) { nhi_pci_shutdown(nhi); icl_nhi_force_power(nhi, false); } static const struct tb_nhi_ops icl_nhi_ops = { .init = icl_nhi_resume, .suspend_noirq = icl_nhi_suspend_noirq, .resume_noirq = icl_nhi_resume, .runtime_suspend = icl_nhi_suspend, .runtime_resume = icl_nhi_resume, .shutdown = icl_nhi_shutdown, .pre_nvm_auth = nhi_pci_start_dma_port, .post_nvm_auth = nhi_pci_complete_dma_port, .request_ring_irq = nhi_pci_ring_request_msix, .release_ring_irq = nhi_pci_ring_release_msix, .is_present = nhi_pci_is_present, .init_interrupts = nhi_pci_init_msi, }; static int nhi_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct device *dev = &pdev->dev; struct tb_nhi_pci *nhi_pci; struct tb_nhi *nhi; int res; if (!nhi_pci_imr_valid(pdev)) return dev_err_probe(dev, -ENODEV, "firmware image not valid, aborting\n"); res = pcim_enable_device(pdev); if (res) return dev_err_probe(dev, res, "cannot enable PCI device, aborting\n"); nhi_pci = devm_kzalloc(dev, sizeof(*nhi_pci), GFP_KERNEL); if (!nhi_pci) return -ENOMEM; nhi = &nhi_pci->nhi; nhi->dev = dev; nhi->ops = (const struct tb_nhi_ops *)id->driver_data ?: &pci_nhi_default_ops; nhi->iobase = pcim_iomap_region(pdev, 0, "thunderbolt"); res = PTR_ERR_OR_ZERO(nhi->iobase); if (res) return dev_err_probe(dev, res, "cannot obtain PCI resources, aborting\n"); nhi_pci_check_quirks(nhi_pci); nhi_pci_check_iommu(nhi_pci); pci_set_master(pdev); return nhi_probe(&nhi_pci->nhi); } static void nhi_pci_remove(struct pci_dev *pdev) { struct tb *tb = pci_get_drvdata(pdev); struct tb_nhi *nhi = tb->nhi; pm_runtime_get_sync(&pdev->dev); pm_runtime_dont_use_autosuspend(&pdev->dev); pm_runtime_forbid(&pdev->dev); tb_domain_remove(tb); wait_for_completion(&nhi->domain_released); nhi_shutdown(nhi); } static struct pci_device_id nhi_ids[] = { /* * We have to specify class, the TB bridges use the same device and * vendor (sub)id on gen 1 and gen 2 controllers. */ { .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, .vendor = PCI_VENDOR_ID_INTEL, .device = PCI_DEVICE_ID_INTEL_LIGHT_RIDGE, .subvendor = 0x2222, .subdevice = 0x1111, }, { .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, .vendor = PCI_VENDOR_ID_INTEL, .device = PCI_DEVICE_ID_INTEL_CACTUS_RIDGE_4C, .subvendor = 0x2222, .subdevice = 0x1111, }, { .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, .vendor = PCI_VENDOR_ID_INTEL, .device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_2C_NHI, .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, }, { .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, .vendor = PCI_VENDOR_ID_INTEL, .device = PCI_DEVICE_ID_INTEL_FALCON_RIDGE_4C_NHI, .subvendor = PCI_ANY_ID, .subdevice = PCI_ANY_ID, }, /* Thunderbolt 3 */ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_2C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_4C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_USBONLY_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_LP_USBONLY_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_2C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_4C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ALPINE_RIDGE_C_USBONLY_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TITAN_RIDGE_2C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TITAN_RIDGE_4C_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ICL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, /* Thunderbolt 4 */ { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_H_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_TGL_H_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_ADL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_RPL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_M_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_P_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_MTL_P_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_LNL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_LNL_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_PTL_M_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_PTL_M_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_PTL_P_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_PTL_P_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_WCL_NHI0), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_80G_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_40G_NHI) }, /* Any USB4 compliant host */ { PCI_DEVICE_CLASS(PCI_CLASS_SERIAL_USB_USB4, ~0) }, { 0,} }; MODULE_DEVICE_TABLE(pci, nhi_ids); MODULE_DESCRIPTION("Thunderbolt/USB4 core driver"); MODULE_LICENSE("GPL"); static struct pci_driver nhi_driver = { .name = "thunderbolt", .id_table = nhi_ids, .probe = nhi_pci_probe, .remove = nhi_pci_remove, .shutdown = nhi_pci_remove, .driver.pm = &nhi_pm_ops, }; static int __init nhi_init(void) { int ret; ret = tb_domain_init(); if (ret) return ret; ret = pci_register_driver(&nhi_driver); if (ret) tb_domain_exit(); return ret; } static void __exit nhi_unload(void) { pci_unregister_driver(&nhi_driver); tb_domain_exit(); } rootfs_initcall(nhi_init); module_exit(nhi_unload);