1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * PCIe bandwidth controller 4 * 5 * Author: Alexandru Gagniuc <mr.nuke.me@gmail.com> 6 * 7 * Copyright (C) 2019 Dell Inc 8 * Copyright (C) 2023-2024 Intel Corporation 9 * 10 * The PCIe bandwidth controller provides a way to alter PCIe Link Speeds 11 * and notify the operating system when the Link Width or Speed changes. The 12 * notification capability is required for all Root Ports and Downstream 13 * Ports supporting Link Width wider than x1 and/or multiple Link Speeds. 14 * 15 * This service port driver hooks into the Bandwidth Notification interrupt 16 * watching for changes or links becoming degraded in operation. It updates 17 * the cached Current Link Speed that is exposed to user space through sysfs. 18 */ 19 20 #define dev_fmt(fmt) "bwctrl: " fmt 21 22 #include <linux/atomic.h> 23 #include <linux/bitops.h> 24 #include <linux/bits.h> 25 #include <linux/cleanup.h> 26 #include <linux/errno.h> 27 #include <linux/interrupt.h> 28 #include <linux/mutex.h> 29 #include <linux/pci.h> 30 #include <linux/pci-bwctrl.h> 31 #include <linux/rwsem.h> 32 #include <linux/slab.h> 33 #include <linux/types.h> 34 35 #include "../pci.h" 36 #include "portdrv.h" 37 38 /** 39 * struct pcie_bwctrl_data - PCIe bandwidth controller 40 * @set_speed_mutex: Serializes link speed changes 41 * @cdev: Thermal cooling device associated with the port 42 */ 43 struct pcie_bwctrl_data { 44 struct mutex set_speed_mutex; 45 struct thermal_cooling_device *cdev; 46 }; 47 48 /* Prevent port removal during Link Speed changes. */ 49 static DECLARE_RWSEM(pcie_bwctrl_setspeed_rwsem); 50 51 static inline u16 pcie_supported_speeds2target_speed(u8 supported_speeds) 52 { 53 return __fls(supported_speeds); 54 } 55 56 /** 57 * pcie_bwctrl_select_speed - Select Target Link Speed 58 * @port: PCIe Port 59 * @speed_req: Requested PCIe Link Speed 60 * 61 * Select Target Link Speed by take into account Supported Link Speeds of 62 * both the Root Port and the Endpoint. 63 * 64 * Return: Target Link Speed (1=2.5GT/s, 2=5GT/s, 3=8GT/s, etc.) 65 */ 66 static u16 pcie_bwctrl_select_speed(struct pci_dev *port, enum pci_bus_speed speed_req) 67 { 68 struct pci_bus *bus = port->subordinate; 69 u8 desired_speeds, supported_speeds; 70 struct pci_dev *dev; 71 72 desired_speeds = GENMASK(pci_bus_speed2lnkctl2(speed_req), 73 __fls(PCI_EXP_LNKCAP2_SLS_2_5GB)); 74 75 supported_speeds = port->supported_speeds; 76 if (bus) { 77 down_read(&pci_bus_sem); 78 dev = list_first_entry_or_null(&bus->devices, struct pci_dev, bus_list); 79 if (dev) 80 supported_speeds &= dev->supported_speeds; 81 up_read(&pci_bus_sem); 82 } 83 if (!supported_speeds) 84 supported_speeds = PCI_EXP_LNKCAP2_SLS_2_5GB; 85 86 return pcie_supported_speeds2target_speed(supported_speeds & desired_speeds); 87 } 88 89 static int pcie_bwctrl_change_speed(struct pci_dev *port, u16 target_speed, bool use_lt) 90 { 91 int ret; 92 93 ret = pcie_capability_clear_and_set_word(port, PCI_EXP_LNKCTL2, 94 PCI_EXP_LNKCTL2_TLS, target_speed); 95 if (ret != PCIBIOS_SUCCESSFUL) 96 return pcibios_err_to_errno(ret); 97 98 return pcie_retrain_link(port, use_lt); 99 } 100 101 /** 102 * pcie_set_target_speed - Set downstream Link Speed for PCIe Port 103 * @port: PCIe Port 104 * @speed_req: Requested PCIe Link Speed 105 * @use_lt: Wait for the LT or DLLLA bit to detect the end of link training 106 * 107 * Attempt to set PCIe Port Link Speed to @speed_req. @speed_req may be 108 * adjusted downwards to the best speed supported by both the Port and PCIe 109 * Device underneath it. 110 * 111 * Return: 112 * * 0 - on success 113 * * -EINVAL - @speed_req is not a PCIe Link Speed 114 * * -ENODEV - @port is not controllable 115 * * -ETIMEDOUT - changing Link Speed took too long 116 * * -EAGAIN - Link Speed was changed but @speed_req was not achieved 117 */ 118 int pcie_set_target_speed(struct pci_dev *port, enum pci_bus_speed speed_req, 119 bool use_lt) 120 { 121 struct pci_bus *bus = port->subordinate; 122 u16 target_speed; 123 int ret; 124 125 if (WARN_ON_ONCE(!pcie_valid_speed(speed_req))) 126 return -EINVAL; 127 128 if (bus && bus->cur_bus_speed == speed_req) 129 return 0; 130 131 target_speed = pcie_bwctrl_select_speed(port, speed_req); 132 133 scoped_guard(rwsem_read, &pcie_bwctrl_setspeed_rwsem) { 134 struct pcie_bwctrl_data *data = port->link_bwctrl; 135 136 /* 137 * port->link_bwctrl is NULL during initial scan when called 138 * e.g. from the Target Speed quirk. 139 */ 140 if (data) 141 mutex_lock(&data->set_speed_mutex); 142 143 ret = pcie_bwctrl_change_speed(port, target_speed, use_lt); 144 145 if (data) 146 mutex_unlock(&data->set_speed_mutex); 147 } 148 149 /* 150 * Despite setting higher speed into the Target Link Speed, empty 151 * bus won't train to 5GT+ speeds. 152 */ 153 if (!ret && bus && bus->cur_bus_speed != speed_req && 154 !list_empty(&bus->devices)) 155 ret = -EAGAIN; 156 157 return ret; 158 } 159 160 static void pcie_bwnotif_enable(struct pcie_device *srv) 161 { 162 struct pci_dev *port = srv->port; 163 u16 link_status; 164 int ret; 165 166 /* Note if LBMS has been seen so far */ 167 ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); 168 if (ret == PCIBIOS_SUCCESSFUL && link_status & PCI_EXP_LNKSTA_LBMS) 169 set_bit(PCI_LINK_LBMS_SEEN, &port->priv_flags); 170 171 pcie_capability_set_word(port, PCI_EXP_LNKCTL, 172 PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 173 pcie_capability_write_word(port, PCI_EXP_LNKSTA, 174 PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS); 175 176 /* 177 * Update after enabling notifications & clearing status bits ensures 178 * link speed is up to date. 179 */ 180 pcie_update_link_speed(port->subordinate, PCIE_BWCTRL_ENABLE); 181 } 182 183 static void pcie_bwnotif_disable(struct pci_dev *port) 184 { 185 pcie_capability_clear_word(port, PCI_EXP_LNKCTL, 186 PCI_EXP_LNKCTL_LBMIE | PCI_EXP_LNKCTL_LABIE); 187 } 188 189 static irqreturn_t pcie_bwnotif_irq(int irq, void *context) 190 { 191 struct pcie_device *srv = context; 192 struct pci_dev *port = srv->port; 193 u16 link_status, events; 194 int ret; 195 196 ret = pcie_capability_read_word(port, PCI_EXP_LNKSTA, &link_status); 197 if (ret != PCIBIOS_SUCCESSFUL) 198 return IRQ_NONE; 199 200 events = link_status & (PCI_EXP_LNKSTA_LBMS | PCI_EXP_LNKSTA_LABS); 201 if (!events) 202 return IRQ_NONE; 203 204 if (events & PCI_EXP_LNKSTA_LBMS) 205 set_bit(PCI_LINK_LBMS_SEEN, &port->priv_flags); 206 207 pcie_capability_write_word(port, PCI_EXP_LNKSTA, events); 208 209 /* 210 * Interrupts will not be triggered from any further Link Speed 211 * change until LBMS is cleared by the write. Therefore, re-read the 212 * speed (inside pcie_update_link_speed()) after LBMS has been 213 * cleared to avoid missing link speed changes. 214 */ 215 pcie_update_link_speed(port->subordinate, PCIE_BWCTRL_IRQ); 216 217 return IRQ_HANDLED; 218 } 219 220 void pcie_reset_lbms(struct pci_dev *port) 221 { 222 clear_bit(PCI_LINK_LBMS_SEEN, &port->priv_flags); 223 pcie_capability_write_word(port, PCI_EXP_LNKSTA, PCI_EXP_LNKSTA_LBMS); 224 } 225 226 static int pcie_bwnotif_probe(struct pcie_device *srv) 227 { 228 struct pci_dev *port = srv->port; 229 int ret; 230 231 if (port->no_bw_notif) 232 return -ENODEV; 233 234 /* Can happen if we run out of bus numbers during enumeration. */ 235 if (!port->subordinate) 236 return -ENODEV; 237 238 struct pcie_bwctrl_data *data = devm_kzalloc(&srv->device, 239 sizeof(*data), GFP_KERNEL); 240 if (!data) 241 return -ENOMEM; 242 243 ret = devm_mutex_init(&srv->device, &data->set_speed_mutex); 244 if (ret) 245 return ret; 246 247 scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { 248 port->link_bwctrl = data; 249 250 ret = request_irq(srv->irq, pcie_bwnotif_irq, 251 IRQF_SHARED, "PCIe bwctrl", srv); 252 if (ret) { 253 port->link_bwctrl = NULL; 254 return ret; 255 } 256 257 pcie_bwnotif_enable(srv); 258 } 259 260 pci_dbg(port, "enabled with IRQ %d\n", srv->irq); 261 262 /* Don't fail on errors. Don't leave IS_ERR() "pointer" into ->cdev */ 263 port->link_bwctrl->cdev = pcie_cooling_device_register(port); 264 if (IS_ERR(port->link_bwctrl->cdev)) 265 port->link_bwctrl->cdev = NULL; 266 267 return 0; 268 } 269 270 static void pcie_bwnotif_remove(struct pcie_device *srv) 271 { 272 struct pcie_bwctrl_data *data = srv->port->link_bwctrl; 273 274 pcie_cooling_device_unregister(data->cdev); 275 276 scoped_guard(rwsem_write, &pcie_bwctrl_setspeed_rwsem) { 277 pcie_bwnotif_disable(srv->port); 278 279 free_irq(srv->irq, srv); 280 281 srv->port->link_bwctrl = NULL; 282 } 283 } 284 285 static int pcie_bwnotif_suspend(struct pcie_device *srv) 286 { 287 pcie_bwnotif_disable(srv->port); 288 return 0; 289 } 290 291 static int pcie_bwnotif_resume(struct pcie_device *srv) 292 { 293 pcie_bwnotif_enable(srv); 294 return 0; 295 } 296 297 static struct pcie_port_service_driver pcie_bwctrl_driver = { 298 .name = "pcie_bwctrl", 299 .port_type = PCIE_ANY_PORT, 300 .service = PCIE_PORT_SERVICE_BWCTRL, 301 .probe = pcie_bwnotif_probe, 302 .suspend = pcie_bwnotif_suspend, 303 .resume = pcie_bwnotif_resume, 304 .remove = pcie_bwnotif_remove, 305 }; 306 307 int __init pcie_bwctrl_init(void) 308 { 309 return pcie_port_service_register(&pcie_bwctrl_driver); 310 } 311