1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * PCI Backend Operations - respond to PCI requests from Frontend 4 * 5 * Author: Ryan Wilson <hap9@epoch.ncsc.mil> 6 */ 7 8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 9 #define dev_fmt pr_fmt 10 11 #include <linux/moduleparam.h> 12 #include <linux/wait.h> 13 #include <linux/bitops.h> 14 #include <xen/events.h> 15 #include <linux/sched.h> 16 #include "pciback.h" 17 18 static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id); 19 20 /* Ensure a device is has the fake IRQ handler "turned on/off" and is 21 * ready to be exported. This MUST be run after xen_pcibk_reset_device 22 * which does the actual PCI device enable/disable. 23 */ 24 static void xen_pcibk_control_isr(struct pci_dev *dev, int reset) 25 { 26 struct xen_pcibk_dev_data *dev_data; 27 int rc; 28 int enable = 0; 29 30 dev_data = pci_get_drvdata(dev); 31 if (!dev_data) 32 return; 33 34 /* We don't deal with bridges */ 35 if (dev->hdr_type != PCI_HEADER_TYPE_NORMAL) 36 return; 37 38 if (reset) { 39 dev_data->enable_intx = 0; 40 dev_data->ack_intr = 0; 41 } 42 enable = dev_data->enable_intx; 43 44 /* Asked to disable, but ISR isn't runnig */ 45 if (!enable && !dev_data->isr_on) 46 return; 47 48 /* Squirrel away the IRQs in the dev_data. We need this 49 * b/c when device transitions to MSI, the dev->irq is 50 * overwritten with the MSI vector. 51 */ 52 if (enable) 53 dev_data->irq = dev->irq; 54 55 /* 56 * SR-IOV devices in all use MSI-X and have no legacy 57 * interrupts, so inhibit creating a fake IRQ handler for them. 58 */ 59 if (dev_data->irq == 0) 60 goto out; 61 62 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s-> %s\n", 63 dev_data->irq_name, 64 dev_data->irq, 65 pci_is_enabled(dev) ? "on" : "off", 66 dev->msi_enabled ? "MSI" : "", 67 dev->msix_enabled ? "MSI/X" : "", 68 dev_data->isr_on ? "enable" : "disable", 69 enable ? "enable" : "disable"); 70 71 if (enable) { 72 /* 73 * The MSI or MSI-X should not have an IRQ handler. Otherwise 74 * if the guest terminates we BUG_ON in free_msi_irqs. 75 */ 76 if (dev->msi_enabled || dev->msix_enabled) 77 goto out; 78 79 rc = request_irq(dev_data->irq, 80 xen_pcibk_guest_interrupt, IRQF_SHARED, 81 dev_data->irq_name, dev); 82 if (rc) { 83 dev_err(&dev->dev, "%s: failed to install fake IRQ " \ 84 "handler for IRQ %d! (rc:%d)\n", 85 dev_data->irq_name, dev_data->irq, rc); 86 goto out; 87 } 88 } else { 89 free_irq(dev_data->irq, dev); 90 dev_data->irq = 0; 91 } 92 dev_data->isr_on = enable; 93 dev_data->ack_intr = enable; 94 out: 95 dev_dbg(&dev->dev, "%s: #%d %s %s%s %s\n", 96 dev_data->irq_name, 97 dev_data->irq, 98 pci_is_enabled(dev) ? "on" : "off", 99 dev->msi_enabled ? "MSI" : "", 100 dev->msix_enabled ? "MSI/X" : "", 101 enable ? (dev_data->isr_on ? "enabled" : "failed to enable") : 102 (dev_data->isr_on ? "failed to disable" : "disabled")); 103 } 104 105 /* Ensure a device is "turned off" and ready to be exported. 106 * (Also see xen_pcibk_config_reset to ensure virtual configuration space is 107 * ready to be re-exported) 108 */ 109 void xen_pcibk_reset_device(struct pci_dev *dev) 110 { 111 u16 cmd; 112 113 xen_pcibk_control_isr(dev, 1 /* reset device */); 114 115 /* Disable devices (but not bridges) */ 116 if (dev->hdr_type == PCI_HEADER_TYPE_NORMAL) { 117 #ifdef CONFIG_PCI_MSI 118 /* The guest could have been abruptly killed without 119 * disabling MSI/MSI-X interrupts.*/ 120 if (dev->msix_enabled) 121 pci_disable_msix(dev); 122 if (dev->msi_enabled) 123 pci_disable_msi(dev); 124 #endif 125 if (pci_is_enabled(dev)) 126 pci_disable_device(dev); 127 128 dev->is_busmaster = 0; 129 } else { 130 pci_read_config_word(dev, PCI_COMMAND, &cmd); 131 if (cmd & (PCI_COMMAND_INVALIDATE)) { 132 cmd &= ~(PCI_COMMAND_INVALIDATE); 133 pci_write_config_word(dev, PCI_COMMAND, cmd); 134 135 dev->is_busmaster = 0; 136 } 137 } 138 } 139 140 #ifdef CONFIG_PCI_MSI 141 static 142 int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev, 143 struct pci_dev *dev, struct xen_pci_op *op) 144 { 145 struct xen_pcibk_dev_data *dev_data; 146 int status; 147 148 if (dev->msi_enabled) 149 status = -EALREADY; 150 else if (dev->msix_enabled) 151 status = -ENXIO; 152 else 153 status = pci_enable_msi(dev); 154 155 if (status) { 156 dev_warn_ratelimited(&dev->dev, "error enabling MSI for guest %u: err %d\n", 157 pdev->xdev->otherend_id, status); 158 op->value = 0; 159 return XEN_PCI_ERR_op_failed; 160 } 161 162 /* The value the guest needs is actually the IDT vector, not the 163 * the local domain's IRQ number. */ 164 165 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 166 167 dev_dbg(&dev->dev, "MSI: %d\n", op->value); 168 169 dev_data = pci_get_drvdata(dev); 170 if (dev_data) 171 dev_data->ack_intr = 0; 172 173 return 0; 174 } 175 176 static 177 int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev, 178 struct pci_dev *dev, struct xen_pci_op *op) 179 { 180 if (dev->msi_enabled) { 181 struct xen_pcibk_dev_data *dev_data; 182 183 pci_disable_msi(dev); 184 185 dev_data = pci_get_drvdata(dev); 186 if (dev_data) 187 dev_data->ack_intr = 1; 188 } 189 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 190 191 dev_dbg(&dev->dev, "MSI: %d\n", op->value); 192 193 return 0; 194 } 195 196 static 197 int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev, 198 struct pci_dev *dev, struct xen_pci_op *op) 199 { 200 struct xen_pcibk_dev_data *dev_data; 201 int i, result; 202 struct msix_entry *entries; 203 u16 cmd; 204 205 dev_dbg(&dev->dev, "enable MSI-X\n"); 206 207 if (op->value > SH_INFO_MAX_VEC) 208 return -EINVAL; 209 210 if (dev->msix_enabled) 211 return -EALREADY; 212 213 /* 214 * PCI_COMMAND_MEMORY must be enabled, otherwise we may not be able 215 * to access the BARs where the MSI-X entries reside. 216 * But VF devices are unique in which the PF needs to be checked. 217 */ 218 pci_read_config_word(pci_physfn(dev), PCI_COMMAND, &cmd); 219 if (dev->msi_enabled || !(cmd & PCI_COMMAND_MEMORY)) 220 return -ENXIO; 221 222 entries = kmalloc_array(op->value, sizeof(*entries), GFP_KERNEL); 223 if (entries == NULL) 224 return -ENOMEM; 225 226 for (i = 0; i < op->value; i++) { 227 entries[i].entry = op->msix_entries[i].entry; 228 entries[i].vector = op->msix_entries[i].vector; 229 } 230 231 result = pci_enable_msix_exact(dev, entries, op->value); 232 if (result == 0) { 233 for (i = 0; i < op->value; i++) { 234 op->msix_entries[i].entry = entries[i].entry; 235 if (entries[i].vector) { 236 op->msix_entries[i].vector = 237 xen_pirq_from_irq(entries[i].vector); 238 dev_dbg(&dev->dev, "MSI-X[%d]: %d\n", i, 239 op->msix_entries[i].vector); 240 } 241 } 242 } else 243 dev_warn_ratelimited(&dev->dev, "error enabling MSI-X for guest %u: err %d!\n", 244 pdev->xdev->otherend_id, result); 245 kfree(entries); 246 247 op->value = result; 248 dev_data = pci_get_drvdata(dev); 249 if (dev_data) 250 dev_data->ack_intr = 0; 251 252 return result > 0 ? 0 : result; 253 } 254 255 static 256 int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev, 257 struct pci_dev *dev, struct xen_pci_op *op) 258 { 259 if (dev->msix_enabled) { 260 struct xen_pcibk_dev_data *dev_data; 261 262 pci_disable_msix(dev); 263 264 dev_data = pci_get_drvdata(dev); 265 if (dev_data) 266 dev_data->ack_intr = 1; 267 } 268 /* 269 * SR-IOV devices (which don't have any legacy IRQ) have 270 * an undefined IRQ value of zero. 271 */ 272 op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; 273 274 dev_dbg(&dev->dev, "MSI-X: %d\n", op->value); 275 276 return 0; 277 } 278 #endif 279 /* 280 * Now the same evtchn is used for both pcifront conf_read_write request 281 * as well as pcie aer front end ack. We use a new work_queue to schedule 282 * xen_pcibk conf_read_write service for avoiding confict with aer_core 283 * do_recovery job which also use the system default work_queue 284 */ 285 void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev) 286 { 287 /* Check that frontend is requesting an operation and that we are not 288 * already processing a request */ 289 if (test_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags) 290 && !test_and_set_bit(_PDEVF_op_active, &pdev->flags)) { 291 schedule_work(&pdev->op_work); 292 } 293 /*_XEN_PCIB_active should have been cleared by pcifront. And also make 294 sure xen_pcibk is waiting for ack by checking _PCIB_op_pending*/ 295 if (!test_bit(_XEN_PCIB_active, (unsigned long *)&pdev->sh_info->flags) 296 && test_bit(_PCIB_op_pending, &pdev->flags)) { 297 wake_up(&xen_pcibk_aer_wait_queue); 298 } 299 } 300 301 /* Performing the configuration space reads/writes must not be done in atomic 302 * context because some of the pci_* functions can sleep (mostly due to ACPI 303 * use of semaphores). This function is intended to be called from a work 304 * queue in process context taking a struct xen_pcibk_device as a parameter */ 305 306 void xen_pcibk_do_op(struct work_struct *data) 307 { 308 struct xen_pcibk_device *pdev = 309 container_of(data, struct xen_pcibk_device, op_work); 310 struct pci_dev *dev; 311 struct xen_pcibk_dev_data *dev_data = NULL; 312 struct xen_pci_op *op = &pdev->op; 313 int test_intx = 0; 314 #ifdef CONFIG_PCI_MSI 315 unsigned int nr = 0; 316 #endif 317 318 *op = pdev->sh_info->op; 319 barrier(); 320 dev = xen_pcibk_get_pci_dev(pdev, op->domain, op->bus, op->devfn); 321 322 if (dev == NULL) 323 op->err = XEN_PCI_ERR_dev_not_found; 324 else { 325 dev_data = pci_get_drvdata(dev); 326 if (dev_data) 327 test_intx = dev_data->enable_intx; 328 switch (op->cmd) { 329 case XEN_PCI_OP_conf_read: 330 op->err = xen_pcibk_config_read(dev, 331 op->offset, op->size, &op->value); 332 break; 333 case XEN_PCI_OP_conf_write: 334 op->err = xen_pcibk_config_write(dev, 335 op->offset, op->size, op->value); 336 break; 337 #ifdef CONFIG_PCI_MSI 338 case XEN_PCI_OP_enable_msi: 339 op->err = xen_pcibk_enable_msi(pdev, dev, op); 340 break; 341 case XEN_PCI_OP_disable_msi: 342 op->err = xen_pcibk_disable_msi(pdev, dev, op); 343 break; 344 case XEN_PCI_OP_enable_msix: 345 nr = op->value; 346 op->err = xen_pcibk_enable_msix(pdev, dev, op); 347 break; 348 case XEN_PCI_OP_disable_msix: 349 op->err = xen_pcibk_disable_msix(pdev, dev, op); 350 break; 351 #endif 352 default: 353 op->err = XEN_PCI_ERR_not_implemented; 354 break; 355 } 356 } 357 if (!op->err && dev && dev_data) { 358 /* Transition detected */ 359 if ((dev_data->enable_intx != test_intx)) 360 xen_pcibk_control_isr(dev, 0 /* no reset */); 361 } 362 pdev->sh_info->op.err = op->err; 363 pdev->sh_info->op.value = op->value; 364 #ifdef CONFIG_PCI_MSI 365 if (op->cmd == XEN_PCI_OP_enable_msix && op->err == 0) { 366 unsigned int i; 367 368 for (i = 0; i < nr; i++) 369 pdev->sh_info->op.msix_entries[i].vector = 370 op->msix_entries[i].vector; 371 } 372 #endif 373 /* Tell the driver domain that we're done. */ 374 wmb(); 375 clear_bit(_XEN_PCIF_active, (unsigned long *)&pdev->sh_info->flags); 376 notify_remote_via_irq(pdev->evtchn_irq); 377 378 /* Mark that we're done. */ 379 smp_mb__before_atomic(); /* /after/ clearing PCIF_active */ 380 clear_bit(_PDEVF_op_active, &pdev->flags); 381 smp_mb__after_atomic(); /* /before/ final check for work */ 382 383 /* Check to see if the driver domain tried to start another request in 384 * between clearing _XEN_PCIF_active and clearing _PDEVF_op_active. 385 */ 386 xen_pcibk_test_and_schedule_op(pdev); 387 } 388 389 irqreturn_t xen_pcibk_handle_event(int irq, void *dev_id) 390 { 391 struct xen_pcibk_device *pdev = dev_id; 392 393 xen_pcibk_test_and_schedule_op(pdev); 394 395 return IRQ_HANDLED; 396 } 397 static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id) 398 { 399 struct pci_dev *dev = (struct pci_dev *)dev_id; 400 struct xen_pcibk_dev_data *dev_data = pci_get_drvdata(dev); 401 402 if (dev_data->isr_on && dev_data->ack_intr) { 403 dev_data->handled++; 404 if ((dev_data->handled % 1000) == 0) { 405 if (xen_test_irq_shared(irq)) { 406 dev_info(&dev->dev, "%s IRQ line is not shared " 407 "with other domains. Turning ISR off\n", 408 dev_data->irq_name); 409 dev_data->ack_intr = 0; 410 } 411 } 412 return IRQ_HANDLED; 413 } 414 return IRQ_NONE; 415 } 416