1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 * 8 * The System z PCI code is a rewrite from a prototype by 9 * the following people (Kudoz!): 10 * Alexander Schmidt 11 * Christoph Raisch 12 * Hannes Hering 13 * Hoang-Nam Nguyen 14 * Jan-Bernd Themann 15 * Stefan Roscher 16 * Thomas Klein 17 */ 18 19 #define pr_fmt(fmt) "zpci: " fmt 20 21 #include <linux/kernel.h> 22 #include <linux/slab.h> 23 #include <linux/err.h> 24 #include <linux/export.h> 25 #include <linux/delay.h> 26 #include <linux/seq_file.h> 27 #include <linux/jump_label.h> 28 #include <linux/pci.h> 29 #include <linux/printk.h> 30 #include <linux/lockdep.h> 31 #include <linux/list_sort.h> 32 33 #include <asm/machine.h> 34 #include <asm/isc.h> 35 #include <asm/airq.h> 36 #include <asm/facility.h> 37 #include <asm/pci_insn.h> 38 #include <asm/pci_clp.h> 39 #include <asm/pci_dma.h> 40 41 #include "pci_bus.h" 42 #include "pci_iov.h" 43 44 /* list of all detected zpci devices */ 45 static LIST_HEAD(zpci_list); 46 static DEFINE_SPINLOCK(zpci_list_lock); 47 static DEFINE_MUTEX(zpci_add_remove_lock); 48 49 static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE); 50 static DEFINE_SPINLOCK(zpci_domain_lock); 51 52 #define ZPCI_IOMAP_ENTRIES \ 53 min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2), \ 54 ZPCI_IOMAP_MAX_ENTRIES) 55 56 unsigned int s390_pci_no_rid; 57 58 static DEFINE_SPINLOCK(zpci_iomap_lock); 59 static unsigned long *zpci_iomap_bitmap; 60 struct zpci_iomap_entry *zpci_iomap_start; 61 EXPORT_SYMBOL_GPL(zpci_iomap_start); 62 63 DEFINE_STATIC_KEY_FALSE(have_mio); 64 65 static struct kmem_cache *zdev_fmb_cache; 66 67 /* AEN structures that must be preserved over KVM module re-insertion */ 68 union zpci_sic_iib *zpci_aipb; 69 EXPORT_SYMBOL_GPL(zpci_aipb); 70 struct airq_iv *zpci_aif_sbv; 71 EXPORT_SYMBOL_GPL(zpci_aif_sbv); 72 73 void zpci_zdev_put(struct zpci_dev *zdev) 74 { 75 if (!zdev) 76 return; 77 mutex_lock(&zpci_add_remove_lock); 78 kref_put_lock(&zdev->kref, zpci_release_device, &zpci_list_lock); 79 mutex_unlock(&zpci_add_remove_lock); 80 } 81 82 struct zpci_dev *get_zdev_by_fid(u32 fid) 83 { 84 struct zpci_dev *tmp, *zdev = NULL; 85 86 spin_lock(&zpci_list_lock); 87 list_for_each_entry(tmp, &zpci_list, entry) { 88 if (tmp->fid == fid) { 89 zdev = tmp; 90 zpci_zdev_get(zdev); 91 break; 92 } 93 } 94 spin_unlock(&zpci_list_lock); 95 return zdev; 96 } 97 98 void zpci_remove_reserved_devices(void) 99 { 100 struct zpci_dev *tmp, *zdev; 101 enum zpci_state state; 102 LIST_HEAD(remove); 103 104 spin_lock(&zpci_list_lock); 105 list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { 106 if (zdev->state == ZPCI_FN_STATE_STANDBY && 107 !clp_get_state(zdev->fid, &state) && 108 state == ZPCI_FN_STATE_RESERVED) 109 list_move_tail(&zdev->entry, &remove); 110 } 111 spin_unlock(&zpci_list_lock); 112 113 list_for_each_entry_safe(zdev, tmp, &remove, entry) 114 zpci_device_reserved(zdev); 115 } 116 117 int pci_domain_nr(struct pci_bus *bus) 118 { 119 return ((struct zpci_bus *) bus->sysdata)->domain_nr; 120 } 121 EXPORT_SYMBOL_GPL(pci_domain_nr); 122 123 int pci_proc_domain(struct pci_bus *bus) 124 { 125 return pci_domain_nr(bus); 126 } 127 EXPORT_SYMBOL_GPL(pci_proc_domain); 128 129 /* Modify PCI: Register I/O address translation parameters */ 130 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 131 u64 base, u64 limit, u64 iota, u8 *status) 132 { 133 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); 134 struct zpci_fib fib = {0}; 135 u8 cc; 136 137 fib.pba = base; 138 /* Work around off by one in ISM virt device */ 139 if (zdev->pft == PCI_FUNC_TYPE_ISM && limit > base) 140 fib.pal = limit + (1 << 12); 141 else 142 fib.pal = limit; 143 fib.iota = iota; 144 fib.gd = zdev->gisa; 145 cc = zpci_mod_fc(req, &fib, status); 146 if (cc) 147 zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status); 148 return cc; 149 } 150 EXPORT_SYMBOL_GPL(zpci_register_ioat); 151 152 /* Modify PCI: Unregister I/O address translation parameters */ 153 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 154 { 155 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); 156 struct zpci_fib fib = {0}; 157 u8 cc, status; 158 159 fib.gd = zdev->gisa; 160 161 cc = zpci_mod_fc(req, &fib, &status); 162 if (cc) 163 zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); 164 return cc; 165 } 166 167 /* Modify PCI: Set PCI function measurement parameters */ 168 int zpci_fmb_enable_device(struct zpci_dev *zdev) 169 { 170 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 171 struct zpci_iommu_ctrs *ctrs; 172 struct zpci_fib fib = {0}; 173 unsigned long flags; 174 u8 cc, status; 175 176 if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) 177 return -EINVAL; 178 179 zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); 180 if (!zdev->fmb) 181 return -ENOMEM; 182 WARN_ON((u64) zdev->fmb & 0xf); 183 184 /* reset software counters */ 185 spin_lock_irqsave(&zdev->dom_lock, flags); 186 ctrs = zpci_get_iommu_ctrs(zdev); 187 if (ctrs) { 188 atomic64_set(&ctrs->mapped_pages, 0); 189 atomic64_set(&ctrs->unmapped_pages, 0); 190 atomic64_set(&ctrs->global_rpcits, 0); 191 atomic64_set(&ctrs->sync_map_rpcits, 0); 192 atomic64_set(&ctrs->sync_rpcits, 0); 193 } 194 spin_unlock_irqrestore(&zdev->dom_lock, flags); 195 196 197 fib.fmb_addr = virt_to_phys(zdev->fmb); 198 fib.gd = zdev->gisa; 199 cc = zpci_mod_fc(req, &fib, &status); 200 if (cc) { 201 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 202 zdev->fmb = NULL; 203 } 204 return cc ? -EIO : 0; 205 } 206 207 /* Modify PCI: Disable PCI function measurement */ 208 int zpci_fmb_disable_device(struct zpci_dev *zdev) 209 { 210 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 211 struct zpci_fib fib = {0}; 212 u8 cc, status; 213 214 if (!zdev->fmb) 215 return -EINVAL; 216 217 fib.gd = zdev->gisa; 218 219 /* Function measurement is disabled if fmb address is zero */ 220 cc = zpci_mod_fc(req, &fib, &status); 221 if (cc == 3) /* Function already gone. */ 222 cc = 0; 223 224 if (!cc) { 225 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 226 zdev->fmb = NULL; 227 } 228 return cc ? -EIO : 0; 229 } 230 231 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 232 { 233 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 234 u64 data; 235 int rc; 236 237 rc = __zpci_load(&data, req, offset); 238 if (!rc) { 239 data = le64_to_cpu((__force __le64) data); 240 data >>= (8 - len) * 8; 241 *val = (u32) data; 242 } else 243 *val = 0xffffffff; 244 return rc; 245 } 246 247 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 248 { 249 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 250 u64 data = val; 251 int rc; 252 253 data <<= (8 - len) * 8; 254 data = (__force u64) cpu_to_le64(data); 255 rc = __zpci_store(data, req, offset); 256 return rc; 257 } 258 259 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 260 resource_size_t size, 261 resource_size_t align) 262 { 263 return 0; 264 } 265 266 void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, 267 pgprot_t prot) 268 { 269 /* 270 * When PCI MIO instructions are unavailable the "physical" address 271 * encodes a hint for accessing the PCI memory space it represents. 272 * Just pass it unchanged such that ioread/iowrite can decode it. 273 */ 274 if (!static_branch_unlikely(&have_mio)) 275 return (void __iomem *)phys_addr; 276 277 return generic_ioremap_prot(phys_addr, size, prot); 278 } 279 EXPORT_SYMBOL(ioremap_prot); 280 281 void iounmap(volatile void __iomem *addr) 282 { 283 if (static_branch_likely(&have_mio)) 284 generic_iounmap(addr); 285 } 286 EXPORT_SYMBOL(iounmap); 287 288 /* Create a virtual mapping cookie for a PCI BAR */ 289 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, 290 unsigned long offset, unsigned long max) 291 { 292 struct zpci_dev *zdev = to_zpci(pdev); 293 int idx; 294 295 idx = zdev->bars[bar].map_idx; 296 spin_lock(&zpci_iomap_lock); 297 /* Detect overrun */ 298 WARN_ON(!++zpci_iomap_start[idx].count); 299 zpci_iomap_start[idx].fh = zdev->fh; 300 zpci_iomap_start[idx].bar = bar; 301 spin_unlock(&zpci_iomap_lock); 302 303 return (void __iomem *) ZPCI_ADDR(idx) + offset; 304 } 305 306 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, 307 unsigned long offset, 308 unsigned long max) 309 { 310 unsigned long barsize = pci_resource_len(pdev, bar); 311 struct zpci_dev *zdev = to_zpci(pdev); 312 void __iomem *iova; 313 314 iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); 315 return iova ? iova + offset : iova; 316 } 317 318 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, 319 unsigned long offset, unsigned long max) 320 { 321 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 322 return NULL; 323 324 if (static_branch_likely(&have_mio)) 325 return pci_iomap_range_mio(pdev, bar, offset, max); 326 else 327 return pci_iomap_range_fh(pdev, bar, offset, max); 328 } 329 EXPORT_SYMBOL(pci_iomap_range); 330 331 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 332 { 333 return pci_iomap_range(dev, bar, 0, maxlen); 334 } 335 EXPORT_SYMBOL(pci_iomap); 336 337 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, 338 unsigned long offset, unsigned long max) 339 { 340 unsigned long barsize = pci_resource_len(pdev, bar); 341 struct zpci_dev *zdev = to_zpci(pdev); 342 void __iomem *iova; 343 344 iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); 345 return iova ? iova + offset : iova; 346 } 347 348 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, 349 unsigned long offset, unsigned long max) 350 { 351 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 352 return NULL; 353 354 if (static_branch_likely(&have_mio)) 355 return pci_iomap_wc_range_mio(pdev, bar, offset, max); 356 else 357 return pci_iomap_range_fh(pdev, bar, offset, max); 358 } 359 EXPORT_SYMBOL(pci_iomap_wc_range); 360 361 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) 362 { 363 return pci_iomap_wc_range(dev, bar, 0, maxlen); 364 } 365 EXPORT_SYMBOL(pci_iomap_wc); 366 367 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) 368 { 369 unsigned int idx = ZPCI_IDX(addr); 370 371 spin_lock(&zpci_iomap_lock); 372 /* Detect underrun */ 373 WARN_ON(!zpci_iomap_start[idx].count); 374 if (!--zpci_iomap_start[idx].count) { 375 zpci_iomap_start[idx].fh = 0; 376 zpci_iomap_start[idx].bar = 0; 377 } 378 spin_unlock(&zpci_iomap_lock); 379 } 380 381 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) 382 { 383 iounmap(addr); 384 } 385 386 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 387 { 388 if (static_branch_likely(&have_mio)) 389 pci_iounmap_mio(pdev, addr); 390 else 391 pci_iounmap_fh(pdev, addr); 392 } 393 EXPORT_SYMBOL(pci_iounmap); 394 395 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 396 int size, u32 *val) 397 { 398 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 399 400 return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; 401 } 402 403 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 404 int size, u32 val) 405 { 406 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 407 408 return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; 409 } 410 411 static struct pci_ops pci_root_ops = { 412 .read = pci_read, 413 .write = pci_write, 414 }; 415 416 static void zpci_map_resources(struct pci_dev *pdev) 417 { 418 struct zpci_dev *zdev = to_zpci(pdev); 419 resource_size_t len; 420 int i; 421 422 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 423 len = pci_resource_len(pdev, i); 424 if (!len) 425 continue; 426 427 if (zpci_use_mio(zdev)) 428 pdev->resource[i].start = 429 (resource_size_t __force) zdev->bars[i].mio_wt; 430 else 431 pdev->resource[i].start = (resource_size_t __force) 432 pci_iomap_range_fh(pdev, i, 0, 0); 433 pdev->resource[i].end = pdev->resource[i].start + len - 1; 434 } 435 436 zpci_iov_map_resources(pdev); 437 } 438 439 static void zpci_unmap_resources(struct pci_dev *pdev) 440 { 441 struct zpci_dev *zdev = to_zpci(pdev); 442 resource_size_t len; 443 int i; 444 445 if (zpci_use_mio(zdev)) 446 return; 447 448 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 449 len = pci_resource_len(pdev, i); 450 if (!len) 451 continue; 452 pci_iounmap_fh(pdev, (void __iomem __force *) 453 pdev->resource[i].start); 454 } 455 } 456 457 static int zpci_alloc_iomap(struct zpci_dev *zdev) 458 { 459 unsigned long entry; 460 461 spin_lock(&zpci_iomap_lock); 462 entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); 463 if (entry == ZPCI_IOMAP_ENTRIES) { 464 spin_unlock(&zpci_iomap_lock); 465 return -ENOSPC; 466 } 467 set_bit(entry, zpci_iomap_bitmap); 468 spin_unlock(&zpci_iomap_lock); 469 return entry; 470 } 471 472 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 473 { 474 spin_lock(&zpci_iomap_lock); 475 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 476 clear_bit(entry, zpci_iomap_bitmap); 477 spin_unlock(&zpci_iomap_lock); 478 } 479 480 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh) 481 { 482 int bar, idx; 483 484 spin_lock(&zpci_iomap_lock); 485 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 486 if (!zdev->bars[bar].size) 487 continue; 488 idx = zdev->bars[bar].map_idx; 489 if (!zpci_iomap_start[idx].count) 490 continue; 491 WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh); 492 } 493 spin_unlock(&zpci_iomap_lock); 494 } 495 496 void zpci_update_fh(struct zpci_dev *zdev, u32 fh) 497 { 498 if (!fh || zdev->fh == fh) 499 return; 500 501 zdev->fh = fh; 502 if (zpci_use_mio(zdev)) 503 return; 504 if (zdev->has_resources && zdev_enabled(zdev)) 505 zpci_do_update_iomap_fh(zdev, fh); 506 } 507 508 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, 509 unsigned long size, unsigned long flags) 510 { 511 struct resource *r; 512 513 r = kzalloc(sizeof(*r), GFP_KERNEL); 514 if (!r) 515 return NULL; 516 517 r->start = start; 518 r->end = r->start + size - 1; 519 r->flags = flags; 520 r->name = zdev->res_name; 521 522 if (request_resource(&iomem_resource, r)) { 523 kfree(r); 524 return NULL; 525 } 526 return r; 527 } 528 529 int zpci_setup_bus_resources(struct zpci_dev *zdev) 530 { 531 unsigned long addr, size, flags; 532 struct resource *res; 533 int i, entry; 534 535 snprintf(zdev->res_name, sizeof(zdev->res_name), 536 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR); 537 538 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 539 if (!zdev->bars[i].size) 540 continue; 541 entry = zpci_alloc_iomap(zdev); 542 if (entry < 0) 543 return entry; 544 zdev->bars[i].map_idx = entry; 545 546 /* only MMIO is supported */ 547 flags = IORESOURCE_MEM; 548 if (zdev->bars[i].val & 8) 549 flags |= IORESOURCE_PREFETCH; 550 if (zdev->bars[i].val & 4) 551 flags |= IORESOURCE_MEM_64; 552 553 if (zpci_use_mio(zdev)) 554 addr = (unsigned long) zdev->bars[i].mio_wt; 555 else 556 addr = ZPCI_ADDR(entry); 557 size = 1UL << zdev->bars[i].size; 558 559 res = __alloc_res(zdev, addr, size, flags); 560 if (!res) { 561 zpci_free_iomap(zdev, entry); 562 return -ENOMEM; 563 } 564 zdev->bars[i].res = res; 565 } 566 zdev->has_resources = 1; 567 568 return 0; 569 } 570 571 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) 572 { 573 struct resource *res; 574 int i; 575 576 pci_lock_rescan_remove(); 577 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 578 res = zdev->bars[i].res; 579 if (!res) 580 continue; 581 582 release_resource(res); 583 pci_bus_remove_resource(zdev->zbus->bus, res); 584 zpci_free_iomap(zdev, zdev->bars[i].map_idx); 585 zdev->bars[i].res = NULL; 586 kfree(res); 587 } 588 zdev->has_resources = 0; 589 pci_unlock_rescan_remove(); 590 } 591 592 int pcibios_device_add(struct pci_dev *pdev) 593 { 594 struct zpci_dev *zdev = to_zpci(pdev); 595 struct resource *res; 596 int i; 597 598 /* The pdev has a reference to the zdev via its bus */ 599 zpci_zdev_get(zdev); 600 if (pdev->is_physfn) 601 pdev->no_vf_scan = 1; 602 603 zpci_map_resources(pdev); 604 605 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 606 res = &pdev->resource[i]; 607 if (res->parent || !res->flags) 608 continue; 609 pci_claim_resource(pdev, i); 610 } 611 612 return 0; 613 } 614 615 void pcibios_release_device(struct pci_dev *pdev) 616 { 617 struct zpci_dev *zdev = to_zpci(pdev); 618 619 zpci_unmap_resources(pdev); 620 zpci_zdev_put(zdev); 621 } 622 623 int pcibios_enable_device(struct pci_dev *pdev, int mask) 624 { 625 struct zpci_dev *zdev = to_zpci(pdev); 626 627 zpci_debug_init_device(zdev, dev_name(&pdev->dev)); 628 zpci_fmb_enable_device(zdev); 629 630 return pci_enable_resources(pdev, mask); 631 } 632 633 void pcibios_disable_device(struct pci_dev *pdev) 634 { 635 struct zpci_dev *zdev = to_zpci(pdev); 636 637 zpci_fmb_disable_device(zdev); 638 zpci_debug_exit_device(zdev); 639 } 640 641 static int __zpci_register_domain(int domain) 642 { 643 spin_lock(&zpci_domain_lock); 644 if (test_bit(domain, zpci_domain)) { 645 spin_unlock(&zpci_domain_lock); 646 pr_err("Domain %04x is already assigned\n", domain); 647 return -EEXIST; 648 } 649 set_bit(domain, zpci_domain); 650 spin_unlock(&zpci_domain_lock); 651 return domain; 652 } 653 654 static int __zpci_alloc_domain(void) 655 { 656 int domain; 657 658 spin_lock(&zpci_domain_lock); 659 /* 660 * We can always auto allocate domains below ZPCI_NR_DEVICES. 661 * There is either a free domain or we have reached the maximum in 662 * which case we would have bailed earlier. 663 */ 664 domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 665 set_bit(domain, zpci_domain); 666 spin_unlock(&zpci_domain_lock); 667 return domain; 668 } 669 670 int zpci_alloc_domain(int domain) 671 { 672 if (zpci_unique_uid) { 673 if (domain) 674 return __zpci_register_domain(domain); 675 pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n"); 676 update_uid_checking(false); 677 } 678 return __zpci_alloc_domain(); 679 } 680 681 void zpci_free_domain(int domain) 682 { 683 spin_lock(&zpci_domain_lock); 684 clear_bit(domain, zpci_domain); 685 spin_unlock(&zpci_domain_lock); 686 } 687 688 689 int zpci_enable_device(struct zpci_dev *zdev) 690 { 691 u32 fh = zdev->fh; 692 int rc = 0; 693 694 if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES)) 695 rc = -EIO; 696 else 697 zpci_update_fh(zdev, fh); 698 return rc; 699 } 700 EXPORT_SYMBOL_GPL(zpci_enable_device); 701 702 int zpci_reenable_device(struct zpci_dev *zdev) 703 { 704 u8 status; 705 int rc; 706 707 rc = zpci_enable_device(zdev); 708 if (rc) 709 return rc; 710 711 rc = zpci_iommu_register_ioat(zdev, &status); 712 if (rc) 713 zpci_disable_device(zdev); 714 715 return rc; 716 } 717 EXPORT_SYMBOL_GPL(zpci_reenable_device); 718 719 int zpci_disable_device(struct zpci_dev *zdev) 720 { 721 u32 fh = zdev->fh; 722 int cc, rc = 0; 723 724 cc = clp_disable_fh(zdev, &fh); 725 if (!cc) { 726 zpci_update_fh(zdev, fh); 727 } else if (cc == CLP_RC_SETPCIFN_ALRDY) { 728 pr_info("Disabling PCI function %08x had no effect as it was already disabled\n", 729 zdev->fid); 730 /* Function is already disabled - update handle */ 731 rc = clp_refresh_fh(zdev->fid, &fh); 732 if (!rc) { 733 zpci_update_fh(zdev, fh); 734 rc = -EINVAL; 735 } 736 } else { 737 rc = -EIO; 738 } 739 return rc; 740 } 741 EXPORT_SYMBOL_GPL(zpci_disable_device); 742 743 /** 744 * zpci_hot_reset_device - perform a reset of the given zPCI function 745 * @zdev: the slot which should be reset 746 * 747 * Performs a low level reset of the zPCI function. The reset is low level in 748 * the sense that the zPCI function can be reset without detaching it from the 749 * common PCI subsystem. The reset may be performed while under control of 750 * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation 751 * table is reinstated at the end of the reset. 752 * 753 * After the reset the functions internal state is reset to an initial state 754 * equivalent to its state during boot when first probing a driver. 755 * Consequently after reset the PCI function requires re-initialization via the 756 * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() 757 * and enabling the function via e.g. pci_enable_device_flags(). The caller 758 * must guard against concurrent reset attempts. 759 * 760 * In most cases this function should not be called directly but through 761 * pci_reset_function() or pci_reset_bus() which handle the save/restore and 762 * locking - asserted by lockdep. 763 * 764 * Return: 0 on success and an error value otherwise 765 */ 766 int zpci_hot_reset_device(struct zpci_dev *zdev) 767 { 768 int rc; 769 770 lockdep_assert_held(&zdev->state_lock); 771 zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); 772 if (zdev_enabled(zdev)) { 773 /* Disables device access, DMAs and IRQs (reset state) */ 774 rc = zpci_disable_device(zdev); 775 /* 776 * Due to a z/VM vs LPAR inconsistency in the error state the 777 * FH may indicate an enabled device but disable says the 778 * device is already disabled don't treat it as an error here. 779 */ 780 if (rc == -EINVAL) 781 rc = 0; 782 if (rc) 783 return rc; 784 } 785 786 rc = zpci_reenable_device(zdev); 787 788 return rc; 789 } 790 791 /** 792 * zpci_create_device() - Create a new zpci_dev and add it to the zbus 793 * @fid: Function ID of the device to be created 794 * @fh: Current Function Handle of the device to be created 795 * @state: Initial state after creation either Standby or Configured 796 * 797 * Allocates a new struct zpci_dev and queries the platform for its details. 798 * If successful the device can subsequently be added to the zPCI subsystem 799 * using zpci_add_device(). 800 * 801 * Returns: the zdev on success or an error pointer otherwise 802 */ 803 struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) 804 { 805 struct zpci_dev *zdev; 806 int rc; 807 808 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); 809 if (!zdev) 810 return ERR_PTR(-ENOMEM); 811 812 /* FID and Function Handle are the static/dynamic identifiers */ 813 zdev->fid = fid; 814 zdev->fh = fh; 815 816 /* Query function properties and update zdev */ 817 rc = clp_query_pci_fn(zdev); 818 if (rc) 819 goto error; 820 zdev->state = state; 821 822 mutex_init(&zdev->state_lock); 823 mutex_init(&zdev->fmb_lock); 824 mutex_init(&zdev->kzdev_lock); 825 826 return zdev; 827 828 error: 829 zpci_dbg(0, "crt fid:%x, rc:%d\n", fid, rc); 830 kfree(zdev); 831 return ERR_PTR(rc); 832 } 833 834 /** 835 * zpci_add_device() - Add a previously created zPCI device to the zPCI subsystem 836 * @zdev: The zPCI device to be added 837 * 838 * A struct zpci_dev is added to the zPCI subsystem and to a virtual PCI bus creating 839 * a new one as necessary. A hotplug slot is created and events start to be handled. 840 * If successful from this point on zpci_zdev_get() and zpci_zdev_put() must be used. 841 * If adding the struct zpci_dev fails the device was not added and should be freed. 842 * 843 * Return: 0 on success, or an error code otherwise 844 */ 845 int zpci_add_device(struct zpci_dev *zdev) 846 { 847 int rc; 848 849 mutex_lock(&zpci_add_remove_lock); 850 zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", zdev->fid, zdev->fh, zdev->state); 851 rc = zpci_init_iommu(zdev); 852 if (rc) 853 goto error; 854 855 rc = zpci_bus_device_register(zdev, &pci_root_ops); 856 if (rc) 857 goto error_destroy_iommu; 858 859 kref_init(&zdev->kref); 860 spin_lock(&zpci_list_lock); 861 list_add_tail(&zdev->entry, &zpci_list); 862 spin_unlock(&zpci_list_lock); 863 mutex_unlock(&zpci_add_remove_lock); 864 return 0; 865 866 error_destroy_iommu: 867 zpci_destroy_iommu(zdev); 868 error: 869 zpci_dbg(0, "add fid:%x, rc:%d\n", zdev->fid, rc); 870 mutex_unlock(&zpci_add_remove_lock); 871 return rc; 872 } 873 874 bool zpci_is_device_configured(struct zpci_dev *zdev) 875 { 876 enum zpci_state state = zdev->state; 877 878 return state != ZPCI_FN_STATE_RESERVED && 879 state != ZPCI_FN_STATE_STANDBY; 880 } 881 882 /** 883 * zpci_scan_configured_device() - Scan a freshly configured zpci_dev 884 * @zdev: The zpci_dev to be configured 885 * @fh: The general function handle supplied by the platform 886 * 887 * Given a device in the configuration state Configured, enables, scans and 888 * adds it to the common code PCI subsystem if possible. If any failure occurs, 889 * the zpci_dev is left disabled. 890 * 891 * Return: 0 on success, or an error code otherwise 892 */ 893 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh) 894 { 895 zpci_update_fh(zdev, fh); 896 return zpci_bus_scan_device(zdev); 897 } 898 899 /** 900 * zpci_deconfigure_device() - Deconfigure a zpci_dev 901 * @zdev: The zpci_dev to configure 902 * 903 * Deconfigure a zPCI function that is currently configured and possibly known 904 * to the common code PCI subsystem. 905 * If any failure occurs the device is left as is. 906 * 907 * Return: 0 on success, or an error code otherwise 908 */ 909 int zpci_deconfigure_device(struct zpci_dev *zdev) 910 { 911 int rc; 912 913 lockdep_assert_held(&zdev->state_lock); 914 if (zdev->state != ZPCI_FN_STATE_CONFIGURED) 915 return 0; 916 917 if (zdev->zbus->bus) 918 zpci_bus_remove_device(zdev, false); 919 920 if (zdev_enabled(zdev)) { 921 rc = zpci_disable_device(zdev); 922 if (rc) 923 return rc; 924 } 925 926 rc = sclp_pci_deconfigure(zdev->fid); 927 zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc); 928 if (rc) 929 return rc; 930 zdev->state = ZPCI_FN_STATE_STANDBY; 931 932 return 0; 933 } 934 935 /** 936 * zpci_device_reserved() - Mark device as reserved 937 * @zdev: the zpci_dev that was reserved 938 * 939 * Handle the case that a given zPCI function was reserved by another system. 940 */ 941 void zpci_device_reserved(struct zpci_dev *zdev) 942 { 943 lockdep_assert_held(&zdev->state_lock); 944 /* We may declare the device reserved multiple times */ 945 if (zdev->state == ZPCI_FN_STATE_RESERVED) 946 return; 947 zdev->state = ZPCI_FN_STATE_RESERVED; 948 zpci_dbg(3, "rsv fid:%x\n", zdev->fid); 949 /* 950 * The underlying device is gone. Allow the zdev to be freed 951 * as soon as all other references are gone by accounting for 952 * the removal as a dropped reference. 953 */ 954 zpci_zdev_put(zdev); 955 } 956 957 void zpci_release_device(struct kref *kref) 958 { 959 struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); 960 961 lockdep_assert_held(&zpci_add_remove_lock); 962 WARN_ON(zdev->state != ZPCI_FN_STATE_RESERVED); 963 /* 964 * We already hold zpci_list_lock thanks to kref_put_lock(). 965 * This makes sure no new reference can be taken from the list. 966 */ 967 list_del(&zdev->entry); 968 spin_unlock(&zpci_list_lock); 969 970 if (zdev->has_hp_slot) 971 zpci_exit_slot(zdev); 972 973 if (zdev->has_resources) 974 zpci_cleanup_bus_resources(zdev); 975 976 zpci_bus_device_unregister(zdev); 977 zpci_destroy_iommu(zdev); 978 zpci_dbg(3, "rem fid:%x\n", zdev->fid); 979 kfree_rcu(zdev, rcu); 980 } 981 982 int zpci_report_error(struct pci_dev *pdev, 983 struct zpci_report_error_header *report) 984 { 985 struct zpci_dev *zdev = to_zpci(pdev); 986 987 return sclp_pci_report(report, zdev->fh, zdev->fid); 988 } 989 EXPORT_SYMBOL(zpci_report_error); 990 991 /** 992 * zpci_clear_error_state() - Clears the zPCI error state of the device 993 * @zdev: The zdev for which the zPCI error state should be reset 994 * 995 * Clear the zPCI error state of the device. If clearing the zPCI error state 996 * fails the device is left in the error state. In this case it may make sense 997 * to call zpci_io_perm_failure() on the associated pdev if it exists. 998 * 999 * Returns: 0 on success, -EIO otherwise 1000 */ 1001 int zpci_clear_error_state(struct zpci_dev *zdev) 1002 { 1003 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR); 1004 struct zpci_fib fib = {0}; 1005 u8 status; 1006 int cc; 1007 1008 cc = zpci_mod_fc(req, &fib, &status); 1009 if (cc) { 1010 zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 1011 return -EIO; 1012 } 1013 1014 return 0; 1015 } 1016 1017 /** 1018 * zpci_reset_load_store_blocked() - Re-enables L/S from error state 1019 * @zdev: The zdev for which to unblock load/store access 1020 * 1021 * Re-enables load/store access for a PCI function in the error state while 1022 * keeping DMA blocked. In this state drivers can poke MMIO space to determine 1023 * if error recovery is possible while catching any rogue DMA access from the 1024 * device. 1025 * 1026 * Returns: 0 on success, -EIO otherwise 1027 */ 1028 int zpci_reset_load_store_blocked(struct zpci_dev *zdev) 1029 { 1030 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK); 1031 struct zpci_fib fib = {0}; 1032 u8 status; 1033 int cc; 1034 1035 cc = zpci_mod_fc(req, &fib, &status); 1036 if (cc) { 1037 zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 1038 return -EIO; 1039 } 1040 1041 return 0; 1042 } 1043 1044 static int zpci_mem_init(void) 1045 { 1046 BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || 1047 __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); 1048 1049 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 1050 __alignof__(struct zpci_fmb), 0, NULL); 1051 if (!zdev_fmb_cache) 1052 goto error_fmb; 1053 1054 zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, 1055 sizeof(*zpci_iomap_start), GFP_KERNEL); 1056 if (!zpci_iomap_start) 1057 goto error_iomap; 1058 1059 zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), 1060 sizeof(*zpci_iomap_bitmap), GFP_KERNEL); 1061 if (!zpci_iomap_bitmap) 1062 goto error_iomap_bitmap; 1063 1064 if (static_branch_likely(&have_mio)) 1065 clp_setup_writeback_mio(); 1066 1067 return 0; 1068 error_iomap_bitmap: 1069 kfree(zpci_iomap_start); 1070 error_iomap: 1071 kmem_cache_destroy(zdev_fmb_cache); 1072 error_fmb: 1073 return -ENOMEM; 1074 } 1075 1076 static void zpci_mem_exit(void) 1077 { 1078 kfree(zpci_iomap_bitmap); 1079 kfree(zpci_iomap_start); 1080 kmem_cache_destroy(zdev_fmb_cache); 1081 } 1082 1083 static unsigned int s390_pci_probe __initdata = 1; 1084 unsigned int s390_pci_force_floating __initdata; 1085 static unsigned int s390_pci_initialized; 1086 1087 char * __init pcibios_setup(char *str) 1088 { 1089 if (!strcmp(str, "off")) { 1090 s390_pci_probe = 0; 1091 return NULL; 1092 } 1093 if (!strcmp(str, "nomio")) { 1094 clear_machine_feature(MFEATURE_PCI_MIO); 1095 return NULL; 1096 } 1097 if (!strcmp(str, "force_floating")) { 1098 s390_pci_force_floating = 1; 1099 return NULL; 1100 } 1101 if (!strcmp(str, "norid")) { 1102 s390_pci_no_rid = 1; 1103 return NULL; 1104 } 1105 return str; 1106 } 1107 1108 bool zpci_is_enabled(void) 1109 { 1110 return s390_pci_initialized; 1111 } 1112 1113 static int zpci_cmp_rid(void *priv, const struct list_head *a, 1114 const struct list_head *b) 1115 { 1116 struct zpci_dev *za = container_of(a, struct zpci_dev, entry); 1117 struct zpci_dev *zb = container_of(b, struct zpci_dev, entry); 1118 1119 /* 1120 * PCI functions without RID available maintain original order 1121 * between themselves but sort before those with RID. 1122 */ 1123 if (za->rid == zb->rid) 1124 return za->rid_available > zb->rid_available; 1125 /* 1126 * PCI functions with RID sort by RID ascending. 1127 */ 1128 return za->rid > zb->rid; 1129 } 1130 1131 static void zpci_add_devices(struct list_head *scan_list) 1132 { 1133 struct zpci_dev *zdev, *tmp; 1134 1135 list_sort(NULL, scan_list, &zpci_cmp_rid); 1136 list_for_each_entry_safe(zdev, tmp, scan_list, entry) { 1137 list_del_init(&zdev->entry); 1138 if (zpci_add_device(zdev)) 1139 kfree(zdev); 1140 } 1141 } 1142 1143 int zpci_scan_devices(void) 1144 { 1145 LIST_HEAD(scan_list); 1146 int rc; 1147 1148 rc = clp_scan_pci_devices(&scan_list); 1149 if (rc) 1150 return rc; 1151 1152 zpci_add_devices(&scan_list); 1153 zpci_bus_scan_busses(); 1154 return 0; 1155 } 1156 1157 static int __init pci_base_init(void) 1158 { 1159 int rc; 1160 1161 if (!s390_pci_probe) 1162 return 0; 1163 1164 if (!test_facility(69) || !test_facility(71)) { 1165 pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); 1166 return 0; 1167 } 1168 1169 if (test_machine_feature(MFEATURE_PCI_MIO)) { 1170 static_branch_enable(&have_mio); 1171 system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT); 1172 } 1173 1174 rc = zpci_debug_init(); 1175 if (rc) 1176 goto out; 1177 1178 rc = zpci_mem_init(); 1179 if (rc) 1180 goto out_mem; 1181 1182 rc = zpci_irq_init(); 1183 if (rc) 1184 goto out_irq; 1185 1186 rc = zpci_scan_devices(); 1187 if (rc) 1188 goto out_find; 1189 1190 rc = zpci_fw_sysfs_init(); 1191 if (rc) 1192 goto out_find; 1193 1194 s390_pci_initialized = 1; 1195 return 0; 1196 1197 out_find: 1198 zpci_irq_exit(); 1199 out_irq: 1200 zpci_mem_exit(); 1201 out_mem: 1202 zpci_debug_exit(); 1203 out: 1204 return rc; 1205 } 1206 subsys_initcall_sync(pci_base_init); 1207