1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 * 8 * The System z PCI code is a rewrite from a prototype by 9 * the following people (Kudoz!): 10 * Alexander Schmidt 11 * Christoph Raisch 12 * Hannes Hering 13 * Hoang-Nam Nguyen 14 * Jan-Bernd Themann 15 * Stefan Roscher 16 * Thomas Klein 17 */ 18 19 #define KMSG_COMPONENT "zpci" 20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/slab.h> 24 #include <linux/err.h> 25 #include <linux/export.h> 26 #include <linux/delay.h> 27 #include <linux/seq_file.h> 28 #include <linux/jump_label.h> 29 #include <linux/pci.h> 30 #include <linux/printk.h> 31 32 #include <asm/isc.h> 33 #include <asm/airq.h> 34 #include <asm/facility.h> 35 #include <asm/pci_insn.h> 36 #include <asm/pci_clp.h> 37 #include <asm/pci_dma.h> 38 39 #include "pci_bus.h" 40 #include "pci_iov.h" 41 42 /* list of all detected zpci devices */ 43 static LIST_HEAD(zpci_list); 44 static DEFINE_SPINLOCK(zpci_list_lock); 45 46 static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE); 47 static DEFINE_SPINLOCK(zpci_domain_lock); 48 49 #define ZPCI_IOMAP_ENTRIES \ 50 min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2), \ 51 ZPCI_IOMAP_MAX_ENTRIES) 52 53 unsigned int s390_pci_no_rid; 54 55 static DEFINE_SPINLOCK(zpci_iomap_lock); 56 static unsigned long *zpci_iomap_bitmap; 57 struct zpci_iomap_entry *zpci_iomap_start; 58 EXPORT_SYMBOL_GPL(zpci_iomap_start); 59 60 DEFINE_STATIC_KEY_FALSE(have_mio); 61 62 static struct kmem_cache *zdev_fmb_cache; 63 64 /* AEN structures that must be preserved over KVM module re-insertion */ 65 union zpci_sic_iib *zpci_aipb; 66 EXPORT_SYMBOL_GPL(zpci_aipb); 67 struct airq_iv *zpci_aif_sbv; 68 EXPORT_SYMBOL_GPL(zpci_aif_sbv); 69 70 struct zpci_dev *get_zdev_by_fid(u32 fid) 71 { 72 struct zpci_dev *tmp, *zdev = NULL; 73 74 spin_lock(&zpci_list_lock); 75 list_for_each_entry(tmp, &zpci_list, entry) { 76 if (tmp->fid == fid) { 77 zdev = tmp; 78 zpci_zdev_get(zdev); 79 break; 80 } 81 } 82 spin_unlock(&zpci_list_lock); 83 return zdev; 84 } 85 86 void zpci_remove_reserved_devices(void) 87 { 88 struct zpci_dev *tmp, *zdev; 89 enum zpci_state state; 90 LIST_HEAD(remove); 91 92 spin_lock(&zpci_list_lock); 93 list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { 94 if (zdev->state == ZPCI_FN_STATE_STANDBY && 95 !clp_get_state(zdev->fid, &state) && 96 state == ZPCI_FN_STATE_RESERVED) 97 list_move_tail(&zdev->entry, &remove); 98 } 99 spin_unlock(&zpci_list_lock); 100 101 list_for_each_entry_safe(zdev, tmp, &remove, entry) 102 zpci_device_reserved(zdev); 103 } 104 105 int pci_domain_nr(struct pci_bus *bus) 106 { 107 return ((struct zpci_bus *) bus->sysdata)->domain_nr; 108 } 109 EXPORT_SYMBOL_GPL(pci_domain_nr); 110 111 int pci_proc_domain(struct pci_bus *bus) 112 { 113 return pci_domain_nr(bus); 114 } 115 EXPORT_SYMBOL_GPL(pci_proc_domain); 116 117 /* Modify PCI: Register I/O address translation parameters */ 118 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 119 u64 base, u64 limit, u64 iota, u8 *status) 120 { 121 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); 122 struct zpci_fib fib = {0}; 123 u8 cc; 124 125 WARN_ON_ONCE(iota & 0x3fff); 126 fib.pba = base; 127 /* Work around off by one in ISM virt device */ 128 if (zdev->pft == PCI_FUNC_TYPE_ISM && limit > base) 129 fib.pal = limit + (1 << 12); 130 else 131 fib.pal = limit; 132 fib.iota = iota | ZPCI_IOTA_RTTO_FLAG; 133 fib.gd = zdev->gisa; 134 cc = zpci_mod_fc(req, &fib, status); 135 if (cc) 136 zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status); 137 return cc; 138 } 139 EXPORT_SYMBOL_GPL(zpci_register_ioat); 140 141 /* Modify PCI: Unregister I/O address translation parameters */ 142 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 143 { 144 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); 145 struct zpci_fib fib = {0}; 146 u8 cc, status; 147 148 fib.gd = zdev->gisa; 149 150 cc = zpci_mod_fc(req, &fib, &status); 151 if (cc) 152 zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); 153 return cc; 154 } 155 156 /* Modify PCI: Set PCI function measurement parameters */ 157 int zpci_fmb_enable_device(struct zpci_dev *zdev) 158 { 159 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 160 struct zpci_iommu_ctrs *ctrs; 161 struct zpci_fib fib = {0}; 162 u8 cc, status; 163 164 if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) 165 return -EINVAL; 166 167 zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); 168 if (!zdev->fmb) 169 return -ENOMEM; 170 WARN_ON((u64) zdev->fmb & 0xf); 171 172 /* reset software counters */ 173 ctrs = zpci_get_iommu_ctrs(zdev); 174 if (ctrs) { 175 atomic64_set(&ctrs->mapped_pages, 0); 176 atomic64_set(&ctrs->unmapped_pages, 0); 177 atomic64_set(&ctrs->global_rpcits, 0); 178 atomic64_set(&ctrs->sync_map_rpcits, 0); 179 atomic64_set(&ctrs->sync_rpcits, 0); 180 } 181 182 183 fib.fmb_addr = virt_to_phys(zdev->fmb); 184 fib.gd = zdev->gisa; 185 cc = zpci_mod_fc(req, &fib, &status); 186 if (cc) { 187 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 188 zdev->fmb = NULL; 189 } 190 return cc ? -EIO : 0; 191 } 192 193 /* Modify PCI: Disable PCI function measurement */ 194 int zpci_fmb_disable_device(struct zpci_dev *zdev) 195 { 196 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 197 struct zpci_fib fib = {0}; 198 u8 cc, status; 199 200 if (!zdev->fmb) 201 return -EINVAL; 202 203 fib.gd = zdev->gisa; 204 205 /* Function measurement is disabled if fmb address is zero */ 206 cc = zpci_mod_fc(req, &fib, &status); 207 if (cc == 3) /* Function already gone. */ 208 cc = 0; 209 210 if (!cc) { 211 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 212 zdev->fmb = NULL; 213 } 214 return cc ? -EIO : 0; 215 } 216 217 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 218 { 219 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 220 u64 data; 221 int rc; 222 223 rc = __zpci_load(&data, req, offset); 224 if (!rc) { 225 data = le64_to_cpu((__force __le64) data); 226 data >>= (8 - len) * 8; 227 *val = (u32) data; 228 } else 229 *val = 0xffffffff; 230 return rc; 231 } 232 233 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 234 { 235 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 236 u64 data = val; 237 int rc; 238 239 data <<= (8 - len) * 8; 240 data = (__force u64) cpu_to_le64(data); 241 rc = __zpci_store(data, req, offset); 242 return rc; 243 } 244 245 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 246 resource_size_t size, 247 resource_size_t align) 248 { 249 return 0; 250 } 251 252 /* combine single writes by using store-block insn */ 253 void __iowrite64_copy(void __iomem *to, const void *from, size_t count) 254 { 255 zpci_memcpy_toio(to, from, count); 256 } 257 258 void __iomem *ioremap_prot(phys_addr_t phys_addr, size_t size, 259 unsigned long prot) 260 { 261 /* 262 * When PCI MIO instructions are unavailable the "physical" address 263 * encodes a hint for accessing the PCI memory space it represents. 264 * Just pass it unchanged such that ioread/iowrite can decode it. 265 */ 266 if (!static_branch_unlikely(&have_mio)) 267 return (void __iomem *)phys_addr; 268 269 return generic_ioremap_prot(phys_addr, size, __pgprot(prot)); 270 } 271 EXPORT_SYMBOL(ioremap_prot); 272 273 void iounmap(volatile void __iomem *addr) 274 { 275 if (static_branch_likely(&have_mio)) 276 generic_iounmap(addr); 277 } 278 EXPORT_SYMBOL(iounmap); 279 280 /* Create a virtual mapping cookie for a PCI BAR */ 281 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, 282 unsigned long offset, unsigned long max) 283 { 284 struct zpci_dev *zdev = to_zpci(pdev); 285 int idx; 286 287 idx = zdev->bars[bar].map_idx; 288 spin_lock(&zpci_iomap_lock); 289 /* Detect overrun */ 290 WARN_ON(!++zpci_iomap_start[idx].count); 291 zpci_iomap_start[idx].fh = zdev->fh; 292 zpci_iomap_start[idx].bar = bar; 293 spin_unlock(&zpci_iomap_lock); 294 295 return (void __iomem *) ZPCI_ADDR(idx) + offset; 296 } 297 298 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, 299 unsigned long offset, 300 unsigned long max) 301 { 302 unsigned long barsize = pci_resource_len(pdev, bar); 303 struct zpci_dev *zdev = to_zpci(pdev); 304 void __iomem *iova; 305 306 iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); 307 return iova ? iova + offset : iova; 308 } 309 310 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, 311 unsigned long offset, unsigned long max) 312 { 313 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 314 return NULL; 315 316 if (static_branch_likely(&have_mio)) 317 return pci_iomap_range_mio(pdev, bar, offset, max); 318 else 319 return pci_iomap_range_fh(pdev, bar, offset, max); 320 } 321 EXPORT_SYMBOL(pci_iomap_range); 322 323 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 324 { 325 return pci_iomap_range(dev, bar, 0, maxlen); 326 } 327 EXPORT_SYMBOL(pci_iomap); 328 329 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, 330 unsigned long offset, unsigned long max) 331 { 332 unsigned long barsize = pci_resource_len(pdev, bar); 333 struct zpci_dev *zdev = to_zpci(pdev); 334 void __iomem *iova; 335 336 iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); 337 return iova ? iova + offset : iova; 338 } 339 340 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, 341 unsigned long offset, unsigned long max) 342 { 343 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 344 return NULL; 345 346 if (static_branch_likely(&have_mio)) 347 return pci_iomap_wc_range_mio(pdev, bar, offset, max); 348 else 349 return pci_iomap_range_fh(pdev, bar, offset, max); 350 } 351 EXPORT_SYMBOL(pci_iomap_wc_range); 352 353 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) 354 { 355 return pci_iomap_wc_range(dev, bar, 0, maxlen); 356 } 357 EXPORT_SYMBOL(pci_iomap_wc); 358 359 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) 360 { 361 unsigned int idx = ZPCI_IDX(addr); 362 363 spin_lock(&zpci_iomap_lock); 364 /* Detect underrun */ 365 WARN_ON(!zpci_iomap_start[idx].count); 366 if (!--zpci_iomap_start[idx].count) { 367 zpci_iomap_start[idx].fh = 0; 368 zpci_iomap_start[idx].bar = 0; 369 } 370 spin_unlock(&zpci_iomap_lock); 371 } 372 373 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) 374 { 375 iounmap(addr); 376 } 377 378 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 379 { 380 if (static_branch_likely(&have_mio)) 381 pci_iounmap_mio(pdev, addr); 382 else 383 pci_iounmap_fh(pdev, addr); 384 } 385 EXPORT_SYMBOL(pci_iounmap); 386 387 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 388 int size, u32 *val) 389 { 390 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 391 392 return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; 393 } 394 395 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 396 int size, u32 val) 397 { 398 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 399 400 return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; 401 } 402 403 static struct pci_ops pci_root_ops = { 404 .read = pci_read, 405 .write = pci_write, 406 }; 407 408 static void zpci_map_resources(struct pci_dev *pdev) 409 { 410 struct zpci_dev *zdev = to_zpci(pdev); 411 resource_size_t len; 412 int i; 413 414 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 415 len = pci_resource_len(pdev, i); 416 if (!len) 417 continue; 418 419 if (zpci_use_mio(zdev)) 420 pdev->resource[i].start = 421 (resource_size_t __force) zdev->bars[i].mio_wt; 422 else 423 pdev->resource[i].start = (resource_size_t __force) 424 pci_iomap_range_fh(pdev, i, 0, 0); 425 pdev->resource[i].end = pdev->resource[i].start + len - 1; 426 } 427 428 zpci_iov_map_resources(pdev); 429 } 430 431 static void zpci_unmap_resources(struct pci_dev *pdev) 432 { 433 struct zpci_dev *zdev = to_zpci(pdev); 434 resource_size_t len; 435 int i; 436 437 if (zpci_use_mio(zdev)) 438 return; 439 440 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 441 len = pci_resource_len(pdev, i); 442 if (!len) 443 continue; 444 pci_iounmap_fh(pdev, (void __iomem __force *) 445 pdev->resource[i].start); 446 } 447 } 448 449 static int zpci_alloc_iomap(struct zpci_dev *zdev) 450 { 451 unsigned long entry; 452 453 spin_lock(&zpci_iomap_lock); 454 entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); 455 if (entry == ZPCI_IOMAP_ENTRIES) { 456 spin_unlock(&zpci_iomap_lock); 457 return -ENOSPC; 458 } 459 set_bit(entry, zpci_iomap_bitmap); 460 spin_unlock(&zpci_iomap_lock); 461 return entry; 462 } 463 464 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 465 { 466 spin_lock(&zpci_iomap_lock); 467 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 468 clear_bit(entry, zpci_iomap_bitmap); 469 spin_unlock(&zpci_iomap_lock); 470 } 471 472 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh) 473 { 474 int bar, idx; 475 476 spin_lock(&zpci_iomap_lock); 477 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 478 if (!zdev->bars[bar].size) 479 continue; 480 idx = zdev->bars[bar].map_idx; 481 if (!zpci_iomap_start[idx].count) 482 continue; 483 WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh); 484 } 485 spin_unlock(&zpci_iomap_lock); 486 } 487 488 void zpci_update_fh(struct zpci_dev *zdev, u32 fh) 489 { 490 if (!fh || zdev->fh == fh) 491 return; 492 493 zdev->fh = fh; 494 if (zpci_use_mio(zdev)) 495 return; 496 if (zdev->has_resources && zdev_enabled(zdev)) 497 zpci_do_update_iomap_fh(zdev, fh); 498 } 499 500 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, 501 unsigned long size, unsigned long flags) 502 { 503 struct resource *r; 504 505 r = kzalloc(sizeof(*r), GFP_KERNEL); 506 if (!r) 507 return NULL; 508 509 r->start = start; 510 r->end = r->start + size - 1; 511 r->flags = flags; 512 r->name = zdev->res_name; 513 514 if (request_resource(&iomem_resource, r)) { 515 kfree(r); 516 return NULL; 517 } 518 return r; 519 } 520 521 int zpci_setup_bus_resources(struct zpci_dev *zdev) 522 { 523 unsigned long addr, size, flags; 524 struct resource *res; 525 int i, entry; 526 527 snprintf(zdev->res_name, sizeof(zdev->res_name), 528 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR); 529 530 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 531 if (!zdev->bars[i].size) 532 continue; 533 entry = zpci_alloc_iomap(zdev); 534 if (entry < 0) 535 return entry; 536 zdev->bars[i].map_idx = entry; 537 538 /* only MMIO is supported */ 539 flags = IORESOURCE_MEM; 540 if (zdev->bars[i].val & 8) 541 flags |= IORESOURCE_PREFETCH; 542 if (zdev->bars[i].val & 4) 543 flags |= IORESOURCE_MEM_64; 544 545 if (zpci_use_mio(zdev)) 546 addr = (unsigned long) zdev->bars[i].mio_wt; 547 else 548 addr = ZPCI_ADDR(entry); 549 size = 1UL << zdev->bars[i].size; 550 551 res = __alloc_res(zdev, addr, size, flags); 552 if (!res) { 553 zpci_free_iomap(zdev, entry); 554 return -ENOMEM; 555 } 556 zdev->bars[i].res = res; 557 } 558 zdev->has_resources = 1; 559 560 return 0; 561 } 562 563 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) 564 { 565 struct resource *res; 566 int i; 567 568 pci_lock_rescan_remove(); 569 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 570 res = zdev->bars[i].res; 571 if (!res) 572 continue; 573 574 release_resource(res); 575 pci_bus_remove_resource(zdev->zbus->bus, res); 576 zpci_free_iomap(zdev, zdev->bars[i].map_idx); 577 zdev->bars[i].res = NULL; 578 kfree(res); 579 } 580 zdev->has_resources = 0; 581 pci_unlock_rescan_remove(); 582 } 583 584 int pcibios_device_add(struct pci_dev *pdev) 585 { 586 struct zpci_dev *zdev = to_zpci(pdev); 587 struct resource *res; 588 int i; 589 590 /* The pdev has a reference to the zdev via its bus */ 591 zpci_zdev_get(zdev); 592 if (pdev->is_physfn) 593 pdev->no_vf_scan = 1; 594 595 pdev->dev.groups = zpci_attr_groups; 596 zpci_map_resources(pdev); 597 598 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 599 res = &pdev->resource[i]; 600 if (res->parent || !res->flags) 601 continue; 602 pci_claim_resource(pdev, i); 603 } 604 605 return 0; 606 } 607 608 void pcibios_release_device(struct pci_dev *pdev) 609 { 610 struct zpci_dev *zdev = to_zpci(pdev); 611 612 zpci_unmap_resources(pdev); 613 zpci_zdev_put(zdev); 614 } 615 616 int pcibios_enable_device(struct pci_dev *pdev, int mask) 617 { 618 struct zpci_dev *zdev = to_zpci(pdev); 619 620 zpci_debug_init_device(zdev, dev_name(&pdev->dev)); 621 zpci_fmb_enable_device(zdev); 622 623 return pci_enable_resources(pdev, mask); 624 } 625 626 void pcibios_disable_device(struct pci_dev *pdev) 627 { 628 struct zpci_dev *zdev = to_zpci(pdev); 629 630 zpci_fmb_disable_device(zdev); 631 zpci_debug_exit_device(zdev); 632 } 633 634 static int __zpci_register_domain(int domain) 635 { 636 spin_lock(&zpci_domain_lock); 637 if (test_bit(domain, zpci_domain)) { 638 spin_unlock(&zpci_domain_lock); 639 pr_err("Domain %04x is already assigned\n", domain); 640 return -EEXIST; 641 } 642 set_bit(domain, zpci_domain); 643 spin_unlock(&zpci_domain_lock); 644 return domain; 645 } 646 647 static int __zpci_alloc_domain(void) 648 { 649 int domain; 650 651 spin_lock(&zpci_domain_lock); 652 /* 653 * We can always auto allocate domains below ZPCI_NR_DEVICES. 654 * There is either a free domain or we have reached the maximum in 655 * which case we would have bailed earlier. 656 */ 657 domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 658 set_bit(domain, zpci_domain); 659 spin_unlock(&zpci_domain_lock); 660 return domain; 661 } 662 663 int zpci_alloc_domain(int domain) 664 { 665 if (zpci_unique_uid) { 666 if (domain) 667 return __zpci_register_domain(domain); 668 pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n"); 669 update_uid_checking(false); 670 } 671 return __zpci_alloc_domain(); 672 } 673 674 void zpci_free_domain(int domain) 675 { 676 spin_lock(&zpci_domain_lock); 677 clear_bit(domain, zpci_domain); 678 spin_unlock(&zpci_domain_lock); 679 } 680 681 682 int zpci_enable_device(struct zpci_dev *zdev) 683 { 684 u32 fh = zdev->fh; 685 int rc = 0; 686 687 if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES)) 688 rc = -EIO; 689 else 690 zpci_update_fh(zdev, fh); 691 return rc; 692 } 693 EXPORT_SYMBOL_GPL(zpci_enable_device); 694 695 int zpci_disable_device(struct zpci_dev *zdev) 696 { 697 u32 fh = zdev->fh; 698 int cc, rc = 0; 699 700 cc = clp_disable_fh(zdev, &fh); 701 if (!cc) { 702 zpci_update_fh(zdev, fh); 703 } else if (cc == CLP_RC_SETPCIFN_ALRDY) { 704 pr_info("Disabling PCI function %08x had no effect as it was already disabled\n", 705 zdev->fid); 706 /* Function is already disabled - update handle */ 707 rc = clp_refresh_fh(zdev->fid, &fh); 708 if (!rc) { 709 zpci_update_fh(zdev, fh); 710 rc = -EINVAL; 711 } 712 } else { 713 rc = -EIO; 714 } 715 return rc; 716 } 717 EXPORT_SYMBOL_GPL(zpci_disable_device); 718 719 /** 720 * zpci_hot_reset_device - perform a reset of the given zPCI function 721 * @zdev: the slot which should be reset 722 * 723 * Performs a low level reset of the zPCI function. The reset is low level in 724 * the sense that the zPCI function can be reset without detaching it from the 725 * common PCI subsystem. The reset may be performed while under control of 726 * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation 727 * table is reinstated at the end of the reset. 728 * 729 * After the reset the functions internal state is reset to an initial state 730 * equivalent to its state during boot when first probing a driver. 731 * Consequently after reset the PCI function requires re-initialization via the 732 * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() 733 * and enabling the function via e.g.pci_enablde_device_flags().The caller 734 * must guard against concurrent reset attempts. 735 * 736 * In most cases this function should not be called directly but through 737 * pci_reset_function() or pci_reset_bus() which handle the save/restore and 738 * locking. 739 * 740 * Return: 0 on success and an error value otherwise 741 */ 742 int zpci_hot_reset_device(struct zpci_dev *zdev) 743 { 744 u8 status; 745 int rc; 746 747 zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); 748 if (zdev_enabled(zdev)) { 749 /* Disables device access, DMAs and IRQs (reset state) */ 750 rc = zpci_disable_device(zdev); 751 /* 752 * Due to a z/VM vs LPAR inconsistency in the error state the 753 * FH may indicate an enabled device but disable says the 754 * device is already disabled don't treat it as an error here. 755 */ 756 if (rc == -EINVAL) 757 rc = 0; 758 if (rc) 759 return rc; 760 } 761 762 rc = zpci_enable_device(zdev); 763 if (rc) 764 return rc; 765 766 if (zdev->dma_table) 767 rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 768 virt_to_phys(zdev->dma_table), &status); 769 if (rc) { 770 zpci_disable_device(zdev); 771 return rc; 772 } 773 774 return 0; 775 } 776 777 /** 778 * zpci_create_device() - Create a new zpci_dev and add it to the zbus 779 * @fid: Function ID of the device to be created 780 * @fh: Current Function Handle of the device to be created 781 * @state: Initial state after creation either Standby or Configured 782 * 783 * Creates a new zpci device and adds it to its, possibly newly created, zbus 784 * as well as zpci_list. 785 * 786 * Returns: the zdev on success or an error pointer otherwise 787 */ 788 struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) 789 { 790 struct zpci_dev *zdev; 791 int rc; 792 793 zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", fid, fh, state); 794 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); 795 if (!zdev) 796 return ERR_PTR(-ENOMEM); 797 798 /* FID and Function Handle are the static/dynamic identifiers */ 799 zdev->fid = fid; 800 zdev->fh = fh; 801 802 /* Query function properties and update zdev */ 803 rc = clp_query_pci_fn(zdev); 804 if (rc) 805 goto error; 806 zdev->state = state; 807 808 kref_init(&zdev->kref); 809 mutex_init(&zdev->lock); 810 mutex_init(&zdev->kzdev_lock); 811 812 rc = zpci_init_iommu(zdev); 813 if (rc) 814 goto error; 815 816 rc = zpci_bus_device_register(zdev, &pci_root_ops); 817 if (rc) 818 goto error_destroy_iommu; 819 820 spin_lock(&zpci_list_lock); 821 list_add_tail(&zdev->entry, &zpci_list); 822 spin_unlock(&zpci_list_lock); 823 824 return zdev; 825 826 error_destroy_iommu: 827 zpci_destroy_iommu(zdev); 828 error: 829 zpci_dbg(0, "add fid:%x, rc:%d\n", fid, rc); 830 kfree(zdev); 831 return ERR_PTR(rc); 832 } 833 834 bool zpci_is_device_configured(struct zpci_dev *zdev) 835 { 836 enum zpci_state state = zdev->state; 837 838 return state != ZPCI_FN_STATE_RESERVED && 839 state != ZPCI_FN_STATE_STANDBY; 840 } 841 842 /** 843 * zpci_scan_configured_device() - Scan a freshly configured zpci_dev 844 * @zdev: The zpci_dev to be configured 845 * @fh: The general function handle supplied by the platform 846 * 847 * Given a device in the configuration state Configured, enables, scans and 848 * adds it to the common code PCI subsystem if possible. If any failure occurs, 849 * the zpci_dev is left disabled. 850 * 851 * Return: 0 on success, or an error code otherwise 852 */ 853 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh) 854 { 855 zpci_update_fh(zdev, fh); 856 return zpci_bus_scan_device(zdev); 857 } 858 859 /** 860 * zpci_deconfigure_device() - Deconfigure a zpci_dev 861 * @zdev: The zpci_dev to configure 862 * 863 * Deconfigure a zPCI function that is currently configured and possibly known 864 * to the common code PCI subsystem. 865 * If any failure occurs the device is left as is. 866 * 867 * Return: 0 on success, or an error code otherwise 868 */ 869 int zpci_deconfigure_device(struct zpci_dev *zdev) 870 { 871 int rc; 872 873 if (zdev->zbus->bus) 874 zpci_bus_remove_device(zdev, false); 875 876 if (zdev_enabled(zdev)) { 877 rc = zpci_disable_device(zdev); 878 if (rc) 879 return rc; 880 } 881 882 rc = sclp_pci_deconfigure(zdev->fid); 883 zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc); 884 if (rc) 885 return rc; 886 zdev->state = ZPCI_FN_STATE_STANDBY; 887 888 return 0; 889 } 890 891 /** 892 * zpci_device_reserved() - Mark device as resverved 893 * @zdev: the zpci_dev that was reserved 894 * 895 * Handle the case that a given zPCI function was reserved by another system. 896 * After a call to this function the zpci_dev can not be found via 897 * get_zdev_by_fid() anymore but may still be accessible via existing 898 * references though it will not be functional anymore. 899 */ 900 void zpci_device_reserved(struct zpci_dev *zdev) 901 { 902 if (zdev->has_hp_slot) 903 zpci_exit_slot(zdev); 904 /* 905 * Remove device from zpci_list as it is going away. This also 906 * makes sure we ignore subsequent zPCI events for this device. 907 */ 908 spin_lock(&zpci_list_lock); 909 list_del(&zdev->entry); 910 spin_unlock(&zpci_list_lock); 911 zdev->state = ZPCI_FN_STATE_RESERVED; 912 zpci_dbg(3, "rsv fid:%x\n", zdev->fid); 913 zpci_zdev_put(zdev); 914 } 915 916 void zpci_release_device(struct kref *kref) 917 { 918 struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); 919 int ret; 920 921 if (zdev->zbus->bus) 922 zpci_bus_remove_device(zdev, false); 923 924 if (zdev_enabled(zdev)) 925 zpci_disable_device(zdev); 926 927 switch (zdev->state) { 928 case ZPCI_FN_STATE_CONFIGURED: 929 ret = sclp_pci_deconfigure(zdev->fid); 930 zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret); 931 fallthrough; 932 case ZPCI_FN_STATE_STANDBY: 933 if (zdev->has_hp_slot) 934 zpci_exit_slot(zdev); 935 spin_lock(&zpci_list_lock); 936 list_del(&zdev->entry); 937 spin_unlock(&zpci_list_lock); 938 zpci_dbg(3, "rsv fid:%x\n", zdev->fid); 939 fallthrough; 940 case ZPCI_FN_STATE_RESERVED: 941 if (zdev->has_resources) 942 zpci_cleanup_bus_resources(zdev); 943 zpci_bus_device_unregister(zdev); 944 zpci_destroy_iommu(zdev); 945 fallthrough; 946 default: 947 break; 948 } 949 zpci_dbg(3, "rem fid:%x\n", zdev->fid); 950 kfree_rcu(zdev, rcu); 951 } 952 953 int zpci_report_error(struct pci_dev *pdev, 954 struct zpci_report_error_header *report) 955 { 956 struct zpci_dev *zdev = to_zpci(pdev); 957 958 return sclp_pci_report(report, zdev->fh, zdev->fid); 959 } 960 EXPORT_SYMBOL(zpci_report_error); 961 962 /** 963 * zpci_clear_error_state() - Clears the zPCI error state of the device 964 * @zdev: The zdev for which the zPCI error state should be reset 965 * 966 * Clear the zPCI error state of the device. If clearing the zPCI error state 967 * fails the device is left in the error state. In this case it may make sense 968 * to call zpci_io_perm_failure() on the associated pdev if it exists. 969 * 970 * Returns: 0 on success, -EIO otherwise 971 */ 972 int zpci_clear_error_state(struct zpci_dev *zdev) 973 { 974 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR); 975 struct zpci_fib fib = {0}; 976 u8 status; 977 int cc; 978 979 cc = zpci_mod_fc(req, &fib, &status); 980 if (cc) { 981 zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 982 return -EIO; 983 } 984 985 return 0; 986 } 987 988 /** 989 * zpci_reset_load_store_blocked() - Re-enables L/S from error state 990 * @zdev: The zdev for which to unblock load/store access 991 * 992 * Re-enables load/store access for a PCI function in the error state while 993 * keeping DMA blocked. In this state drivers can poke MMIO space to determine 994 * if error recovery is possible while catching any rogue DMA access from the 995 * device. 996 * 997 * Returns: 0 on success, -EIO otherwise 998 */ 999 int zpci_reset_load_store_blocked(struct zpci_dev *zdev) 1000 { 1001 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK); 1002 struct zpci_fib fib = {0}; 1003 u8 status; 1004 int cc; 1005 1006 cc = zpci_mod_fc(req, &fib, &status); 1007 if (cc) { 1008 zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 1009 return -EIO; 1010 } 1011 1012 return 0; 1013 } 1014 1015 static int zpci_mem_init(void) 1016 { 1017 BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || 1018 __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); 1019 1020 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 1021 __alignof__(struct zpci_fmb), 0, NULL); 1022 if (!zdev_fmb_cache) 1023 goto error_fmb; 1024 1025 zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, 1026 sizeof(*zpci_iomap_start), GFP_KERNEL); 1027 if (!zpci_iomap_start) 1028 goto error_iomap; 1029 1030 zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), 1031 sizeof(*zpci_iomap_bitmap), GFP_KERNEL); 1032 if (!zpci_iomap_bitmap) 1033 goto error_iomap_bitmap; 1034 1035 if (static_branch_likely(&have_mio)) 1036 clp_setup_writeback_mio(); 1037 1038 return 0; 1039 error_iomap_bitmap: 1040 kfree(zpci_iomap_start); 1041 error_iomap: 1042 kmem_cache_destroy(zdev_fmb_cache); 1043 error_fmb: 1044 return -ENOMEM; 1045 } 1046 1047 static void zpci_mem_exit(void) 1048 { 1049 kfree(zpci_iomap_bitmap); 1050 kfree(zpci_iomap_start); 1051 kmem_cache_destroy(zdev_fmb_cache); 1052 } 1053 1054 static unsigned int s390_pci_probe __initdata = 1; 1055 unsigned int s390_pci_force_floating __initdata; 1056 static unsigned int s390_pci_initialized; 1057 1058 char * __init pcibios_setup(char *str) 1059 { 1060 if (!strcmp(str, "off")) { 1061 s390_pci_probe = 0; 1062 return NULL; 1063 } 1064 if (!strcmp(str, "nomio")) { 1065 S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO; 1066 return NULL; 1067 } 1068 if (!strcmp(str, "force_floating")) { 1069 s390_pci_force_floating = 1; 1070 return NULL; 1071 } 1072 if (!strcmp(str, "norid")) { 1073 s390_pci_no_rid = 1; 1074 return NULL; 1075 } 1076 return str; 1077 } 1078 1079 bool zpci_is_enabled(void) 1080 { 1081 return s390_pci_initialized; 1082 } 1083 1084 static int __init pci_base_init(void) 1085 { 1086 int rc; 1087 1088 if (!s390_pci_probe) 1089 return 0; 1090 1091 if (!test_facility(69) || !test_facility(71)) { 1092 pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); 1093 return 0; 1094 } 1095 1096 if (MACHINE_HAS_PCI_MIO) { 1097 static_branch_enable(&have_mio); 1098 system_ctl_set_bit(2, CR2_MIO_ADDRESSING_BIT); 1099 } 1100 1101 rc = zpci_debug_init(); 1102 if (rc) 1103 goto out; 1104 1105 rc = zpci_mem_init(); 1106 if (rc) 1107 goto out_mem; 1108 1109 rc = zpci_irq_init(); 1110 if (rc) 1111 goto out_irq; 1112 1113 rc = clp_scan_pci_devices(); 1114 if (rc) 1115 goto out_find; 1116 zpci_bus_scan_busses(); 1117 1118 s390_pci_initialized = 1; 1119 return 0; 1120 1121 out_find: 1122 zpci_irq_exit(); 1123 out_irq: 1124 zpci_mem_exit(); 1125 out_mem: 1126 zpci_debug_exit(); 1127 out: 1128 return rc; 1129 } 1130 subsys_initcall_sync(pci_base_init); 1131