1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright IBM Corp. 2012 4 * 5 * Author(s): 6 * Jan Glauber <jang@linux.vnet.ibm.com> 7 * 8 * The System z PCI code is a rewrite from a prototype by 9 * the following people (Kudoz!): 10 * Alexander Schmidt 11 * Christoph Raisch 12 * Hannes Hering 13 * Hoang-Nam Nguyen 14 * Jan-Bernd Themann 15 * Stefan Roscher 16 * Thomas Klein 17 */ 18 19 #define KMSG_COMPONENT "zpci" 20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 21 22 #include <linux/kernel.h> 23 #include <linux/slab.h> 24 #include <linux/err.h> 25 #include <linux/export.h> 26 #include <linux/delay.h> 27 #include <linux/seq_file.h> 28 #include <linux/jump_label.h> 29 #include <linux/pci.h> 30 #include <linux/printk.h> 31 32 #include <asm/isc.h> 33 #include <asm/airq.h> 34 #include <asm/facility.h> 35 #include <asm/pci_insn.h> 36 #include <asm/pci_clp.h> 37 #include <asm/pci_dma.h> 38 39 #include "pci_bus.h" 40 #include "pci_iov.h" 41 42 /* list of all detected zpci devices */ 43 static LIST_HEAD(zpci_list); 44 static DEFINE_SPINLOCK(zpci_list_lock); 45 46 static DECLARE_BITMAP(zpci_domain, ZPCI_DOMAIN_BITMAP_SIZE); 47 static DEFINE_SPINLOCK(zpci_domain_lock); 48 49 #define ZPCI_IOMAP_ENTRIES \ 50 min(((unsigned long) ZPCI_NR_DEVICES * PCI_STD_NUM_BARS / 2), \ 51 ZPCI_IOMAP_MAX_ENTRIES) 52 53 unsigned int s390_pci_no_rid; 54 55 static DEFINE_SPINLOCK(zpci_iomap_lock); 56 static unsigned long *zpci_iomap_bitmap; 57 struct zpci_iomap_entry *zpci_iomap_start; 58 EXPORT_SYMBOL_GPL(zpci_iomap_start); 59 60 DEFINE_STATIC_KEY_FALSE(have_mio); 61 62 static struct kmem_cache *zdev_fmb_cache; 63 64 /* AEN structures that must be preserved over KVM module re-insertion */ 65 union zpci_sic_iib *zpci_aipb; 66 EXPORT_SYMBOL_GPL(zpci_aipb); 67 struct airq_iv *zpci_aif_sbv; 68 EXPORT_SYMBOL_GPL(zpci_aif_sbv); 69 70 struct zpci_dev *get_zdev_by_fid(u32 fid) 71 { 72 struct zpci_dev *tmp, *zdev = NULL; 73 74 spin_lock(&zpci_list_lock); 75 list_for_each_entry(tmp, &zpci_list, entry) { 76 if (tmp->fid == fid) { 77 zdev = tmp; 78 zpci_zdev_get(zdev); 79 break; 80 } 81 } 82 spin_unlock(&zpci_list_lock); 83 return zdev; 84 } 85 86 void zpci_remove_reserved_devices(void) 87 { 88 struct zpci_dev *tmp, *zdev; 89 enum zpci_state state; 90 LIST_HEAD(remove); 91 92 spin_lock(&zpci_list_lock); 93 list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) { 94 if (zdev->state == ZPCI_FN_STATE_STANDBY && 95 !clp_get_state(zdev->fid, &state) && 96 state == ZPCI_FN_STATE_RESERVED) 97 list_move_tail(&zdev->entry, &remove); 98 } 99 spin_unlock(&zpci_list_lock); 100 101 list_for_each_entry_safe(zdev, tmp, &remove, entry) 102 zpci_device_reserved(zdev); 103 } 104 105 int pci_domain_nr(struct pci_bus *bus) 106 { 107 return ((struct zpci_bus *) bus->sysdata)->domain_nr; 108 } 109 EXPORT_SYMBOL_GPL(pci_domain_nr); 110 111 int pci_proc_domain(struct pci_bus *bus) 112 { 113 return pci_domain_nr(bus); 114 } 115 EXPORT_SYMBOL_GPL(pci_proc_domain); 116 117 /* Modify PCI: Register I/O address translation parameters */ 118 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas, 119 u64 base, u64 limit, u64 iota, u8 *status) 120 { 121 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT); 122 struct zpci_fib fib = {0}; 123 u8 cc; 124 125 WARN_ON_ONCE(iota & 0x3fff); 126 fib.pba = base; 127 fib.pal = limit; 128 fib.iota = iota | ZPCI_IOTA_RTTO_FLAG; 129 fib.gd = zdev->gisa; 130 cc = zpci_mod_fc(req, &fib, status); 131 if (cc) 132 zpci_dbg(3, "reg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, *status); 133 return cc; 134 } 135 EXPORT_SYMBOL_GPL(zpci_register_ioat); 136 137 /* Modify PCI: Unregister I/O address translation parameters */ 138 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas) 139 { 140 u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT); 141 struct zpci_fib fib = {0}; 142 u8 cc, status; 143 144 fib.gd = zdev->gisa; 145 146 cc = zpci_mod_fc(req, &fib, &status); 147 if (cc) 148 zpci_dbg(3, "unreg ioat fid:%x, cc:%d, status:%d\n", zdev->fid, cc, status); 149 return cc; 150 } 151 152 /* Modify PCI: Set PCI function measurement parameters */ 153 int zpci_fmb_enable_device(struct zpci_dev *zdev) 154 { 155 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 156 struct zpci_fib fib = {0}; 157 u8 cc, status; 158 159 if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length) 160 return -EINVAL; 161 162 zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL); 163 if (!zdev->fmb) 164 return -ENOMEM; 165 WARN_ON((u64) zdev->fmb & 0xf); 166 167 /* reset software counters */ 168 atomic64_set(&zdev->allocated_pages, 0); 169 atomic64_set(&zdev->mapped_pages, 0); 170 atomic64_set(&zdev->unmapped_pages, 0); 171 172 fib.fmb_addr = virt_to_phys(zdev->fmb); 173 fib.gd = zdev->gisa; 174 cc = zpci_mod_fc(req, &fib, &status); 175 if (cc) { 176 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 177 zdev->fmb = NULL; 178 } 179 return cc ? -EIO : 0; 180 } 181 182 /* Modify PCI: Disable PCI function measurement */ 183 int zpci_fmb_disable_device(struct zpci_dev *zdev) 184 { 185 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE); 186 struct zpci_fib fib = {0}; 187 u8 cc, status; 188 189 if (!zdev->fmb) 190 return -EINVAL; 191 192 fib.gd = zdev->gisa; 193 194 /* Function measurement is disabled if fmb address is zero */ 195 cc = zpci_mod_fc(req, &fib, &status); 196 if (cc == 3) /* Function already gone. */ 197 cc = 0; 198 199 if (!cc) { 200 kmem_cache_free(zdev_fmb_cache, zdev->fmb); 201 zdev->fmb = NULL; 202 } 203 return cc ? -EIO : 0; 204 } 205 206 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len) 207 { 208 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 209 u64 data; 210 int rc; 211 212 rc = __zpci_load(&data, req, offset); 213 if (!rc) { 214 data = le64_to_cpu((__force __le64) data); 215 data >>= (8 - len) * 8; 216 *val = (u32) data; 217 } else 218 *val = 0xffffffff; 219 return rc; 220 } 221 222 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len) 223 { 224 u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len); 225 u64 data = val; 226 int rc; 227 228 data <<= (8 - len) * 8; 229 data = (__force u64) cpu_to_le64(data); 230 rc = __zpci_store(data, req, offset); 231 return rc; 232 } 233 234 resource_size_t pcibios_align_resource(void *data, const struct resource *res, 235 resource_size_t size, 236 resource_size_t align) 237 { 238 return 0; 239 } 240 241 /* combine single writes by using store-block insn */ 242 void __iowrite64_copy(void __iomem *to, const void *from, size_t count) 243 { 244 zpci_memcpy_toio(to, from, count); 245 } 246 247 static void __iomem *__ioremap(phys_addr_t addr, size_t size, pgprot_t prot) 248 { 249 unsigned long offset, vaddr; 250 struct vm_struct *area; 251 phys_addr_t last_addr; 252 253 last_addr = addr + size - 1; 254 if (!size || last_addr < addr) 255 return NULL; 256 257 if (!static_branch_unlikely(&have_mio)) 258 return (void __iomem *) addr; 259 260 offset = addr & ~PAGE_MASK; 261 addr &= PAGE_MASK; 262 size = PAGE_ALIGN(size + offset); 263 area = get_vm_area(size, VM_IOREMAP); 264 if (!area) 265 return NULL; 266 267 vaddr = (unsigned long) area->addr; 268 if (ioremap_page_range(vaddr, vaddr + size, addr, prot)) { 269 free_vm_area(area); 270 return NULL; 271 } 272 return (void __iomem *) ((unsigned long) area->addr + offset); 273 } 274 275 void __iomem *ioremap_prot(phys_addr_t addr, size_t size, unsigned long prot) 276 { 277 return __ioremap(addr, size, __pgprot(prot)); 278 } 279 EXPORT_SYMBOL(ioremap_prot); 280 281 void __iomem *ioremap(phys_addr_t addr, size_t size) 282 { 283 return __ioremap(addr, size, PAGE_KERNEL); 284 } 285 EXPORT_SYMBOL(ioremap); 286 287 void __iomem *ioremap_wc(phys_addr_t addr, size_t size) 288 { 289 return __ioremap(addr, size, pgprot_writecombine(PAGE_KERNEL)); 290 } 291 EXPORT_SYMBOL(ioremap_wc); 292 293 void __iomem *ioremap_wt(phys_addr_t addr, size_t size) 294 { 295 return __ioremap(addr, size, pgprot_writethrough(PAGE_KERNEL)); 296 } 297 EXPORT_SYMBOL(ioremap_wt); 298 299 void iounmap(volatile void __iomem *addr) 300 { 301 if (static_branch_likely(&have_mio)) 302 vunmap((__force void *) ((unsigned long) addr & PAGE_MASK)); 303 } 304 EXPORT_SYMBOL(iounmap); 305 306 /* Create a virtual mapping cookie for a PCI BAR */ 307 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar, 308 unsigned long offset, unsigned long max) 309 { 310 struct zpci_dev *zdev = to_zpci(pdev); 311 int idx; 312 313 idx = zdev->bars[bar].map_idx; 314 spin_lock(&zpci_iomap_lock); 315 /* Detect overrun */ 316 WARN_ON(!++zpci_iomap_start[idx].count); 317 zpci_iomap_start[idx].fh = zdev->fh; 318 zpci_iomap_start[idx].bar = bar; 319 spin_unlock(&zpci_iomap_lock); 320 321 return (void __iomem *) ZPCI_ADDR(idx) + offset; 322 } 323 324 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar, 325 unsigned long offset, 326 unsigned long max) 327 { 328 unsigned long barsize = pci_resource_len(pdev, bar); 329 struct zpci_dev *zdev = to_zpci(pdev); 330 void __iomem *iova; 331 332 iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize); 333 return iova ? iova + offset : iova; 334 } 335 336 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar, 337 unsigned long offset, unsigned long max) 338 { 339 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 340 return NULL; 341 342 if (static_branch_likely(&have_mio)) 343 return pci_iomap_range_mio(pdev, bar, offset, max); 344 else 345 return pci_iomap_range_fh(pdev, bar, offset, max); 346 } 347 EXPORT_SYMBOL(pci_iomap_range); 348 349 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen) 350 { 351 return pci_iomap_range(dev, bar, 0, maxlen); 352 } 353 EXPORT_SYMBOL(pci_iomap); 354 355 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar, 356 unsigned long offset, unsigned long max) 357 { 358 unsigned long barsize = pci_resource_len(pdev, bar); 359 struct zpci_dev *zdev = to_zpci(pdev); 360 void __iomem *iova; 361 362 iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize); 363 return iova ? iova + offset : iova; 364 } 365 366 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar, 367 unsigned long offset, unsigned long max) 368 { 369 if (bar >= PCI_STD_NUM_BARS || !pci_resource_len(pdev, bar)) 370 return NULL; 371 372 if (static_branch_likely(&have_mio)) 373 return pci_iomap_wc_range_mio(pdev, bar, offset, max); 374 else 375 return pci_iomap_range_fh(pdev, bar, offset, max); 376 } 377 EXPORT_SYMBOL(pci_iomap_wc_range); 378 379 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen) 380 { 381 return pci_iomap_wc_range(dev, bar, 0, maxlen); 382 } 383 EXPORT_SYMBOL(pci_iomap_wc); 384 385 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr) 386 { 387 unsigned int idx = ZPCI_IDX(addr); 388 389 spin_lock(&zpci_iomap_lock); 390 /* Detect underrun */ 391 WARN_ON(!zpci_iomap_start[idx].count); 392 if (!--zpci_iomap_start[idx].count) { 393 zpci_iomap_start[idx].fh = 0; 394 zpci_iomap_start[idx].bar = 0; 395 } 396 spin_unlock(&zpci_iomap_lock); 397 } 398 399 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr) 400 { 401 iounmap(addr); 402 } 403 404 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr) 405 { 406 if (static_branch_likely(&have_mio)) 407 pci_iounmap_mio(pdev, addr); 408 else 409 pci_iounmap_fh(pdev, addr); 410 } 411 EXPORT_SYMBOL(pci_iounmap); 412 413 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, 414 int size, u32 *val) 415 { 416 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 417 418 return (zdev) ? zpci_cfg_load(zdev, where, val, size) : -ENODEV; 419 } 420 421 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, 422 int size, u32 val) 423 { 424 struct zpci_dev *zdev = zdev_from_bus(bus, devfn); 425 426 return (zdev) ? zpci_cfg_store(zdev, where, val, size) : -ENODEV; 427 } 428 429 static struct pci_ops pci_root_ops = { 430 .read = pci_read, 431 .write = pci_write, 432 }; 433 434 static void zpci_map_resources(struct pci_dev *pdev) 435 { 436 struct zpci_dev *zdev = to_zpci(pdev); 437 resource_size_t len; 438 int i; 439 440 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 441 len = pci_resource_len(pdev, i); 442 if (!len) 443 continue; 444 445 if (zpci_use_mio(zdev)) 446 pdev->resource[i].start = 447 (resource_size_t __force) zdev->bars[i].mio_wt; 448 else 449 pdev->resource[i].start = (resource_size_t __force) 450 pci_iomap_range_fh(pdev, i, 0, 0); 451 pdev->resource[i].end = pdev->resource[i].start + len - 1; 452 } 453 454 zpci_iov_map_resources(pdev); 455 } 456 457 static void zpci_unmap_resources(struct pci_dev *pdev) 458 { 459 struct zpci_dev *zdev = to_zpci(pdev); 460 resource_size_t len; 461 int i; 462 463 if (zpci_use_mio(zdev)) 464 return; 465 466 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 467 len = pci_resource_len(pdev, i); 468 if (!len) 469 continue; 470 pci_iounmap_fh(pdev, (void __iomem __force *) 471 pdev->resource[i].start); 472 } 473 } 474 475 static int zpci_alloc_iomap(struct zpci_dev *zdev) 476 { 477 unsigned long entry; 478 479 spin_lock(&zpci_iomap_lock); 480 entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES); 481 if (entry == ZPCI_IOMAP_ENTRIES) { 482 spin_unlock(&zpci_iomap_lock); 483 return -ENOSPC; 484 } 485 set_bit(entry, zpci_iomap_bitmap); 486 spin_unlock(&zpci_iomap_lock); 487 return entry; 488 } 489 490 static void zpci_free_iomap(struct zpci_dev *zdev, int entry) 491 { 492 spin_lock(&zpci_iomap_lock); 493 memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry)); 494 clear_bit(entry, zpci_iomap_bitmap); 495 spin_unlock(&zpci_iomap_lock); 496 } 497 498 static void zpci_do_update_iomap_fh(struct zpci_dev *zdev, u32 fh) 499 { 500 int bar, idx; 501 502 spin_lock(&zpci_iomap_lock); 503 for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) { 504 if (!zdev->bars[bar].size) 505 continue; 506 idx = zdev->bars[bar].map_idx; 507 if (!zpci_iomap_start[idx].count) 508 continue; 509 WRITE_ONCE(zpci_iomap_start[idx].fh, zdev->fh); 510 } 511 spin_unlock(&zpci_iomap_lock); 512 } 513 514 void zpci_update_fh(struct zpci_dev *zdev, u32 fh) 515 { 516 if (!fh || zdev->fh == fh) 517 return; 518 519 zdev->fh = fh; 520 if (zpci_use_mio(zdev)) 521 return; 522 if (zdev->has_resources && zdev_enabled(zdev)) 523 zpci_do_update_iomap_fh(zdev, fh); 524 } 525 526 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start, 527 unsigned long size, unsigned long flags) 528 { 529 struct resource *r; 530 531 r = kzalloc(sizeof(*r), GFP_KERNEL); 532 if (!r) 533 return NULL; 534 535 r->start = start; 536 r->end = r->start + size - 1; 537 r->flags = flags; 538 r->name = zdev->res_name; 539 540 if (request_resource(&iomem_resource, r)) { 541 kfree(r); 542 return NULL; 543 } 544 return r; 545 } 546 547 int zpci_setup_bus_resources(struct zpci_dev *zdev) 548 { 549 unsigned long addr, size, flags; 550 struct resource *res; 551 int i, entry; 552 553 snprintf(zdev->res_name, sizeof(zdev->res_name), 554 "PCI Bus %04x:%02x", zdev->uid, ZPCI_BUS_NR); 555 556 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 557 if (!zdev->bars[i].size) 558 continue; 559 entry = zpci_alloc_iomap(zdev); 560 if (entry < 0) 561 return entry; 562 zdev->bars[i].map_idx = entry; 563 564 /* only MMIO is supported */ 565 flags = IORESOURCE_MEM; 566 if (zdev->bars[i].val & 8) 567 flags |= IORESOURCE_PREFETCH; 568 if (zdev->bars[i].val & 4) 569 flags |= IORESOURCE_MEM_64; 570 571 if (zpci_use_mio(zdev)) 572 addr = (unsigned long) zdev->bars[i].mio_wt; 573 else 574 addr = ZPCI_ADDR(entry); 575 size = 1UL << zdev->bars[i].size; 576 577 res = __alloc_res(zdev, addr, size, flags); 578 if (!res) { 579 zpci_free_iomap(zdev, entry); 580 return -ENOMEM; 581 } 582 zdev->bars[i].res = res; 583 } 584 zdev->has_resources = 1; 585 586 return 0; 587 } 588 589 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev) 590 { 591 struct resource *res; 592 int i; 593 594 pci_lock_rescan_remove(); 595 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 596 res = zdev->bars[i].res; 597 if (!res) 598 continue; 599 600 release_resource(res); 601 pci_bus_remove_resource(zdev->zbus->bus, res); 602 zpci_free_iomap(zdev, zdev->bars[i].map_idx); 603 zdev->bars[i].res = NULL; 604 kfree(res); 605 } 606 zdev->has_resources = 0; 607 pci_unlock_rescan_remove(); 608 } 609 610 int pcibios_device_add(struct pci_dev *pdev) 611 { 612 struct zpci_dev *zdev = to_zpci(pdev); 613 struct resource *res; 614 int i; 615 616 /* The pdev has a reference to the zdev via its bus */ 617 zpci_zdev_get(zdev); 618 if (pdev->is_physfn) 619 pdev->no_vf_scan = 1; 620 621 pdev->dev.groups = zpci_attr_groups; 622 pdev->dev.dma_ops = &s390_pci_dma_ops; 623 zpci_map_resources(pdev); 624 625 for (i = 0; i < PCI_STD_NUM_BARS; i++) { 626 res = &pdev->resource[i]; 627 if (res->parent || !res->flags) 628 continue; 629 pci_claim_resource(pdev, i); 630 } 631 632 return 0; 633 } 634 635 void pcibios_release_device(struct pci_dev *pdev) 636 { 637 struct zpci_dev *zdev = to_zpci(pdev); 638 639 zpci_unmap_resources(pdev); 640 zpci_zdev_put(zdev); 641 } 642 643 int pcibios_enable_device(struct pci_dev *pdev, int mask) 644 { 645 struct zpci_dev *zdev = to_zpci(pdev); 646 647 zpci_debug_init_device(zdev, dev_name(&pdev->dev)); 648 zpci_fmb_enable_device(zdev); 649 650 return pci_enable_resources(pdev, mask); 651 } 652 653 void pcibios_disable_device(struct pci_dev *pdev) 654 { 655 struct zpci_dev *zdev = to_zpci(pdev); 656 657 zpci_fmb_disable_device(zdev); 658 zpci_debug_exit_device(zdev); 659 } 660 661 static int __zpci_register_domain(int domain) 662 { 663 spin_lock(&zpci_domain_lock); 664 if (test_bit(domain, zpci_domain)) { 665 spin_unlock(&zpci_domain_lock); 666 pr_err("Domain %04x is already assigned\n", domain); 667 return -EEXIST; 668 } 669 set_bit(domain, zpci_domain); 670 spin_unlock(&zpci_domain_lock); 671 return domain; 672 } 673 674 static int __zpci_alloc_domain(void) 675 { 676 int domain; 677 678 spin_lock(&zpci_domain_lock); 679 /* 680 * We can always auto allocate domains below ZPCI_NR_DEVICES. 681 * There is either a free domain or we have reached the maximum in 682 * which case we would have bailed earlier. 683 */ 684 domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES); 685 set_bit(domain, zpci_domain); 686 spin_unlock(&zpci_domain_lock); 687 return domain; 688 } 689 690 int zpci_alloc_domain(int domain) 691 { 692 if (zpci_unique_uid) { 693 if (domain) 694 return __zpci_register_domain(domain); 695 pr_warn("UID checking was active but no UID is provided: switching to automatic domain allocation\n"); 696 update_uid_checking(false); 697 } 698 return __zpci_alloc_domain(); 699 } 700 701 void zpci_free_domain(int domain) 702 { 703 spin_lock(&zpci_domain_lock); 704 clear_bit(domain, zpci_domain); 705 spin_unlock(&zpci_domain_lock); 706 } 707 708 709 int zpci_enable_device(struct zpci_dev *zdev) 710 { 711 u32 fh = zdev->fh; 712 int rc = 0; 713 714 if (clp_enable_fh(zdev, &fh, ZPCI_NR_DMA_SPACES)) 715 rc = -EIO; 716 else 717 zpci_update_fh(zdev, fh); 718 return rc; 719 } 720 EXPORT_SYMBOL_GPL(zpci_enable_device); 721 722 int zpci_disable_device(struct zpci_dev *zdev) 723 { 724 u32 fh = zdev->fh; 725 int cc, rc = 0; 726 727 cc = clp_disable_fh(zdev, &fh); 728 if (!cc) { 729 zpci_update_fh(zdev, fh); 730 } else if (cc == CLP_RC_SETPCIFN_ALRDY) { 731 pr_info("Disabling PCI function %08x had no effect as it was already disabled\n", 732 zdev->fid); 733 /* Function is already disabled - update handle */ 734 rc = clp_refresh_fh(zdev->fid, &fh); 735 if (!rc) { 736 zpci_update_fh(zdev, fh); 737 rc = -EINVAL; 738 } 739 } else { 740 rc = -EIO; 741 } 742 return rc; 743 } 744 EXPORT_SYMBOL_GPL(zpci_disable_device); 745 746 /** 747 * zpci_hot_reset_device - perform a reset of the given zPCI function 748 * @zdev: the slot which should be reset 749 * 750 * Performs a low level reset of the zPCI function. The reset is low level in 751 * the sense that the zPCI function can be reset without detaching it from the 752 * common PCI subsystem. The reset may be performed while under control of 753 * either DMA or IOMMU APIs in which case the existing DMA/IOMMU translation 754 * table is reinstated at the end of the reset. 755 * 756 * After the reset the functions internal state is reset to an initial state 757 * equivalent to its state during boot when first probing a driver. 758 * Consequently after reset the PCI function requires re-initialization via the 759 * common PCI code including re-enabling IRQs via pci_alloc_irq_vectors() 760 * and enabling the function via e.g.pci_enablde_device_flags().The caller 761 * must guard against concurrent reset attempts. 762 * 763 * In most cases this function should not be called directly but through 764 * pci_reset_function() or pci_reset_bus() which handle the save/restore and 765 * locking. 766 * 767 * Return: 0 on success and an error value otherwise 768 */ 769 int zpci_hot_reset_device(struct zpci_dev *zdev) 770 { 771 u8 status; 772 int rc; 773 774 zpci_dbg(3, "rst fid:%x, fh:%x\n", zdev->fid, zdev->fh); 775 if (zdev_enabled(zdev)) { 776 /* Disables device access, DMAs and IRQs (reset state) */ 777 rc = zpci_disable_device(zdev); 778 /* 779 * Due to a z/VM vs LPAR inconsistency in the error state the 780 * FH may indicate an enabled device but disable says the 781 * device is already disabled don't treat it as an error here. 782 */ 783 if (rc == -EINVAL) 784 rc = 0; 785 if (rc) 786 return rc; 787 } 788 789 rc = zpci_enable_device(zdev); 790 if (rc) 791 return rc; 792 793 if (zdev->dma_table) 794 rc = zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma, 795 virt_to_phys(zdev->dma_table), &status); 796 else 797 rc = zpci_dma_init_device(zdev); 798 if (rc) { 799 zpci_disable_device(zdev); 800 return rc; 801 } 802 803 return 0; 804 } 805 806 /** 807 * zpci_create_device() - Create a new zpci_dev and add it to the zbus 808 * @fid: Function ID of the device to be created 809 * @fh: Current Function Handle of the device to be created 810 * @state: Initial state after creation either Standby or Configured 811 * 812 * Creates a new zpci device and adds it to its, possibly newly created, zbus 813 * as well as zpci_list. 814 * 815 * Returns: the zdev on success or an error pointer otherwise 816 */ 817 struct zpci_dev *zpci_create_device(u32 fid, u32 fh, enum zpci_state state) 818 { 819 struct zpci_dev *zdev; 820 int rc; 821 822 zpci_dbg(1, "add fid:%x, fh:%x, c:%d\n", fid, fh, state); 823 zdev = kzalloc(sizeof(*zdev), GFP_KERNEL); 824 if (!zdev) 825 return ERR_PTR(-ENOMEM); 826 827 /* FID and Function Handle are the static/dynamic identifiers */ 828 zdev->fid = fid; 829 zdev->fh = fh; 830 831 /* Query function properties and update zdev */ 832 rc = clp_query_pci_fn(zdev); 833 if (rc) 834 goto error; 835 zdev->state = state; 836 837 kref_init(&zdev->kref); 838 mutex_init(&zdev->lock); 839 mutex_init(&zdev->kzdev_lock); 840 841 rc = zpci_init_iommu(zdev); 842 if (rc) 843 goto error; 844 845 rc = zpci_bus_device_register(zdev, &pci_root_ops); 846 if (rc) 847 goto error_destroy_iommu; 848 849 spin_lock(&zpci_list_lock); 850 list_add_tail(&zdev->entry, &zpci_list); 851 spin_unlock(&zpci_list_lock); 852 853 return zdev; 854 855 error_destroy_iommu: 856 zpci_destroy_iommu(zdev); 857 error: 858 zpci_dbg(0, "add fid:%x, rc:%d\n", fid, rc); 859 kfree(zdev); 860 return ERR_PTR(rc); 861 } 862 863 bool zpci_is_device_configured(struct zpci_dev *zdev) 864 { 865 enum zpci_state state = zdev->state; 866 867 return state != ZPCI_FN_STATE_RESERVED && 868 state != ZPCI_FN_STATE_STANDBY; 869 } 870 871 /** 872 * zpci_scan_configured_device() - Scan a freshly configured zpci_dev 873 * @zdev: The zpci_dev to be configured 874 * @fh: The general function handle supplied by the platform 875 * 876 * Given a device in the configuration state Configured, enables, scans and 877 * adds it to the common code PCI subsystem if possible. If any failure occurs, 878 * the zpci_dev is left disabled. 879 * 880 * Return: 0 on success, or an error code otherwise 881 */ 882 int zpci_scan_configured_device(struct zpci_dev *zdev, u32 fh) 883 { 884 zpci_update_fh(zdev, fh); 885 return zpci_bus_scan_device(zdev); 886 } 887 888 /** 889 * zpci_deconfigure_device() - Deconfigure a zpci_dev 890 * @zdev: The zpci_dev to configure 891 * 892 * Deconfigure a zPCI function that is currently configured and possibly known 893 * to the common code PCI subsystem. 894 * If any failure occurs the device is left as is. 895 * 896 * Return: 0 on success, or an error code otherwise 897 */ 898 int zpci_deconfigure_device(struct zpci_dev *zdev) 899 { 900 int rc; 901 902 if (zdev->zbus->bus) 903 zpci_bus_remove_device(zdev, false); 904 905 if (zdev->dma_table) { 906 rc = zpci_dma_exit_device(zdev); 907 if (rc) 908 return rc; 909 } 910 if (zdev_enabled(zdev)) { 911 rc = zpci_disable_device(zdev); 912 if (rc) 913 return rc; 914 } 915 916 rc = sclp_pci_deconfigure(zdev->fid); 917 zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, rc); 918 if (rc) 919 return rc; 920 zdev->state = ZPCI_FN_STATE_STANDBY; 921 922 return 0; 923 } 924 925 /** 926 * zpci_device_reserved() - Mark device as resverved 927 * @zdev: the zpci_dev that was reserved 928 * 929 * Handle the case that a given zPCI function was reserved by another system. 930 * After a call to this function the zpci_dev can not be found via 931 * get_zdev_by_fid() anymore but may still be accessible via existing 932 * references though it will not be functional anymore. 933 */ 934 void zpci_device_reserved(struct zpci_dev *zdev) 935 { 936 if (zdev->has_hp_slot) 937 zpci_exit_slot(zdev); 938 /* 939 * Remove device from zpci_list as it is going away. This also 940 * makes sure we ignore subsequent zPCI events for this device. 941 */ 942 spin_lock(&zpci_list_lock); 943 list_del(&zdev->entry); 944 spin_unlock(&zpci_list_lock); 945 zdev->state = ZPCI_FN_STATE_RESERVED; 946 zpci_dbg(3, "rsv fid:%x\n", zdev->fid); 947 zpci_zdev_put(zdev); 948 } 949 950 void zpci_release_device(struct kref *kref) 951 { 952 struct zpci_dev *zdev = container_of(kref, struct zpci_dev, kref); 953 int ret; 954 955 if (zdev->zbus->bus) 956 zpci_bus_remove_device(zdev, false); 957 958 if (zdev->dma_table) 959 zpci_dma_exit_device(zdev); 960 if (zdev_enabled(zdev)) 961 zpci_disable_device(zdev); 962 963 switch (zdev->state) { 964 case ZPCI_FN_STATE_CONFIGURED: 965 ret = sclp_pci_deconfigure(zdev->fid); 966 zpci_dbg(3, "deconf fid:%x, rc:%d\n", zdev->fid, ret); 967 fallthrough; 968 case ZPCI_FN_STATE_STANDBY: 969 if (zdev->has_hp_slot) 970 zpci_exit_slot(zdev); 971 spin_lock(&zpci_list_lock); 972 list_del(&zdev->entry); 973 spin_unlock(&zpci_list_lock); 974 zpci_dbg(3, "rsv fid:%x\n", zdev->fid); 975 fallthrough; 976 case ZPCI_FN_STATE_RESERVED: 977 if (zdev->has_resources) 978 zpci_cleanup_bus_resources(zdev); 979 zpci_bus_device_unregister(zdev); 980 zpci_destroy_iommu(zdev); 981 fallthrough; 982 default: 983 break; 984 } 985 zpci_dbg(3, "rem fid:%x\n", zdev->fid); 986 kfree_rcu(zdev, rcu); 987 } 988 989 int zpci_report_error(struct pci_dev *pdev, 990 struct zpci_report_error_header *report) 991 { 992 struct zpci_dev *zdev = to_zpci(pdev); 993 994 return sclp_pci_report(report, zdev->fh, zdev->fid); 995 } 996 EXPORT_SYMBOL(zpci_report_error); 997 998 /** 999 * zpci_clear_error_state() - Clears the zPCI error state of the device 1000 * @zdev: The zdev for which the zPCI error state should be reset 1001 * 1002 * Clear the zPCI error state of the device. If clearing the zPCI error state 1003 * fails the device is left in the error state. In this case it may make sense 1004 * to call zpci_io_perm_failure() on the associated pdev if it exists. 1005 * 1006 * Returns: 0 on success, -EIO otherwise 1007 */ 1008 int zpci_clear_error_state(struct zpci_dev *zdev) 1009 { 1010 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_ERROR); 1011 struct zpci_fib fib = {0}; 1012 u8 status; 1013 int cc; 1014 1015 cc = zpci_mod_fc(req, &fib, &status); 1016 if (cc) { 1017 zpci_dbg(3, "ces fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 1018 return -EIO; 1019 } 1020 1021 return 0; 1022 } 1023 1024 /** 1025 * zpci_reset_load_store_blocked() - Re-enables L/S from error state 1026 * @zdev: The zdev for which to unblock load/store access 1027 * 1028 * Re-enables load/store access for a PCI function in the error state while 1029 * keeping DMA blocked. In this state drivers can poke MMIO space to determine 1030 * if error recovery is possible while catching any rogue DMA access from the 1031 * device. 1032 * 1033 * Returns: 0 on success, -EIO otherwise 1034 */ 1035 int zpci_reset_load_store_blocked(struct zpci_dev *zdev) 1036 { 1037 u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_RESET_BLOCK); 1038 struct zpci_fib fib = {0}; 1039 u8 status; 1040 int cc; 1041 1042 cc = zpci_mod_fc(req, &fib, &status); 1043 if (cc) { 1044 zpci_dbg(3, "rls fid:%x, cc:%d, status:%x\n", zdev->fid, cc, status); 1045 return -EIO; 1046 } 1047 1048 return 0; 1049 } 1050 1051 static int zpci_mem_init(void) 1052 { 1053 BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) || 1054 __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb)); 1055 1056 zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb), 1057 __alignof__(struct zpci_fmb), 0, NULL); 1058 if (!zdev_fmb_cache) 1059 goto error_fmb; 1060 1061 zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES, 1062 sizeof(*zpci_iomap_start), GFP_KERNEL); 1063 if (!zpci_iomap_start) 1064 goto error_iomap; 1065 1066 zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES), 1067 sizeof(*zpci_iomap_bitmap), GFP_KERNEL); 1068 if (!zpci_iomap_bitmap) 1069 goto error_iomap_bitmap; 1070 1071 if (static_branch_likely(&have_mio)) 1072 clp_setup_writeback_mio(); 1073 1074 return 0; 1075 error_iomap_bitmap: 1076 kfree(zpci_iomap_start); 1077 error_iomap: 1078 kmem_cache_destroy(zdev_fmb_cache); 1079 error_fmb: 1080 return -ENOMEM; 1081 } 1082 1083 static void zpci_mem_exit(void) 1084 { 1085 kfree(zpci_iomap_bitmap); 1086 kfree(zpci_iomap_start); 1087 kmem_cache_destroy(zdev_fmb_cache); 1088 } 1089 1090 static unsigned int s390_pci_probe __initdata = 1; 1091 unsigned int s390_pci_force_floating __initdata; 1092 static unsigned int s390_pci_initialized; 1093 1094 char * __init pcibios_setup(char *str) 1095 { 1096 if (!strcmp(str, "off")) { 1097 s390_pci_probe = 0; 1098 return NULL; 1099 } 1100 if (!strcmp(str, "nomio")) { 1101 S390_lowcore.machine_flags &= ~MACHINE_FLAG_PCI_MIO; 1102 return NULL; 1103 } 1104 if (!strcmp(str, "force_floating")) { 1105 s390_pci_force_floating = 1; 1106 return NULL; 1107 } 1108 if (!strcmp(str, "norid")) { 1109 s390_pci_no_rid = 1; 1110 return NULL; 1111 } 1112 return str; 1113 } 1114 1115 bool zpci_is_enabled(void) 1116 { 1117 return s390_pci_initialized; 1118 } 1119 1120 static int __init pci_base_init(void) 1121 { 1122 int rc; 1123 1124 if (!s390_pci_probe) 1125 return 0; 1126 1127 if (!test_facility(69) || !test_facility(71)) { 1128 pr_info("PCI is not supported because CPU facilities 69 or 71 are not available\n"); 1129 return 0; 1130 } 1131 1132 if (MACHINE_HAS_PCI_MIO) { 1133 static_branch_enable(&have_mio); 1134 ctl_set_bit(2, 5); 1135 } 1136 1137 rc = zpci_debug_init(); 1138 if (rc) 1139 goto out; 1140 1141 rc = zpci_mem_init(); 1142 if (rc) 1143 goto out_mem; 1144 1145 rc = zpci_irq_init(); 1146 if (rc) 1147 goto out_irq; 1148 1149 rc = zpci_dma_init(); 1150 if (rc) 1151 goto out_dma; 1152 1153 rc = clp_scan_pci_devices(); 1154 if (rc) 1155 goto out_find; 1156 zpci_bus_scan_busses(); 1157 1158 s390_pci_initialized = 1; 1159 return 0; 1160 1161 out_find: 1162 zpci_dma_exit(); 1163 out_dma: 1164 zpci_irq_exit(); 1165 out_irq: 1166 zpci_mem_exit(); 1167 out_mem: 1168 zpci_debug_exit(); 1169 out: 1170 return rc; 1171 } 1172 subsys_initcall_sync(pci_base_init); 1173