xref: /linux/arch/s390/pci/pci.c (revision ebf68996de0ab250c5d520eb2291ab65643e9a1e)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright IBM Corp. 2012
4  *
5  * Author(s):
6  *   Jan Glauber <jang@linux.vnet.ibm.com>
7  *
8  * The System z PCI code is a rewrite from a prototype by
9  * the following people (Kudoz!):
10  *   Alexander Schmidt
11  *   Christoph Raisch
12  *   Hannes Hering
13  *   Hoang-Nam Nguyen
14  *   Jan-Bernd Themann
15  *   Stefan Roscher
16  *   Thomas Klein
17  */
18 
19 #define KMSG_COMPONENT "zpci"
20 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
21 
22 #include <linux/kernel.h>
23 #include <linux/slab.h>
24 #include <linux/err.h>
25 #include <linux/export.h>
26 #include <linux/delay.h>
27 #include <linux/seq_file.h>
28 #include <linux/jump_label.h>
29 #include <linux/pci.h>
30 
31 #include <asm/isc.h>
32 #include <asm/airq.h>
33 #include <asm/facility.h>
34 #include <asm/pci_insn.h>
35 #include <asm/pci_clp.h>
36 #include <asm/pci_dma.h>
37 
38 /* list of all detected zpci devices */
39 static LIST_HEAD(zpci_list);
40 static DEFINE_SPINLOCK(zpci_list_lock);
41 
42 static DECLARE_BITMAP(zpci_domain, ZPCI_NR_DEVICES);
43 static DEFINE_SPINLOCK(zpci_domain_lock);
44 
45 #define ZPCI_IOMAP_ENTRIES						\
46 	min(((unsigned long) ZPCI_NR_DEVICES * PCI_BAR_COUNT / 2),	\
47 	    ZPCI_IOMAP_MAX_ENTRIES)
48 
49 static DEFINE_SPINLOCK(zpci_iomap_lock);
50 static unsigned long *zpci_iomap_bitmap;
51 struct zpci_iomap_entry *zpci_iomap_start;
52 EXPORT_SYMBOL_GPL(zpci_iomap_start);
53 
54 DEFINE_STATIC_KEY_FALSE(have_mio);
55 
56 static struct kmem_cache *zdev_fmb_cache;
57 
58 struct zpci_dev *get_zdev_by_fid(u32 fid)
59 {
60 	struct zpci_dev *tmp, *zdev = NULL;
61 
62 	spin_lock(&zpci_list_lock);
63 	list_for_each_entry(tmp, &zpci_list, entry) {
64 		if (tmp->fid == fid) {
65 			zdev = tmp;
66 			break;
67 		}
68 	}
69 	spin_unlock(&zpci_list_lock);
70 	return zdev;
71 }
72 
73 void zpci_remove_reserved_devices(void)
74 {
75 	struct zpci_dev *tmp, *zdev;
76 	enum zpci_state state;
77 	LIST_HEAD(remove);
78 
79 	spin_lock(&zpci_list_lock);
80 	list_for_each_entry_safe(zdev, tmp, &zpci_list, entry) {
81 		if (zdev->state == ZPCI_FN_STATE_STANDBY &&
82 		    !clp_get_state(zdev->fid, &state) &&
83 		    state == ZPCI_FN_STATE_RESERVED)
84 			list_move_tail(&zdev->entry, &remove);
85 	}
86 	spin_unlock(&zpci_list_lock);
87 
88 	list_for_each_entry_safe(zdev, tmp, &remove, entry)
89 		zpci_remove_device(zdev);
90 }
91 
92 static struct zpci_dev *get_zdev_by_bus(struct pci_bus *bus)
93 {
94 	return (bus && bus->sysdata) ? (struct zpci_dev *) bus->sysdata : NULL;
95 }
96 
97 int pci_domain_nr(struct pci_bus *bus)
98 {
99 	return ((struct zpci_dev *) bus->sysdata)->domain;
100 }
101 EXPORT_SYMBOL_GPL(pci_domain_nr);
102 
103 int pci_proc_domain(struct pci_bus *bus)
104 {
105 	return pci_domain_nr(bus);
106 }
107 EXPORT_SYMBOL_GPL(pci_proc_domain);
108 
109 /* Modify PCI: Register I/O address translation parameters */
110 int zpci_register_ioat(struct zpci_dev *zdev, u8 dmaas,
111 		       u64 base, u64 limit, u64 iota)
112 {
113 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_REG_IOAT);
114 	struct zpci_fib fib = {0};
115 	u8 status;
116 
117 	WARN_ON_ONCE(iota & 0x3fff);
118 	fib.pba = base;
119 	fib.pal = limit;
120 	fib.iota = iota | ZPCI_IOTA_RTTO_FLAG;
121 	return zpci_mod_fc(req, &fib, &status) ? -EIO : 0;
122 }
123 
124 /* Modify PCI: Unregister I/O address translation parameters */
125 int zpci_unregister_ioat(struct zpci_dev *zdev, u8 dmaas)
126 {
127 	u64 req = ZPCI_CREATE_REQ(zdev->fh, dmaas, ZPCI_MOD_FC_DEREG_IOAT);
128 	struct zpci_fib fib = {0};
129 	u8 cc, status;
130 
131 	cc = zpci_mod_fc(req, &fib, &status);
132 	if (cc == 3) /* Function already gone. */
133 		cc = 0;
134 	return cc ? -EIO : 0;
135 }
136 
137 /* Modify PCI: Set PCI function measurement parameters */
138 int zpci_fmb_enable_device(struct zpci_dev *zdev)
139 {
140 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
141 	struct zpci_fib fib = {0};
142 	u8 cc, status;
143 
144 	if (zdev->fmb || sizeof(*zdev->fmb) < zdev->fmb_length)
145 		return -EINVAL;
146 
147 	zdev->fmb = kmem_cache_zalloc(zdev_fmb_cache, GFP_KERNEL);
148 	if (!zdev->fmb)
149 		return -ENOMEM;
150 	WARN_ON((u64) zdev->fmb & 0xf);
151 
152 	/* reset software counters */
153 	atomic64_set(&zdev->allocated_pages, 0);
154 	atomic64_set(&zdev->mapped_pages, 0);
155 	atomic64_set(&zdev->unmapped_pages, 0);
156 
157 	fib.fmb_addr = virt_to_phys(zdev->fmb);
158 	cc = zpci_mod_fc(req, &fib, &status);
159 	if (cc) {
160 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
161 		zdev->fmb = NULL;
162 	}
163 	return cc ? -EIO : 0;
164 }
165 
166 /* Modify PCI: Disable PCI function measurement */
167 int zpci_fmb_disable_device(struct zpci_dev *zdev)
168 {
169 	u64 req = ZPCI_CREATE_REQ(zdev->fh, 0, ZPCI_MOD_FC_SET_MEASURE);
170 	struct zpci_fib fib = {0};
171 	u8 cc, status;
172 
173 	if (!zdev->fmb)
174 		return -EINVAL;
175 
176 	/* Function measurement is disabled if fmb address is zero */
177 	cc = zpci_mod_fc(req, &fib, &status);
178 	if (cc == 3) /* Function already gone. */
179 		cc = 0;
180 
181 	if (!cc) {
182 		kmem_cache_free(zdev_fmb_cache, zdev->fmb);
183 		zdev->fmb = NULL;
184 	}
185 	return cc ? -EIO : 0;
186 }
187 
188 static int zpci_cfg_load(struct zpci_dev *zdev, int offset, u32 *val, u8 len)
189 {
190 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
191 	u64 data;
192 	int rc;
193 
194 	rc = __zpci_load(&data, req, offset);
195 	if (!rc) {
196 		data = le64_to_cpu((__force __le64) data);
197 		data >>= (8 - len) * 8;
198 		*val = (u32) data;
199 	} else
200 		*val = 0xffffffff;
201 	return rc;
202 }
203 
204 static int zpci_cfg_store(struct zpci_dev *zdev, int offset, u32 val, u8 len)
205 {
206 	u64 req = ZPCI_CREATE_REQ(zdev->fh, ZPCI_PCIAS_CFGSPC, len);
207 	u64 data = val;
208 	int rc;
209 
210 	data <<= (8 - len) * 8;
211 	data = (__force u64) cpu_to_le64(data);
212 	rc = __zpci_store(data, req, offset);
213 	return rc;
214 }
215 
216 resource_size_t pcibios_align_resource(void *data, const struct resource *res,
217 				       resource_size_t size,
218 				       resource_size_t align)
219 {
220 	return 0;
221 }
222 
223 /* combine single writes by using store-block insn */
224 void __iowrite64_copy(void __iomem *to, const void *from, size_t count)
225 {
226        zpci_memcpy_toio(to, from, count);
227 }
228 
229 void __iomem *ioremap(unsigned long ioaddr, unsigned long size)
230 {
231 	struct vm_struct *area;
232 	unsigned long offset;
233 
234 	if (!size)
235 		return NULL;
236 
237 	if (!static_branch_unlikely(&have_mio))
238 		return (void __iomem *) ioaddr;
239 
240 	offset = ioaddr & ~PAGE_MASK;
241 	ioaddr &= PAGE_MASK;
242 	size = PAGE_ALIGN(size + offset);
243 	area = get_vm_area(size, VM_IOREMAP);
244 	if (!area)
245 		return NULL;
246 
247 	if (ioremap_page_range((unsigned long) area->addr,
248 			       (unsigned long) area->addr + size,
249 			       ioaddr, PAGE_KERNEL)) {
250 		vunmap(area->addr);
251 		return NULL;
252 	}
253 	return (void __iomem *) ((unsigned long) area->addr + offset);
254 }
255 EXPORT_SYMBOL(ioremap);
256 
257 void iounmap(volatile void __iomem *addr)
258 {
259 	if (static_branch_likely(&have_mio))
260 		vunmap((__force void *) ((unsigned long) addr & PAGE_MASK));
261 }
262 EXPORT_SYMBOL(iounmap);
263 
264 /* Create a virtual mapping cookie for a PCI BAR */
265 static void __iomem *pci_iomap_range_fh(struct pci_dev *pdev, int bar,
266 					unsigned long offset, unsigned long max)
267 {
268 	struct zpci_dev *zdev =	to_zpci(pdev);
269 	int idx;
270 
271 	idx = zdev->bars[bar].map_idx;
272 	spin_lock(&zpci_iomap_lock);
273 	/* Detect overrun */
274 	WARN_ON(!++zpci_iomap_start[idx].count);
275 	zpci_iomap_start[idx].fh = zdev->fh;
276 	zpci_iomap_start[idx].bar = bar;
277 	spin_unlock(&zpci_iomap_lock);
278 
279 	return (void __iomem *) ZPCI_ADDR(idx) + offset;
280 }
281 
282 static void __iomem *pci_iomap_range_mio(struct pci_dev *pdev, int bar,
283 					 unsigned long offset,
284 					 unsigned long max)
285 {
286 	unsigned long barsize = pci_resource_len(pdev, bar);
287 	struct zpci_dev *zdev = to_zpci(pdev);
288 	void __iomem *iova;
289 
290 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wt, barsize);
291 	return iova ? iova + offset : iova;
292 }
293 
294 void __iomem *pci_iomap_range(struct pci_dev *pdev, int bar,
295 			      unsigned long offset, unsigned long max)
296 {
297 	if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT)
298 		return NULL;
299 
300 	if (static_branch_likely(&have_mio))
301 		return pci_iomap_range_mio(pdev, bar, offset, max);
302 	else
303 		return pci_iomap_range_fh(pdev, bar, offset, max);
304 }
305 EXPORT_SYMBOL(pci_iomap_range);
306 
307 void __iomem *pci_iomap(struct pci_dev *dev, int bar, unsigned long maxlen)
308 {
309 	return pci_iomap_range(dev, bar, 0, maxlen);
310 }
311 EXPORT_SYMBOL(pci_iomap);
312 
313 static void __iomem *pci_iomap_wc_range_mio(struct pci_dev *pdev, int bar,
314 					    unsigned long offset, unsigned long max)
315 {
316 	unsigned long barsize = pci_resource_len(pdev, bar);
317 	struct zpci_dev *zdev = to_zpci(pdev);
318 	void __iomem *iova;
319 
320 	iova = ioremap((unsigned long) zdev->bars[bar].mio_wb, barsize);
321 	return iova ? iova + offset : iova;
322 }
323 
324 void __iomem *pci_iomap_wc_range(struct pci_dev *pdev, int bar,
325 				 unsigned long offset, unsigned long max)
326 {
327 	if (!pci_resource_len(pdev, bar) || bar >= PCI_BAR_COUNT)
328 		return NULL;
329 
330 	if (static_branch_likely(&have_mio))
331 		return pci_iomap_wc_range_mio(pdev, bar, offset, max);
332 	else
333 		return pci_iomap_range_fh(pdev, bar, offset, max);
334 }
335 EXPORT_SYMBOL(pci_iomap_wc_range);
336 
337 void __iomem *pci_iomap_wc(struct pci_dev *dev, int bar, unsigned long maxlen)
338 {
339 	return pci_iomap_wc_range(dev, bar, 0, maxlen);
340 }
341 EXPORT_SYMBOL(pci_iomap_wc);
342 
343 static void pci_iounmap_fh(struct pci_dev *pdev, void __iomem *addr)
344 {
345 	unsigned int idx = ZPCI_IDX(addr);
346 
347 	spin_lock(&zpci_iomap_lock);
348 	/* Detect underrun */
349 	WARN_ON(!zpci_iomap_start[idx].count);
350 	if (!--zpci_iomap_start[idx].count) {
351 		zpci_iomap_start[idx].fh = 0;
352 		zpci_iomap_start[idx].bar = 0;
353 	}
354 	spin_unlock(&zpci_iomap_lock);
355 }
356 
357 static void pci_iounmap_mio(struct pci_dev *pdev, void __iomem *addr)
358 {
359 	iounmap(addr);
360 }
361 
362 void pci_iounmap(struct pci_dev *pdev, void __iomem *addr)
363 {
364 	if (static_branch_likely(&have_mio))
365 		pci_iounmap_mio(pdev, addr);
366 	else
367 		pci_iounmap_fh(pdev, addr);
368 }
369 EXPORT_SYMBOL(pci_iounmap);
370 
371 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where,
372 		    int size, u32 *val)
373 {
374 	struct zpci_dev *zdev = get_zdev_by_bus(bus);
375 	int ret;
376 
377 	if (!zdev || devfn != ZPCI_DEVFN)
378 		ret = -ENODEV;
379 	else
380 		ret = zpci_cfg_load(zdev, where, val, size);
381 
382 	return ret;
383 }
384 
385 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where,
386 		     int size, u32 val)
387 {
388 	struct zpci_dev *zdev = get_zdev_by_bus(bus);
389 	int ret;
390 
391 	if (!zdev || devfn != ZPCI_DEVFN)
392 		ret = -ENODEV;
393 	else
394 		ret = zpci_cfg_store(zdev, where, val, size);
395 
396 	return ret;
397 }
398 
399 static struct pci_ops pci_root_ops = {
400 	.read = pci_read,
401 	.write = pci_write,
402 };
403 
404 #ifdef CONFIG_PCI_IOV
405 static struct resource iov_res = {
406 	.name	= "PCI IOV res",
407 	.start	= 0,
408 	.end	= -1,
409 	.flags	= IORESOURCE_MEM,
410 };
411 #endif
412 
413 static void zpci_map_resources(struct pci_dev *pdev)
414 {
415 	struct zpci_dev *zdev = to_zpci(pdev);
416 	resource_size_t len;
417 	int i;
418 
419 	for (i = 0; i < PCI_BAR_COUNT; i++) {
420 		len = pci_resource_len(pdev, i);
421 		if (!len)
422 			continue;
423 
424 		if (static_branch_likely(&have_mio))
425 			pdev->resource[i].start =
426 				(resource_size_t __force) zdev->bars[i].mio_wb;
427 		else
428 			pdev->resource[i].start =
429 				(resource_size_t __force) pci_iomap(pdev, i, 0);
430 		pdev->resource[i].end = pdev->resource[i].start + len - 1;
431 	}
432 
433 #ifdef CONFIG_PCI_IOV
434 	i = PCI_IOV_RESOURCES;
435 
436 	for (; i < PCI_SRIOV_NUM_BARS + PCI_IOV_RESOURCES; i++) {
437 		len = pci_resource_len(pdev, i);
438 		if (!len)
439 			continue;
440 		pdev->resource[i].parent = &iov_res;
441 	}
442 #endif
443 }
444 
445 static void zpci_unmap_resources(struct pci_dev *pdev)
446 {
447 	resource_size_t len;
448 	int i;
449 
450 	if (static_branch_likely(&have_mio))
451 		return;
452 
453 	for (i = 0; i < PCI_BAR_COUNT; i++) {
454 		len = pci_resource_len(pdev, i);
455 		if (!len)
456 			continue;
457 		pci_iounmap(pdev, (void __iomem __force *)
458 			    pdev->resource[i].start);
459 	}
460 }
461 
462 static int zpci_alloc_iomap(struct zpci_dev *zdev)
463 {
464 	unsigned long entry;
465 
466 	spin_lock(&zpci_iomap_lock);
467 	entry = find_first_zero_bit(zpci_iomap_bitmap, ZPCI_IOMAP_ENTRIES);
468 	if (entry == ZPCI_IOMAP_ENTRIES) {
469 		spin_unlock(&zpci_iomap_lock);
470 		return -ENOSPC;
471 	}
472 	set_bit(entry, zpci_iomap_bitmap);
473 	spin_unlock(&zpci_iomap_lock);
474 	return entry;
475 }
476 
477 static void zpci_free_iomap(struct zpci_dev *zdev, int entry)
478 {
479 	spin_lock(&zpci_iomap_lock);
480 	memset(&zpci_iomap_start[entry], 0, sizeof(struct zpci_iomap_entry));
481 	clear_bit(entry, zpci_iomap_bitmap);
482 	spin_unlock(&zpci_iomap_lock);
483 }
484 
485 static struct resource *__alloc_res(struct zpci_dev *zdev, unsigned long start,
486 				    unsigned long size, unsigned long flags)
487 {
488 	struct resource *r;
489 
490 	r = kzalloc(sizeof(*r), GFP_KERNEL);
491 	if (!r)
492 		return NULL;
493 
494 	r->start = start;
495 	r->end = r->start + size - 1;
496 	r->flags = flags;
497 	r->name = zdev->res_name;
498 
499 	if (request_resource(&iomem_resource, r)) {
500 		kfree(r);
501 		return NULL;
502 	}
503 	return r;
504 }
505 
506 static int zpci_setup_bus_resources(struct zpci_dev *zdev,
507 				    struct list_head *resources)
508 {
509 	unsigned long addr, size, flags;
510 	struct resource *res;
511 	int i, entry;
512 
513 	snprintf(zdev->res_name, sizeof(zdev->res_name),
514 		 "PCI Bus %04x:%02x", zdev->domain, ZPCI_BUS_NR);
515 
516 	for (i = 0; i < PCI_BAR_COUNT; i++) {
517 		if (!zdev->bars[i].size)
518 			continue;
519 		entry = zpci_alloc_iomap(zdev);
520 		if (entry < 0)
521 			return entry;
522 		zdev->bars[i].map_idx = entry;
523 
524 		/* only MMIO is supported */
525 		flags = IORESOURCE_MEM;
526 		if (zdev->bars[i].val & 8)
527 			flags |= IORESOURCE_PREFETCH;
528 		if (zdev->bars[i].val & 4)
529 			flags |= IORESOURCE_MEM_64;
530 
531 		if (static_branch_likely(&have_mio))
532 			addr = (unsigned long) zdev->bars[i].mio_wb;
533 		else
534 			addr = ZPCI_ADDR(entry);
535 		size = 1UL << zdev->bars[i].size;
536 
537 		res = __alloc_res(zdev, addr, size, flags);
538 		if (!res) {
539 			zpci_free_iomap(zdev, entry);
540 			return -ENOMEM;
541 		}
542 		zdev->bars[i].res = res;
543 		pci_add_resource(resources, res);
544 	}
545 
546 	return 0;
547 }
548 
549 static void zpci_cleanup_bus_resources(struct zpci_dev *zdev)
550 {
551 	int i;
552 
553 	for (i = 0; i < PCI_BAR_COUNT; i++) {
554 		if (!zdev->bars[i].size || !zdev->bars[i].res)
555 			continue;
556 
557 		zpci_free_iomap(zdev, zdev->bars[i].map_idx);
558 		release_resource(zdev->bars[i].res);
559 		kfree(zdev->bars[i].res);
560 	}
561 }
562 
563 int pcibios_add_device(struct pci_dev *pdev)
564 {
565 	struct resource *res;
566 	int i;
567 
568 	if (pdev->is_physfn)
569 		pdev->no_vf_scan = 1;
570 
571 	pdev->dev.groups = zpci_attr_groups;
572 	pdev->dev.dma_ops = &s390_pci_dma_ops;
573 	zpci_map_resources(pdev);
574 
575 	for (i = 0; i < PCI_BAR_COUNT; i++) {
576 		res = &pdev->resource[i];
577 		if (res->parent || !res->flags)
578 			continue;
579 		pci_claim_resource(pdev, i);
580 	}
581 
582 	return 0;
583 }
584 
585 void pcibios_release_device(struct pci_dev *pdev)
586 {
587 	zpci_unmap_resources(pdev);
588 }
589 
590 int pcibios_enable_device(struct pci_dev *pdev, int mask)
591 {
592 	struct zpci_dev *zdev = to_zpci(pdev);
593 
594 	zpci_debug_init_device(zdev, dev_name(&pdev->dev));
595 	zpci_fmb_enable_device(zdev);
596 
597 	return pci_enable_resources(pdev, mask);
598 }
599 
600 void pcibios_disable_device(struct pci_dev *pdev)
601 {
602 	struct zpci_dev *zdev = to_zpci(pdev);
603 
604 	zpci_fmb_disable_device(zdev);
605 	zpci_debug_exit_device(zdev);
606 }
607 
608 #ifdef CONFIG_HIBERNATE_CALLBACKS
609 static int zpci_restore(struct device *dev)
610 {
611 	struct pci_dev *pdev = to_pci_dev(dev);
612 	struct zpci_dev *zdev = to_zpci(pdev);
613 	int ret = 0;
614 
615 	if (zdev->state != ZPCI_FN_STATE_ONLINE)
616 		goto out;
617 
618 	ret = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
619 	if (ret)
620 		goto out;
621 
622 	zpci_map_resources(pdev);
623 	zpci_register_ioat(zdev, 0, zdev->start_dma, zdev->end_dma,
624 			   (u64) zdev->dma_table);
625 
626 out:
627 	return ret;
628 }
629 
630 static int zpci_freeze(struct device *dev)
631 {
632 	struct pci_dev *pdev = to_pci_dev(dev);
633 	struct zpci_dev *zdev = to_zpci(pdev);
634 
635 	if (zdev->state != ZPCI_FN_STATE_ONLINE)
636 		return 0;
637 
638 	zpci_unregister_ioat(zdev, 0);
639 	zpci_unmap_resources(pdev);
640 	return clp_disable_fh(zdev);
641 }
642 
643 struct dev_pm_ops pcibios_pm_ops = {
644 	.thaw_noirq = zpci_restore,
645 	.freeze_noirq = zpci_freeze,
646 	.restore_noirq = zpci_restore,
647 	.poweroff_noirq = zpci_freeze,
648 };
649 #endif /* CONFIG_HIBERNATE_CALLBACKS */
650 
651 static int zpci_alloc_domain(struct zpci_dev *zdev)
652 {
653 	if (zpci_unique_uid) {
654 		zdev->domain = (u16) zdev->uid;
655 		if (zdev->domain >= ZPCI_NR_DEVICES)
656 			return 0;
657 
658 		spin_lock(&zpci_domain_lock);
659 		if (test_bit(zdev->domain, zpci_domain)) {
660 			spin_unlock(&zpci_domain_lock);
661 			return -EEXIST;
662 		}
663 		set_bit(zdev->domain, zpci_domain);
664 		spin_unlock(&zpci_domain_lock);
665 		return 0;
666 	}
667 
668 	spin_lock(&zpci_domain_lock);
669 	zdev->domain = find_first_zero_bit(zpci_domain, ZPCI_NR_DEVICES);
670 	if (zdev->domain == ZPCI_NR_DEVICES) {
671 		spin_unlock(&zpci_domain_lock);
672 		return -ENOSPC;
673 	}
674 	set_bit(zdev->domain, zpci_domain);
675 	spin_unlock(&zpci_domain_lock);
676 	return 0;
677 }
678 
679 static void zpci_free_domain(struct zpci_dev *zdev)
680 {
681 	if (zdev->domain >= ZPCI_NR_DEVICES)
682 		return;
683 
684 	spin_lock(&zpci_domain_lock);
685 	clear_bit(zdev->domain, zpci_domain);
686 	spin_unlock(&zpci_domain_lock);
687 }
688 
689 void pcibios_remove_bus(struct pci_bus *bus)
690 {
691 	struct zpci_dev *zdev = get_zdev_by_bus(bus);
692 
693 	zpci_exit_slot(zdev);
694 	zpci_cleanup_bus_resources(zdev);
695 	zpci_destroy_iommu(zdev);
696 	zpci_free_domain(zdev);
697 
698 	spin_lock(&zpci_list_lock);
699 	list_del(&zdev->entry);
700 	spin_unlock(&zpci_list_lock);
701 
702 	zpci_dbg(3, "rem fid:%x\n", zdev->fid);
703 	kfree(zdev);
704 }
705 
706 static int zpci_scan_bus(struct zpci_dev *zdev)
707 {
708 	LIST_HEAD(resources);
709 	int ret;
710 
711 	ret = zpci_setup_bus_resources(zdev, &resources);
712 	if (ret)
713 		goto error;
714 
715 	zdev->bus = pci_scan_root_bus(NULL, ZPCI_BUS_NR, &pci_root_ops,
716 				      zdev, &resources);
717 	if (!zdev->bus) {
718 		ret = -EIO;
719 		goto error;
720 	}
721 	zdev->bus->max_bus_speed = zdev->max_bus_speed;
722 	pci_bus_add_devices(zdev->bus);
723 	return 0;
724 
725 error:
726 	zpci_cleanup_bus_resources(zdev);
727 	pci_free_resource_list(&resources);
728 	return ret;
729 }
730 
731 int zpci_enable_device(struct zpci_dev *zdev)
732 {
733 	int rc;
734 
735 	rc = clp_enable_fh(zdev, ZPCI_NR_DMA_SPACES);
736 	if (rc)
737 		goto out;
738 
739 	rc = zpci_dma_init_device(zdev);
740 	if (rc)
741 		goto out_dma;
742 
743 	zdev->state = ZPCI_FN_STATE_ONLINE;
744 	return 0;
745 
746 out_dma:
747 	clp_disable_fh(zdev);
748 out:
749 	return rc;
750 }
751 EXPORT_SYMBOL_GPL(zpci_enable_device);
752 
753 int zpci_disable_device(struct zpci_dev *zdev)
754 {
755 	zpci_dma_exit_device(zdev);
756 	return clp_disable_fh(zdev);
757 }
758 EXPORT_SYMBOL_GPL(zpci_disable_device);
759 
760 int zpci_create_device(struct zpci_dev *zdev)
761 {
762 	int rc;
763 
764 	rc = zpci_alloc_domain(zdev);
765 	if (rc)
766 		goto out;
767 
768 	rc = zpci_init_iommu(zdev);
769 	if (rc)
770 		goto out_free;
771 
772 	mutex_init(&zdev->lock);
773 	if (zdev->state == ZPCI_FN_STATE_CONFIGURED) {
774 		rc = zpci_enable_device(zdev);
775 		if (rc)
776 			goto out_destroy_iommu;
777 	}
778 	rc = zpci_scan_bus(zdev);
779 	if (rc)
780 		goto out_disable;
781 
782 	spin_lock(&zpci_list_lock);
783 	list_add_tail(&zdev->entry, &zpci_list);
784 	spin_unlock(&zpci_list_lock);
785 
786 	zpci_init_slot(zdev);
787 
788 	return 0;
789 
790 out_disable:
791 	if (zdev->state == ZPCI_FN_STATE_ONLINE)
792 		zpci_disable_device(zdev);
793 out_destroy_iommu:
794 	zpci_destroy_iommu(zdev);
795 out_free:
796 	zpci_free_domain(zdev);
797 out:
798 	return rc;
799 }
800 
801 void zpci_remove_device(struct zpci_dev *zdev)
802 {
803 	if (!zdev->bus)
804 		return;
805 
806 	pci_stop_root_bus(zdev->bus);
807 	pci_remove_root_bus(zdev->bus);
808 }
809 
810 int zpci_report_error(struct pci_dev *pdev,
811 		      struct zpci_report_error_header *report)
812 {
813 	struct zpci_dev *zdev = to_zpci(pdev);
814 
815 	return sclp_pci_report(report, zdev->fh, zdev->fid);
816 }
817 EXPORT_SYMBOL(zpci_report_error);
818 
819 static int zpci_mem_init(void)
820 {
821 	BUILD_BUG_ON(!is_power_of_2(__alignof__(struct zpci_fmb)) ||
822 		     __alignof__(struct zpci_fmb) < sizeof(struct zpci_fmb));
823 
824 	zdev_fmb_cache = kmem_cache_create("PCI_FMB_cache", sizeof(struct zpci_fmb),
825 					   __alignof__(struct zpci_fmb), 0, NULL);
826 	if (!zdev_fmb_cache)
827 		goto error_fmb;
828 
829 	zpci_iomap_start = kcalloc(ZPCI_IOMAP_ENTRIES,
830 				   sizeof(*zpci_iomap_start), GFP_KERNEL);
831 	if (!zpci_iomap_start)
832 		goto error_iomap;
833 
834 	zpci_iomap_bitmap = kcalloc(BITS_TO_LONGS(ZPCI_IOMAP_ENTRIES),
835 				    sizeof(*zpci_iomap_bitmap), GFP_KERNEL);
836 	if (!zpci_iomap_bitmap)
837 		goto error_iomap_bitmap;
838 
839 	return 0;
840 error_iomap_bitmap:
841 	kfree(zpci_iomap_start);
842 error_iomap:
843 	kmem_cache_destroy(zdev_fmb_cache);
844 error_fmb:
845 	return -ENOMEM;
846 }
847 
848 static void zpci_mem_exit(void)
849 {
850 	kfree(zpci_iomap_bitmap);
851 	kfree(zpci_iomap_start);
852 	kmem_cache_destroy(zdev_fmb_cache);
853 }
854 
855 static unsigned int s390_pci_probe __initdata = 1;
856 static unsigned int s390_pci_no_mio __initdata;
857 unsigned int s390_pci_force_floating __initdata;
858 static unsigned int s390_pci_initialized;
859 
860 char * __init pcibios_setup(char *str)
861 {
862 	if (!strcmp(str, "off")) {
863 		s390_pci_probe = 0;
864 		return NULL;
865 	}
866 	if (!strcmp(str, "nomio")) {
867 		s390_pci_no_mio = 1;
868 		return NULL;
869 	}
870 	if (!strcmp(str, "force_floating")) {
871 		s390_pci_force_floating = 1;
872 		return NULL;
873 	}
874 	return str;
875 }
876 
877 bool zpci_is_enabled(void)
878 {
879 	return s390_pci_initialized;
880 }
881 
882 static int __init pci_base_init(void)
883 {
884 	int rc;
885 
886 	if (!s390_pci_probe)
887 		return 0;
888 
889 	if (!test_facility(69) || !test_facility(71))
890 		return 0;
891 
892 	if (test_facility(153) && !s390_pci_no_mio)
893 		static_branch_enable(&have_mio);
894 
895 	rc = zpci_debug_init();
896 	if (rc)
897 		goto out;
898 
899 	rc = zpci_mem_init();
900 	if (rc)
901 		goto out_mem;
902 
903 	rc = zpci_irq_init();
904 	if (rc)
905 		goto out_irq;
906 
907 	rc = zpci_dma_init();
908 	if (rc)
909 		goto out_dma;
910 
911 	rc = clp_scan_pci_devices();
912 	if (rc)
913 		goto out_find;
914 
915 	s390_pci_initialized = 1;
916 	return 0;
917 
918 out_find:
919 	zpci_dma_exit();
920 out_dma:
921 	zpci_irq_exit();
922 out_irq:
923 	zpci_mem_exit();
924 out_mem:
925 	zpci_debug_exit();
926 out:
927 	return rc;
928 }
929 subsys_initcall_sync(pci_base_init);
930 
931 void zpci_rescan(void)
932 {
933 	if (zpci_is_enabled())
934 		clp_rescan_pci_devices_simple();
935 }
936