1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe host controller driver
4 *
5 * Copyright (C) 2013 Samsung Electronics Co., Ltd.
6 * https://www.samsung.com
7 *
8 * Author: Jingoo Han <jg1.han@samsung.com>
9 */
10
11 #include <linux/align.h>
12 #include <linux/iopoll.h>
13 #include <linux/irqchip/chained_irq.h>
14 #include <linux/irqchip/irq-msi-lib.h>
15 #include <linux/irqdomain.h>
16 #include <linux/msi.h>
17 #include <linux/of_address.h>
18 #include <linux/of_pci.h>
19 #include <linux/pci_regs.h>
20 #include <linux/platform_device.h>
21
22 #include "../../pci.h"
23 #include "pcie-designware.h"
24
25 static struct pci_ops dw_pcie_ops;
26 static struct pci_ops dw_pcie_ecam_ops;
27 static struct pci_ops dw_child_pcie_ops;
28
29 #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \
30 MSI_FLAG_USE_DEF_CHIP_OPS | \
31 MSI_FLAG_NO_AFFINITY | \
32 MSI_FLAG_PCI_MSI_MASK_PARENT)
33 #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \
34 MSI_FLAG_PCI_MSIX | \
35 MSI_GENERIC_FLAGS_MASK)
36
37 #define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M)
38
39 static const struct msi_parent_ops dw_pcie_msi_parent_ops = {
40 .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED,
41 .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED,
42 .bus_select_token = DOMAIN_BUS_PCI_MSI,
43 .chip_flags = MSI_CHIP_FLAG_SET_ACK,
44 .prefix = "DW-",
45 .init_dev_msi_info = msi_lib_init_dev_msi_info,
46 };
47
48 /* MSI int handler */
dw_handle_msi_irq(struct dw_pcie_rp * pp)49 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
50 {
51 int i, pos;
52 unsigned long val;
53 u32 status, num_ctrls;
54 irqreturn_t ret = IRQ_NONE;
55 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
56
57 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
58
59 for (i = 0; i < num_ctrls; i++) {
60 status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS +
61 (i * MSI_REG_CTRL_BLOCK_SIZE));
62 if (!status)
63 continue;
64
65 ret = IRQ_HANDLED;
66 val = status;
67 pos = 0;
68 while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL,
69 pos)) != MAX_MSI_IRQS_PER_CTRL) {
70 generic_handle_domain_irq(pp->irq_domain,
71 (i * MAX_MSI_IRQS_PER_CTRL) +
72 pos);
73 pos++;
74 }
75 }
76
77 return ret;
78 }
79
80 /* Chained MSI interrupt service routine */
dw_chained_msi_isr(struct irq_desc * desc)81 static void dw_chained_msi_isr(struct irq_desc *desc)
82 {
83 struct irq_chip *chip = irq_desc_get_chip(desc);
84 struct dw_pcie_rp *pp;
85
86 chained_irq_enter(chip, desc);
87
88 pp = irq_desc_get_handler_data(desc);
89 dw_handle_msi_irq(pp);
90
91 chained_irq_exit(chip, desc);
92 }
93
dw_pci_setup_msi_msg(struct irq_data * d,struct msi_msg * msg)94 static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
95 {
96 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
97 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
98 u64 msi_target;
99
100 msi_target = (u64)pp->msi_data;
101
102 msg->address_lo = lower_32_bits(msi_target);
103 msg->address_hi = upper_32_bits(msi_target);
104
105 msg->data = d->hwirq;
106
107 dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
108 (int)d->hwirq, msg->address_hi, msg->address_lo);
109 }
110
dw_pci_bottom_mask(struct irq_data * d)111 static void dw_pci_bottom_mask(struct irq_data *d)
112 {
113 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
114 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
115 unsigned int res, bit, ctrl;
116 unsigned long flags;
117
118 raw_spin_lock_irqsave(&pp->lock, flags);
119
120 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
121 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
122 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
123
124 pp->irq_mask[ctrl] |= BIT(bit);
125 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
126
127 raw_spin_unlock_irqrestore(&pp->lock, flags);
128 }
129
dw_pci_bottom_unmask(struct irq_data * d)130 static void dw_pci_bottom_unmask(struct irq_data *d)
131 {
132 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
133 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
134 unsigned int res, bit, ctrl;
135 unsigned long flags;
136
137 raw_spin_lock_irqsave(&pp->lock, flags);
138
139 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
140 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
141 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
142
143 pp->irq_mask[ctrl] &= ~BIT(bit);
144 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
145
146 raw_spin_unlock_irqrestore(&pp->lock, flags);
147 }
148
dw_pci_bottom_ack(struct irq_data * d)149 static void dw_pci_bottom_ack(struct irq_data *d)
150 {
151 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
152 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
153 unsigned int res, bit, ctrl;
154
155 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
156 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
157 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
158
159 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit));
160 }
161
162 static struct irq_chip dw_pci_msi_bottom_irq_chip = {
163 .name = "DWPCI-MSI",
164 .irq_ack = dw_pci_bottom_ack,
165 .irq_compose_msi_msg = dw_pci_setup_msi_msg,
166 .irq_mask = dw_pci_bottom_mask,
167 .irq_unmask = dw_pci_bottom_unmask,
168 };
169
dw_pcie_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)170 static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
171 unsigned int virq, unsigned int nr_irqs,
172 void *args)
173 {
174 struct dw_pcie_rp *pp = domain->host_data;
175 unsigned long flags;
176 u32 i;
177 int bit;
178
179 raw_spin_lock_irqsave(&pp->lock, flags);
180
181 bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
182 order_base_2(nr_irqs));
183
184 raw_spin_unlock_irqrestore(&pp->lock, flags);
185
186 if (bit < 0)
187 return -ENOSPC;
188
189 for (i = 0; i < nr_irqs; i++)
190 irq_domain_set_info(domain, virq + i, bit + i,
191 pp->msi_irq_chip,
192 pp, handle_edge_irq,
193 NULL, NULL);
194
195 return 0;
196 }
197
dw_pcie_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)198 static void dw_pcie_irq_domain_free(struct irq_domain *domain,
199 unsigned int virq, unsigned int nr_irqs)
200 {
201 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
202 struct dw_pcie_rp *pp = domain->host_data;
203 unsigned long flags;
204
205 raw_spin_lock_irqsave(&pp->lock, flags);
206
207 bitmap_release_region(pp->msi_irq_in_use, d->hwirq,
208 order_base_2(nr_irqs));
209
210 raw_spin_unlock_irqrestore(&pp->lock, flags);
211 }
212
213 static const struct irq_domain_ops dw_pcie_msi_domain_ops = {
214 .alloc = dw_pcie_irq_domain_alloc,
215 .free = dw_pcie_irq_domain_free,
216 };
217
dw_pcie_allocate_domains(struct dw_pcie_rp * pp)218 int dw_pcie_allocate_domains(struct dw_pcie_rp *pp)
219 {
220 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
221 struct irq_domain_info info = {
222 .fwnode = dev_fwnode(pci->dev),
223 .ops = &dw_pcie_msi_domain_ops,
224 .size = pp->num_vectors,
225 .host_data = pp,
226 };
227
228 pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops);
229 if (!pp->irq_domain) {
230 dev_err(pci->dev, "Failed to create IRQ domain\n");
231 return -ENOMEM;
232 }
233
234 return 0;
235 }
236
dw_pcie_free_msi(struct dw_pcie_rp * pp)237 void dw_pcie_free_msi(struct dw_pcie_rp *pp)
238 {
239 u32 ctrl;
240
241 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
242 if (pp->msi_irq[ctrl] > 0)
243 irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
244 NULL, NULL);
245 }
246
247 irq_domain_remove(pp->irq_domain);
248 }
249 EXPORT_SYMBOL_GPL(dw_pcie_free_msi);
250
dw_pcie_msi_init(struct dw_pcie_rp * pp)251 void dw_pcie_msi_init(struct dw_pcie_rp *pp)
252 {
253 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
254 u64 msi_target = (u64)pp->msi_data;
255 u32 ctrl, num_ctrls;
256
257 if (!pci_msi_enabled() || !pp->has_msi_ctrl)
258 return;
259
260 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
261
262 /* Initialize IRQ Status array */
263 for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
264 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK +
265 (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
266 pp->irq_mask[ctrl]);
267 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE +
268 (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
269 ~0);
270 }
271
272 /* Program the msi_data */
273 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target));
274 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target));
275 }
276 EXPORT_SYMBOL_GPL(dw_pcie_msi_init);
277
dw_pcie_parse_split_msi_irq(struct dw_pcie_rp * pp)278 static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp)
279 {
280 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
281 struct device *dev = pci->dev;
282 struct platform_device *pdev = to_platform_device(dev);
283 u32 ctrl, max_vectors;
284 int irq;
285
286 /* Parse any "msiX" IRQs described in the devicetree */
287 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
288 char msi_name[] = "msiX";
289
290 msi_name[3] = '0' + ctrl;
291 irq = platform_get_irq_byname_optional(pdev, msi_name);
292 if (irq == -ENXIO)
293 break;
294 if (irq < 0)
295 return dev_err_probe(dev, irq,
296 "Failed to parse MSI IRQ '%s'\n",
297 msi_name);
298
299 pp->msi_irq[ctrl] = irq;
300 }
301
302 /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */
303 if (ctrl == 0)
304 return -ENXIO;
305
306 max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL;
307 if (pp->num_vectors > max_vectors) {
308 dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n",
309 max_vectors);
310 pp->num_vectors = max_vectors;
311 }
312 if (!pp->num_vectors)
313 pp->num_vectors = max_vectors;
314
315 return 0;
316 }
317
dw_pcie_msi_host_init(struct dw_pcie_rp * pp)318 int dw_pcie_msi_host_init(struct dw_pcie_rp *pp)
319 {
320 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
321 struct device *dev = pci->dev;
322 struct platform_device *pdev = to_platform_device(dev);
323 u64 *msi_vaddr = NULL;
324 int ret;
325 u32 ctrl, num_ctrls;
326
327 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
328 pp->irq_mask[ctrl] = ~0;
329
330 if (!pp->msi_irq[0]) {
331 ret = dw_pcie_parse_split_msi_irq(pp);
332 if (ret < 0 && ret != -ENXIO)
333 return ret;
334 }
335
336 if (!pp->num_vectors)
337 pp->num_vectors = MSI_DEF_NUM_VECTORS;
338 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
339
340 if (!pp->msi_irq[0]) {
341 pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi");
342 if (pp->msi_irq[0] < 0) {
343 pp->msi_irq[0] = platform_get_irq(pdev, 0);
344 if (pp->msi_irq[0] < 0)
345 return pp->msi_irq[0];
346 }
347 }
348
349 dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors);
350
351 pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
352
353 ret = dw_pcie_allocate_domains(pp);
354 if (ret)
355 return ret;
356
357 for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
358 if (pp->msi_irq[ctrl] > 0)
359 irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
360 dw_chained_msi_isr, pp);
361 }
362
363 /*
364 * Even though the iMSI-RX Module supports 64-bit addresses some
365 * peripheral PCIe devices may lack 64-bit message support. In
366 * order not to miss MSI TLPs from those devices the MSI target
367 * address has to be within the lowest 4GB.
368 *
369 * Note until there is a better alternative found the reservation is
370 * done by allocating from the artificially limited DMA-coherent
371 * memory.
372 */
373 ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
374 if (!ret)
375 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
376 GFP_KERNEL);
377
378 if (!msi_vaddr) {
379 dev_warn(dev, "Failed to allocate 32-bit MSI address\n");
380 dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
381 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
382 GFP_KERNEL);
383 if (!msi_vaddr) {
384 dev_err(dev, "Failed to allocate MSI address\n");
385 dw_pcie_free_msi(pp);
386 return -ENOMEM;
387 }
388 }
389
390 return 0;
391 }
392 EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init);
393
dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp * pp)394 static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
395 {
396 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
397 struct resource_entry *win;
398 struct resource *res;
399
400 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM);
401 if (win) {
402 res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL);
403 if (!res)
404 return;
405
406 /*
407 * Allocate MSG TLP region of size 'region_align' at the end of
408 * the host bridge window.
409 */
410 res->start = win->res->end - pci->region_align + 1;
411 res->end = win->res->end;
412 res->name = "msg";
413 res->flags = win->res->flags | IORESOURCE_BUSY;
414
415 if (!devm_request_resource(pci->dev, win->res, res))
416 pp->msg_res = res;
417 }
418 }
419
dw_pcie_config_ecam_iatu(struct dw_pcie_rp * pp)420 static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
421 {
422 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
423 struct dw_pcie_ob_atu_cfg atu = {0};
424 resource_size_t bus_range_max;
425 struct resource_entry *bus;
426 int ret;
427
428 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
429
430 /*
431 * Root bus under the host bridge doesn't require any iATU configuration
432 * as DBI region will be used to access root bus config space.
433 * Immediate bus under Root Bus, needs type 0 iATU configuration and
434 * remaining buses need type 1 iATU configuration.
435 */
436 atu.index = 0;
437 atu.type = PCIE_ATU_TYPE_CFG0;
438 atu.parent_bus_addr = pp->cfg0_base + SZ_1M;
439 /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */
440 atu.size = SZ_1M;
441 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
442 ret = dw_pcie_prog_outbound_atu(pci, &atu);
443 if (ret)
444 return ret;
445
446 bus_range_max = resource_size(bus->res);
447
448 if (bus_range_max < 2)
449 return 0;
450
451 /* Configure remaining buses in type 1 iATU configuration */
452 atu.index = 1;
453 atu.type = PCIE_ATU_TYPE_CFG1;
454 atu.parent_bus_addr = pp->cfg0_base + SZ_2M;
455 atu.size = (SZ_1M * bus_range_max) - SZ_2M;
456 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
457
458 return dw_pcie_prog_outbound_atu(pci, &atu);
459 }
460
dw_pcie_create_ecam_window(struct dw_pcie_rp * pp,struct resource * res)461 static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
462 {
463 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
464 struct device *dev = pci->dev;
465 struct resource_entry *bus;
466
467 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
468 if (!bus)
469 return -ENODEV;
470
471 pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
472 if (IS_ERR(pp->cfg))
473 return PTR_ERR(pp->cfg);
474
475 return 0;
476 }
477
dw_pcie_ecam_enabled(struct dw_pcie_rp * pp,struct resource * config_res)478 static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res)
479 {
480 struct resource *bus_range;
481 u64 nr_buses;
482
483 /* Vendor glue drivers may implement their own ECAM mechanism */
484 if (pp->native_ecam)
485 return false;
486
487 /*
488 * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to
489 * be aligned on a 2^(n+20) byte boundary, where n is the number of bits
490 * used for representing 'bus' in BDF. Since the DWC cores always use 8
491 * bits for representing 'bus', the base address has to be aligned to
492 * 2^28 byte boundary, which is 256 MiB.
493 */
494 if (!IS_256MB_ALIGNED(config_res->start))
495 return false;
496
497 bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res;
498 if (!bus_range)
499 return false;
500
501 nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT;
502
503 return nr_buses >= resource_size(bus_range);
504 }
505
dw_pcie_host_get_resources(struct dw_pcie_rp * pp)506 static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp)
507 {
508 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
509 struct device *dev = pci->dev;
510 struct platform_device *pdev = to_platform_device(dev);
511 struct resource_entry *win;
512 struct resource *res;
513 int ret;
514
515 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
516 if (!res) {
517 dev_err(dev, "Missing \"config\" reg space\n");
518 return -ENODEV;
519 }
520
521 pp->cfg0_size = resource_size(res);
522 pp->cfg0_base = res->start;
523
524 pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res);
525 if (pp->ecam_enabled) {
526 ret = dw_pcie_create_ecam_window(pp, res);
527 if (ret)
528 return ret;
529
530 pp->bridge->ops = &dw_pcie_ecam_ops;
531 pp->bridge->sysdata = pp->cfg;
532 pp->cfg->priv = pp;
533 } else {
534 pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
535 if (IS_ERR(pp->va_cfg0_base))
536 return PTR_ERR(pp->va_cfg0_base);
537
538 /* Set default bus ops */
539 pp->bridge->ops = &dw_pcie_ops;
540 pp->bridge->child_ops = &dw_child_pcie_ops;
541 pp->bridge->sysdata = pp;
542 }
543
544 ret = dw_pcie_get_resources(pci);
545 if (ret) {
546 if (pp->cfg)
547 pci_ecam_free(pp->cfg);
548 return ret;
549 }
550
551 /* Get the I/O range from DT */
552 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO);
553 if (win) {
554 pp->io_size = resource_size(win->res);
555 pp->io_bus_addr = win->res->start - win->offset;
556 pp->io_base = pci_pio_to_address(win->res->start);
557 }
558
559 /*
560 * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to
561 * call dw_pcie_parent_bus_offset() after setting pp->io_base.
562 */
563 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config",
564 pp->cfg0_base);
565 return 0;
566 }
567
dw_pcie_host_init(struct dw_pcie_rp * pp)568 int dw_pcie_host_init(struct dw_pcie_rp *pp)
569 {
570 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
571 struct device *dev = pci->dev;
572 struct device_node *np = dev->of_node;
573 struct pci_host_bridge *bridge;
574 int ret;
575
576 raw_spin_lock_init(&pp->lock);
577
578 bridge = devm_pci_alloc_host_bridge(dev, 0);
579 if (!bridge)
580 return -ENOMEM;
581
582 pp->bridge = bridge;
583
584 ret = dw_pcie_host_get_resources(pp);
585 if (ret)
586 return ret;
587
588 if (pp->ops->init) {
589 ret = pp->ops->init(pp);
590 if (ret)
591 goto err_free_ecam;
592 }
593
594 if (pci_msi_enabled()) {
595 pp->has_msi_ctrl = !(pp->ops->msi_init ||
596 of_property_present(np, "msi-parent") ||
597 of_property_present(np, "msi-map"));
598
599 /*
600 * For the has_msi_ctrl case the default assignment is handled
601 * in the dw_pcie_msi_host_init().
602 */
603 if (!pp->has_msi_ctrl && !pp->num_vectors) {
604 pp->num_vectors = MSI_DEF_NUM_VECTORS;
605 } else if (pp->num_vectors > MAX_MSI_IRQS) {
606 dev_err(dev, "Invalid number of vectors\n");
607 ret = -EINVAL;
608 goto err_deinit_host;
609 }
610
611 if (pp->ops->msi_init) {
612 ret = pp->ops->msi_init(pp);
613 if (ret < 0)
614 goto err_deinit_host;
615 } else if (pp->has_msi_ctrl) {
616 ret = dw_pcie_msi_host_init(pp);
617 if (ret < 0)
618 goto err_deinit_host;
619 }
620 }
621
622 dw_pcie_version_detect(pci);
623
624 dw_pcie_iatu_detect(pci);
625
626 if (pci->num_lanes < 1)
627 pci->num_lanes = dw_pcie_link_get_max_link_width(pci);
628
629 ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes);
630 if (ret)
631 goto err_free_msi;
632
633 if (pp->ecam_enabled) {
634 ret = dw_pcie_config_ecam_iatu(pp);
635 if (ret) {
636 dev_err(dev, "Failed to configure iATU in ECAM mode\n");
637 goto err_free_msi;
638 }
639 }
640
641 /*
642 * Allocate the resource for MSG TLP before programming the iATU
643 * outbound window in dw_pcie_setup_rc(). Since the allocation depends
644 * on the value of 'region_align', this has to be done after
645 * dw_pcie_iatu_detect().
646 *
647 * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to
648 * make use of the generic MSG TLP implementation.
649 */
650 if (pp->use_atu_msg)
651 dw_pcie_host_request_msg_tlp_res(pp);
652
653 ret = dw_pcie_edma_detect(pci);
654 if (ret)
655 goto err_free_msi;
656
657 ret = dw_pcie_setup_rc(pp);
658 if (ret)
659 goto err_remove_edma;
660
661 if (!dw_pcie_link_up(pci)) {
662 ret = dw_pcie_start_link(pci);
663 if (ret)
664 goto err_remove_edma;
665 }
666
667 /*
668 * Note: Skip the link up delay only when a Link Up IRQ is present.
669 * If there is no Link Up IRQ, we should not bypass the delay
670 * because that would require users to manually rescan for devices.
671 */
672 if (!pp->use_linkup_irq)
673 /* Ignore errors, the link may come up later */
674 dw_pcie_wait_for_link(pci);
675
676 ret = pci_host_probe(bridge);
677 if (ret)
678 goto err_stop_link;
679
680 if (pp->ops->post_init)
681 pp->ops->post_init(pp);
682
683 dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE);
684
685 return 0;
686
687 err_stop_link:
688 dw_pcie_stop_link(pci);
689
690 err_remove_edma:
691 dw_pcie_edma_remove(pci);
692
693 err_free_msi:
694 if (pp->has_msi_ctrl)
695 dw_pcie_free_msi(pp);
696
697 err_deinit_host:
698 if (pp->ops->deinit)
699 pp->ops->deinit(pp);
700
701 err_free_ecam:
702 if (pp->cfg)
703 pci_ecam_free(pp->cfg);
704
705 return ret;
706 }
707 EXPORT_SYMBOL_GPL(dw_pcie_host_init);
708
dw_pcie_host_deinit(struct dw_pcie_rp * pp)709 void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
710 {
711 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
712
713 dwc_pcie_debugfs_deinit(pci);
714
715 pci_stop_root_bus(pp->bridge->bus);
716 pci_remove_root_bus(pp->bridge->bus);
717
718 dw_pcie_stop_link(pci);
719
720 dw_pcie_edma_remove(pci);
721
722 if (pp->has_msi_ctrl)
723 dw_pcie_free_msi(pp);
724
725 if (pp->ops->deinit)
726 pp->ops->deinit(pp);
727
728 if (pp->cfg)
729 pci_ecam_free(pp->cfg);
730 }
731 EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
732
dw_pcie_other_conf_map_bus(struct pci_bus * bus,unsigned int devfn,int where)733 static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
734 unsigned int devfn, int where)
735 {
736 struct dw_pcie_rp *pp = bus->sysdata;
737 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
738 struct dw_pcie_ob_atu_cfg atu = { 0 };
739 int type, ret;
740 u32 busdev;
741
742 /*
743 * Checking whether the link is up here is a last line of defense
744 * against platforms that forward errors on the system bus as
745 * SError upon PCI configuration transactions issued when the link
746 * is down. This check is racy by definition and does not stop
747 * the system from triggering an SError if the link goes down
748 * after this check is performed.
749 */
750 if (!dw_pcie_link_up(pci))
751 return NULL;
752
753 busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
754 PCIE_ATU_FUNC(PCI_FUNC(devfn));
755
756 if (pci_is_root_bus(bus->parent))
757 type = PCIE_ATU_TYPE_CFG0;
758 else
759 type = PCIE_ATU_TYPE_CFG1;
760
761 atu.type = type;
762 atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset;
763 atu.pci_addr = busdev;
764 atu.size = pp->cfg0_size;
765
766 ret = dw_pcie_prog_outbound_atu(pci, &atu);
767 if (ret)
768 return NULL;
769
770 return pp->va_cfg0_base + where;
771 }
772
dw_pcie_rd_other_conf(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)773 static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
774 int where, int size, u32 *val)
775 {
776 struct dw_pcie_rp *pp = bus->sysdata;
777 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
778 struct dw_pcie_ob_atu_cfg atu = { 0 };
779 int ret;
780
781 ret = pci_generic_config_read(bus, devfn, where, size, val);
782 if (ret != PCIBIOS_SUCCESSFUL)
783 return ret;
784
785 if (pp->cfg0_io_shared) {
786 atu.type = PCIE_ATU_TYPE_IO;
787 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
788 atu.pci_addr = pp->io_bus_addr;
789 atu.size = pp->io_size;
790
791 ret = dw_pcie_prog_outbound_atu(pci, &atu);
792 if (ret)
793 return PCIBIOS_SET_FAILED;
794 }
795
796 return PCIBIOS_SUCCESSFUL;
797 }
798
dw_pcie_wr_other_conf(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)799 static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
800 int where, int size, u32 val)
801 {
802 struct dw_pcie_rp *pp = bus->sysdata;
803 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
804 struct dw_pcie_ob_atu_cfg atu = { 0 };
805 int ret;
806
807 ret = pci_generic_config_write(bus, devfn, where, size, val);
808 if (ret != PCIBIOS_SUCCESSFUL)
809 return ret;
810
811 if (pp->cfg0_io_shared) {
812 atu.type = PCIE_ATU_TYPE_IO;
813 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
814 atu.pci_addr = pp->io_bus_addr;
815 atu.size = pp->io_size;
816
817 ret = dw_pcie_prog_outbound_atu(pci, &atu);
818 if (ret)
819 return PCIBIOS_SET_FAILED;
820 }
821
822 return PCIBIOS_SUCCESSFUL;
823 }
824
825 static struct pci_ops dw_child_pcie_ops = {
826 .map_bus = dw_pcie_other_conf_map_bus,
827 .read = dw_pcie_rd_other_conf,
828 .write = dw_pcie_wr_other_conf,
829 };
830
dw_pcie_own_conf_map_bus(struct pci_bus * bus,unsigned int devfn,int where)831 void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
832 {
833 struct dw_pcie_rp *pp = bus->sysdata;
834 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
835
836 if (PCI_SLOT(devfn) > 0)
837 return NULL;
838
839 return pci->dbi_base + where;
840 }
841 EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus);
842
dw_pcie_ecam_conf_map_bus(struct pci_bus * bus,unsigned int devfn,int where)843 static void __iomem *dw_pcie_ecam_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
844 {
845 struct pci_config_window *cfg = bus->sysdata;
846 struct dw_pcie_rp *pp = cfg->priv;
847 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
848 unsigned int busn = bus->number;
849
850 if (busn > 0)
851 return pci_ecam_map_bus(bus, devfn, where);
852
853 if (PCI_SLOT(devfn) > 0)
854 return NULL;
855
856 return pci->dbi_base + where;
857 }
858
859 static struct pci_ops dw_pcie_ops = {
860 .map_bus = dw_pcie_own_conf_map_bus,
861 .read = pci_generic_config_read,
862 .write = pci_generic_config_write,
863 };
864
865 static struct pci_ops dw_pcie_ecam_ops = {
866 .map_bus = dw_pcie_ecam_conf_map_bus,
867 .read = pci_generic_config_read,
868 .write = pci_generic_config_write,
869 };
870
dw_pcie_iatu_setup(struct dw_pcie_rp * pp)871 static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
872 {
873 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
874 struct dw_pcie_ob_atu_cfg atu = { 0 };
875 struct resource_entry *entry;
876 int i, ret;
877
878 /* Note the very first outbound ATU is used for CFG IOs */
879 if (!pci->num_ob_windows) {
880 dev_err(pci->dev, "No outbound iATU found\n");
881 return -EINVAL;
882 }
883
884 /*
885 * Ensure all out/inbound windows are disabled before proceeding with
886 * the MEM/IO (dma-)ranges setups.
887 */
888 for (i = 0; i < pci->num_ob_windows; i++)
889 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i);
890
891 for (i = 0; i < pci->num_ib_windows; i++)
892 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i);
893
894 i = 0;
895 resource_list_for_each_entry(entry, &pp->bridge->windows) {
896 if (resource_type(entry->res) != IORESOURCE_MEM)
897 continue;
898
899 if (pci->num_ob_windows <= ++i)
900 break;
901
902 atu.index = i;
903 atu.type = PCIE_ATU_TYPE_MEM;
904 atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset;
905 atu.pci_addr = entry->res->start - entry->offset;
906
907 /* Adjust iATU size if MSG TLP region was allocated before */
908 if (pp->msg_res && pp->msg_res->parent == entry->res)
909 atu.size = resource_size(entry->res) -
910 resource_size(pp->msg_res);
911 else
912 atu.size = resource_size(entry->res);
913
914 ret = dw_pcie_prog_outbound_atu(pci, &atu);
915 if (ret) {
916 dev_err(pci->dev, "Failed to set MEM range %pr\n",
917 entry->res);
918 return ret;
919 }
920 }
921
922 if (pp->io_size) {
923 if (pci->num_ob_windows > ++i) {
924 atu.index = i;
925 atu.type = PCIE_ATU_TYPE_IO;
926 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
927 atu.pci_addr = pp->io_bus_addr;
928 atu.size = pp->io_size;
929
930 ret = dw_pcie_prog_outbound_atu(pci, &atu);
931 if (ret) {
932 dev_err(pci->dev, "Failed to set IO range %pr\n",
933 entry->res);
934 return ret;
935 }
936 } else {
937 pp->cfg0_io_shared = true;
938 }
939 }
940
941 if (pci->num_ob_windows <= i)
942 dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n",
943 pci->num_ob_windows);
944
945 pp->msg_atu_index = i;
946
947 i = 0;
948 resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) {
949 if (resource_type(entry->res) != IORESOURCE_MEM)
950 continue;
951
952 if (pci->num_ib_windows <= i)
953 break;
954
955 ret = dw_pcie_prog_inbound_atu(pci, i++, PCIE_ATU_TYPE_MEM,
956 entry->res->start,
957 entry->res->start - entry->offset,
958 resource_size(entry->res));
959 if (ret) {
960 dev_err(pci->dev, "Failed to set DMA range %pr\n",
961 entry->res);
962 return ret;
963 }
964 }
965
966 if (pci->num_ib_windows <= i)
967 dev_warn(pci->dev, "Dma-ranges exceed inbound iATU size (%u)\n",
968 pci->num_ib_windows);
969
970 return 0;
971 }
972
dw_pcie_program_presets(struct dw_pcie_rp * pp,enum pci_bus_speed speed)973 static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed)
974 {
975 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
976 u8 lane_eq_offset, lane_reg_size, cap_id;
977 u8 *presets;
978 u32 cap;
979 int i;
980
981 if (speed == PCIE_SPEED_8_0GT) {
982 presets = (u8 *)pp->presets.eq_presets_8gts;
983 lane_eq_offset = PCI_SECPCI_LE_CTRL;
984 cap_id = PCI_EXT_CAP_ID_SECPCI;
985 /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/
986 lane_reg_size = 0x2;
987 } else if (speed == PCIE_SPEED_16_0GT) {
988 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1];
989 lane_eq_offset = PCI_PL_16GT_LE_CTRL;
990 cap_id = PCI_EXT_CAP_ID_PL_16GT;
991 lane_reg_size = 0x1;
992 } else if (speed == PCIE_SPEED_32_0GT) {
993 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1];
994 lane_eq_offset = PCI_PL_32GT_LE_CTRL;
995 cap_id = PCI_EXT_CAP_ID_PL_32GT;
996 lane_reg_size = 0x1;
997 } else if (speed == PCIE_SPEED_64_0GT) {
998 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1];
999 lane_eq_offset = PCI_PL_64GT_LE_CTRL;
1000 cap_id = PCI_EXT_CAP_ID_PL_64GT;
1001 lane_reg_size = 0x1;
1002 } else {
1003 return;
1004 }
1005
1006 if (presets[0] == PCI_EQ_RESV)
1007 return;
1008
1009 cap = dw_pcie_find_ext_capability(pci, cap_id);
1010 if (!cap)
1011 return;
1012
1013 /*
1014 * Write preset values to the registers byte-by-byte for the given
1015 * number of lanes and register size.
1016 */
1017 for (i = 0; i < pci->num_lanes * lane_reg_size; i++)
1018 dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]);
1019 }
1020
dw_pcie_config_presets(struct dw_pcie_rp * pp)1021 static void dw_pcie_config_presets(struct dw_pcie_rp *pp)
1022 {
1023 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1024 enum pci_bus_speed speed = pcie_link_speed[pci->max_link_speed];
1025
1026 /*
1027 * Lane equalization settings need to be applied for all data rates the
1028 * controller supports and for all supported lanes.
1029 */
1030
1031 if (speed >= PCIE_SPEED_8_0GT)
1032 dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT);
1033
1034 if (speed >= PCIE_SPEED_16_0GT)
1035 dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT);
1036
1037 if (speed >= PCIE_SPEED_32_0GT)
1038 dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT);
1039
1040 if (speed >= PCIE_SPEED_64_0GT)
1041 dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT);
1042 }
1043
dw_pcie_setup_rc(struct dw_pcie_rp * pp)1044 int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
1045 {
1046 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1047 u32 val;
1048 int ret;
1049
1050 /*
1051 * Enable DBI read-only registers for writing/updating configuration.
1052 * Write permission gets disabled towards the end of this function.
1053 */
1054 dw_pcie_dbi_ro_wr_en(pci);
1055
1056 dw_pcie_setup(pci);
1057
1058 dw_pcie_msi_init(pp);
1059
1060 /* Setup RC BARs */
1061 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004);
1062 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
1063
1064 /* Setup interrupt pins */
1065 val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
1066 val &= 0xffff00ff;
1067 val |= 0x00000100;
1068 dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
1069
1070 /* Setup bus numbers */
1071 val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
1072 val &= 0xff000000;
1073 val |= 0x00ff0100;
1074 dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val);
1075
1076 /* Setup command register */
1077 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
1078 val &= 0xffff0000;
1079 val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
1080 PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
1081 dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
1082
1083 dw_pcie_config_presets(pp);
1084 /*
1085 * If the platform provides its own child bus config accesses, it means
1086 * the platform uses its own address translation component rather than
1087 * ATU, so we should not program the ATU here.
1088 */
1089 if (pp->bridge->child_ops == &dw_child_pcie_ops) {
1090 ret = dw_pcie_iatu_setup(pp);
1091 if (ret)
1092 return ret;
1093 }
1094
1095 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
1096
1097 /* Program correct class for RC */
1098 dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
1099
1100 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
1101 val |= PORT_LOGIC_SPEED_CHANGE;
1102 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
1103
1104 dw_pcie_dbi_ro_wr_dis(pci);
1105
1106 return 0;
1107 }
1108 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
1109
dw_pcie_pme_turn_off(struct dw_pcie * pci)1110 static int dw_pcie_pme_turn_off(struct dw_pcie *pci)
1111 {
1112 struct dw_pcie_ob_atu_cfg atu = { 0 };
1113 void __iomem *mem;
1114 int ret;
1115
1116 if (pci->num_ob_windows <= pci->pp.msg_atu_index)
1117 return -ENOSPC;
1118
1119 if (!pci->pp.msg_res)
1120 return -ENOSPC;
1121
1122 atu.code = PCIE_MSG_CODE_PME_TURN_OFF;
1123 atu.routing = PCIE_MSG_TYPE_R_BC;
1124 atu.type = PCIE_ATU_TYPE_MSG;
1125 atu.size = resource_size(pci->pp.msg_res);
1126 atu.index = pci->pp.msg_atu_index;
1127
1128 atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset;
1129
1130 ret = dw_pcie_prog_outbound_atu(pci, &atu);
1131 if (ret)
1132 return ret;
1133
1134 mem = ioremap(pci->pp.msg_res->start, pci->region_align);
1135 if (!mem)
1136 return -ENOMEM;
1137
1138 /* A dummy write is converted to a Msg TLP */
1139 writel(0, mem);
1140
1141 iounmap(mem);
1142
1143 return 0;
1144 }
1145
dw_pcie_suspend_noirq(struct dw_pcie * pci)1146 int dw_pcie_suspend_noirq(struct dw_pcie *pci)
1147 {
1148 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1149 u32 val;
1150 int ret;
1151
1152 /*
1153 * If L1SS is supported, then do not put the link into L2 as some
1154 * devices such as NVMe expect low resume latency.
1155 */
1156 if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
1157 return 0;
1158
1159 if (pci->pp.ops->pme_turn_off) {
1160 pci->pp.ops->pme_turn_off(&pci->pp);
1161 } else {
1162 ret = dw_pcie_pme_turn_off(pci);
1163 if (ret)
1164 return ret;
1165 }
1166
1167 ret = read_poll_timeout(dw_pcie_get_ltssm, val,
1168 val == DW_PCIE_LTSSM_L2_IDLE ||
1169 val <= DW_PCIE_LTSSM_DETECT_WAIT,
1170 PCIE_PME_TO_L2_TIMEOUT_US/10,
1171 PCIE_PME_TO_L2_TIMEOUT_US, false, pci);
1172 if (ret) {
1173 /* Only log message when LTSSM isn't in DETECT or POLL */
1174 dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val);
1175 return ret;
1176 }
1177
1178 /*
1179 * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least
1180 * 100ns after L2/L3 Ready before turning off refclock and
1181 * main power. This is harmless when no endpoint is connected.
1182 */
1183 udelay(1);
1184
1185 dw_pcie_stop_link(pci);
1186 if (pci->pp.ops->deinit)
1187 pci->pp.ops->deinit(&pci->pp);
1188
1189 pci->suspended = true;
1190
1191 return ret;
1192 }
1193 EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
1194
dw_pcie_resume_noirq(struct dw_pcie * pci)1195 int dw_pcie_resume_noirq(struct dw_pcie *pci)
1196 {
1197 int ret;
1198
1199 if (!pci->suspended)
1200 return 0;
1201
1202 pci->suspended = false;
1203
1204 if (pci->pp.ops->init) {
1205 ret = pci->pp.ops->init(&pci->pp);
1206 if (ret) {
1207 dev_err(pci->dev, "Host init failed: %d\n", ret);
1208 return ret;
1209 }
1210 }
1211
1212 dw_pcie_setup_rc(&pci->pp);
1213
1214 ret = dw_pcie_start_link(pci);
1215 if (ret)
1216 return ret;
1217
1218 ret = dw_pcie_wait_for_link(pci);
1219 if (ret)
1220 return ret;
1221
1222 return ret;
1223 }
1224 EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
1225