1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Synopsys DesignWare PCIe host controller driver
4 *
5 * Copyright (C) 2013 Samsung Electronics Co., Ltd.
6 * https://www.samsung.com
7 *
8 * Author: Jingoo Han <jg1.han@samsung.com>
9 */
10
11 #include <linux/align.h>
12 #include <linux/iopoll.h>
13 #include <linux/irqchip/chained_irq.h>
14 #include <linux/irqchip/irq-msi-lib.h>
15 #include <linux/irqdomain.h>
16 #include <linux/msi.h>
17 #include <linux/of_address.h>
18 #include <linux/of_pci.h>
19 #include <linux/pci_regs.h>
20 #include <linux/platform_device.h>
21
22 #include "../../pci.h"
23 #include "pcie-designware.h"
24
25 static struct pci_ops dw_pcie_ops;
26 static struct pci_ops dw_child_pcie_ops;
27
28 #define DW_PCIE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \
29 MSI_FLAG_USE_DEF_CHIP_OPS | \
30 MSI_FLAG_NO_AFFINITY | \
31 MSI_FLAG_PCI_MSI_MASK_PARENT)
32 #define DW_PCIE_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \
33 MSI_FLAG_PCI_MSIX | \
34 MSI_GENERIC_FLAGS_MASK)
35
36 #define IS_256MB_ALIGNED(x) IS_ALIGNED(x, SZ_256M)
37
38 static const struct msi_parent_ops dw_pcie_msi_parent_ops = {
39 .required_flags = DW_PCIE_MSI_FLAGS_REQUIRED,
40 .supported_flags = DW_PCIE_MSI_FLAGS_SUPPORTED,
41 .bus_select_token = DOMAIN_BUS_PCI_MSI,
42 .chip_flags = MSI_CHIP_FLAG_SET_ACK,
43 .prefix = "DW-",
44 .init_dev_msi_info = msi_lib_init_dev_msi_info,
45 };
46
47 /* MSI int handler */
dw_handle_msi_irq(struct dw_pcie_rp * pp)48 irqreturn_t dw_handle_msi_irq(struct dw_pcie_rp *pp)
49 {
50 int i, pos;
51 unsigned long val;
52 u32 status, num_ctrls;
53 irqreturn_t ret = IRQ_NONE;
54 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
55
56 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
57
58 for (i = 0; i < num_ctrls; i++) {
59 status = dw_pcie_readl_dbi(pci, PCIE_MSI_INTR0_STATUS +
60 (i * MSI_REG_CTRL_BLOCK_SIZE));
61 if (!status)
62 continue;
63
64 ret = IRQ_HANDLED;
65 val = status;
66 pos = 0;
67 while ((pos = find_next_bit(&val, MAX_MSI_IRQS_PER_CTRL,
68 pos)) != MAX_MSI_IRQS_PER_CTRL) {
69 generic_handle_domain_irq(pp->irq_domain,
70 (i * MAX_MSI_IRQS_PER_CTRL) +
71 pos);
72 pos++;
73 }
74 }
75
76 return ret;
77 }
78
79 /* Chained MSI interrupt service routine */
dw_chained_msi_isr(struct irq_desc * desc)80 static void dw_chained_msi_isr(struct irq_desc *desc)
81 {
82 struct irq_chip *chip = irq_desc_get_chip(desc);
83 struct dw_pcie_rp *pp;
84
85 chained_irq_enter(chip, desc);
86
87 pp = irq_desc_get_handler_data(desc);
88 dw_handle_msi_irq(pp);
89
90 chained_irq_exit(chip, desc);
91 }
92
dw_pci_setup_msi_msg(struct irq_data * d,struct msi_msg * msg)93 static void dw_pci_setup_msi_msg(struct irq_data *d, struct msi_msg *msg)
94 {
95 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
96 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
97 u64 msi_target;
98
99 msi_target = (u64)pp->msi_data;
100
101 msg->address_lo = lower_32_bits(msi_target);
102 msg->address_hi = upper_32_bits(msi_target);
103
104 msg->data = d->hwirq;
105
106 dev_dbg(pci->dev, "msi#%d address_hi %#x address_lo %#x\n",
107 (int)d->hwirq, msg->address_hi, msg->address_lo);
108 }
109
dw_pci_bottom_mask(struct irq_data * d)110 static void dw_pci_bottom_mask(struct irq_data *d)
111 {
112 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
113 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
114 unsigned int res, bit, ctrl;
115 unsigned long flags;
116
117 raw_spin_lock_irqsave(&pp->lock, flags);
118
119 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
120 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
121 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
122
123 pp->irq_mask[ctrl] |= BIT(bit);
124 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
125
126 raw_spin_unlock_irqrestore(&pp->lock, flags);
127 }
128
dw_pci_bottom_unmask(struct irq_data * d)129 static void dw_pci_bottom_unmask(struct irq_data *d)
130 {
131 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
132 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
133 unsigned int res, bit, ctrl;
134 unsigned long flags;
135
136 raw_spin_lock_irqsave(&pp->lock, flags);
137
138 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
139 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
140 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
141
142 pp->irq_mask[ctrl] &= ~BIT(bit);
143 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK + res, pp->irq_mask[ctrl]);
144
145 raw_spin_unlock_irqrestore(&pp->lock, flags);
146 }
147
dw_pci_bottom_ack(struct irq_data * d)148 static void dw_pci_bottom_ack(struct irq_data *d)
149 {
150 struct dw_pcie_rp *pp = irq_data_get_irq_chip_data(d);
151 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
152 unsigned int res, bit, ctrl;
153
154 ctrl = d->hwirq / MAX_MSI_IRQS_PER_CTRL;
155 res = ctrl * MSI_REG_CTRL_BLOCK_SIZE;
156 bit = d->hwirq % MAX_MSI_IRQS_PER_CTRL;
157
158 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_STATUS + res, BIT(bit));
159 }
160
161 static struct irq_chip dw_pci_msi_bottom_irq_chip = {
162 .name = "DWPCI-MSI",
163 .irq_ack = dw_pci_bottom_ack,
164 .irq_compose_msi_msg = dw_pci_setup_msi_msg,
165 .irq_mask = dw_pci_bottom_mask,
166 .irq_unmask = dw_pci_bottom_unmask,
167 };
168
dw_pcie_irq_domain_alloc(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs,void * args)169 static int dw_pcie_irq_domain_alloc(struct irq_domain *domain,
170 unsigned int virq, unsigned int nr_irqs,
171 void *args)
172 {
173 struct dw_pcie_rp *pp = domain->host_data;
174 unsigned long flags;
175 u32 i;
176 int bit;
177
178 raw_spin_lock_irqsave(&pp->lock, flags);
179
180 bit = bitmap_find_free_region(pp->msi_irq_in_use, pp->num_vectors,
181 order_base_2(nr_irqs));
182
183 raw_spin_unlock_irqrestore(&pp->lock, flags);
184
185 if (bit < 0)
186 return -ENOSPC;
187
188 for (i = 0; i < nr_irqs; i++)
189 irq_domain_set_info(domain, virq + i, bit + i,
190 pp->msi_irq_chip,
191 pp, handle_edge_irq,
192 NULL, NULL);
193
194 return 0;
195 }
196
dw_pcie_irq_domain_free(struct irq_domain * domain,unsigned int virq,unsigned int nr_irqs)197 static void dw_pcie_irq_domain_free(struct irq_domain *domain,
198 unsigned int virq, unsigned int nr_irqs)
199 {
200 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
201 struct dw_pcie_rp *pp = domain->host_data;
202 unsigned long flags;
203
204 raw_spin_lock_irqsave(&pp->lock, flags);
205
206 bitmap_release_region(pp->msi_irq_in_use, d->hwirq,
207 order_base_2(nr_irqs));
208
209 raw_spin_unlock_irqrestore(&pp->lock, flags);
210 }
211
212 static const struct irq_domain_ops dw_pcie_msi_domain_ops = {
213 .alloc = dw_pcie_irq_domain_alloc,
214 .free = dw_pcie_irq_domain_free,
215 };
216
dw_pcie_allocate_domains(struct dw_pcie_rp * pp)217 int dw_pcie_allocate_domains(struct dw_pcie_rp *pp)
218 {
219 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
220 struct irq_domain_info info = {
221 .fwnode = dev_fwnode(pci->dev),
222 .ops = &dw_pcie_msi_domain_ops,
223 .size = pp->num_vectors,
224 .host_data = pp,
225 };
226
227 pp->irq_domain = msi_create_parent_irq_domain(&info, &dw_pcie_msi_parent_ops);
228 if (!pp->irq_domain) {
229 dev_err(pci->dev, "Failed to create IRQ domain\n");
230 return -ENOMEM;
231 }
232
233 return 0;
234 }
235
dw_pcie_free_msi(struct dw_pcie_rp * pp)236 void dw_pcie_free_msi(struct dw_pcie_rp *pp)
237 {
238 u32 ctrl;
239
240 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
241 if (pp->msi_irq[ctrl] > 0)
242 irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
243 NULL, NULL);
244 }
245
246 irq_domain_remove(pp->irq_domain);
247 }
248 EXPORT_SYMBOL_GPL(dw_pcie_free_msi);
249
dw_pcie_msi_init(struct dw_pcie_rp * pp)250 void dw_pcie_msi_init(struct dw_pcie_rp *pp)
251 {
252 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
253 u64 msi_target = (u64)pp->msi_data;
254 u32 ctrl, num_ctrls;
255
256 if (!pci_msi_enabled() || !pp->has_msi_ctrl)
257 return;
258
259 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
260
261 /* Initialize IRQ Status array */
262 for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
263 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_MASK +
264 (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
265 pp->irq_mask[ctrl]);
266 dw_pcie_writel_dbi(pci, PCIE_MSI_INTR0_ENABLE +
267 (ctrl * MSI_REG_CTRL_BLOCK_SIZE),
268 ~0);
269 }
270
271 /* Program the msi_data */
272 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_LO, lower_32_bits(msi_target));
273 dw_pcie_writel_dbi(pci, PCIE_MSI_ADDR_HI, upper_32_bits(msi_target));
274 }
275 EXPORT_SYMBOL_GPL(dw_pcie_msi_init);
276
dw_pcie_parse_split_msi_irq(struct dw_pcie_rp * pp)277 static int dw_pcie_parse_split_msi_irq(struct dw_pcie_rp *pp)
278 {
279 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
280 struct device *dev = pci->dev;
281 struct platform_device *pdev = to_platform_device(dev);
282 u32 ctrl, max_vectors;
283 int irq;
284
285 /* Parse any "msiX" IRQs described in the devicetree */
286 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++) {
287 char msi_name[] = "msiX";
288
289 msi_name[3] = '0' + ctrl;
290 irq = platform_get_irq_byname_optional(pdev, msi_name);
291 if (irq == -ENXIO)
292 break;
293 if (irq < 0)
294 return dev_err_probe(dev, irq,
295 "Failed to parse MSI IRQ '%s'\n",
296 msi_name);
297
298 pp->msi_irq[ctrl] = irq;
299 }
300
301 /* If no "msiX" IRQs, caller should fallback to "msi" IRQ */
302 if (ctrl == 0)
303 return -ENXIO;
304
305 max_vectors = ctrl * MAX_MSI_IRQS_PER_CTRL;
306 if (pp->num_vectors > max_vectors) {
307 dev_warn(dev, "Exceeding number of MSI vectors, limiting to %u\n",
308 max_vectors);
309 pp->num_vectors = max_vectors;
310 }
311 if (!pp->num_vectors)
312 pp->num_vectors = max_vectors;
313
314 return 0;
315 }
316
dw_pcie_msi_host_init(struct dw_pcie_rp * pp)317 int dw_pcie_msi_host_init(struct dw_pcie_rp *pp)
318 {
319 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
320 struct device *dev = pci->dev;
321 struct platform_device *pdev = to_platform_device(dev);
322 u64 *msi_vaddr = NULL;
323 int ret;
324 u32 ctrl, num_ctrls;
325
326 for (ctrl = 0; ctrl < MAX_MSI_CTRLS; ctrl++)
327 pp->irq_mask[ctrl] = ~0;
328
329 if (!pp->msi_irq[0]) {
330 ret = dw_pcie_parse_split_msi_irq(pp);
331 if (ret < 0 && ret != -ENXIO)
332 return ret;
333 }
334
335 if (!pp->num_vectors)
336 pp->num_vectors = MSI_DEF_NUM_VECTORS;
337 num_ctrls = pp->num_vectors / MAX_MSI_IRQS_PER_CTRL;
338
339 if (!pp->msi_irq[0]) {
340 pp->msi_irq[0] = platform_get_irq_byname_optional(pdev, "msi");
341 if (pp->msi_irq[0] < 0) {
342 pp->msi_irq[0] = platform_get_irq(pdev, 0);
343 if (pp->msi_irq[0] < 0)
344 return pp->msi_irq[0];
345 }
346 }
347
348 dev_dbg(dev, "Using %d MSI vectors\n", pp->num_vectors);
349
350 pp->msi_irq_chip = &dw_pci_msi_bottom_irq_chip;
351
352 ret = dw_pcie_allocate_domains(pp);
353 if (ret)
354 return ret;
355
356 for (ctrl = 0; ctrl < num_ctrls; ctrl++) {
357 if (pp->msi_irq[ctrl] > 0)
358 irq_set_chained_handler_and_data(pp->msi_irq[ctrl],
359 dw_chained_msi_isr, pp);
360 }
361
362 /*
363 * Even though the iMSI-RX Module supports 64-bit addresses some
364 * peripheral PCIe devices may lack 64-bit message support. In
365 * order not to miss MSI TLPs from those devices the MSI target
366 * address has to be within the lowest 4GB.
367 *
368 * Note until there is a better alternative found the reservation is
369 * done by allocating from the artificially limited DMA-coherent
370 * memory.
371 */
372 ret = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
373 if (!ret)
374 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
375 GFP_KERNEL);
376
377 if (!msi_vaddr) {
378 dev_warn(dev, "Failed to allocate 32-bit MSI address\n");
379 dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
380 msi_vaddr = dmam_alloc_coherent(dev, sizeof(u64), &pp->msi_data,
381 GFP_KERNEL);
382 if (!msi_vaddr) {
383 dev_err(dev, "Failed to allocate MSI address\n");
384 dw_pcie_free_msi(pp);
385 return -ENOMEM;
386 }
387 }
388
389 return 0;
390 }
391 EXPORT_SYMBOL_GPL(dw_pcie_msi_host_init);
392
dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp * pp)393 static void dw_pcie_host_request_msg_tlp_res(struct dw_pcie_rp *pp)
394 {
395 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
396 struct resource_entry *win;
397 struct resource *res;
398
399 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_MEM);
400 if (win) {
401 res = devm_kzalloc(pci->dev, sizeof(*res), GFP_KERNEL);
402 if (!res)
403 return;
404
405 /*
406 * Allocate MSG TLP region of size 'region_align' at the end of
407 * the host bridge window.
408 */
409 res->start = win->res->end - pci->region_align + 1;
410 res->end = win->res->end;
411 res->name = "msg";
412 res->flags = win->res->flags | IORESOURCE_BUSY;
413
414 if (!devm_request_resource(pci->dev, win->res, res))
415 pp->msg_res = res;
416 }
417 }
418
dw_pcie_config_ecam_iatu(struct dw_pcie_rp * pp)419 static int dw_pcie_config_ecam_iatu(struct dw_pcie_rp *pp)
420 {
421 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
422 struct dw_pcie_ob_atu_cfg atu = {0};
423 resource_size_t bus_range_max;
424 struct resource_entry *bus;
425 int ret;
426
427 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
428
429 /*
430 * Root bus under the host bridge doesn't require any iATU configuration
431 * as DBI region will be used to access root bus config space.
432 * Immediate bus under Root Bus, needs type 0 iATU configuration and
433 * remaining buses need type 1 iATU configuration.
434 */
435 atu.index = 0;
436 atu.type = PCIE_ATU_TYPE_CFG0;
437 atu.parent_bus_addr = pp->cfg0_base + SZ_1M;
438 /* 1MiB is to cover 1 (bus) * 32 (devices) * 8 (functions) */
439 atu.size = SZ_1M;
440 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
441 ret = dw_pcie_prog_outbound_atu(pci, &atu);
442 if (ret)
443 return ret;
444
445 bus_range_max = resource_size(bus->res);
446
447 if (bus_range_max < 2)
448 return 0;
449
450 /* Configure remaining buses in type 1 iATU configuration */
451 atu.index = 1;
452 atu.type = PCIE_ATU_TYPE_CFG1;
453 atu.parent_bus_addr = pp->cfg0_base + SZ_2M;
454 atu.size = (SZ_1M * bus_range_max) - SZ_2M;
455 atu.ctrl2 = PCIE_ATU_CFG_SHIFT_MODE_ENABLE;
456
457 return dw_pcie_prog_outbound_atu(pci, &atu);
458 }
459
dw_pcie_create_ecam_window(struct dw_pcie_rp * pp,struct resource * res)460 static int dw_pcie_create_ecam_window(struct dw_pcie_rp *pp, struct resource *res)
461 {
462 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
463 struct device *dev = pci->dev;
464 struct resource_entry *bus;
465
466 bus = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS);
467 if (!bus)
468 return -ENODEV;
469
470 pp->cfg = pci_ecam_create(dev, res, bus->res, &pci_generic_ecam_ops);
471 if (IS_ERR(pp->cfg))
472 return PTR_ERR(pp->cfg);
473
474 pci->dbi_base = pp->cfg->win;
475 pci->dbi_phys_addr = res->start;
476
477 return 0;
478 }
479
dw_pcie_ecam_enabled(struct dw_pcie_rp * pp,struct resource * config_res)480 static bool dw_pcie_ecam_enabled(struct dw_pcie_rp *pp, struct resource *config_res)
481 {
482 struct resource *bus_range;
483 u64 nr_buses;
484
485 /* Vendor glue drivers may implement their own ECAM mechanism */
486 if (pp->native_ecam)
487 return false;
488
489 /*
490 * PCIe spec r6.0, sec 7.2.2 mandates the base address used for ECAM to
491 * be aligned on a 2^(n+20) byte boundary, where n is the number of bits
492 * used for representing 'bus' in BDF. Since the DWC cores always use 8
493 * bits for representing 'bus', the base address has to be aligned to
494 * 2^28 byte boundary, which is 256 MiB.
495 */
496 if (!IS_256MB_ALIGNED(config_res->start))
497 return false;
498
499 bus_range = resource_list_first_type(&pp->bridge->windows, IORESOURCE_BUS)->res;
500 if (!bus_range)
501 return false;
502
503 nr_buses = resource_size(config_res) >> PCIE_ECAM_BUS_SHIFT;
504
505 return nr_buses >= resource_size(bus_range);
506 }
507
dw_pcie_host_get_resources(struct dw_pcie_rp * pp)508 static int dw_pcie_host_get_resources(struct dw_pcie_rp *pp)
509 {
510 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
511 struct device *dev = pci->dev;
512 struct platform_device *pdev = to_platform_device(dev);
513 struct resource_entry *win;
514 struct resource *res;
515 int ret;
516
517 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "config");
518 if (!res) {
519 dev_err(dev, "Missing \"config\" reg space\n");
520 return -ENODEV;
521 }
522
523 pp->cfg0_size = resource_size(res);
524 pp->cfg0_base = res->start;
525
526 pp->ecam_enabled = dw_pcie_ecam_enabled(pp, res);
527 if (pp->ecam_enabled) {
528 ret = dw_pcie_create_ecam_window(pp, res);
529 if (ret)
530 return ret;
531
532 pp->bridge->ops = (struct pci_ops *)&pci_generic_ecam_ops.pci_ops;
533 pp->bridge->sysdata = pp->cfg;
534 pp->cfg->priv = pp;
535 } else {
536 pp->va_cfg0_base = devm_pci_remap_cfg_resource(dev, res);
537 if (IS_ERR(pp->va_cfg0_base))
538 return PTR_ERR(pp->va_cfg0_base);
539
540 /* Set default bus ops */
541 pp->bridge->ops = &dw_pcie_ops;
542 pp->bridge->child_ops = &dw_child_pcie_ops;
543 pp->bridge->sysdata = pp;
544 }
545
546 ret = dw_pcie_get_resources(pci);
547 if (ret) {
548 if (pp->cfg)
549 pci_ecam_free(pp->cfg);
550 return ret;
551 }
552
553 /* Get the I/O range from DT */
554 win = resource_list_first_type(&pp->bridge->windows, IORESOURCE_IO);
555 if (win) {
556 pp->io_size = resource_size(win->res);
557 pp->io_bus_addr = win->res->start - win->offset;
558 pp->io_base = pci_pio_to_address(win->res->start);
559 }
560
561 /*
562 * visconti_pcie_cpu_addr_fixup() uses pp->io_base, so we have to
563 * call dw_pcie_parent_bus_offset() after setting pp->io_base.
564 */
565 pci->parent_bus_offset = dw_pcie_parent_bus_offset(pci, "config",
566 pp->cfg0_base);
567 return 0;
568 }
569
dw_pcie_host_init(struct dw_pcie_rp * pp)570 int dw_pcie_host_init(struct dw_pcie_rp *pp)
571 {
572 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
573 struct device *dev = pci->dev;
574 struct device_node *np = dev->of_node;
575 struct pci_host_bridge *bridge;
576 int ret;
577
578 raw_spin_lock_init(&pp->lock);
579
580 bridge = devm_pci_alloc_host_bridge(dev, 0);
581 if (!bridge)
582 return -ENOMEM;
583
584 pp->bridge = bridge;
585
586 ret = dw_pcie_host_get_resources(pp);
587 if (ret)
588 return ret;
589
590 if (pp->ops->init) {
591 ret = pp->ops->init(pp);
592 if (ret)
593 goto err_free_ecam;
594 }
595
596 if (pci_msi_enabled()) {
597 pp->has_msi_ctrl = !(pp->ops->msi_init ||
598 of_property_present(np, "msi-parent") ||
599 of_property_present(np, "msi-map"));
600
601 /*
602 * For the has_msi_ctrl case the default assignment is handled
603 * in the dw_pcie_msi_host_init().
604 */
605 if (!pp->has_msi_ctrl && !pp->num_vectors) {
606 pp->num_vectors = MSI_DEF_NUM_VECTORS;
607 } else if (pp->num_vectors > MAX_MSI_IRQS) {
608 dev_err(dev, "Invalid number of vectors\n");
609 ret = -EINVAL;
610 goto err_deinit_host;
611 }
612
613 if (pp->ops->msi_init) {
614 ret = pp->ops->msi_init(pp);
615 if (ret < 0)
616 goto err_deinit_host;
617 } else if (pp->has_msi_ctrl) {
618 ret = dw_pcie_msi_host_init(pp);
619 if (ret < 0)
620 goto err_deinit_host;
621 }
622 }
623
624 dw_pcie_version_detect(pci);
625
626 dw_pcie_iatu_detect(pci);
627
628 if (pci->num_lanes < 1)
629 pci->num_lanes = dw_pcie_link_get_max_link_width(pci);
630
631 ret = of_pci_get_equalization_presets(dev, &pp->presets, pci->num_lanes);
632 if (ret)
633 goto err_free_msi;
634
635 if (pp->ecam_enabled) {
636 ret = dw_pcie_config_ecam_iatu(pp);
637 if (ret) {
638 dev_err(dev, "Failed to configure iATU in ECAM mode\n");
639 goto err_free_msi;
640 }
641 }
642
643 /*
644 * Allocate the resource for MSG TLP before programming the iATU
645 * outbound window in dw_pcie_setup_rc(). Since the allocation depends
646 * on the value of 'region_align', this has to be done after
647 * dw_pcie_iatu_detect().
648 *
649 * Glue drivers need to set 'use_atu_msg' before dw_pcie_host_init() to
650 * make use of the generic MSG TLP implementation.
651 */
652 if (pp->use_atu_msg)
653 dw_pcie_host_request_msg_tlp_res(pp);
654
655 ret = dw_pcie_edma_detect(pci);
656 if (ret)
657 goto err_free_msi;
658
659 ret = dw_pcie_setup_rc(pp);
660 if (ret)
661 goto err_remove_edma;
662
663 if (!dw_pcie_link_up(pci)) {
664 ret = dw_pcie_start_link(pci);
665 if (ret)
666 goto err_remove_edma;
667 }
668
669 /*
670 * Note: Skip the link up delay only when a Link Up IRQ is present.
671 * If there is no Link Up IRQ, we should not bypass the delay
672 * because that would require users to manually rescan for devices.
673 */
674 if (!pp->use_linkup_irq)
675 /* Ignore errors, the link may come up later */
676 dw_pcie_wait_for_link(pci);
677
678 ret = pci_host_probe(bridge);
679 if (ret)
680 goto err_stop_link;
681
682 if (pp->ops->post_init)
683 pp->ops->post_init(pp);
684
685 dwc_pcie_debugfs_init(pci, DW_PCIE_RC_TYPE);
686
687 return 0;
688
689 err_stop_link:
690 dw_pcie_stop_link(pci);
691
692 err_remove_edma:
693 dw_pcie_edma_remove(pci);
694
695 err_free_msi:
696 if (pp->has_msi_ctrl)
697 dw_pcie_free_msi(pp);
698
699 err_deinit_host:
700 if (pp->ops->deinit)
701 pp->ops->deinit(pp);
702
703 err_free_ecam:
704 if (pp->cfg)
705 pci_ecam_free(pp->cfg);
706
707 return ret;
708 }
709 EXPORT_SYMBOL_GPL(dw_pcie_host_init);
710
dw_pcie_host_deinit(struct dw_pcie_rp * pp)711 void dw_pcie_host_deinit(struct dw_pcie_rp *pp)
712 {
713 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
714
715 dwc_pcie_debugfs_deinit(pci);
716
717 pci_stop_root_bus(pp->bridge->bus);
718 pci_remove_root_bus(pp->bridge->bus);
719
720 dw_pcie_stop_link(pci);
721
722 dw_pcie_edma_remove(pci);
723
724 if (pp->has_msi_ctrl)
725 dw_pcie_free_msi(pp);
726
727 if (pp->ops->deinit)
728 pp->ops->deinit(pp);
729
730 if (pp->cfg)
731 pci_ecam_free(pp->cfg);
732 }
733 EXPORT_SYMBOL_GPL(dw_pcie_host_deinit);
734
dw_pcie_other_conf_map_bus(struct pci_bus * bus,unsigned int devfn,int where)735 static void __iomem *dw_pcie_other_conf_map_bus(struct pci_bus *bus,
736 unsigned int devfn, int where)
737 {
738 struct dw_pcie_rp *pp = bus->sysdata;
739 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
740 struct dw_pcie_ob_atu_cfg atu = { 0 };
741 int type, ret;
742 u32 busdev;
743
744 /*
745 * Checking whether the link is up here is a last line of defense
746 * against platforms that forward errors on the system bus as
747 * SError upon PCI configuration transactions issued when the link
748 * is down. This check is racy by definition and does not stop
749 * the system from triggering an SError if the link goes down
750 * after this check is performed.
751 */
752 if (!dw_pcie_link_up(pci))
753 return NULL;
754
755 busdev = PCIE_ATU_BUS(bus->number) | PCIE_ATU_DEV(PCI_SLOT(devfn)) |
756 PCIE_ATU_FUNC(PCI_FUNC(devfn));
757
758 if (pci_is_root_bus(bus->parent))
759 type = PCIE_ATU_TYPE_CFG0;
760 else
761 type = PCIE_ATU_TYPE_CFG1;
762
763 atu.type = type;
764 atu.parent_bus_addr = pp->cfg0_base - pci->parent_bus_offset;
765 atu.pci_addr = busdev;
766 atu.size = pp->cfg0_size;
767
768 ret = dw_pcie_prog_outbound_atu(pci, &atu);
769 if (ret)
770 return NULL;
771
772 return pp->va_cfg0_base + where;
773 }
774
dw_pcie_rd_other_conf(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)775 static int dw_pcie_rd_other_conf(struct pci_bus *bus, unsigned int devfn,
776 int where, int size, u32 *val)
777 {
778 struct dw_pcie_rp *pp = bus->sysdata;
779 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
780 struct dw_pcie_ob_atu_cfg atu = { 0 };
781 int ret;
782
783 ret = pci_generic_config_read(bus, devfn, where, size, val);
784 if (ret != PCIBIOS_SUCCESSFUL)
785 return ret;
786
787 if (pp->cfg0_io_shared) {
788 atu.type = PCIE_ATU_TYPE_IO;
789 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
790 atu.pci_addr = pp->io_bus_addr;
791 atu.size = pp->io_size;
792
793 ret = dw_pcie_prog_outbound_atu(pci, &atu);
794 if (ret)
795 return PCIBIOS_SET_FAILED;
796 }
797
798 return PCIBIOS_SUCCESSFUL;
799 }
800
dw_pcie_wr_other_conf(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)801 static int dw_pcie_wr_other_conf(struct pci_bus *bus, unsigned int devfn,
802 int where, int size, u32 val)
803 {
804 struct dw_pcie_rp *pp = bus->sysdata;
805 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
806 struct dw_pcie_ob_atu_cfg atu = { 0 };
807 int ret;
808
809 ret = pci_generic_config_write(bus, devfn, where, size, val);
810 if (ret != PCIBIOS_SUCCESSFUL)
811 return ret;
812
813 if (pp->cfg0_io_shared) {
814 atu.type = PCIE_ATU_TYPE_IO;
815 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
816 atu.pci_addr = pp->io_bus_addr;
817 atu.size = pp->io_size;
818
819 ret = dw_pcie_prog_outbound_atu(pci, &atu);
820 if (ret)
821 return PCIBIOS_SET_FAILED;
822 }
823
824 return PCIBIOS_SUCCESSFUL;
825 }
826
827 static struct pci_ops dw_child_pcie_ops = {
828 .map_bus = dw_pcie_other_conf_map_bus,
829 .read = dw_pcie_rd_other_conf,
830 .write = dw_pcie_wr_other_conf,
831 };
832
dw_pcie_own_conf_map_bus(struct pci_bus * bus,unsigned int devfn,int where)833 void __iomem *dw_pcie_own_conf_map_bus(struct pci_bus *bus, unsigned int devfn, int where)
834 {
835 struct dw_pcie_rp *pp = bus->sysdata;
836 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
837
838 if (PCI_SLOT(devfn) > 0)
839 return NULL;
840
841 return pci->dbi_base + where;
842 }
843 EXPORT_SYMBOL_GPL(dw_pcie_own_conf_map_bus);
844
845 static struct pci_ops dw_pcie_ops = {
846 .map_bus = dw_pcie_own_conf_map_bus,
847 .read = pci_generic_config_read,
848 .write = pci_generic_config_write,
849 };
850
dw_pcie_iatu_setup(struct dw_pcie_rp * pp)851 static int dw_pcie_iatu_setup(struct dw_pcie_rp *pp)
852 {
853 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
854 struct dw_pcie_ob_atu_cfg atu = { 0 };
855 struct resource_entry *entry;
856 int i, ret;
857
858 /* Note the very first outbound ATU is used for CFG IOs */
859 if (!pci->num_ob_windows) {
860 dev_err(pci->dev, "No outbound iATU found\n");
861 return -EINVAL;
862 }
863
864 /*
865 * Ensure all out/inbound windows are disabled before proceeding with
866 * the MEM/IO (dma-)ranges setups.
867 */
868 for (i = 0; i < pci->num_ob_windows; i++)
869 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_OB, i);
870
871 for (i = 0; i < pci->num_ib_windows; i++)
872 dw_pcie_disable_atu(pci, PCIE_ATU_REGION_DIR_IB, i);
873
874 i = 0;
875 resource_list_for_each_entry(entry, &pp->bridge->windows) {
876 if (resource_type(entry->res) != IORESOURCE_MEM)
877 continue;
878
879 if (pci->num_ob_windows <= ++i)
880 break;
881
882 atu.index = i;
883 atu.type = PCIE_ATU_TYPE_MEM;
884 atu.parent_bus_addr = entry->res->start - pci->parent_bus_offset;
885 atu.pci_addr = entry->res->start - entry->offset;
886
887 /* Adjust iATU size if MSG TLP region was allocated before */
888 if (pp->msg_res && pp->msg_res->parent == entry->res)
889 atu.size = resource_size(entry->res) -
890 resource_size(pp->msg_res);
891 else
892 atu.size = resource_size(entry->res);
893
894 ret = dw_pcie_prog_outbound_atu(pci, &atu);
895 if (ret) {
896 dev_err(pci->dev, "Failed to set MEM range %pr\n",
897 entry->res);
898 return ret;
899 }
900 }
901
902 if (pp->io_size) {
903 if (pci->num_ob_windows > ++i) {
904 atu.index = i;
905 atu.type = PCIE_ATU_TYPE_IO;
906 atu.parent_bus_addr = pp->io_base - pci->parent_bus_offset;
907 atu.pci_addr = pp->io_bus_addr;
908 atu.size = pp->io_size;
909
910 ret = dw_pcie_prog_outbound_atu(pci, &atu);
911 if (ret) {
912 dev_err(pci->dev, "Failed to set IO range %pr\n",
913 entry->res);
914 return ret;
915 }
916 } else {
917 pp->cfg0_io_shared = true;
918 }
919 }
920
921 if (pci->num_ob_windows <= i)
922 dev_warn(pci->dev, "Ranges exceed outbound iATU size (%d)\n",
923 pci->num_ob_windows);
924
925 pp->msg_atu_index = i;
926
927 i = 0;
928 resource_list_for_each_entry(entry, &pp->bridge->dma_ranges) {
929 if (resource_type(entry->res) != IORESOURCE_MEM)
930 continue;
931
932 if (pci->num_ib_windows <= i)
933 break;
934
935 ret = dw_pcie_prog_inbound_atu(pci, i++, PCIE_ATU_TYPE_MEM,
936 entry->res->start,
937 entry->res->start - entry->offset,
938 resource_size(entry->res));
939 if (ret) {
940 dev_err(pci->dev, "Failed to set DMA range %pr\n",
941 entry->res);
942 return ret;
943 }
944 }
945
946 if (pci->num_ib_windows <= i)
947 dev_warn(pci->dev, "Dma-ranges exceed inbound iATU size (%u)\n",
948 pci->num_ib_windows);
949
950 return 0;
951 }
952
dw_pcie_program_presets(struct dw_pcie_rp * pp,enum pci_bus_speed speed)953 static void dw_pcie_program_presets(struct dw_pcie_rp *pp, enum pci_bus_speed speed)
954 {
955 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
956 u8 lane_eq_offset, lane_reg_size, cap_id;
957 u8 *presets;
958 u32 cap;
959 int i;
960
961 if (speed == PCIE_SPEED_8_0GT) {
962 presets = (u8 *)pp->presets.eq_presets_8gts;
963 lane_eq_offset = PCI_SECPCI_LE_CTRL;
964 cap_id = PCI_EXT_CAP_ID_SECPCI;
965 /* For data rate of 8 GT/S each lane equalization control is 16bits wide*/
966 lane_reg_size = 0x2;
967 } else if (speed == PCIE_SPEED_16_0GT) {
968 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_16GTS - 1];
969 lane_eq_offset = PCI_PL_16GT_LE_CTRL;
970 cap_id = PCI_EXT_CAP_ID_PL_16GT;
971 lane_reg_size = 0x1;
972 } else if (speed == PCIE_SPEED_32_0GT) {
973 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_32GTS - 1];
974 lane_eq_offset = PCI_PL_32GT_LE_CTRL;
975 cap_id = PCI_EXT_CAP_ID_PL_32GT;
976 lane_reg_size = 0x1;
977 } else if (speed == PCIE_SPEED_64_0GT) {
978 presets = pp->presets.eq_presets_Ngts[EQ_PRESET_TYPE_64GTS - 1];
979 lane_eq_offset = PCI_PL_64GT_LE_CTRL;
980 cap_id = PCI_EXT_CAP_ID_PL_64GT;
981 lane_reg_size = 0x1;
982 } else {
983 return;
984 }
985
986 if (presets[0] == PCI_EQ_RESV)
987 return;
988
989 cap = dw_pcie_find_ext_capability(pci, cap_id);
990 if (!cap)
991 return;
992
993 /*
994 * Write preset values to the registers byte-by-byte for the given
995 * number of lanes and register size.
996 */
997 for (i = 0; i < pci->num_lanes * lane_reg_size; i++)
998 dw_pcie_writeb_dbi(pci, cap + lane_eq_offset + i, presets[i]);
999 }
1000
dw_pcie_config_presets(struct dw_pcie_rp * pp)1001 static void dw_pcie_config_presets(struct dw_pcie_rp *pp)
1002 {
1003 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1004 enum pci_bus_speed speed = pcie_link_speed[pci->max_link_speed];
1005
1006 /*
1007 * Lane equalization settings need to be applied for all data rates the
1008 * controller supports and for all supported lanes.
1009 */
1010
1011 if (speed >= PCIE_SPEED_8_0GT)
1012 dw_pcie_program_presets(pp, PCIE_SPEED_8_0GT);
1013
1014 if (speed >= PCIE_SPEED_16_0GT)
1015 dw_pcie_program_presets(pp, PCIE_SPEED_16_0GT);
1016
1017 if (speed >= PCIE_SPEED_32_0GT)
1018 dw_pcie_program_presets(pp, PCIE_SPEED_32_0GT);
1019
1020 if (speed >= PCIE_SPEED_64_0GT)
1021 dw_pcie_program_presets(pp, PCIE_SPEED_64_0GT);
1022 }
1023
dw_pcie_setup_rc(struct dw_pcie_rp * pp)1024 int dw_pcie_setup_rc(struct dw_pcie_rp *pp)
1025 {
1026 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
1027 u32 val;
1028 int ret;
1029
1030 /*
1031 * Enable DBI read-only registers for writing/updating configuration.
1032 * Write permission gets disabled towards the end of this function.
1033 */
1034 dw_pcie_dbi_ro_wr_en(pci);
1035
1036 dw_pcie_setup(pci);
1037
1038 dw_pcie_msi_init(pp);
1039
1040 /* Setup RC BARs */
1041 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0x00000004);
1042 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_1, 0x00000000);
1043
1044 /* Setup interrupt pins */
1045 val = dw_pcie_readl_dbi(pci, PCI_INTERRUPT_LINE);
1046 val &= 0xffff00ff;
1047 val |= 0x00000100;
1048 dw_pcie_writel_dbi(pci, PCI_INTERRUPT_LINE, val);
1049
1050 /* Setup bus numbers */
1051 val = dw_pcie_readl_dbi(pci, PCI_PRIMARY_BUS);
1052 val &= 0xff000000;
1053 val |= 0x00ff0100;
1054 dw_pcie_writel_dbi(pci, PCI_PRIMARY_BUS, val);
1055
1056 /* Setup command register */
1057 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
1058 val &= 0xffff0000;
1059 val |= PCI_COMMAND_IO | PCI_COMMAND_MEMORY |
1060 PCI_COMMAND_MASTER | PCI_COMMAND_SERR;
1061 dw_pcie_writel_dbi(pci, PCI_COMMAND, val);
1062
1063 dw_pcie_config_presets(pp);
1064 /*
1065 * If the platform provides its own child bus config accesses, it means
1066 * the platform uses its own address translation component rather than
1067 * ATU, so we should not program the ATU here.
1068 */
1069 if (pp->bridge->child_ops == &dw_child_pcie_ops) {
1070 ret = dw_pcie_iatu_setup(pp);
1071 if (ret)
1072 return ret;
1073 }
1074
1075 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
1076
1077 /* Program correct class for RC */
1078 dw_pcie_writew_dbi(pci, PCI_CLASS_DEVICE, PCI_CLASS_BRIDGE_PCI);
1079
1080 val = dw_pcie_readl_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL);
1081 val |= PORT_LOGIC_SPEED_CHANGE;
1082 dw_pcie_writel_dbi(pci, PCIE_LINK_WIDTH_SPEED_CONTROL, val);
1083
1084 dw_pcie_dbi_ro_wr_dis(pci);
1085
1086 return 0;
1087 }
1088 EXPORT_SYMBOL_GPL(dw_pcie_setup_rc);
1089
dw_pcie_pme_turn_off(struct dw_pcie * pci)1090 static int dw_pcie_pme_turn_off(struct dw_pcie *pci)
1091 {
1092 struct dw_pcie_ob_atu_cfg atu = { 0 };
1093 void __iomem *mem;
1094 int ret;
1095
1096 if (pci->num_ob_windows <= pci->pp.msg_atu_index)
1097 return -ENOSPC;
1098
1099 if (!pci->pp.msg_res)
1100 return -ENOSPC;
1101
1102 atu.code = PCIE_MSG_CODE_PME_TURN_OFF;
1103 atu.routing = PCIE_MSG_TYPE_R_BC;
1104 atu.type = PCIE_ATU_TYPE_MSG;
1105 atu.size = resource_size(pci->pp.msg_res);
1106 atu.index = pci->pp.msg_atu_index;
1107
1108 atu.parent_bus_addr = pci->pp.msg_res->start - pci->parent_bus_offset;
1109
1110 ret = dw_pcie_prog_outbound_atu(pci, &atu);
1111 if (ret)
1112 return ret;
1113
1114 mem = ioremap(pci->pp.msg_res->start, pci->region_align);
1115 if (!mem)
1116 return -ENOMEM;
1117
1118 /* A dummy write is converted to a Msg TLP */
1119 writel(0, mem);
1120
1121 iounmap(mem);
1122
1123 return 0;
1124 }
1125
dw_pcie_suspend_noirq(struct dw_pcie * pci)1126 int dw_pcie_suspend_noirq(struct dw_pcie *pci)
1127 {
1128 u8 offset = dw_pcie_find_capability(pci, PCI_CAP_ID_EXP);
1129 u32 val;
1130 int ret;
1131
1132 /*
1133 * If L1SS is supported, then do not put the link into L2 as some
1134 * devices such as NVMe expect low resume latency.
1135 */
1136 if (dw_pcie_readw_dbi(pci, offset + PCI_EXP_LNKCTL) & PCI_EXP_LNKCTL_ASPM_L1)
1137 return 0;
1138
1139 if (pci->pp.ops->pme_turn_off) {
1140 pci->pp.ops->pme_turn_off(&pci->pp);
1141 } else {
1142 ret = dw_pcie_pme_turn_off(pci);
1143 if (ret)
1144 return ret;
1145 }
1146
1147 ret = read_poll_timeout(dw_pcie_get_ltssm, val,
1148 val == DW_PCIE_LTSSM_L2_IDLE ||
1149 val <= DW_PCIE_LTSSM_DETECT_WAIT,
1150 PCIE_PME_TO_L2_TIMEOUT_US/10,
1151 PCIE_PME_TO_L2_TIMEOUT_US, false, pci);
1152 if (ret) {
1153 /* Only log message when LTSSM isn't in DETECT or POLL */
1154 dev_err(pci->dev, "Timeout waiting for L2 entry! LTSSM: 0x%x\n", val);
1155 return ret;
1156 }
1157
1158 /*
1159 * Per PCIe r6.0, sec 5.3.3.2.1, software should wait at least
1160 * 100ns after L2/L3 Ready before turning off refclock and
1161 * main power. This is harmless when no endpoint is connected.
1162 */
1163 udelay(1);
1164
1165 dw_pcie_stop_link(pci);
1166 if (pci->pp.ops->deinit)
1167 pci->pp.ops->deinit(&pci->pp);
1168
1169 pci->suspended = true;
1170
1171 return ret;
1172 }
1173 EXPORT_SYMBOL_GPL(dw_pcie_suspend_noirq);
1174
dw_pcie_resume_noirq(struct dw_pcie * pci)1175 int dw_pcie_resume_noirq(struct dw_pcie *pci)
1176 {
1177 int ret;
1178
1179 if (!pci->suspended)
1180 return 0;
1181
1182 pci->suspended = false;
1183
1184 if (pci->pp.ops->init) {
1185 ret = pci->pp.ops->init(&pci->pp);
1186 if (ret) {
1187 dev_err(pci->dev, "Host init failed: %d\n", ret);
1188 return ret;
1189 }
1190 }
1191
1192 dw_pcie_setup_rc(&pci->pp);
1193
1194 ret = dw_pcie_start_link(pci);
1195 if (ret)
1196 return ret;
1197
1198 ret = dw_pcie_wait_for_link(pci);
1199 if (ret)
1200 return ret;
1201
1202 return ret;
1203 }
1204 EXPORT_SYMBOL_GPL(dw_pcie_resume_noirq);
1205